From 9568eee0d200d791e569ca0560772c1475014270 Mon Sep 17 00:00:00 2001 From: 9pace Date: Mon, 19 Oct 2026 13:03:36 +0000 Subject: [PATCH 1/2] fix: retry Lambda timeouts in the catalog check and reprocess steps --- .../__snapshots__/construct-hub.test.ts.snap | 20 +++++++++---------- .../__snapshots__/snapshot.test.ts.snap | 4 ++-- src/backend/ingestion/index.ts | 7 +++++++ src/backend/orchestration/index.ts | 9 ++++++++- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 64d354374..c3c53844d 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -3035,7 +3035,7 @@ Direct link to the function: /lambda/home#/functions/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, - "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Type":"Task","Resource":"arn:", + "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1},{"ErrorEquals":["Sandbox.Timedout","Error"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2}],"Type":"Task","Resource":"arn:", { "Ref": "AWS::Partition", }, @@ -4972,7 +4972,7 @@ Direct link to function: /lambda/home#/functions/", "GroupId", ], }, - ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", + ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Sandbox.Timedout"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", { "Fn::GetAtt": [ "ConstructHubOrchestrationNeedsCatalogUpdate5D7370DC", @@ -18050,7 +18050,7 @@ Direct link to the function: /lambda/home#/functions/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, - "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Type":"Task","Resource":"arn:", + "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1},{"ErrorEquals":["Sandbox.Timedout","Error"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2}],"Type":"Task","Resource":"arn:", { "Ref": "AWS::Partition", }, @@ -20067,7 +20067,7 @@ Direct link to function: /lambda/home#/functions/", "GroupId", ], }, - ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", + ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Sandbox.Timedout"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", { "Fn::GetAtt": [ "ConstructHubOrchestrationNeedsCatalogUpdate5D7370DC", @@ -32943,7 +32943,7 @@ Direct link to the function: /lambda/home#/functions/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, - "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Type":"Task","Resource":"arn:", + "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1},{"ErrorEquals":["Sandbox.Timedout","Error"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2}],"Type":"Task","Resource":"arn:", { "Ref": "AWS::Partition", }, @@ -34880,7 +34880,7 @@ Direct link to function: /lambda/home#/functions/", "GroupId", ], }, - ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", + ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Sandbox.Timedout"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", { "Fn::GetAtt": [ "ConstructHubOrchestrationNeedsCatalogUpdate5D7370DC", @@ -47812,7 +47812,7 @@ Direct link to the function: /lambda/home#/functions/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, - "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Type":"Task","Resource":"arn:", + "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1},{"ErrorEquals":["Sandbox.Timedout","Error"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2}],"Type":"Task","Resource":"arn:", { "Ref": "AWS::Partition", }, @@ -49771,7 +49771,7 @@ Direct link to function: /lambda/home#/functions/", "GroupId", ], }, - ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", + ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Sandbox.Timedout"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", { "Fn::GetAtt": [ "ConstructHubOrchestrationNeedsCatalogUpdate5D7370DC", @@ -62890,7 +62890,7 @@ Direct link to the function: /lambda/home#/functions/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, - "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Type":"Task","Resource":"arn:", + "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1},{"ErrorEquals":["Sandbox.Timedout","Error"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2}],"Type":"Task","Resource":"arn:", { "Ref": "AWS::Partition", }, @@ -64982,7 +64982,7 @@ Direct link to function: /lambda/home#/functions/", "GroupId", ], }, - ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", + ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Sandbox.Timedout"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", { "Fn::GetAtt": [ "ConstructHubOrchestrationNeedsCatalogUpdate5D7370DC", diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 4836e0db0..590155110 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -3508,7 +3508,7 @@ Direct link to the function: /lambda/home#/functions/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, - "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Type":"Task","Resource":"arn:", + "","Prefix":"data/","MaxKeys":150}},"Process Result":{"Type":"Map","ResultPath":null,"End":true,"InputPath":"$.response.Contents[*][?(@.Key =~ /^.*/metadata.json$/)]","Iterator":{"StartAt":"Send for reprocessing","States":{"Send for reprocessing":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1},{"ErrorEquals":["Sandbox.Timedout","Error"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2}],"Type":"Task","Resource":"arn:", { "Ref": "AWS::Partition", }, @@ -5580,7 +5580,7 @@ Direct link to function: /lambda/home#/functions/", "GroupId", ], }, - ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", + ""]}},"Overrides":{"ContainerOverrides":[{"Name":"Resource","Command.$":"$","Environment":[{"Name":"SFN_TASK_TOKEN","Value.$":"$$.Task.Token"},{"Name":"RUN_LSOF_ON_HEARTBEAT","Value":""}]}]},"LaunchType":"FARGATE","PlatformVersion":"1.4.0"}},"Check whether catalog needs udpating":{"Next":"Is catalog update needed?","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["Sandbox.Timedout"],"IntervalSeconds":5,"MaxAttempts":3,"BackoffRate":2},{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"IntervalSeconds":60,"MaxAttempts":30,"BackoffRate":1.1}],"Catch":[{"ErrorEquals":["Lambda.TooManyRequestsException","Lambda.Unknown"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.TaskFailed"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"},{"ErrorEquals":["States.ALL"],"ResultPath":"$.error","Next":"Send to Dead Letter Queue"}],"Type":"Task","ResultPath":"$.catalogNeedsUpdating","Resource":"", { "Fn::GetAtt": [ "ConstructHubOrchestrationNeedsCatalogUpdate5D7370DC", diff --git a/src/backend/ingestion/index.ts b/src/backend/ingestion/index.ts index 27612ccb6..009cbb966 100644 --- a/src/backend/ingestion/index.ts +++ b/src/backend/ingestion/index.ts @@ -505,6 +505,13 @@ class ReprocessIngestionWorkflow extends Construct { interval: Duration.minutes(1), maxAttempts: 30, }) + // A dropped or stalled S3 read fails one package; the next attempt gets a new connection. + .addRetry({ + errors: ['Sandbox.Timedout', 'Error'], + interval: Duration.seconds(5), + backoffRate: 2, + maxAttempts: 3, + }) ); const isThereMore = new Choice(this, 'Is there more?') diff --git a/src/backend/orchestration/index.ts b/src/backend/orchestration/index.ts index 83e101f79..0fb79ba1d 100644 --- a/src/backend/orchestration/index.ts +++ b/src/backend/orchestration/index.ts @@ -326,10 +326,17 @@ export class Orchestration extends Construct { resultPath: '$.catalogNeedsUpdating', } ) + .addRetry({ + // a Lambda timeout arrives as Sandbox.Timedout + errors: ['Sandbox.Timedout'], + interval: Duration.seconds(5), + backoffRate: 2, + maxAttempts: 3, + }) .addRetry({ errors: [ 'Lambda.TooManyRequestsException', - 'Lambda.Unknown', // happens when a lambda times out. + 'Lambda.Unknown', // how Lambda timeouts used to be reported ], ...THROTTLE_RETRY_POLICY, }) From da7cbbae88c5f64284090d2836107822d12e344e Mon Sep 17 00:00:00 2001 From: 9pace Date: Mon, 19 Oct 2026 13:22:36 +0000 Subject: [PATCH 2/2] chore: keep original Lambda.Unknown comment --- src/backend/orchestration/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/orchestration/index.ts b/src/backend/orchestration/index.ts index 0fb79ba1d..a1a5393c6 100644 --- a/src/backend/orchestration/index.ts +++ b/src/backend/orchestration/index.ts @@ -336,7 +336,7 @@ export class Orchestration extends Construct { .addRetry({ errors: [ 'Lambda.TooManyRequestsException', - 'Lambda.Unknown', // how Lambda timeouts used to be reported + 'Lambda.Unknown', // happens when a lambda times out. ], ...THROTTLE_RETRY_POLICY, })