Skip to content

Commit fdbc2e3

Browse files
carderneTrigger.dev RepoOps
carderne
authored and
Trigger.dev RepoOps
committed
fix(run-engine): log scalar execution state
Mono-RevId: 0232ea18f00ec94e669b755bd916b839a667ccb5
1 parent 685aa94 commit fdbc2e3

5 files changed

Lines changed: 81 additions & 38 deletions

File tree

‎internal-packages/run-engine/src/engine/index.ts‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2692,7 +2692,8 @@ export class RunEngine {
26922692
{
26932693
runId,
26942694
snapshotId,
2695-
latestSnapshot: latestSnapshot,
2695+
latestSnapshotId: latestSnapshot.id,
2696+
latestSnapshotExecutionStatus: latestSnapshot.executionStatus,
26962697
}
26972698
);
26982699

@@ -2701,7 +2702,8 @@ export class RunEngine {
27012702

27022703
this.logger.log("RunEngine.#handleStalledSnapshot() handling stalled snapshot", {
27032704
runId,
2704-
snapshot: latestSnapshot,
2705+
snapshotId: latestSnapshot.id,
2706+
executionStatus: latestSnapshot.executionStatus,
27052707
});
27062708

27072709
switch (latestSnapshot.executionStatus) {
@@ -2728,7 +2730,8 @@ export class RunEngine {
27282730
"RunEngine.#handleStalledSnapshot() PENDING_EXECUTING run not found",
27292731
{
27302732
runId,
2731-
snapshot: latestSnapshot,
2733+
snapshotId: latestSnapshot.id,
2734+
executionStatus: latestSnapshot.executionStatus,
27322735
}
27332736
);
27342737

@@ -2827,7 +2830,9 @@ export class RunEngine {
28272830

28282831
this.logger.info("handleStalledSnapshot SUSPENDED continueRunIfUnblocked", {
28292832
runId,
2830-
result,
2833+
continuationStatus: result.status,
2834+
reason: result.status === "skipped" ? result.reason : undefined,
2835+
waitpointCount: "waitpoints" in result ? result.waitpoints.length : undefined,
28312836
snapshotId: latestSnapshot.id,
28322837
});
28332838

@@ -2840,7 +2845,8 @@ export class RunEngine {
28402845
if (result.waitpoints.length === 0) {
28412846
this.logger.info("handleStalledSnapshot SUSPENDED blocked but no waitpoints", {
28422847
runId,
2843-
result,
2848+
continuationStatus: result.status,
2849+
waitpointCount: result.waitpoints.length,
28442850
snapshotId: latestSnapshot.id,
28452851
});
28462852
// If the run is blocked but there are no waitpoints, we don't restart the heartbeat
@@ -2856,7 +2862,8 @@ export class RunEngine {
28562862
"handleStalledSnapshot SUSPENDED blocked but no run or batch waitpoints",
28572863
{
28582864
runId,
2859-
result,
2865+
continuationStatus: result.status,
2866+
waitpointCount: result.waitpoints.length,
28602867
snapshotId: latestSnapshot.id,
28612868
}
28622869
);
@@ -2877,7 +2884,8 @@ export class RunEngine {
28772884
"handleStalledSnapshot SUSPENDED blocked with waitpoints, max retries reached",
28782885
{
28792886
runId,
2880-
result,
2887+
continuationStatus: result.status,
2888+
waitpointCount: result.waitpoints.length,
28812889
snapshotId: latestSnapshot.id,
28822890
restartAttempt: $restartAttempt,
28832891
maxCount,
@@ -2898,7 +2906,8 @@ export class RunEngine {
28982906
"handleStalledSnapshot SUSPENDED blocked with waitpoints, restarting heartbeat",
28992907
{
29002908
runId,
2901-
result,
2909+
continuationStatus: result.status,
2910+
waitpointCount: result.waitpoints.length,
29022911
snapshotId: latestSnapshot.id,
29032912
delayMs,
29042913
restartAttempt: $restartAttempt,
@@ -2998,12 +3007,14 @@ export class RunEngine {
29983007
if (!gotRequeued) {
29993008
this.logger.error("RunEngine.handleRepairSnapshot QUEUED repair failed", {
30003009
runId,
3001-
snapshot: latestSnapshot,
3010+
snapshotId: latestSnapshot.id,
3011+
executionStatus: latestSnapshot.executionStatus,
30023012
});
30033013
} else {
30043014
this.logger.log("RunEngine.handleRepairSnapshot QUEUED repair successful", {
30053015
runId,
3006-
snapshot: latestSnapshot,
3016+
snapshotId: latestSnapshot.id,
3017+
executionStatus: latestSnapshot.executionStatus,
30073018
});
30083019
}
30093020

‎internal-packages/run-engine/src/engine/systems/checkpointSystem.ts‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,11 @@ export class CheckpointSystem {
6767

6868
if (!isValidSnapshot) {
6969
this.$.logger.info("Tried to createCheckpoint on an invalid snapshot", {
70-
snapshot,
70+
runId,
7171
snapshotId,
72+
latestSnapshotId: snapshot.id,
73+
latestSnapshotExecutionStatus: snapshot.executionStatus,
74+
latestSnapshotPreviousSnapshotId: snapshot.previousSnapshotId,
7275
});
7376

7477
this.$.eventBus.emit("incomingCheckpointDiscarded", {
@@ -94,7 +97,9 @@ export class CheckpointSystem {
9497

9598
if (!isCheckpointable(snapshot.executionStatus)) {
9699
this.$.logger.error("Tried to createCheckpoint on a run in an invalid state", {
97-
snapshot,
100+
runId,
101+
snapshotId: snapshot.id,
102+
executionStatus: snapshot.executionStatus,
98103
});
99104

100105
this.$.eventBus.emit("incomingCheckpointDiscarded", {
@@ -130,7 +135,9 @@ export class CheckpointSystem {
130135

131136
if (!run) {
132137
this.$.logger.error("Run not found for createCheckpoint", {
133-
snapshot,
138+
runId,
139+
snapshotId: snapshot.id,
140+
executionStatus: snapshot.executionStatus,
134141
});
135142

136143
throw new ServiceValidationError("Run not found", 404);
@@ -140,7 +147,9 @@ export class CheckpointSystem {
140147

141148
if (!env) {
142149
this.$.logger.error("Environment not found for createCheckpoint", {
143-
snapshot,
150+
runId,
151+
snapshotId: snapshot.id,
152+
executionStatus: snapshot.executionStatus,
144153
runtimeEnvironmentId: run.runtimeEnvironmentId,
145154
});
146155

@@ -208,8 +217,11 @@ export class CheckpointSystem {
208217
});
209218

210219
this.$.logger.debug("Releasing concurrency for run because it was checkpointed", {
211-
snapshot,
212-
newSnapshot,
220+
runId,
221+
snapshotId: snapshot.id,
222+
executionStatus: snapshot.executionStatus,
223+
newSnapshotId: newSnapshot.id,
224+
newExecutionStatus: newSnapshot.executionStatus,
213225
});
214226

215227
if (run.organizationId) {
@@ -246,8 +258,11 @@ export class CheckpointSystem {
246258
});
247259

248260
this.$.logger.debug("Releasing concurrency for run because it was checkpointed", {
249-
snapshot,
250-
newSnapshot,
261+
runId,
262+
snapshotId: snapshot.id,
263+
executionStatus: snapshot.executionStatus,
264+
newSnapshotId: newSnapshot.id,
265+
newExecutionStatus: newSnapshot.executionStatus,
251266
});
252267

253268
if (run.organizationId) {
@@ -335,7 +350,9 @@ export class CheckpointSystem {
335350

336351
if (!run) {
337352
this.$.logger.error("Run not found for createCheckpoint", {
338-
snapshot,
353+
runId,
354+
snapshotId: snapshot.id,
355+
executionStatus: snapshot.executionStatus,
339356
});
340357

341358
throw new ServiceValidationError("Run not found", 404);

‎internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,8 @@ export class ExecutionSnapshotSystem {
564564
this.$.logger.log("heartbeatRun: no longer the latest snapshot, stopping the heartbeat.", {
565565
runId,
566566
snapshotId,
567-
latestSnapshot,
567+
latestSnapshotId: latestSnapshot.id,
568+
latestSnapshotExecutionStatus: latestSnapshot.executionStatus,
568569
workerId,
569570
runnerId,
570571
});
@@ -576,7 +577,7 @@ export class ExecutionSnapshotSystem {
576577
this.$.logger.debug("heartbeatRun: worker ID does not match the latest snapshot", {
577578
runId,
578579
snapshotId,
579-
latestSnapshot,
580+
latestSnapshotWorkerId: latestSnapshot.workerId,
580581
workerId,
581582
runnerId,
582583
});

‎internal-packages/run-engine/src/engine/systems/ttlSystem.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,14 +63,17 @@ export class TtlSystem {
6363

6464
if (run.status !== "PENDING") {
6565
this.$.logger.debug("Run cannot be expired because it's not in PENDING status", {
66-
run,
66+
runId: run.id,
67+
status: run.status,
6768
});
6869
return;
6970
}
7071

7172
if (run.lockedAt) {
7273
this.$.logger.debug("Run cannot be expired because it's locked, so will run", {
73-
run,
74+
runId: run.id,
75+
status: run.status,
76+
lockedAt: run.lockedAt,
7477
});
7578
return;
7679
}

‎internal-packages/run-engine/src/engine/systems/waitpointSystem.ts‎

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -555,7 +555,7 @@ export class WaitpointSystem {
555555
if (blockingWaitpoints.some((w) => w.waitpoint.status !== "COMPLETED")) {
556556
this.$.logger.debug(`continueRunIfUnblocked: blocking waitpoints still exist`, {
557557
runId,
558-
blockingWaitpoints,
558+
blockingWaitpointCount: blockingWaitpoints.length,
559559
});
560560

561561
return {
@@ -599,7 +599,7 @@ export class WaitpointSystem {
599599
case "RUN_CREATED": {
600600
this.$.logger.info(`continueRunIfUnblocked: run is run created, skipping`, {
601601
runId,
602-
snapshot,
602+
snapshotId: snapshot.id,
603603
executionStatus: snapshot.executionStatus,
604604
});
605605

@@ -611,7 +611,7 @@ export class WaitpointSystem {
611611
case "DELAYED": {
612612
this.$.logger.debug(`continueRunIfUnblocked: run is delayed, skipping`, {
613613
runId,
614-
snapshot,
614+
snapshotId: snapshot.id,
615615
executionStatus: snapshot.executionStatus,
616616
});
617617

@@ -623,7 +623,7 @@ export class WaitpointSystem {
623623
case "QUEUED": {
624624
this.$.logger.info(`continueRunIfUnblocked: run is queued, skipping`, {
625625
runId,
626-
snapshot,
626+
snapshotId: snapshot.id,
627627
executionStatus: snapshot.executionStatus,
628628
});
629629

@@ -635,7 +635,7 @@ export class WaitpointSystem {
635635
case "PENDING_EXECUTING": {
636636
this.$.logger.info(`continueRunIfUnblocked: run is pending executing, skipping`, {
637637
runId,
638-
snapshot,
638+
snapshotId: snapshot.id,
639639
executionStatus: snapshot.executionStatus,
640640
});
641641

@@ -647,7 +647,7 @@ export class WaitpointSystem {
647647
case "QUEUED_EXECUTING": {
648648
this.$.logger.info(`continueRunIfUnblocked: run is already queued executing, skipping`, {
649649
runId,
650-
snapshot,
650+
snapshotId: snapshot.id,
651651
executionStatus: snapshot.executionStatus,
652652
});
653653

@@ -659,7 +659,7 @@ export class WaitpointSystem {
659659
case "EXECUTING": {
660660
this.$.logger.info(`continueRunIfUnblocked: run is already executing, skipping`, {
661661
runId,
662-
snapshot,
662+
snapshotId: snapshot.id,
663663
executionStatus: snapshot.executionStatus,
664664
});
665665

@@ -672,7 +672,7 @@ export class WaitpointSystem {
672672
case "FINISHED": {
673673
this.$.logger.debug(`continueRunIfUnblocked: run is finished, skipping`, {
674674
runId,
675-
snapshot,
675+
snapshotId: snapshot.id,
676676
executionStatus: snapshot.executionStatus,
677677
});
678678
return {
@@ -710,8 +710,10 @@ export class WaitpointSystem {
710710
`continueRunIfUnblocked: run was still executing, sending notification`,
711711
{
712712
runId,
713-
snapshot,
714-
newSnapshot,
713+
snapshotId: snapshot.id,
714+
snapshotExecutionStatus: snapshot.executionStatus,
715+
newSnapshotId: newSnapshot.id,
716+
newSnapshotExecutionStatus: newSnapshot.executionStatus,
715717
}
716718
);
717719

@@ -731,7 +733,12 @@ export class WaitpointSystem {
731733
if (snapshot.runStatus === "CANCELED") {
732734
this.$.logger.warn(
733735
`continueRunIfUnblocked: run was canceled while suspended, skipping`,
734-
{ runId, snapshot }
736+
{
737+
runId,
738+
snapshotId: snapshot.id,
739+
executionStatus: snapshot.executionStatus,
740+
runStatus: snapshot.runStatus,
741+
}
735742
);
736743
return {
737744
status: "skipped",
@@ -741,7 +748,9 @@ export class WaitpointSystem {
741748

742749
this.$.logger.error(`continueRunIfUnblocked: run is suspended, but has no checkpoint`, {
743750
runId,
744-
snapshot,
751+
snapshotId: snapshot.id,
752+
executionStatus: snapshot.executionStatus,
753+
runStatus: snapshot.runStatus,
745754
});
746755
throw new Error(
747756
`continueRunIfUnblocked: run is suspended, but has no checkpoint: ${runId}`
@@ -767,8 +776,10 @@ export class WaitpointSystem {
767776

768777
this.$.logger.debug(`continueRunIfUnblocked: run goes to QUEUED`, {
769778
runId,
770-
snapshot,
771-
newSnapshot,
779+
snapshotId: snapshot.id,
780+
snapshotExecutionStatus: snapshot.executionStatus,
781+
newSnapshotId: newSnapshot.id,
782+
newSnapshotExecutionStatus: newSnapshot.executionStatus,
772783
});
773784

774785
break;
@@ -787,7 +798,7 @@ export class WaitpointSystem {
787798

788799
this.$.logger.debug(`continueRunIfUnblocked: removed blocking waitpoints`, {
789800
runId,
790-
blockingWaitpoints,
801+
blockingWaitpointCount: blockingWaitpoints.length,
791802
});
792803
}
793804

0 commit comments

Comments
 (0)