diff --git a/brain/knowledge/flows-execution/action-run.md b/brain/knowledge/flows-execution/action-run.md index ad9c7bfa96b9..e122925d2926 100644 --- a/brain/knowledge/flows-execution/action-run.md +++ b/brain/knowledge/flows-execution/action-run.md @@ -15,7 +15,7 @@ At this stage action runs are **execution only — nothing is persisted**. The c - **Priority** `high`, not `critical`, so action runs never outrank the builder interactions a human is actively waiting on. ### Gotchas -- **The budget is one end-to-end deadline, not two timeouts with a margin.** `actionRunService` stamps `expiresAt = now + 120s` onto the job and waits 130s. The extra 10s only covers the sandbox kill and the pubsub hop home — it is *not* an allowance for queueing, resolution or provisioning, because those spend the same 120s: `createSandboxRuntime.execute` clamps the run to what is left of `expiresAt` after provisioning **and boot**, and throws `SANDBOX_EXECUTION_TIMEOUT` without starting the engine if nothing is left. **Do not re-shape this as "sandbox 120s + a fudge factor".** That was the original bug: the watcher's clock starts at enqueue and the sandbox's at run-start, so a cold piece install (easily >10s) expired the caller first while the action kept running and writing — and the retry that invites duplicates the write. The worker keeps its own 120s cap for a deadline it cannot trust (missing from an older API, or inflated by clock skew). +- **The budget is one end-to-end deadline, not two timeouts with a margin.** `actionRunService` stamps `expiresAt = now + AP_FLOW_TIMEOUT_SECONDS` onto the job and waits that plus 10s. The extra 10s only covers the sandbox kill and the pubsub hop home — it is *not* an allowance for queueing, resolution or provisioning, because those spend the same budget: `createSandboxRuntime.execute` clamps the run to what is left of `expiresAt` after provisioning **and boot**, and throws `SANDBOX_EXECUTION_TIMEOUT` without starting the engine if nothing is left. **Do not re-shape this as "sandbox budget + a fudge factor".** That was the original bug: the watcher's clock starts at enqueue and the sandbox's at run-start, so a cold piece install (easily >10s) expired the caller first while the action kept running and writing — and the retry that invites duplicates the write. The worker no longer carries a cap of its own; `execute-action.ts` derives `timeoutInSeconds` from `expiresAt` alone. - **Derive the clamp where the timer is armed. Every phase between the two is a hole.** The only thing enforcing the run budget is the SIGKILL timer armed *inside* `sandbox.execute`; `remainingTimeoutInSeconds` was called before `sandbox.start()`, so the engine got its full remaining budget counted from a later instant and user code ran until `expiresAt + bootMs`. Boot is the norm, not the tail — `canReuseSandbox` is false for `SANDBOX_PROCESS` unless `AP_REUSE_SANDBOX=true`, so every production action run pays bind retries, two *unbounded* isolate `execPromise` calls and a 30s connect cap, comfortably past the 10s grace. The pre-boot check survives only as a fast path (skip a pointless spawn); the authoritative clamp is recomputed after the `sandboxStart` timed block. Note the recomputed throw lands inside the `try`, so it invalidates a freshly booted box instead of parking it warm — one cold boot on a rare path, accepted over restructuring boot out of the `try`. - **Do not bound provision or the connect wait by the deadline without reworking the error taxonomy first.** It looks like the obvious next step — `spawnWithKill` already takes `timeoutMs` and `waitForConnection`'s 30s is a bare literal — but both raise a plain `Error`, which fails `isSandboxTimeout` in `execute-action.ts` and is rethrown, so the caller gets `INTERNAL_ERROR` / *"the engine crashed while loading or executing the piece"* — the exact misreport the next gotcha forbids, in place of an honest `neverStarted`. Capping the connect wait also does not bound boot: it is the *last* of three boot phases, behind the unbounded isolate calls. Neither buys any safety, because a setup overrun already yields `neverStarted` with no user code executed. - A watcher timeout maps to `TIMEOUT`, **never** `INTERNAL_ERROR` — reporting "the engine crashed while loading or executing the piece" when nothing ran sends the calling agent off debugging the piece instead of retrying. @@ -39,6 +39,12 @@ At this stage action runs are **execution only — nothing is persisted**. The c - **The sweep is convergent, not coordinated, because a worker cannot lock.** N sweepers over one shared `./cache` is the steady state (compose `replicas: 5`; Helm's default `rollout` mounts one RWO PVC into every replica), and a worker process has no Redis, so `distributedLock` and system jobs are both unavailable — see [workers § Gotchas](../execution-runtime/workers.md). Safety is structural instead: every step is idempotent (`force: true`, ENOENT-tolerant, mtime re-checked immediately before `rm`) and eviction recomputes its target from the live `readdir` rather than accumulating across deletions, so two simultaneous sweepers pick the same oldest set, delete it once, and neither evicts past the cap. Do not add state that accumulates across deletions — the removed byte accounting was exactly that, and it over-evicted whenever a peer deleted a dir first. Timer jitter is unnecessary for the same reason: concurrent sweeps cost duplicated `stat` calls, nothing more. - **Directories from every earlier layout are deliberately left to leak.** Bare-`sha256`, `mcp-flow-version-id` and `ar_`-prefixed dirs only exist on machines that ran intermediate commits of the branch that added them — none of those layouts ever reached `main`, so there is nothing in the field to migrate and no reclaim path was written. The sweeper does *not* reclaim them: the name-sniffing branch that did had no TTL and no mtime re-check, and `mcp-flow-version-id` is still a live constant on `main` (`DEFAULT_MCP_DATA.flowVersionId`), so the day anything provisions code under it that branch would `rm -rf` it every 30 minutes. On a dev box that ran those commits, `rm -rf cache/v12/codes` is the cleanup. - **A cache hit must prove the artifact exists — do not "simplify" the `compiledArtifactPresent` check out of `code-builder`.** `cacheState`'s memo is module-scoped with no invalidation API, and a hit returns without touching disk. Delete a step dir and the memo still reports `cacheHit: true`, nothing rebuilds, and the engine `require`s a missing `index.js` — failing that snippet on that worker until the process restarts. Process-local invalidation would not be enough either: the reference `docker-compose.yml` gives `app` and five `worker` replicas the same `./cache` bind mount, so one container's sweeper deletes a directory another has memoised. The one `stat` per hit is the only thing that makes deletion — by the sweeper, by an operator, by a reset volume — recoverable. +- **The budget is `AP_FLOW_TIMEOUT_SECONDS`, not a knob of its own, and the layers must stay ordered.** An action run is one step of automation, so it gets the budget a flow step gets — `actionRunService` and `runFlowAsTool` both read the same prop. The order that has to hold, outermost first: **worker RPC timeout for `LONG_RUNNING_RPC_METHODS` (budget + `LONG_RUNNING_RPC_MARGIN_MS`) > watcher wait (budget + `WATCHER_GRACE_MS`) > action budget > engine run**. The margin is not decoration — the app spends time *outside* the action on the same call (the `pieceInputFiller` model call, which has no timeout of its own), so a margin-free deadline expires the caller while the action is still legitimately running. Do not give any layer its own env var; move the whole stack with the one knob. +- **This raised every ceiling; it did not restore an old one.** Issue #15127 reports that 0.86 ran agent tools under `AP_FLOW_TIMEOUT_SECONDS` — the history does not support that. `runPieceTool` and the configured-piece-tool path were *introduced* in #14613, and before it an agent's piece action went over the MCP client into `actionRunService` at the same 120s. 0.88 added a second, lower ceiling (a 60s RPC timeout in the worker) that made the new feature unusable from birth. Worth knowing before anyone repeats "we just restored 0.86" in a changelog. +- **socket.io's ack timeout is per-call, so a long tool call needs a bigger timeout, not a different transport.** `socket.timeout(ms)` sets `flags.timeout`, `_registerAckCallback` reads it, and `emit` clears `flags` afterwards — so `createRpcClient` taking `RpcTimeout = number | ((method: string) => number)` is the entire fix, worker-side only. A deferred variant (ack immediately, deliver the result later on an `rpc-result` event keyed by a `callId`) was built and reverted: it bought nothing, needed both sides upgraded to work, and introduced a crash — the result promise has no handler attached until *after* the ack resolves, so a disconnect inside the ack window rejects it unhandled and Node's default `--unhandled-rejections=throw` kills the worker (which, in a `WORKER_AND_APP` container, takes the whole instance down via `docker-entrypoint.sh`). Disconnect is already handled for free: `Socket.onclose` calls `_clearAcks`, which rejects every `emitWithAck` ack with `socket has been disconnected`. +- **Keep the `handler threw` prefix in the RPC error text.** `createConfiguredPieceTools` greps for it to choose between telling the model "that action failed" and "it may already have run, do not call it again" — the second is what stops an agent re-running a side effect it cannot see the result of. +- **The code cache's active-execution window must cover the run budget, or a long code action deletes its own build.** `ACTION_RUN_CACHE_ACTIVE_WINDOW_MS` (15 min) exempts fresh dirs from eviction, and mtime is stamped at *provision* and never refreshed during the run — so a run longer than the window ages out of its own exemption while executing, and if the tree is over `ACTION_RUN_CACHE_MAX_DIRS` the sweeper removes the directory under it, failing the run on a missing `index.js`. Unreachable while the budget was a hard 120s; reachable the moment the budget followed `AP_FLOW_TIMEOUT_SECONDS`, which operators do set above 15 minutes. `sweep` takes `activeWindowMs`, and the worker passes `max(window, budget)`. Only action runs are exposed — the sweeper's entire scope is `codes/action-runs/`, so flow-version code caches are untouched. +- **`EXECUTE_ACTION` is exempt from the project concurrency limiter, and raising the budget scaled that.** `RATE_LIMIT_WORKER_JOB_TYPES` is `[EXECUTE_FLOW]` only, and `rate-limiter-interceptor` is gated on `AP_PROJECT_RATE_LIMITER_ENABLED` (default `false`). So an action run holds a `WORKER_JOBS` slot for the full budget with no per-project cap — a slow endpoint driven through MCP `ap_run_action` can occupy every slot on an instance for that long. Adding `EXECUTE_ACTION` to the list is not a one-liner: `shouldContinue` narrows to `ExecuteFlowJobData` and reads `environment`, which `ExecuteActionJobData` does not have. - **`EXECUTE_ACTION` is not project-group routable.** `PROJECT_GROUP_ROUTABLE_JOB_TYPES` is `{EXECUTE_FLOW, EXECUTE_WEBHOOK}`, so action runs land on the platform/shared queue even when the project has dedicated workers — unlike the temporary-flow path they replaced, which routed as `EXECUTE_FLOW`. Matters when a project's own workers are the only ones that can reach its network. - **`actionRunMode` disables the two flow-only behaviours** in `piece-executor.ts`: the progress reporter becomes a no-op (no flow run to stream to), and waitpoints are rejected by `assertActionRunCannotSuspend` as a plain `Error` (USER-level) so the step ends FAILED, not INTERNAL_ERROR — "this action only works inside a flow" is a usage error, not an engine bug, and must not page oncall. - **`createWaitpointHook` must throw *synchronously*, from the returned function's body and not from inside an `async` body. Do not refactor it into a single `async` closure.** This is why the hook is split into a sync wrapper plus `submitWaitpoint`. The deprecated `pause()` shim at `packages/pieces/framework/src/lib/context/versioning.ts` (`buildLegacyPauseHook`, kept until 2026-10-12) does `context.run.createWaitpoint({...}).catch(() => process.exit(1))`. If the rejection arrives as a rejected promise, that `.catch` attaches and **kills the worker process**; thrown synchronously, it propagates as a FAILED step instead. Any piece still on `context.run.pause()` hits this path. diff --git a/brain/knowledge/flows-execution/flow-runs.md b/brain/knowledge/flows-execution/flow-runs.md index 9f279fab2976..7752694c7ff1 100644 --- a/brain/knowledge/flows-execution/flow-runs.md +++ b/brain/knowledge/flows-execution/flow-runs.md @@ -42,7 +42,7 @@ CE has full run tracking. Cloud may enforce retention windows; bulk-retry admin Entry point: `flowRunService`, defined in `flow-run-service.ts` and wired through `flow-run-module.ts`. - `packages/server/api/src/app/flows/flow-run/` — controller, service, entity, hooks, side effects, runs queue, AI usage extractor/tracker -- `packages/server/api/src/app/flows/flow-run/waitpoint/` — resume routes, the `/confirm` page, and its theme hooks +- `packages/server/api/src/app/waitpoints/` — the waitpoint module: entity, service, resume routes, the `/confirm` page, its theme hooks, and the `RESUME_DELAY_WAITPOINT` handler - `packages/core/execution/src/lib/flow-run/` — `FlowRun` type, request dtos, execution types (`StepOutput`, `FlowExecution`), zstd log serializer - `packages/server/engine/src/lib/helper/logging-utils.ts` — produces the truncated-input placeholder the web run-details tab detects - `packages/server/api/src/app/ee/billing-usage-report/` — daily EE job emitting per-platform run counts to PostHog (`TOTAL_RUNS_PER_DAY`, captured and flushed in platform batches) diff --git a/bun.lock b/bun.lock index a89cc6830146..59f2123b2f84 100644 --- a/bun.lock +++ b/bun.lock @@ -116,7 +116,7 @@ }, "packages/core/execution": { "name": "@activepieces/core-execution", - "version": "0.17.0", + "version": "0.18.0", "dependencies": { "@activepieces/core-piece-types": "workspace:*", "@activepieces/core-utils": "workspace:*", diff --git a/docs/install/configure-operate/production-setup.mdx b/docs/install/configure-operate/production-setup.mdx index 64927234fddd..495e2cd250db 100644 --- a/docs/install/configure-operate/production-setup.mdx +++ b/docs/install/configure-operate/production-setup.mdx @@ -68,6 +68,7 @@ Past roughly 80 workers the worker fleet stops being the only dial: throughput k | Limit | Default | Env var | |---|---|---| | Flow run timeout | 600 s | `AP_FLOW_TIMEOUT_SECONDS` | +| Single action run timeout | 600 s | `AP_FLOW_TIMEOUT_SECONDS` | | Sync webhook response | 30 s | `AP_WEBHOOK_TIMEOUT_SECONDS` | | Max webhook payload | 25 MB | `AP_MAX_WEBHOOK_PAYLOAD_SIZE_MB` | | Step file size | 25 MB | `AP_MAX_FILE_SIZE_MB` | diff --git a/docs/install/reference/breaking-changes.mdx b/docs/install/reference/breaking-changes.mdx index 3d3d81d07756..0e911bdcfbca 100644 --- a/docs/install/reference/breaking-changes.mdx +++ b/docs/install/reference/breaking-changes.mdx @@ -24,6 +24,18 @@ The `SMTP Blacklist Sender` checkbox is replaced by a `Blocked Sender Addresses` Nothing to configure or migrate, and no existing step stops working. Review any flow whose Create or Update Contact step relied on the old behaviour: a step that was silently wiping attributes will now leave them intact, and a step you were relying on to blacklist contacts must have the checkbox explicitly ticked. To block a sender for a contact, list that sender's address in `Blocked Sender Addresses` — it must be an active sender in your Brevo account, or Brevo answers *"One of the sender is invalid or inactive"*. +#### `AP_FLOW_TIMEOUT_SECONDS` now also bounds a single action run + +An action run — an agent's configured piece tool, a chat action, or MCP `ap_run_action` — used to be capped at a hardcoded 120 seconds, and a flow run started as an MCP tool at a hardcoded 300 seconds. Neither could be changed. Any agent tool whose action took longer than a minute failed outright with `RPC [executePieceTool] failed (timeout: 60000ms)`, because a second hardcoded limit in the worker cut it off first. + +Both hardcoded limits are gone. All of them now follow `AP_FLOW_TIMEOUT_SECONDS`, the same setting that bounds a flow run, which defaults to 600 seconds. An action run therefore gets the same budget as a flow step. + +The limit going up is the change to be aware of: a slow or hung action now occupies a worker slot for up to `AP_FLOW_TIMEOUT_SECONDS` instead of 120 seconds, and action runs are not covered by the per-project concurrency limiter. On a small worker pool, several slow actions can hold execution capacity for longer than they used to. + +#### What you need to do + +Nothing to configure or migrate, and no new environment variable — a default installation gets the fix with no action. Two things to consider if they apply to you. If you raised `AP_FLOW_TIMEOUT_SECONDS` for long flows, note that agent tools, chat actions and MCP action runs now inherit that same ceiling; lower it if you do not want them held that long. If you put Activepieces behind a reverse proxy and use MCP `ap_run_action`, an action that runs past the proxy's read timeout (nginx defaults to 60 seconds, Cloudflare to 100) returns a gateway error to the MCP client while the action keeps running — raise `proxy_read_timeout` if you rely on long action runs over MCP. + #### A Run Agent step whose turn dies now fails its flow run A Run Agent step whose turn died — no AI provider configured for the chosen vendor, a revoked API key, a model the provider no longer serves — used to report success. The step returned the failed agent result rather than raising it, so the step read SUCCEEDED, the run completed SUCCEEDED, and later steps acted on a payload that carried the failure inside it. Such a run is now FAILED, and the agent's error message is the step's error. diff --git a/docs/install/reference/environment-variables.mdx b/docs/install/reference/environment-variables.mdx index 576abd8ac3a2..280d6bf17b09 100644 --- a/docs/install/reference/environment-variables.mdx +++ b/docs/install/reference/environment-variables.mdx @@ -148,7 +148,7 @@ run timeouts, and the network egress posture for user code. Read | `AP_PREWARM_CACHE_ON_STARTUP` | Pre-fill the worker's local piece and code cache on startup by resolving every enabled flow on the platform, instead of filling it lazily on each flow's first run. Enabling it removes the one-time cold-start latency of the first run after a worker (re)starts, but the warm-up itself costs memory and CPU proportional to the number of enabled flows — on instances with many flows it can pin the worker at its CPU limit for the duration of the warm-up and spike memory enough to OOM-kill small workers, especially when all workers restart at once during an upgrade. Keep it disabled unless your instance has a modest number of enabled flows, your workers have memory headroom, and first-run latency after deploys matters to you. | `false` | | `AP_SANDBOX_MEMORY_LIMIT` | Maximum memory (KB) a single sandboxed engine process can use. Each process runs at most one execution at a time. | `1048576` | | `AP_SANDBOX_PROPAGATED_ENV_VARS` | Comma-separated environment variables propagated into sandboxed code. For pieces, keep everything in the authentication object so it works across instances. | `None` | -| `AP_FLOW_TIMEOUT_SECONDS` | Maximum runtime for a single flow run, in seconds. | `600` | +| `AP_FLOW_TIMEOUT_SECONDS` | Maximum runtime for a single flow run, in seconds. Also bounds a single action run — an agent's piece tool, a chat action, or MCP `ap_run_action` — so raising it lets those hold a worker slot for the same length of time. | `600` | | `AP_TRIGGER_TIMEOUT_SECONDS` | Maximum runtime for a trigger's polling, in seconds. | `60` | | `AP_DEFAULT_CONCURRENT_JOBS_LIMIT` | Default maximum concurrent runs per project. Can be overridden per project in settings. | `5` | | `AP_PROJECT_RATE_LIMITER_ENABLED` | Enforce per-project rate limits to prevent excessive usage. | `false` | diff --git a/docs/install/reference/limits.mdx b/docs/install/reference/limits.mdx index 5c759ce91c20..1bcba634905d 100644 --- a/docs/install/reference/limits.mdx +++ b/docs/install/reference/limits.mdx @@ -39,6 +39,7 @@ it. | Limit | Cloud | Env var | Self-hosted default | |---|---|---|---| | Flow run timeout | 10 min | `AP_FLOW_TIMEOUT_SECONDS` | `600` | +| Single action run timeout | 10 min | `AP_FLOW_TIMEOUT_SECONDS` | `600` | | Worker process memory | 1 GB | the worker container's memory cap | `1 GB` | | Paused flow lifetime | 30 days | `AP_PAUSED_FLOW_TIMEOUT_DAYS` | `30` | | Execution data retention | 30 days | `AP_EXECUTION_DATA_RETENTION_DAYS` | `30` | diff --git a/packages/core/execution/package.json b/packages/core/execution/package.json index a9ba86da9cd3..2a77d8a8bb31 100644 --- a/packages/core/execution/package.json +++ b/packages/core/execution/package.json @@ -1,6 +1,6 @@ { "name": "@activepieces/core-execution", - "version": "0.17.0", + "version": "0.18.0", "type": "commonjs", "main": "./dist/src/index.js", "scripts": { diff --git a/packages/core/execution/src/lib/engine/rpc.ts b/packages/core/execution/src/lib/engine/rpc.ts index a52aacd125fb..501798364653 100644 --- a/packages/core/execution/src/lib/engine/rpc.ts +++ b/packages/core/execution/src/lib/engine/rpc.ts @@ -1,4 +1,4 @@ -import { ActivepiecesError, isObject, spreadIfNotUndefined } from '@activepieces/core-utils' +import { ActivepiecesError, isObject, spreadIfNotUndefined, toError } from '@activepieces/core-utils' const RPC_EVENT = 'rpc' const NOTIFY_EVENT = 'rpc-notify' @@ -17,11 +17,12 @@ type NotifySocket = Pick export function createRpcClient( socket: RpcSocket, - timeoutMs: number, + timeout: RpcTimeout, ): T { return new Proxy({} as T, { get(_target, method: string) { return async (payload: unknown) => { + const timeoutMs = typeof timeout === 'function' ? timeout(method) : timeout try { const result = await socket.timeout(timeoutMs).emitWithAck(RPC_EVENT, { method, payload }) if (isRpcErrorEnvelope(result)) { @@ -36,8 +37,7 @@ export function createRpcClient( if (error instanceof Error && error.message.startsWith('RPC [')) { throw error } - const message = error instanceof Error ? error.message : String(error) - throw new Error(`RPC [${method}] failed (timeout: ${timeoutMs}ms): ${message}`) + throw new Error(`RPC [${method}] failed (timeout: ${timeoutMs}ms): ${toError(error).message}`) } } }, @@ -58,7 +58,7 @@ export function createRpcServer( catch (error) { log?.error({ error, rpc: { method: msg.method } }, 'RPC handler threw') ack({ - __rpcError: error instanceof Error ? error.message : String(error), + __rpcError: toError(error).message, ...spreadIfNotUndefined('__rpcApError', apErrorOf(error)), }) } @@ -105,6 +105,8 @@ function isRpcErrorEnvelope(value: unknown): value is { __rpcError: string, __rp return isObject(value) && '__rpcError' in value } +export type RpcTimeout = number | ((method: string) => number) + export type RpcApError = { code: string entityType?: string diff --git a/packages/core/execution/src/lib/workers/worker-contract.ts b/packages/core/execution/src/lib/workers/worker-contract.ts index e234056a8f03..7dd10f021ae2 100644 --- a/packages/core/execution/src/lib/workers/worker-contract.ts +++ b/packages/core/execution/src/lib/workers/worker-contract.ts @@ -370,3 +370,10 @@ export type SendPersonalizationProgressRequest = { phase: string message: string } + +export const LONG_RUNNING_RPC_METHODS: readonly string[] = [ + 'executePieceTool', + 'executeFlowTool', + 'executeKnowledgeBaseTool', + 'executeAgentTool', +] diff --git a/packages/core/execution/test/automation/engine/rpc.test.ts b/packages/core/execution/test/automation/engine/rpc.test.ts index 52e5d0c06a33..5b1ffc14a885 100644 --- a/packages/core/execution/test/automation/engine/rpc.test.ts +++ b/packages/core/execution/test/automation/engine/rpc.test.ts @@ -4,6 +4,7 @@ import { apErrorOf, createRpcClient, createRpcServer } from '../../../src/lib/en type TestContract = { boom: (input: unknown) => Promise + slow: (input: unknown) => Promise } function loopbackSocket() { @@ -28,6 +29,22 @@ function clientFor(boom: () => unknown): TestContract { return createRpcClient(socket, 1_000) } +describe('rpc timeout', () => { + it('gives each method its own budget, so a tool that runs for minutes does not widen the budget every other call uses', async () => { + const timedOutSocket = { + ...loopbackSocket(), + timeout: () => ({ emitWithAck: () => Promise.reject(new Error('operation has timed out')) }), + } + const client = createRpcClient(timedOutSocket, method => method === 'slow' ? 600_000 : 1_000) + + const quick = await client.boom({}).catch((error: unknown) => error) + const long = await client.slow({}).catch((error: unknown) => error) + + expect(String(quick)).toContain('failed (timeout: 1000ms)') + expect(String(long)).toContain('failed (timeout: 600000ms)') + }) +}) + describe('rpc error envelope', () => { it('carries the code and entity of an ActivepiecesError across the boundary', async () => { const client = clientFor(() => { diff --git a/packages/server/api/src/app/action-run/action-run.service.ts b/packages/server/api/src/app/action-run/action-run.service.ts index 1c4ab05cb534..97c4bb629068 100644 --- a/packages/server/api/src/app/action-run/action-run.service.ts +++ b/packages/server/api/src/app/action-run/action-run.service.ts @@ -1,13 +1,13 @@ import { ActivepiecesError, apId, ErrorCode, isNil, Result, tryCatch } from '@activepieces/core-utils' import { ActionRunStep, CodeAction, FlowActionType, FlowRunStatus, PieceAction, WorkerJobType } from '@activepieces/shared' import { FastifyBaseLogger } from 'fastify' +import { system } from '../helper/system/system' +import { AppSystemProp } from '../helper/system/system-props' import { getPiecePackageWithoutArchive } from '../pieces/metadata/piece-metadata-service' import { jobQueue } from '../workers/job-queue/job-queue' import { userInteractionWatcher } from '../workers/user-interaction-watcher' import { ActionRunOutcome, deriveActionRunOutcome, EngineActionResponse } from './action-run-outcome' -const ACTION_RUN_BUDGET_MS = 120 * 1000 - export const actionRunService = (log: FastifyBaseLogger) => ({ async run({ projectId, platformId, step }: RunParams): Promise { const validatedStep = parseStep(step) @@ -25,7 +25,7 @@ export const actionRunService = (log: FastifyBaseLogger) => ({ platformId, step: validatedStep, piece, - expiresAt: Date.now() + ACTION_RUN_BUDGET_MS, + expiresAt: Date.now() + system.getNumberOrThrow(AppSystemProp.FLOW_TIMEOUT_SECONDS) * 1000, }, log, id)) const outcome = deriveActionRunOutcome({ result }) diff --git a/packages/server/api/src/app/app.ts b/packages/server/api/src/app/app.ts index 7efe6f5466bc..b701ac68c502 100644 --- a/packages/server/api/src/app/app.ts +++ b/packages/server/api/src/app/app.ts @@ -73,7 +73,6 @@ import { flagHooks } from './flags/flags.hooks' import { flowBackgroundJobs } from './flows/flow/flow.jobs' import { humanInputModule } from './flows/flow/human-input/human-input.module' import { flowRunModule } from './flows/flow-run/flow-run-module' -import { resumePageHooks } from './flows/flow-run/waitpoint/resume-page-hooks' import { pieceUpgradeModule } from './flows/flow-version/piece-upgrade.module' import { flowModule } from './flows/flow.module' import { folderModule } from './flows/folder/folder.module' @@ -110,6 +109,7 @@ import { triggerModule } from './trigger/trigger.module' import { platformUserModule } from './user/platform/platform-user-module' import { invitationModule } from './user-invitations/user-invitation.module' import { variableModule } from './variable/variable.module' +import { resumePageHooks } from './waitpoints/resume-page-hooks' import { webhookModule } from './webhooks/webhook-module' import { engineResponseWatcher } from './workers/engine-response-watcher' diff --git a/packages/server/api/src/app/database/database-connection.ts b/packages/server/api/src/app/database/database-connection.ts index d50be4cfa4f8..cac1f00d2917 100644 --- a/packages/server/api/src/app/database/database-connection.ts +++ b/packages/server/api/src/app/database/database-connection.ts @@ -37,7 +37,6 @@ import { FileEntity } from '../file/file.entity' import { FlagEntity } from '../flags/flag.entity' import { FlowEntity } from '../flows/flow/flow.entity' import { FlowRunEntity } from '../flows/flow-run/flow-run-entity' -import { WaitpointEntity } from '../flows/flow-run/waitpoint/waitpoint-entity' import { FlowVersionEntity } from '../flows/flow-version/flow-version-entity' import { FolderEntity } from '../flows/folder/folder.entity' import { system } from '../helper/system/system' @@ -65,6 +64,7 @@ import { TriggerSourceEntity } from '../trigger/trigger-source/trigger-source-en import { UserEntity } from '../user/user-entity' import { UserInvitationEntity } from '../user-invitations/user-invitation.entity' import { VariableEntity } from '../variable/variable.entity' +import { WaitpointEntity } from '../waitpoints/waitpoint-entity' import { DatabaseType } from './database-type' import { createPGliteDataSource } from './pglite-connection' import { createPostgresDataSource } from './postgres-connection' diff --git a/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts b/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts index d4f551a6bbd2..59981152c043 100644 --- a/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts +++ b/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts @@ -10,7 +10,6 @@ import { filesService } from '../../file/files-service' import { flowService } from '../../flows/flow/flow.service' import { engineRunCallbackService } from '../../flows/flow-run/engine-run-callback-service' import { flowRunService } from '../../flows/flow-run/flow-run-service' -import { resumeService } from '../../flows/flow-run/waitpoint/resume-service' import { rejectedPromiseHandler } from '../../helper/promise-handler' import { system } from '../../helper/system/system' import { AppSystemProp } from '../../helper/system/system-props' @@ -18,6 +17,7 @@ import { knowledgeBaseService } from '../../knowledge-base/knowledge-base.servic import { runFlowAsTool } from '../../mcp/mcp-server-builder' import { platformService } from '../../platform/platform.service' import { userService } from '../../user/user-service' +import { resumeService } from '../../waitpoints/resume-service' import { smtpEmailSender } from '../helper/email/email-sender/smtp-email-sender' import { emailService } from '../helper/email/email-service' import { agentApprovalGate } from './agent-approval-gate' diff --git a/packages/server/api/src/app/flows/flow-run/flow-run-module.ts b/packages/server/api/src/app/flows/flow-run/flow-run-module.ts index e1f907171cd0..58a32f3112a9 100644 --- a/packages/server/api/src/app/flows/flow-run/flow-run-module.ts +++ b/packages/server/api/src/app/flows/flow-run/flow-run-module.ts @@ -1,5 +1,4 @@ -import { isNil } from '@activepieces/core-utils' -import { FlowRunStatus, TelemetryEventName } from '@activepieces/shared' +import { TelemetryEventName } from '@activepieces/shared' import dayjs from 'dayjs' import { FastifyPluginAsync } from 'fastify' import { Between, EntityManager } from 'typeorm' @@ -9,13 +8,13 @@ import { SystemJobData, SystemJobName } from '../../helper/system-jobs/common' import { systemJobHandlers } from '../../helper/system-jobs/job-handlers' import { systemJobsSchedule } from '../../helper/system-jobs/system-job' import { telemetry } from '../../helper/telemetry.utils' +import { resumeController } from '../../waitpoints/resume-controller' +import { handleResumeDelayWaitpoint } from '../../waitpoints/resume-delay-handler' +import { waitpointController } from '../../waitpoints/waitpoint-controller' import { engineResponseWatcher } from '../../workers/engine-response-watcher' import { flowRunController } from './flow-run-controller' import { FlowRunEntity } from './flow-run-entity' -import { flowRunRepo, flowRunService } from './flow-run-service' -import { resumeController } from './waitpoint/resume-controller' -import { resumeService } from './waitpoint/resume-service' -import { waitpointController } from './waitpoint/waitpoint-controller' +import { flowRunRepo } from './flow-run-service' const RUN_TELEMETRY_STATEMENT_TIMEOUT_MS = 5 * 60 * 1000 // 5 minutes @@ -76,24 +75,7 @@ export const flowRunModule: FastifyPluginAsync = async (app) => { }, }) systemJobHandlers.registerJobHandler(SystemJobName.RESUME_DELAY_WAITPOINT, async (data: SystemJobData) => { - const flowRun = await flowRunService(app.log).getOne({ id: data.flowRunId, projectId: data.projectId }) - if (isNil(flowRun)) { - app.log.info({ flowRun: { id: data.flowRunId }, waitpoint: { id: data.waitpointId } }, - '[RESUME_DELAY_WAITPOINT] Flow run no longer exists (expired/deleted), skipping') - return - } - if (flowRun.status !== FlowRunStatus.PAUSED) { - app.log.info({ flowRun: { id: data.flowRunId }, waitpoint: { id: data.waitpointId }, status: flowRun.status }, - '[RESUME_DELAY_WAITPOINT] Flow not PAUSED, skipping') - return - } - app.log.info({ flowRun: { id: data.flowRunId }, waitpoint: { id: data.waitpointId } }, - '[RESUME_DELAY_WAITPOINT] Resuming flow') - await resumeService(app.log).resumeFromWaitpoint({ - flowRunId: data.flowRunId, - waitpointId: data.waitpointId, - resumePayload: null, - }) + await handleResumeDelayWaitpoint({ data, log: app.log }) }) await engineResponseWatcher(app.log).init() } diff --git a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts index de79a607533a..a4349fe5b7d0 100644 --- a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts +++ b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts @@ -15,6 +15,7 @@ import { system } from '../../helper/system/system' import { AppSystemProp } from '../../helper/system/system-props' import { assertRunCreditsNotExceeded, shouldBlockRunOnCredits } from '../../platform/billing-provider' import { projectService } from '../../project/project-service' +import { waitpointService } from '../../waitpoints/waitpoint-service' import { jobQueue, JobType } from '../../workers/job-queue/job-queue' import { payloadOffloader } from '../../workers/payload-offloader' import { flowService } from '../flow/flow.service' @@ -23,7 +24,6 @@ import { sampleDataService } from '../step-run/sample-data.service' import { FlowRunEntity } from './flow-run-entity' import { flowRunSideEffects } from './flow-run-side-effects' import { runsMetadataQueue } from './flow-runs-queue' -import { waitpointService } from './waitpoint/waitpoint-service' const CANCELLABLE_STATUSES: FlowRunStatus[] = [FlowRunStatus.PAUSED, FlowRunStatus.QUEUED] diff --git a/packages/server/api/src/app/flows/flow-run/flow-run-side-effects.ts b/packages/server/api/src/app/flows/flow-run/flow-run-side-effects.ts index 7de2cdde0702..70a2173c1b0f 100644 --- a/packages/server/api/src/app/flows/flow-run/flow-run-side-effects.ts +++ b/packages/server/api/src/app/flows/flow-run/flow-run-side-effects.ts @@ -5,8 +5,8 @@ import { ApplicationEventName, } from '@activepieces/shared' import { FastifyBaseLogger } from 'fastify' import { applicationEvents } from '../../helper/application-events' +import { waitpointService } from '../../waitpoints/waitpoint-service' import { flowRunHooks } from './flow-run-hooks' -import { waitpointService } from './waitpoint/waitpoint-service' export const flowRunSideEffects = (log: FastifyBaseLogger) => ({ async onFinish({ flowRun, platformId }: FlowRunSideEffectParams): Promise { diff --git a/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts b/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts index b0d7116df14e..90ab3cd62f90 100644 --- a/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts +++ b/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts @@ -8,14 +8,14 @@ import { exceptionHandler } from '../../helper/exception-handler' import { system } from '../../helper/system/system' import { AppSystemProp } from '../../helper/system/system-props' import { projectService } from '../../project/project-service' +import { resumeService } from '../../waitpoints/resume-service' +import { waitpointService } from '../../waitpoints/waitpoint-service' +import { WaitpointStatus } from '../../waitpoints/waitpoint-types' import { QueueName, redisMetadataKey, RunsMetadataJobData, RunsMetadataQueueConfig, runsMetadataQueueFactory, RunsMetadataUpsertData } from '../../workers/job' import { flowService } from '../flow/flow.service' import { flowRunRepo } from './flow-run-service' import { flowRunSideEffects } from './flow-run-side-effects' import { buildRunTimeline } from './run-timeline' -import { resumeService } from './waitpoint/resume-service' -import { waitpointService } from './waitpoint/waitpoint-service' -import { WaitpointStatus } from './waitpoint/waitpoint-types' let runsMetadataWorker: Worker | undefined = undefined diff --git a/packages/server/api/src/app/flows/flow/flow.jobs.ts b/packages/server/api/src/app/flows/flow/flow.jobs.ts index 27d772129b73..f71fa3caa30b 100644 --- a/packages/server/api/src/app/flows/flow/flow.jobs.ts +++ b/packages/server/api/src/app/flows/flow/flow.jobs.ts @@ -3,8 +3,8 @@ import { FastifyBaseLogger } from 'fastify' import { repoFactory } from '../../core/db/repo-factory' import { SystemJobData, SystemJobName } from '../../helper/system-jobs/common' import { systemJobsSchedule } from '../../helper/system-jobs/system-job' +import { WaitpointEntity } from '../../waitpoints/waitpoint-entity' import { flowRunRepo } from '../flow-run/flow-run-service' -import { WaitpointEntity } from '../flow-run/waitpoint/waitpoint-entity' import { flowVersionRepo } from '../flow-version/flow-version.service' import { flowExecutionCache } from './flow-execution-cache' import { flowSideEffects } from './flow-service-side-effects' diff --git a/packages/server/api/src/app/mcp/mcp-server-builder.ts b/packages/server/api/src/app/mcp/mcp-server-builder.ts index 8b9bdea4073e..fd9e44e05916 100644 --- a/packages/server/api/src/app/mcp/mcp-server-builder.ts +++ b/packages/server/api/src/app/mcp/mcp-server-builder.ts @@ -4,6 +4,8 @@ import { McpServer, ResourceTemplate } from '@modelcontextprotocol/sdk/server/mc import { FastifyBaseLogger } from 'fastify' import { z } from 'zod' import { rejectedPromiseHandler } from '../helper/promise-handler' +import { system } from '../helper/system/system' +import { AppSystemProp } from '../helper/system/system-props' import { telemetry } from '../helper/telemetry.utils' import { WebhookFlowVersionToRun, webhookService } from '../webhooks/webhook.service' import { ALLOW_ALL, PermissionChecker, resolvePermissionChecker } from './mcp-permissions' @@ -12,7 +14,6 @@ import { activepiecesTools, ALL_CONTROLLABLE_TOOL_NAMES, LOCKED_TOOL_NAMES, PLAT import { apSetProjectContextTool } from './tools/ap-set-project-context' const PLATFORM_LEVEL_TOOL_SET = new Set(PLATFORM_LEVEL_TOOL_NAMES) -const MCP_TIMEOUT_MS = 5 * 60 * 1000 // 5 minutes const MCP_SERVER_INSTRUCTIONS = `## Activepieces MCP Server @@ -188,7 +189,7 @@ export async function runFlowAsTool({ flowId, flowDisplayName, payload, returnsR payload, execute: true, failParentOnFailure: false, - timeoutMs: MCP_TIMEOUT_MS, + timeoutMs: system.getNumberOrThrow(AppSystemProp.FLOW_TIMEOUT_SECONDS) * 1000, }) const isOkay = Math.floor(response.status / 100) === 2 diff --git a/packages/server/api/src/app/flows/flow-run/waitpoint/resume-controller.ts b/packages/server/api/src/app/waitpoints/resume-controller.ts similarity index 98% rename from packages/server/api/src/app/flows/flow-run/waitpoint/resume-controller.ts rename to packages/server/api/src/app/waitpoints/resume-controller.ts index f63d21891cdc..6afdd22b3c80 100644 --- a/packages/server/api/src/app/flows/flow-run/waitpoint/resume-controller.ts +++ b/packages/server/api/src/app/waitpoints/resume-controller.ts @@ -4,10 +4,10 @@ import { FastifyBaseLogger, FastifyReply } from 'fastify' import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod' import Mustache from 'mustache' import { z } from 'zod' -import { securityAccess } from '../../../core/security/authorization/fastify-security' -import { projectService } from '../../../project/project-service' -import { flowVersionService } from '../../flow-version/flow-version.service' -import { findFlowRunOrThrow } from '../flow-run-service' +import { securityAccess } from '../core/security/authorization/fastify-security' +import { findFlowRunOrThrow } from '../flows/flow-run/flow-run-service' +import { flowVersionService } from '../flows/flow-version/flow-version.service' +import { projectService } from '../project/project-service' import { resumePageHooks, ResumePageTheme } from './resume-page-hooks' import { resumeService } from './resume-service' import { waitpointService } from './waitpoint-service' diff --git a/packages/server/api/src/app/waitpoints/resume-delay-handler.ts b/packages/server/api/src/app/waitpoints/resume-delay-handler.ts new file mode 100644 index 000000000000..eac43bc26927 --- /dev/null +++ b/packages/server/api/src/app/waitpoints/resume-delay-handler.ts @@ -0,0 +1,32 @@ +import { isNil } from '@activepieces/core-utils' +import { FlowRunStatus } from '@activepieces/shared' +import { FastifyBaseLogger } from 'fastify' +import { flowRunService } from '../flows/flow-run/flow-run-service' +import { SystemJobData, SystemJobName } from '../helper/system-jobs/common' +import { resumeService } from './resume-service' + +export async function handleResumeDelayWaitpoint({ data, log }: HandleResumeDelayWaitpointParams): Promise { + const flowRun = await flowRunService(log).getOne({ id: data.flowRunId, projectId: data.projectId }) + if (isNil(flowRun)) { + log.info({ flowRun: { id: data.flowRunId }, waitpoint: { id: data.waitpointId } }, + '[RESUME_DELAY_WAITPOINT] Flow run no longer exists (expired/deleted), skipping') + return + } + if (flowRun.status !== FlowRunStatus.PAUSED) { + log.info({ flowRun: { id: data.flowRunId }, waitpoint: { id: data.waitpointId }, status: flowRun.status }, + '[RESUME_DELAY_WAITPOINT] Flow not PAUSED, skipping') + return + } + log.info({ flowRun: { id: data.flowRunId }, waitpoint: { id: data.waitpointId } }, + '[RESUME_DELAY_WAITPOINT] Resuming flow') + await resumeService(log).resumeFromWaitpoint({ + flowRunId: data.flowRunId, + waitpointId: data.waitpointId, + resumePayload: null, + }) +} + +type HandleResumeDelayWaitpointParams = { + data: SystemJobData + log: FastifyBaseLogger +} diff --git a/packages/server/api/src/app/flows/flow-run/waitpoint/resume-page-hooks.ts b/packages/server/api/src/app/waitpoints/resume-page-hooks.ts similarity index 72% rename from packages/server/api/src/app/flows/flow-run/waitpoint/resume-page-hooks.ts rename to packages/server/api/src/app/waitpoints/resume-page-hooks.ts index a51d4e5bf5d8..833cb4cb5348 100644 --- a/packages/server/api/src/app/flows/flow-run/waitpoint/resume-page-hooks.ts +++ b/packages/server/api/src/app/waitpoints/resume-page-hooks.ts @@ -1,5 +1,5 @@ -import { defaultTheme, generateTheme } from '../../../flags/theme' -import { hooksFactory } from '../../../helper/hooks-factory' +import { defaultTheme, generateTheme } from '../flags/theme' +import { hooksFactory } from '../helper/hooks-factory' export const resumePageHooks = hooksFactory.create(() => ({ async getTheme(): Promise { diff --git a/packages/server/api/src/app/flows/flow-run/waitpoint/resume-service.ts b/packages/server/api/src/app/waitpoints/resume-service.ts similarity index 95% rename from packages/server/api/src/app/flows/flow-run/waitpoint/resume-service.ts rename to packages/server/api/src/app/waitpoints/resume-service.ts index 6c053c616755..f0584ffa2a87 100644 --- a/packages/server/api/src/app/flows/flow-run/waitpoint/resume-service.ts +++ b/packages/server/api/src/app/waitpoints/resume-service.ts @@ -2,11 +2,11 @@ import { apId, FlowRunId, isNil } from '@activepieces/core-utils' import { EngineHttpResponse, ExecutionType, FlowRun, FlowRunStatus, isFlowRunStateTerminal, ResumeReason, RunEnvironment, StreamStepProgress } from '@activepieces/shared' import { FastifyBaseLogger } from 'fastify' import { StatusCodes } from 'http-status-codes' -import { distributedLock } from '../../../database/redis-connections' -import { projectService } from '../../../project/project-service' -import { engineResponseWatcher } from '../../../workers/engine-response-watcher' -import { addToQueue, findFlowRunOrThrow, flowRunService, WEBHOOK_TIMEOUT_MS } from '../flow-run-service' -import { flowRunSideEffects } from '../flow-run-side-effects' +import { distributedLock } from '../database/redis-connections' +import { addToQueue, findFlowRunOrThrow, flowRunService, WEBHOOK_TIMEOUT_MS } from '../flows/flow-run/flow-run-service' +import { flowRunSideEffects } from '../flows/flow-run/flow-run-side-effects' +import { projectService } from '../project/project-service' +import { engineResponseWatcher } from '../workers/engine-response-watcher' import { waitpointService } from './waitpoint-service' import { Waitpoint, WaitpointResumePayload, WaitpointStatus } from './waitpoint-types' diff --git a/packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-controller.ts b/packages/server/api/src/app/waitpoints/waitpoint-controller.ts similarity index 90% rename from packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-controller.ts rename to packages/server/api/src/app/waitpoints/waitpoint-controller.ts index 4468fa088d9b..3348a0ceab62 100644 --- a/packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-controller.ts +++ b/packages/server/api/src/app/waitpoints/waitpoint-controller.ts @@ -1,8 +1,8 @@ import { CreateWaitpointRequest, CreateWaitpointResponse } from '@activepieces/shared' import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod' import { StatusCodes } from 'http-status-codes' -import { securityAccess } from '../../../core/security/authorization/fastify-security' -import { domainHelper } from '../../../helper/domain-helper' +import { securityAccess } from '../core/security/authorization/fastify-security' +import { domainHelper } from '../helper/domain-helper' import { waitpointService } from './waitpoint-service' export const waitpointController: FastifyPluginAsyncZod = async (app) => { diff --git a/packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-entity.ts b/packages/server/api/src/app/waitpoints/waitpoint-entity.ts similarity index 96% rename from packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-entity.ts rename to packages/server/api/src/app/waitpoints/waitpoint-entity.ts index d5dd020852a1..0e3a58484338 100644 --- a/packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-entity.ts +++ b/packages/server/api/src/app/waitpoints/waitpoint-entity.ts @@ -1,6 +1,6 @@ import { PauseType, Project } from '@activepieces/shared' import { EntitySchema } from 'typeorm' -import { ApIdSchema, BaseColumnSchemaPart } from '../../../database/database-common' +import { ApIdSchema, BaseColumnSchemaPart } from '../database/database-common' import { Waitpoint, WaitpointStatus, WaitpointVersionEnum } from './waitpoint-types' type WaitpointSchema = Waitpoint & { diff --git a/packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-service.ts b/packages/server/api/src/app/waitpoints/waitpoint-service.ts similarity index 96% rename from packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-service.ts rename to packages/server/api/src/app/waitpoints/waitpoint-service.ts index 6b3c34f5413f..5d37de0d57bc 100644 --- a/packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-service.ts +++ b/packages/server/api/src/app/waitpoints/waitpoint-service.ts @@ -2,10 +2,10 @@ import { apId, isNil } from '@activepieces/core-utils' import { FlowRunStatus } from '@activepieces/shared' import dayjs from 'dayjs' import { FastifyBaseLogger } from 'fastify' -import { repoFactory } from '../../../core/db/repo-factory' -import { transaction } from '../../../core/db/transaction' -import { SystemJobName } from '../../../helper/system-jobs/common' -import { systemJobsSchedule } from '../../../helper/system-jobs/system-job' +import { repoFactory } from '../core/db/repo-factory' +import { transaction } from '../core/db/transaction' +import { SystemJobName } from '../helper/system-jobs/common' +import { systemJobsSchedule } from '../helper/system-jobs/system-job' import { WaitpointEntity } from './waitpoint-entity' import { CompleteParams, CompleteResult, CreateForPauseParams, CreateForPauseResult, FindPendingByVersionParams, HandleResumeSignalParams, Waitpoint, WaitpointStatus } from './waitpoint-types' diff --git a/packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-types.ts b/packages/server/api/src/app/waitpoints/waitpoint-types.ts similarity index 100% rename from packages/server/api/src/app/flows/flow-run/waitpoint/waitpoint-types.ts rename to packages/server/api/src/app/waitpoints/waitpoint-types.ts diff --git a/packages/server/api/src/app/workers/migrations/refill-paused-jobs.ts b/packages/server/api/src/app/workers/migrations/refill-paused-jobs.ts index 3389bb058d37..5eaadd285afa 100644 --- a/packages/server/api/src/app/workers/migrations/refill-paused-jobs.ts +++ b/packages/server/api/src/app/workers/migrations/refill-paused-jobs.ts @@ -6,11 +6,11 @@ import { In, MoreThan } from 'typeorm' import { repoFactory } from '../../core/db/repo-factory' import { redisConnections } from '../../database/redis-connections' import { flowRunRepo } from '../../flows/flow-run/flow-run-service' -import { WaitpointEntity } from '../../flows/flow-run/waitpoint/waitpoint-entity' import { system } from '../../helper/system/system' import { AppSystemProp } from '../../helper/system/system-props' import { SystemJobName } from '../../helper/system-jobs/common' import { systemJobsSchedule } from '../../helper/system-jobs/system-job' +import { WaitpointEntity } from '../../waitpoints/waitpoint-entity' import { jobQueue } from '../job-queue/job-queue' const REFILL_PAUSED_RUNS_KEY = 'refill_paused_runs_v7' diff --git a/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts index a0c9ab11dcea..e57c8cfa7a4d 100644 --- a/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts +++ b/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts @@ -4,7 +4,7 @@ import { FastifyInstance } from 'fastify' import { distributedStore } from '../../../../../src/app/database/redis-connections' import { batchDeleteByFlowId } from '../../../../../src/app/flows/flow/flow.jobs' import { flowRunSideEffects } from '../../../../../src/app/flows/flow-run/flow-run-side-effects' -import { waitpointService } from '../../../../../src/app/flows/flow-run/waitpoint/waitpoint-service' +import { waitpointService } from '../../../../../src/app/waitpoints/waitpoint-service' import { pubsub } from '../../../../../src/app/helper/pubsub' import { engineResponseWatcher } from '../../../../../src/app/workers/engine-response-watcher' import { redisMetadataKey, RunsMetadataUpsertData } from '../../../../../src/app/workers/job' diff --git a/packages/server/api/test/integration/ce/flows/flow-run/resume-service.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/resume-service.test.ts index a5f595297289..926f968200b4 100644 --- a/packages/server/api/test/integration/ce/flows/flow-run/resume-service.test.ts +++ b/packages/server/api/test/integration/ce/flows/flow-run/resume-service.test.ts @@ -1,9 +1,9 @@ import { apId } from '@activepieces/core-utils' import { FlowRunStatus, FlowVersionState, PauseType, RunEnvironment } from '@activepieces/shared' import { FastifyInstance } from 'fastify' -import { resumeService } from '../../../../../src/app/flows/flow-run/waitpoint/resume-service' -import { waitpointService } from '../../../../../src/app/flows/flow-run/waitpoint/waitpoint-service' -import { WaitpointStatus } from '../../../../../src/app/flows/flow-run/waitpoint/waitpoint-types' +import { resumeService } from '../../../../../src/app/waitpoints/resume-service' +import { waitpointService } from '../../../../../src/app/waitpoints/waitpoint-service' +import { WaitpointStatus } from '../../../../../src/app/waitpoints/waitpoint-types' import { db } from '../../../../helpers/db' import { createMockFlow, createMockFlowRun, createMockFlowVersion } from '../../../../helpers/mocks' import { createTestContext, TestContext } from '../../../../helpers/test-context' diff --git a/packages/server/api/test/integration/ce/flows/flow-run/waitpoint.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/waitpoint.test.ts index ac2f00d26082..a283a8475a22 100644 --- a/packages/server/api/test/integration/ce/flows/flow-run/waitpoint.test.ts +++ b/packages/server/api/test/integration/ce/flows/flow-run/waitpoint.test.ts @@ -1,9 +1,9 @@ import { apId } from '@activepieces/core-utils' import { FlowRunStatus, FlowVersionState, PauseType, RunEnvironment } from '@activepieces/shared' import { FastifyInstance } from 'fastify' -import { waitpointService } from '../../../../../src/app/flows/flow-run/waitpoint/waitpoint-service' import * as systemJobModule from '../../../../../src/app/helper/system-jobs/system-job' -import { WaitpointStatus } from '../../../../../src/app/flows/flow-run/waitpoint/waitpoint-types' +import { waitpointService } from '../../../../../src/app/waitpoints/waitpoint-service' +import { WaitpointStatus } from '../../../../../src/app/waitpoints/waitpoint-types' import { db } from '../../../../helpers/db' import { createMockFlow, createMockFlowRun, createMockFlowVersion } from '../../../../helpers/mocks' import { createTestContext, TestContext } from '../../../../helpers/test-context' diff --git a/packages/server/api/test/unit/app/action-run/action-run-budget.test.ts b/packages/server/api/test/unit/app/action-run/action-run-budget.test.ts new file mode 100644 index 000000000000..12bf478ae4fc --- /dev/null +++ b/packages/server/api/test/unit/app/action-run/action-run-budget.test.ts @@ -0,0 +1,60 @@ +import { CodeAction, FlowActionType, FlowRunStatus } from '@activepieces/shared' +import { FastifyBaseLogger } from 'fastify' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const submitAndWaitForResponse = vi.fn() + +vi.mock('../../../../src/app/workers/user-interaction-watcher', () => ({ + userInteractionWatcher: { submitAndWaitForResponse: (...args: unknown[]) => submitAndWaitForResponse(...args) }, +})) +vi.mock('../../../../src/app/workers/job-queue/job-queue', () => ({ + jobQueue: () => ({ cancelAndReportNeverStarted: vi.fn() }), +})) +vi.mock('../../../../src/app/pieces/metadata/piece-metadata-service', () => ({ + getPiecePackageWithoutArchive: vi.fn(), +})) + +const { actionRunService } = await import('../../../../src/app/action-run/action-run.service') +const { system } = await import('../../../../src/app/helper/system/system') +const { AppSystemProp } = await import('../../../../src/app/helper/system/system-props') + +const mockLog = { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as unknown as FastifyBaseLogger + +const codeStep: CodeAction = { + name: 'step_1', + type: FlowActionType.CODE, + valid: true, + displayName: 'Code', + lastUpdatedDate: new Date(0).toISOString(), + settings: { + sourceCode: { packageJson: '{}', code: 'export const code = async () => 1' }, + input: {}, + }, +} + +async function runAndReadBudgetMs(): Promise { + const startedAt = Date.now() + await actionRunService(mockLog).run({ projectId: 'proj-1', platformId: 'plat-1', step: codeStep }) + const [request] = submitAndWaitForResponse.mock.calls[0] + return request.expiresAt - startedAt +} + +describe('action run budget', () => { + beforeEach(() => { + vi.restoreAllMocks() + submitAndWaitForResponse.mockReset() + submitAndWaitForResponse.mockResolvedValue({ success: true, output: 1, status: FlowRunStatus.SUCCEEDED }) + }) + + it('gives an action run the same budget a flow run gets', async () => { + vi.spyOn(system, 'getNumberOrThrow').mockImplementation(prop => prop === AppSystemProp.FLOW_TIMEOUT_SECONDS ? 600 : 0) + + expect(await runAndReadBudgetMs()).toBeGreaterThanOrEqual(600_000) + }) + + it('follows AP_FLOW_TIMEOUT_SECONDS when a self-hoster raises it, which is what unblocks a tool that runs longer than the old fixed 120s', async () => { + vi.spyOn(system, 'getNumberOrThrow').mockImplementation(prop => prop === AppSystemProp.FLOW_TIMEOUT_SECONDS ? 1_800 : 0) + + expect(await runAndReadBudgetMs()).toBeGreaterThanOrEqual(1_800_000) + }) +}) diff --git a/packages/server/api/test/unit/app/ee/agent/agent-rpc-handlers.test.ts b/packages/server/api/test/unit/app/ee/agent/agent-rpc-handlers.test.ts index 00006eeb6f64..896a17cd3b6b 100644 --- a/packages/server/api/test/unit/app/ee/agent/agent-rpc-handlers.test.ts +++ b/packages/server/api/test/unit/app/ee/agent/agent-rpc-handlers.test.ts @@ -9,7 +9,7 @@ vi.mock('../../../../../src/app/flows/flow-run/flow-run-service', () => ({ flowRunService: () => ({ getOneOrThrow: mockGetFlowRun }), })) -vi.mock('../../../../../src/app/flows/flow-run/waitpoint/resume-service', () => ({ +vi.mock('../../../../../src/app/waitpoints/resume-service', () => ({ resumeService: () => ({ resumeFromWaitpoint: mockResumeFromWaitpoint }), })) diff --git a/packages/server/sandbox/src/index.ts b/packages/server/sandbox/src/index.ts index 8ed0d830b620..8f48acbc316b 100644 --- a/packages/server/sandbox/src/index.ts +++ b/packages/server/sandbox/src/index.ts @@ -1,6 +1,6 @@ export { createSandboxRuntime } from './lib/sandbox' export { createResolver } from './lib/resolver' -export { actionRunCache, ACTION_RUN_CACHE_FIRST_SWEEP_DELAY_MS, ACTION_RUN_CACHE_SWEEP_INTERVAL_MS } from './lib/cache/action-run-cache' +export { actionRunCache, ACTION_RUN_CACHE_ACTIVE_WINDOW_MS, ACTION_RUN_CACHE_FIRST_SWEEP_DELAY_MS, ACTION_RUN_CACHE_SWEEP_INTERVAL_MS } from './lib/cache/action-run-cache' export { cacheUtils } from './lib/cache/cache-paths' export type { diff --git a/packages/server/sandbox/src/lib/cache/action-run-cache.ts b/packages/server/sandbox/src/lib/cache/action-run-cache.ts index 281d99d2b405..18cd7af37fcc 100644 --- a/packages/server/sandbox/src/lib/cache/action-run-cache.ts +++ b/packages/server/sandbox/src/lib/cache/action-run-cache.ts @@ -33,7 +33,7 @@ export const actionRunCache = { return true }, - async sweep({ basePath, log }: SweepParams): Promise { + async sweep({ basePath, log, activeWindowMs = ACTION_RUN_CACHE_ACTIVE_WINDOW_MS }: SweepParams): Promise { const startedAt = Date.now() const actionRunsPath = cacheUtils(basePath).getActionRunCodeCachePath() const { data: entries, error } = await tryCatch(() => readdir(actionRunsPath, { withFileTypes: true })) @@ -52,7 +52,7 @@ export const actionRunCache = { const expired = await removeExpired({ dirPaths: managed, expiredAt: Date.now() - ACTION_RUN_CACHE_TTL_MS }) - const activeAt = Date.now() - ACTION_RUN_CACHE_ACTIVE_WINDOW_MS + const activeAt = Date.now() - activeWindowMs const evictable = expired.survivors.filter((entry) => entry.mtimeMs < activeAt) const overBy = expired.survivors.length - ACTION_RUN_CACHE_MAX_DIRS const evicted = await removeOldest(overBy > 0 @@ -155,6 +155,7 @@ type NamespaceParams = { type SweepParams = { basePath: string log: ApLogger + activeWindowMs?: number } type DirEntry = { diff --git a/packages/server/sandbox/test/lib/cache/action-run-cache.test.ts b/packages/server/sandbox/test/lib/cache/action-run-cache.test.ts index 66332f914ba8..a70b54351944 100644 --- a/packages/server/sandbox/test/lib/cache/action-run-cache.test.ts +++ b/packages/server/sandbox/test/lib/cache/action-run-cache.test.ts @@ -243,6 +243,22 @@ describe('actionRunCache.sweep', () => { } }) + it('widens the window to cover a run budget longer than 15 minutes, so a long code action is not evicted while it executes', async () => { + const basePath = uniqueBasePath() + const runningDirs = await seedOldestFirst({ + basePath, + total: ACTION_RUN_CACHE_MAX_DIRS + 7, + ageOffsetMs: ACTION_RUN_CACHE_ACTIVE_WINDOW_MS + 60_000, + label: 'c', + }) + + await actionRunCache.sweep({ basePath, log: noopLog, activeWindowMs: 60 * 60 * 1000 }) + + for (const dirPath of runningDirs) { + await expect(exists(dirPath)).resolves.toBe(true) + } + }) + it('is a no-op on a cache that was never created, and is idempotent', async () => { const basePath = uniqueBasePath() diff --git a/packages/server/worker/src/lib/worker.ts b/packages/server/worker/src/lib/worker.ts index 9d9e99b84e79..a587298eb5fc 100644 --- a/packages/server/worker/src/lib/worker.ts +++ b/packages/server/worker/src/lib/worker.ts @@ -1,9 +1,9 @@ import { createServer } from 'http' import os from 'os' -import { ActivepiecesError, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils' -import { ACTION_RUN_CACHE_FIRST_SWEEP_DELAY_MS, ACTION_RUN_CACHE_SWEEP_INTERVAL_MS, actionRunCache, cacheUtils, createResolver, createSandboxRuntime, Runtime } from '@activepieces/sandbox' +import { ActivepiecesError, isNil, spreadIfDefined, tryCatch, tryCatchSync } from '@activepieces/core-utils' +import { ACTION_RUN_CACHE_ACTIVE_WINDOW_MS, ACTION_RUN_CACHE_FIRST_SWEEP_DELAY_MS, ACTION_RUN_CACHE_SWEEP_INTERVAL_MS, actionRunCache, cacheUtils, createResolver, createSandboxRuntime, Runtime } from '@activepieces/sandbox' import { apVersionUtil, createLogger, onCallService, systemUsage, UNKNOWN_VERSION, wideEvent } from '@activepieces/server-utils' -import { ApEdition, ApiToWorkerContract, ConsumeJobRequest, createNotifyServer, createRpcClient, EngineResponseStatus, ExecutionMode, JobData, SandboxInformation, WebsocketServerEvent, WorkerJobType, WorkerMachineHealthcheckRequest, WorkerProps, WorkerSettingsResponse, WorkerToApiContract } from '@activepieces/shared' +import { ApEdition, ApiToWorkerContract, ConsumeJobRequest, createNotifyServer, createRpcClient, EngineResponseStatus, ExecutionMode, JobData, LONG_RUNNING_RPC_METHODS, SandboxInformation, WebsocketServerEvent, WorkerJobType, WorkerMachineHealthcheckRequest, WorkerProps, WorkerSettingsResponse, WorkerToApiContract } from '@activepieces/shared' import { nanoid } from 'nanoid' import { io, Socket } from 'socket.io-client' import { createApiToWorkerHandlers } from './api-notify-service' @@ -79,6 +79,9 @@ const SERVER_PING_TIMEOUT_MS = 5_000 const MACHINE_INFO_TIMEOUT_MS = 15_000 const POLL_LIVENESS_TIMEOUT_MS = 180_000 const POLL_WATCHDOG_INTERVAL_MS = 30_000 +const RPC_TIMEOUT_MS = 60_000 +const LONG_RUNNING_RPC_MARGIN_MS = 120_000 +const FALLBACK_FLOW_TIMEOUT_SECONDS = 600 let pollLoopLiveness: PollLoopLiveness[] = [] let pollWatchdogInterval: NodeJS.Timeout | null = null @@ -98,7 +101,7 @@ export const worker = { reconnection: true, }) - const apiClient = createRpcClient(socket, 60_000) + const apiClient = createRpcClient(socket, rpcTimeoutMsFor) socket.on('connect', async () => { logger.info('Connected to API server via Socket.IO') @@ -333,6 +336,22 @@ function abortInFlightRuntime(): void { }) } +function flowTimeoutMs(): number { + const { data: settings } = tryCatchSync(() => workerSettings.getSettings()) + return (settings?.FLOW_TIMEOUT_SECONDS ?? FALLBACK_FLOW_TIMEOUT_SECONDS) * 1000 +} + +function rpcTimeoutMsFor(method: string): number { + if (!LONG_RUNNING_RPC_METHODS.includes(method)) { + return RPC_TIMEOUT_MS + } + const { data: settings } = tryCatchSync(() => workerSettings.getSettings()) + if (isNil(settings)) { + logger.warn({ rpc: { method } }, 'Worker settings have not arrived, timing a long-running RPC by the default flow timeout') + } + return flowTimeoutMs() + LONG_RUNNING_RPC_MARGIN_MS +} + async function executeJob(apiClient: WorkerToApiContract, job: ConsumeJobRequest, runtime: Runtime, workerIndex: number): Promise { const rawData = job.jobData const jobData = JobData.parse(rawData) @@ -619,7 +638,11 @@ function stopCacheSweeper(): void { } async function sweepActionRunCache(): Promise { - const { error } = await tryCatch(() => actionRunCache.sweep({ basePath: sandboxConfig.getCacheBasePath(), log: logger })) + const { error } = await tryCatch(() => actionRunCache.sweep({ + basePath: sandboxConfig.getCacheBasePath(), + log: logger, + activeWindowMs: Math.max(ACTION_RUN_CACHE_ACTIVE_WINDOW_MS, flowTimeoutMs()), + })) if (error) { logger.warn({ error }, 'Action-run code cache sweep failed') }