diff --git a/.agents/skills/design/SKILL.md b/.agents/skills/design/SKILL.md index 5530fbb21c75..5be064bace2f 100644 --- a/.agents/skills/design/SKILL.md +++ b/.agents/skills/design/SKILL.md @@ -50,12 +50,12 @@ Read `README.md` in this folder **first** — it is the canonical reference. Thi ## Hard rules (never violate) 1. **Primary is purple `hsl(257 74% 57%)` / `#8142E3`** — the shipping value from `packages/web/src/styles.css`. Not the `#9747FF` swatch some Figma files show. **Primary stays purple in dark mode** too (brand continuity) — use `.dark.blue-primary` to opt back into the repo's blue-in-dark behaviour. -2. **Body text is 14px (`text-sm`), not 16**. Activepieces feels dense and tool-like. Headings use `-0.01em` to `-0.02em` tracking. +2. **Body text is 14px (`text-sm`), not 16**. Activepieces feels dense and tool-like. Headings use `-0.01em` to `-0.02em` tracking. The **agent editor's Configure panel** pairs 15px controls with 13px labels and meta, one step up from `text-sm` so a settings column reads at arm's length; treat that pair as local to that panel, not as a second ramp. 3. **Sentence case everywhere**: headings, buttons, menu items, page titles. Proper nouns only for feature names (Flows, Runs, Pieces, MCP, Agents, Connections). 4. **Lucide icons only**, 1.5–2px stroke, rounded caps. Default size `16` (`size-4`). Icon + text → `gap-2` (8px). No emoji in the product UI. No Unicode glyphs (✓ × ←) — always a Lucide component. 5. **Borders are 1px**, color `neutral-200` (light) / `white/14` (dark). Never thicker. (Note: repo ships `white/10` in dark — we bump to `14%` so dividers stay readable against `neutral-900` surfaces.) 6. **No negative margins.** Use `gap-*`, `p-*`, `space-*`. Explicitly banned in the repo's AGENTS.md. -7. **Cards: white fill, 1px border, `radius-lg` (10px), NO shadow by default.** Shadows only on floating surfaces (popovers, menus, dialogs). +7. **Cards: white fill, 1px border, `radius-lg` (10px), NO shadow by default.** Shadows only on floating surfaces (popovers, menus, dialogs). One named exception: the **agents list and its first-run hero** use a 19px radius and a two-layer lift (`0 1px 2px` + `0 4px 12px -2px`, deepening on hover), because that surface is a showcase rather than a dense tool view. It is the only place that does, and new cards elsewhere stay 10px and flat. 8. **Main content is a floating card**: the sidebar blends with the outer shell (`neutral-50` in light, `neutral-950` in dark); the content area sits inside with `radius-xl` (12px), 1px border, `shadow-xs`, and 8px inset from the viewport edges. Matches the shipping app layout. 9. **Builder canvas has the dotted look** — background `#FBFBFB` (light) / `#171717` (dark), radial-gradient dots `#b2b2b2 1px` at `16px 16px`. This is signature. 10. **Hover states darken only** — primary buttons go to `/90`, secondary `/80`, ghost `bg-gray-300/30`. No scale, no translate, no elevation change on hover or press. diff --git a/brain/knowledge/ai-intelligence/ai-providers.md b/brain/knowledge/ai-intelligence/ai-providers.md index 7bc31579bdbe..e2d2efd80f5e 100644 --- a/brain/knowledge/ai-intelligence/ai-providers.md +++ b/brain/knowledge/ai-intelligence/ai-providers.md @@ -61,11 +61,21 @@ renders and preserves every real price; the cheapest in the set is 0.01. - **Two keys may legitimately hold the same secret, and you could not detect it anyway.** One API key scoped to two rows with different `modelScope`/`projectScope` allow-lists is a supported setup, not a mistake. Blocking it is also impractical: `encryptUtils.encryptObject` uses a fresh random IV per write, so the same credential stores as different ciphertext every time — deduping would need a separate HMAC column. Key *names* are the thing worth constraining, since a picker showing two rows called "Anthropic key" is unpickable. - **`mockAndSaveAIProvider` uses `save`, not upsert** — the old `(platformId, provider)` ON CONFLICT target died with the unique index; seeding the same provider twice now creates two keys, which is usually what a test wants. - ACTIVEPIECES auto-provision needs `OPENROUTER_PROVISION_KEY` env var set AND `aiCreditsEnabled` true. +- **`secret: true` on a credential field is not about masking, it is what stops a stored credential being prefilled back into the form.** In `PROVIDER_CREDENTIAL_FIELDS` the flag feeds `secretKeysOf`, and the dialog's default-values builder blanks exactly those keys when reopening an existing key; the password-style `Input` is a side effect of the same flag, not its purpose. So dropping it to change how a field *renders* silently turns a stored secret into a prefilled value. Render differently instead: `type` (`dictionary`, `textarea`) is checked ahead of `secret` in `CredentialFieldInput`, so a field can keep the flag and still draw as something other than a masked input — that is how the Vertex service-account JSON gets a monospace textarea while staying blanked on edit. Both the initialiser and the required-field check filter only `dictionary`, so any other `type` is initialised and validated like a normal credential. - **Adding a provider is a leaf change, and the credential fields are the only part that is not.** A new vendor touches six places: the `AIProviderName` enum (`packages/core/utils/.../permission.ts`), its auth/config schemas plus the two unions and `ProviderConfigUnion` in `packages/core/shared/.../management/ai-providers/index.ts` (all in the per-provider region, well above the generic request/response schemas at the bottom), a strategy file registered in `ai/providers/index.ts`, the model factory switch, name/logo/markdown in `packages/web/src/features/agents/ai-providers.ts`, and translation keys. None of that is the credential form: extra fields beyond `apiKey` (Azure's `resourceName`, Bedrock's region) are declared in **one** file — `PROVIDER_CREDENTIAL_FIELDS` in `.../setup/ai/providers-tab/provider-credentials.ts`, which falls back to `DEFAULT_CREDENTIAL_FIELDS` (a single `apiKey`) for any provider with no entry, so a plain API-key vendor needs no UI work at all. That file replaced the deleted `universal-pieces/upsert-provider-config-form.tsx` in the multi-key redesign, so a provider authored against an older branch loses its custom fields on merge **silently** — git resolves delete-vs-modify by taking the deletion, no conflict marker, and the provider just becomes unconfigurable in the admin UI. A vendor with no `/models` endpoint also belongs in `MANUAL_MODEL_PROVIDERS` in that same file (`CUSTOM`, `CLOUDFLARE_GATEWAY`) so the admin enters model ids by hand. Everything else about a provider is orthogonal to multi-key: that is a table-level change (drop `UNIQUE (platformId, provider)`, add the four scope columns, keep a unique partial index for `activepieces` only), so a provider inherits multi-key with no provider-side code, and the admin providers-tab enumerates `SUPPORTED_AI_PROVIDERS` from `packages/web/src/features/agents/ai-providers.ts` rather than a catalog of its own, so a new vendor appears there on its own. When basing provider work off a branch that predates the redesign, re-check `provider-credentials.ts` after the merge. - **The OpenAI-compatible vendors (xAI, DeepSeek, Z.ai, Qwen, MiniMax, Moonshot) share one strategy rather than a file each**, via `openAiCompatibleVendor({ name, provider })` in `ai/providers/`, with defaults in `OPENAI_COMPATIBLE_VENDOR_BASE_URLS` and an optional per-key `baseUrl` override because four of them run separate China and international endpoints. Their `listModels` GETs `{baseUrl}/models`, which the vendor docs mostly do not document — **confirmed working against live keys for DeepSeek, Z.ai, MiniMax and Moonshot** (Qwen still unverified), so don't redo that research. If a future vendor turns out to lack `/models`, the fallback is the manual-models path rather than a bespoke strategy. Unlike its siblings this factory uses `safeHttp.axios`, not `httpClient` from `pieces-common`: the base URL is admin-supplied, so it must go through the SSRF filter. +- **Adding a provider compiles fine and still breaks flows, because the two model factories fail differently.** `createLanguageModel` in `@activepieces/ai-providers` ends its switch with `const exhaustiveCheck: never = provider`, so a missing provider is a *compile* error. The deliberately duplicated `buildLanguageModel` inside the AI piece (`pieces/community/ai/.../common/ai-sdk.ts`, kept on ai@6 because a v4 model cannot cross into the engine) ends with `default: throw new Error(...)` — so a provider wired only into the first one type-checks, ships, connects happily in the admin UI, and then throws `Provider is not supported` the moment someone uses it in a flow step. That is the `.claude/rules/self-hosting.md` antipattern exactly. Wire **both** factories, and note `buildNativeImageModel` in that same file is a third switch, and its miss is worse than a throw: `createAIModel` does `if (imageModel) return imageModel` and otherwise **falls through to `buildLanguageModel`**, so a provider absent from that switch answers an image request with a *language* model rather than erroring. `NO_IMAGE_GENERATION_PROVIDERS` is what turns that into the clear `does not support image models` message, so a provider belongs there until images are actually wired — the entry is load bearing, not cosmetic. +- **Overriding `fetch` on a provider silently switches off key-health reporting.** `...observed` *is* a `fetch` (`observedProviderFetch`), so any later `fetch` in the same options object replaces it rather than adding to it — spread order decides, and nothing errors or logs. A path that needs its own `fetch` (the Responses header-stripping one, the Cloudflare gateway's `Authorization`-deleting one) must take `observedProviderFetch(options.onOutcome)` as a delegate and call it, not call `globalThis.fetch`. Resolve the delegate at call time rather than capturing it at model-construction time; late binding is both more correct and the only way a test can stub the global. This is the same family as the merge hazard above — the failure is always one provider, or one path within a provider, quietly not reporting. +- **A provider branch merges cleanly and still opts out of whatever the switch gained meanwhile.** `createLanguageModel` applies cross-cutting behaviour by spreading it into *every* case — `...observed` (`observedProviderFetch`, added by the key-health work) is the current one. Git treats a long-lived branch's new `case` as an added block, so it lands with none of the spreads its siblings picked up, and the merge reports no conflict: the new provider is simply the one that never reports health. Nothing fails, nothing logs. After merging main into any provider branch, diff your `case` against a neighbouring one and check the spreads match, rather than trusting a clean merge. +- **Only two `Record` maps are exhaustive, so the enum addition barely fights back.** `AI_PROVIDER_CAPABILITIES` (piece-types) and `aiProviders` (`ai/providers/index.ts`) must gain an entry; `PROVIDER_CREDENTIAL_FIELDS`, `PROVIDER_EMBEDDING_MODELS`, `ALLOWED_CHAT_MODELS_BY_PROVIDER`, `PROVIDER_MAX_CONTEXT_TOKENS`, `DEFAULT_EMBEDDING_MODELS` and `WEB_SEARCH_MODE_BY_PROVIDER` are all `Partial>` and degrade silently to defaults. `grep -rn 'Record` and pick the version whose `dependencies` equal the target package's own pins. Faster still, `ls node_modules/.bun/ | grep '@'` usually already lists every generation bun has resolved. +- **The OpenAI-compatible provider can speak the Responses API, and `@ai-sdk/openai-compatible` is not how you get there.** That SDK exposes only `chatModel` (`/chat/completions`) and `completionModel` — no responses model in any version. The route is `@ai-sdk/openai`, which accepts a custom `baseURL` + `headers` and exposes `.responses()`; the Cloudflare gateway's openai branch already did this before the `apiStyle: 'chat' | 'responses'` config field existed. Header precedence is the subtlety: the SDK builds `{ Authorization: Bearer , ...options.headers }`, so a caller header wins on `Authorization` (the Bedrock case, where the key *is* the bearer token) but a custom `apiKeyHeader` such as `x-api-key` leaves the SDK's default `Authorization` riding along. `withUserAgentSuffix` drops `undefined` values, but `OpenAIProviderSettings.headers` is typed `Record`, so the way to strip it is the header-stripping `fetch` the Cloudflare branch uses — no cast needed, and worth reaching for straight away rather than writing the duplicate header up as an accepted limitation. Strip only when `apiKeyHeader` is not itself Authorization, compared **case-insensitively**: HTTP header names are, and an admin can type `authorization`. Confirmed shape, since it is easy to assert on the wrong key: the SDK lowercases everything, so a mis-set provider really does emit `{authorization: 'Bearer ', 'x-api-key': ''}` — the credential twice. +- **AWS's OpenAI-compatible surface is two endpoints, and "Bedrock Mantle" is a real AWS product name, not a customer's nickname.** `bedrock-runtime.{region}.amazonaws.com/openai/v1` (AWS-recommended) and `bedrock-mantle.{region}.api.aws/v1` both serve Chat Completions *and* Responses, both authenticate with a Bedrock API key as `Authorization: Bearer`, and both take an unmodified OpenAI SDK. Mantle alone adds server-side tools and web search, `background=true` async inference, and Projects/Workspaces; bedrock-runtime alone has Guardrails, cross-Region inference and intelligent prompt routing. So a ticket claiming "Mantle only supports responses" is half right — it serves chat/completions too, and the real loss is the Responses-only capabilities. +- **Vertex is a different door onto the same models, not a different vendor.** The `GOOGLE` provider is the Gemini Developer API (`generativelanguage.googleapis.com`, static API key); `VERTEX` is Vertex AI (`{region}-aiplatform.googleapis.com/v1beta1/projects/{project}/locations/{region}/publishers/google`, service-account OAuth2 with ~1h tokens) — the same Gemini models, but billed to the customer's GCP account and inheriting IAM, VPC-SC, CMEK, data residency and audit logs, plus Model Garden's Claude/Llama/Mistral. That rotating token is why the CUSTOM provider can never reach Vertex: `buildOpenAICompatibleHeaders` injects one static header, which is why the workaround used to be a LiteLLM proxy. `@ai-sdk/google-vertex` does the JWT-to-token exchange itself, so the native provider needs no proxy. Vertex models are entered by hand (`MANUAL_MODEL_PROVIDERS`) on purpose: availability is project-, region- and Model-Garden-specific, and a `publishers/google` listing would miss exactly the third-party models that motivate choosing Vertex. **But "same models" only holds for Gemini.** Model Garden also serves Anthropic, Meta, Mistral and xAI, and those do not share Gemini's API surface — `@ai-sdk/google-vertex` ships separate `/anthropic`, `/maas` and `/xai` entry points for exactly that reason, so one `createVertex(...)` call does not cover Vertex. Because models are typed in by hand, a Claude id reaches the factory as an ordinary string and, unrouted, gets built with the Gemini client and fails at the endpoint. `create-language-model.ts` routes ids containing `claude` to `createVertexAnthropic`; `vertexClientFor` picks the client from the id shape, so all three are routed. Because the shapes are unambiguous the check is a lookup, not a tuned heuristic — and note the publisher prefix has to be tested *before* the `claude` one, or a MaaS path containing "claude" routes to the Anthropic client. Routing is cheap because the id shapes are distinct, not because a heuristic was tuned: `gemini-2.5-pro` is bare, Anthropic carries an `@date` (`claude-3-5-sonnet@20241022`), and MaaS is publisher-prefixed and suffixed (`meta/llama-4-scout-17b-16e-instruct-maas`). The publisher prefix is a stronger discriminator than the `includes('claude')` match currently shipped, and it looks like an argument for listing models rather than taking them by hand — but it is not, see below: the listing is scoped to a single publisher, so Model Garden ids stay hand-typed and the string matching stays with them. **Listing was considered and rejected**, and the reason is worth keeping so it does not get reopened, and **the response shape does not need a GCP account to learn** — every Google API publishes an unauthenticated discovery document: `curl -s 'https://aiplatform.googleapis.com/$discovery/rest?version=v1beta1'` returns ~5MB of JSON whose `schemas` are the authoritative request/response types. `publishers.models.list` returns `{ publisherModels: PublisherModel[], nextPageToken }`, and `PublisherModel` is `{ name: 'publishers/google/models/', versionId, versionState (STABLE|UNSTABLE), launchStage (GA|PUBLIC_PREVIEW|PRIVATE_PREVIEW|EXPERIMENTAL), openSourceCategory, supportedActions, predictSchemata, frameworks, parent }`. **There is no `displayName` on it** — the one in `google-vertexai`'s piece comes from `@google/genai`'s `Model` type, which is a different shape, so copying that code gives every row a blank label with nothing failing. `displayName` exists only on the nested `parent`, which is the base model a tuned model derives from. The decisive fact is in that same doc: `publishers.models.list` takes `parent` matching `^publishers/[^/]+$` — **one publisher per call**. Listing everything therefore means hardcoding the set of publishers to enumerate, which is the same shape of hardcoding it was meant to remove, and Model Garden models would still need hand entry, leaving two mechanisms where there is now one. `VERTEX` stays in `MANUAL_MODEL_PROVIDERS` alongside `CUSTOM` and `CLOUDFLARE_GATEWAY`, which fits: Vertex model availability really is per-project. Reach for the discovery doc before a live probe on any `*.googleapis.com` integration; a curl against the real endpoint needs **billing attached to the project even for a read** (`403 BILLING_DISABLED`), so it is the slower path and it is not free to set up. The `x-goog-user-project` header is not optional when probing with a personal login: without it aiplatform answers `403 SERVICE_DISABLED` naming `consumer: projects/32555940559`, which is Google's shared gcloud CLI project rather than yours, and the message talks about quota projects rather than the missing header. This is a user-credential quirk only — a service account carries its own `project_id`, so the provider itself never hits it. **The multi-vendor story is text only.** Vertex's image models are Google's own Imagen (`imagen-3.0-*`, `imagen-4.0-*`), not Model Garden's third parties, and `createVertex(...)` also exposes `imageModel`, `video`, `speech`, `transcription` and `textEmbeddingModel` — none of which we wire today. Imagen is wired (`createVertex(...).imageModel()`, with a `VERTEX` case in the piece's image switch, so `VERTEX` is no longer in `NO_IMAGE_GENERATION_PROVIDERS`); video, speech, transcription and embeddings are not. Embeddings still have no `DEFAULT_EMBEDDING_MODELS` entry, so that path paths fail with an explanatory message instead of reaching a switch that has no case for it. +- **Chat-tier resolution answers "which model" from three places, and only one of them is the key.** `resolveModelIdForProvider` (`ee/agent/agent-helpers.ts`) now prefers the resolved key's own configured text models when its config carries a `models` array — that is the `MANUAL_MODEL_PROVIDERS` case (Vertex, Custom, Cloudflare Gateway), where the admin typed the exact ids the key exposes. **An empty catalog is not a missing one**, and collapsing the two is how the first cut of this went wrong: a helper returning `string[] | undefined` treated "lists zero text models" the same as "has no catalog", so a key configured with only image models fell through to the curated list and resolved to a Gemini id it never offered. An admin-listed catalog is the whole truth about a key, so an empty one now refuses the turn with a message rather than guessing. Otherwise it falls back to the static `ALLOWED_CHAT_MODELS_BY_PROVIDER`, and if the provider is missing from that too it returns the raw tier id with its vendor prefix stripped — which is how a Vertex key once resolved to `claude-sonnet-4-6` and handed it to the Gemini client. That map is `Partial<`, so omitting a chat-capable provider compiles cleanly and fails only at the endpoint. **The key's own `modelScope`/`modelIds` allow-list is applied last, to whichever candidate list was chosen**, and a resolution with nothing left refuses the turn rather than returning a model the key forbids. `GetProviderConfigResponse` carries both fields for that, populated at all three construction sites (`getChatProvider`, `getConfigOrThrow`, `enrichWithKeysIfNeeded`); adding fields there is backward compatible because the AI piece reads the response as a typed shape rather than parsing it. I first recorded this as too big for a provider PR — "a wire contract the engine and the AI piece both consume" — having never measured it. It was about thirty lines across four files. Measure before recording something as out of scope; the estimate outlives the guess. **The resolver is exported and has five call sites across four files, and only two of them are runs.** `agent-rpc-handlers` resolves the chat turn's model *and* its fast model, and `chat-personalization-service` resolves both for research — those four must be given the resolved row's `config`, `modelScope` and `modelIds`, or the key's constraints are bypassed on exactly the paths a user takes. `resolveTierModel` inside `agent-helpers` is the fifth. `agent-service` and `agent-draft-ai` also call it but hold only a provider name and set a stored default on a draft, so they are correctly left thin. Adding a parameter here means grepping `resolveModelIdForProvider|resolveFastModelId` across `packages/server`, not editing the call site in front of you — threading only `resolveTierModel` looks complete, passes every test, and still leaves the real chat path unconstrained. One trap when touching this function: a tier id (`fast`, `smart`) is **not** a model id, so the preferred pick must be the resolved native id unless the selection is itself among the candidates — passing `selectedModel` straight through makes `resolveFastModelId` return the first curated model instead of the fast tier's. - **A failed credential validation tells the admin nothing, for every provider except Cloudflare Gateway.** `aiProviderService.validateProviderCredentials` gates the upstream message behind `includeHttpErrorInMessage`, which is `provider === CLOUDFLARE_GATEWAY` and nothing else, so everyone else gets a bare `Failed to validate credentials for `. The cause is not lost — it is logged one line earlier (`log.error({ error }, '[aiProviderService#validateProviderCredentials] ...')`) and passed as the `httpErrorResponse` error param — but **web never renders `httpErrorResponse`**, so the only way to diagnose a rejected key is the server log. Grep the log for `validateProviderCredentials` before assuming the provider integration is broken — that text is the whole diagnosis, and it is often not about credentials at all. Confirmed case: a brand-new xAI team with no credits purchased answers `GET /v1/models` with `403 permission-denied — Your newly created team doesn't have any credits or licenses yet`, naming the console page that fixes it, and we render that as "Failed to validate credentials for xAI" — sending the admin off to regenerate a key that was never wrong. Vendors also phrase real key failures inconsistently (xAI uses `400 Incorrect API key provided`, not a 401). The corollary: **a provider that saves without error is not a working provider.** A no-credits 403 and a bad key are indistinguishable in the UI, so only an actual generation proves a key end to end. This is an admin-only surface (`platformAdminOnly`), so there is little reason to keep hiding it. - **A provider's logo is an asset someone has to upload, not something the code ships.** `AiProviderInfo.logoUrl` in `packages/web/src/features/agents/ai-providers.ts` is a plain string rendered into an ``, and every provider points at `https://cdn.activepieces.com/pieces/.png` — nothing is bundled. Adding a provider therefore carries a cross-team dependency with no compile-time or test signal: a slug with no asset behind it renders a broken-image icon in the platform admin list, and only a live request tells you. Check the URL with `curl -o /dev/null -w '%{http_code}'` before assuming it works — a vendor that already ships as a *piece* usually has its logo there already (`deepseek.png`, `grok-xai.png` did), so start the upload request only for the genuinely missing ones. A Vite asset import also satisfies `logoUrl` (see `GoogleIcon` in `platform/security/sso/index.tsx`) and removes the runtime CDN dependency for air-gapped installs, but it diverges from every other provider — treat it as a fallback, not the default. -- **`AIProviderConfig` is an *untagged* `z.union`, so a new provider's config schema must sit ahead of the empty ones — and "empty" includes a schema whose every field is optional.** Zod strips unknown keys and a union returns the first member that parses, so `AnthropicProviderConfig` (`z.object({})`) matches *any* object: list it before a `{ baseUrl?: string }` config and a configured base URL is silently reduced to `{}` — no error, no log, the admin's override just stops existing on the next read. The file carries an `Order matters` comment, but it says "empty ones last", which reads as though only a literal `z.object({})` is at risk. The safe rule is to insert any new config immediately after the last schema with a *required* field (today `BedrockProviderConfig`). `ProviderConfigUnion` is discriminated on `provider` and so is immune; only the two untagged unions (`AIProviderConfig`, `AIProviderAuthConfig`) bite. Both live twice — `packages/core/shared/.../management/ai-providers/index.ts` (zod classic) and `packages/core/piece-types/.../ai-providers.ts` (zod/mini, the copy pieces use) — and every provider edit has to land in both. **There is a third copy the shared package does not own:** `createFormSchema` in the admin dialog (`.../setup/ai/universal-pieces/upsert-provider-dialog.tsx`) re-declares a per-provider schema, branching explicitly on Azure / Cloudflare / Custom / Bedrock and falling through to a generic case whose `config` is a union of three empty objects. **A provider with a non-empty config and no branch there loses that config entirely** — `zodResolver` hands react-hook-form the *parsed* value, so the strip happens before submit and the setting is never sent, with no error anywhere. Fixing the shared union does not fix this one; grep for every union of config schemas when adding a provider. The dialog only diverges from the correct `ProviderConfigUnion` to make auth optional in edit mode, so collapsing it onto the shared discriminated union is the real repair. (Testing that file directly is awkward: importing it pulls in a transitive dep that touches `document` at import time, which a node-env vitest cannot load — the schema factory would have to move out of the component file first.) +- **`AIProviderConfig` is an *untagged* `z.union`, so a new provider's config schema must sit ahead of the empty ones — and "empty" includes a schema whose every field is optional.** Zod strips unknown keys and a union returns the first member that parses, so `AnthropicProviderConfig` (`z.object({})`) matches *any* object: list it before a `{ baseUrl?: string }` config and a configured base URL is silently reduced to `{}` — no error, no log, the admin's override just stops existing on the next read. The file carries an `Order matters` comment, but it says "empty ones last", which reads as though only a literal `z.object({})` is at risk. The rule is **not** "after the last schema with a required field" — that phrasing reads as "append after `BedrockProviderConfig`" and is how a Vertex config got silently reduced to `{ region }`, losing `project` and `models` with no error. What matters is the *subset relation*, not the count: `BedrockProviderConfig` is `{ region }`, a strict subset of `{ project, region, models }`, so it matches a Vertex object first and strips the rest. Order by specificity — any schema whose required keys are a subset of yours must sit **after** you. Check with a one-line parse before trusting the order: `AIProviderConfig.parse(yourConfig)` must return every field it was given. `ProviderConfigUnion` is discriminated on `provider` and so is immune; only the two untagged unions (`AIProviderConfig`, `AIProviderAuthConfig`) bite. Both live twice — `packages/core/shared/.../management/ai-providers/index.ts` (zod classic) and `packages/core/piece-types/.../ai-providers.ts` (zod/mini, the copy pieces use) — and every provider edit has to land in both. **The third copy moved rather than disappearing.** `.../setup/ai/universal-pieces/` is gone and the connect dialog now builds from `PROVIDER_CREDENTIAL_FIELDS`, but `config-detail.tsx` declares its own `ManualProviderConfig = z.union([...])` that gates whether the manual-models panel sends `config` at all. A provider missing from it fails `safeParse`, its models are dropped from the payload, and they silently vanish on reload — which for a `MANUAL_MODEL_PROVIDERS` member means the only way to choose a model does not work. So it is still **three** unions per provider: the two shared copies plus this one, and it is the easiest to miss because it lives in a component and nothing references the provider by name. The rest of this sentence describes that deleted file and is kept only as history: `createFormSchema` in the admin dialog (`.../setup/ai/universal-pieces/upsert-provider-dialog.tsx`) re-declares a per-provider schema, branching explicitly on Azure / Cloudflare / Custom / Bedrock and falling through to a generic case whose `config` is a union of three empty objects. **A provider with a non-empty config and no branch there loses that config entirely** — `zodResolver` hands react-hook-form the *parsed* value, so the strip happens before submit and the setting is never sent, with no error anywhere. Fixing the shared union does not fix this one; grep for every union of config schemas when adding a provider. The dialog only diverges from the correct `ProviderConfigUnion` to make auth optional in edit mode, so collapsing it onto the shared discriminated union is the real repair. (Testing that file directly is awkward: importing it pulls in a transitive dep that touches `document` at import time, which a node-env vitest cannot load — the schema factory would have to move out of the component file first.) - **Every catalog field is optional, and two providers never match at all.** Azure's `listModels` returns *deployment names* (arbitrary admin-chosen strings), and CUSTOM / CLOUDFLARE_GATEWAY ids are hand-typed, so `modelCatalog.lookup` returns `undefined` for them by design. Any UI reading diff --git a/brain/knowledge/connections-auth/app-connections.md b/brain/knowledge/connections-auth/app-connections.md index eb532b915a1b..d5a4f80cf83a 100644 --- a/brain/knowledge/connections-auth/app-connections.md +++ b/brain/knowledge/connections-auth/app-connections.md @@ -28,6 +28,7 @@ Encrypted credential records (OAuth2 tokens, API keys, basic/custom auth, OIDC p - `AP_ENFORCE_CONNECTION_PIECE_BINDING` (default `false`) makes a step resolve only connections whose `pieceName` equals the step's own piece; a mismatch raises a USER-level `ConnectionPieceMismatchError`. The check lives in the **engine's** `connection-resolver`, not the worker endpoint. Set the var on the **app** container — the engine cannot read `process.env` (sandbox env is an allowlist), so the flag rides `WorkerSettings` → `SandboxSettings` → sandbox env, the same path as `AP_DEV_PIECES`. Code / loop / router steps have no piece, so a missing name is a denial — they lose connection access entirely, and enabling the flag breaks flows that feed a connection into custom JS. - `metadata.accountIdentifier` (the "which account is this" label) must be **rewritten on every upsert, never left untouched** — `spreadIfDefined` omits the column and TypeORM `upsert(connection, ['id'])` then leaves the old value in place, so a reconnect that fails to resolve would keep labelling the connection with an account it no longer authenticates as. `mergeConnectionMetadata` also strips the key from caller-supplied `metadata`, because `metadata` is a caller-owned jsonb bag: without that, any `WRITE_APP_CONNECTION` holder can forge the label. Note `POST /:id` (update) still replaces the whole bag. +- **A pasted service-account JSON is attacker-controlled config, and `token_uri` inside it is an SSRF vector.** A Google service-account key file carries its own `token_uri`/`auth_uri`, and `google-auth-library` (`GoogleAuth`, `JWT`) *honours* them — so `{ ...JSON.parse(raw) }` handed to `googleAuthOptions.credentials` lets whoever pasted the file redirect the OAuth token exchange to any host, link-local metadata included, behind a valid-looking `type: service_account`. Forward only the fields you use (`client_email`, `private_key`, `project_id` if you check it); never spread the parsed object. This bit the Vertex AI provider in review, and **`packages/pieces/community/google-vertexai/.../common.ts` still does `{ ...raw, private_key }`** — worse there than in a provider, because a piece connection needs only connection-write, not platform admin. The same `new GoogleAuth`/`new JWT` shape recurs across the google-* pieces, so grep before assuming one is narrow. Note `.claude/rules/safe-http.md` does **not** catch this: the request is made inside the auth library, never through `safeHttp.axios`, so the rule's "admin config reaching outbound HTTP" clause has to be applied by hand. ### Key files Entry point: `appConnectionService`, exported from the app-connection service and reached through `appConnectionModule`, registered in `packages/server/api/src/app/app.ts`. diff --git a/brain/knowledge/decisions/000029-the-engine-never-imports-a-piece-a-fresh-child-process-does.md b/brain/knowledge/decisions/000029-the-engine-never-imports-a-piece-a-fresh-child-process-does.md deleted file mode 100644 index f79ea7bba5ac..000000000000 --- a/brain/knowledge/decisions/000029-the-engine-never-imports-a-piece-a-fresh-child-process-does.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -status: accepted ---- - -# The engine never imports a piece, a fresh child process does - -## Decision - -Nothing in the engine process may `import()` a piece package. A piece is loaded only inside a child process spawned per call and killed when that call returns (`packages/server/engine/src/lib/core/piece/piece-child.ts`, shipped as its own esbuild entry `piece-child.js`). The parent talks to it with exactly two requests — `describe` (piece metadata as JSON plus the list of paths that are functions) and `call` (`['actions', 'send_http', 'run']` and its arguments) — over `piece-runner.ts`. - -## Context - -The engine is long-lived and served many operations, each `import()`ing pieces into the same process. Resident piece modules (and their duplicated `@activepieces/shared` copies) never came back — measured as hundreds of MB of a single engine heap. Loading a piece to read its metadata, or just to discover an auth `validate` hook does not exist, cost the same permanent memory as running it. Measured after the change with `smoke-test/verify-memory.sh` (webhook → data-mapper → return-response, 2000 runs): the engine ends **28 MB below** its warm baseline, i.e. V8 gives the heap back because nothing from the pieces stays resident. - -## Why - -Process exit is the only reliable way to free a required module graph; a cache or a `delete require.cache` does not free native handles or the transitive graph. Everything the engine needs about a piece is data (props, auth, trigger type, `contextInfo`), so it can cross a process boundary — only *behaviour* has to run where the piece is loaded. Rejected: keeping metadata loading in-process and isolating only `run` (metadata loading is what most operations do, so the leak would remain), and a persistent piece process per version (it re-creates the leak with extra lifecycle). - -The child is a real bundled engine entry, not an inline `--eval` script, because file materialization must live with the engine's own file processor: `ApStreamingFile.body` is a `Readable` and cannot be structured-cloned. - -## Consequences - -- **The piece's context cannot be proxied back to the engine — the child has to build it.** Two hard constraints kill any marker/RPC bridge, and both fail silently: (1) parts of the context are **synchronous by contract** — `CreateWaitpointResult.buildResumeUrl` returns a `string` and pieces call it without `await`, which an RPC can only answer with a Promise; (2) pieces **mutate objects after handing them to a hook** — `return-response-and-wait-for-next-webhook` passes `response` to `createWaitpoint` and only then writes the resume URL into its `headers`, and in-process the engine sees that through the shared reference. A snapshot does not. Since almost every context function is just HTTP over scalars (`apiUrl`, `engineToken`, `projectId`, `flowId`) the child builds them itself; only the collectors the engine reads afterwards (`hookResponse` tags/stop/respond/paused/responseToSend, trigger `listeners` and `scheduleOptions`) travel back, as plain data on the result. -- `describe` costs one extra spawn per piece per engine process (memoized by `name@version`), so a 10-step flow on one piece is 11 spawns, not 20. -- **A piece call costs ~79 ms** (spawn + import the piece + build the context + run + IPC), measured on a warm cache against the built child bundle. The benchmark flow's three calls show up as `RUN=400ms` in `FlowRun.timeline` with `PROVISION=0ms, BOOT=0ms`. That is the price of never letting a piece into the engine heap; if the sync-webhook path ever needs it back, the upgrade is one child per *flow run* instead of per *call* — the process still dies at the end of the run, so nothing accumulates, and an N-step flow pays one spawn instead of N. -- `fileProcessor` returns a `__apFileSource` marker and the child calls `materializeFile`, so nothing is fetched until the piece actually runs — a validation failure now opens zero connections. The cost: an unreachable file URL fails *in the child when the step runs* rather than in the parent's prop validation (same message, later stage), because you cannot check a remote file without fetching it. -- Piece metadata reaches the engine JSON-round-tripped, so any *function* on a property (dropdown `options`, dynamic `props`) is addressable only by path, never callable in-process. -- A sandbox gets the engine by **file-by-file copy**, not by copying a directory: `engineInstaller` (`packages/server/sandbox`) copies each bundle into the cache dir that isolate mounts at `/root/common`. A new engine entry point must be added to that list or it is simply absent at runtime — the Docker image, which copies all of `dist/packages/engine`, looks perfectly fine and hides it. -- The child is a second esbuild entry, so anything that builds it (including `vitest.config.ts`, which builds it for tests) must reuse the same `alias` map as `esbuild.config.mjs` — miss it and tests bundle `@activepieces/*` from `dist` while production bundles from `src`, so a green suite proves nothing about the shipped child. -- **The child inherits the engine's node flags, and its OOM is detected rather than prevented.** `engineNodeArgs` sets `--max-old-space-size` as the *only* bound on engine memory (isolate passes no `--mem`/`--cg-mem`), so spawning the child without it would leave it on V8's default heap — spawn it with `[...process.execArgv, entry]`. Engine + child can then together reach `AP_SANDBOX_MEMORY_LIMIT`, and that is accepted: the worker is the sandbox, so it dies and restarts. What must not happen is the failure being anonymous — `piece-runner` classifies the child's exit the way `sandbox.ts` classifies the engine's (V8 heap message, exit 134, SIGABRT, SIGKILL) and raises a user-level `PieceMemoryLimitError`. A budget split between the two processes was tried and reverted: it bought little, and a floor on each share silently overshot the limit on small configurations. -- The child inherits no in-process guards the parent installs (e.g. the SSRF monkeypatches in `network/ssrf-guard.ts`). Whatever must apply to piece code has to be installed in `src/piece-child.ts`. diff --git a/brain/knowledge/engineering/ci-pr-review-hygiene.md b/brain/knowledge/engineering/ci-pr-review-hygiene.md index ff6b73394fe8..481dc56d1e11 100644 --- a/brain/knowledge/engineering/ci-pr-review-hygiene.md +++ b/brain/knowledge/engineering/ci-pr-review-hygiene.md @@ -30,6 +30,9 @@ Enforcement is the **`Codeowners review` repository ruleset** (active on the def - **A workflow that opens a PR must authenticate with `secrets.CROWDIN_PRS`, not `GITHUB_TOKEN`.** Despite the name, that PAT is this repo's open-a-PR-as-a-bot token: `crowdin-pr-merger.yml`, `reusable-finalize-translations-pr.yml` and — the tell — `release-self-hosted.yml`, which has nothing to do with Crowdin and uses it for both `actions/checkout`'s `token:` and `gh pr create`'s `GH_TOKEN`. Those jobs declare only `permissions: contents: read`, because the PAT does the pushing and the PR-opening; raising `GITHUB_TOKEN` to `contents: write` / `pull-requests: write` instead is treating the symptom, since *Allow GitHub Actions to create and approve pull requests* is evidently off for the org (not readable without `admin:org`). The failure mode is nasty because it is half-done and unattended: the branch pushes fine and only `pulls.create` fails, leaving an orphan `auto/*` branch every scheduled run. Copy `release-self-hosted.yml`, and have the job delete its own branch on failure so a bad week retries clean instead of accumulating. - **Workflow actions are pinned to major-version tags, not SHAs** (`actions/checkout@v5`, `oven-sh/setup-bun@v2`). The only SHA pins live in the CodeQL security workflow. Reviewers — human and AI — regularly suggest SHA-pinning a single new workflow; decline it. Moving to SHA pinning is a repo-wide policy call, and a half-pinned `.github/` is worse than a consistent one. - **`redis-memory-server` compiles Redis from source during `bun install`, so its version must stay pinned.** It is in `trustedDependencies`, and with no version configured it defaults to `stable` — whatever `download.redis.io/redis-stable.tar.gz` points at today. When that moved to Redis 8.10.0 (2026-07-29), the bundled module tree (redisearch, redistimeseries, LibMR) started failing to build on runners and took `bun install` down across every branch: 8.10.0 vendors the module sources into the tarball and changes the default make goal to `build`, which compiles every module under `modules/*/src` regardless of `BUILD_WITH_MODULES`. It reads as flakiness because `ci.yml` caches `~/.bun/install/cache` but not the compiled binary, so each run recompiles and only sometimes survives. Root `package.json` pins `redisMemoryServer.version` to **8.8.1**, the newest release that still builds core-only — treat it as a ceiling, bump it deliberately, and never go back to `stable`. +- **`validate-publishable-packages` compares against npm, not against `main`, so touching a published piece without bumping it fails CI on its own.** The error is `[packagePrePublishValidation] package version not incremented, path=packages/pieces/community/, version=X`. Editing *any* file in a published package is enough — a one-line change to the AI piece's model factory tripped it while `@activepieces/piece-ai` sat at `0.9.0` on npm. Check with `curl -s https://registry.npmjs.org/@activepieces/piece- | jq -r ."dist-tags".latest`, and follow the piece's own history for the size of the bump: capability additions have gone minor, fixes patch. **The version lives in two files** — `package.json` *and* `bun.lock`, which records each workspace's version — so bump then `bun install`, or the lockfile check fails instead. Distinct from the merge-drift trap below: this one fires before any merge, and only for packages that are actually published. +- **Standalone `prettier --check` disagrees with the `prettier/prettier` eslint rule in this repo, so it is a false guide — run `eslint` on the file.** From `packages/web`, `../../node_modules/.bin/eslint 'src/path/to/file.ts'` reproduces CI exactly and `--fix` resolves it. Standalone prettier flags files that are clean on `main` and that CI passes, whether invoked through `npx` or the pinned 2.8.4 with `--config .prettierrc` — so "prettier says it's unformatted" proves nothing, and chasing it wastes the time the eslint run would have taken. Only `packages/web` is prettier-enforced: the server and `packages/core/*` are 4-space, semicolon-free, and running prettier over them would rewrite the file wholesale. +- **`bun install` on a recent bun adds `"configVersion": 0` to `bun.lock`, which is not on `main`.** It rides along in any commit that touches the lockfile and reads as an unrelated change; drop the line and re-run `bun install --frozen-lockfile` to confirm the lockfile is still consistent without it. - **A version bump that merges cleanly can still be wrong — check what `main`'s number *means*, not whether it conflicts.** Two branches bumping the same package to the same number do not conflict, so git takes it silently; but if `main`'s copy of `0.5.0` is another PR's content and yours adds further exports on top, you ship new exports under an already-published version and nothing catches it. Seen merging [#15001](https://github.com/activepieces/activepieces/pull/15001) after the six-providers PR landed: `core-piece-types` and `pieces-framework` auto-merged at `0.5.0` / `0.37.0` and both needed a further bump. Only a *conflicting* version (like `core/shared` `0.140.0` vs `0.141.0`) forces you to think; the clean ones are the dangerous ones. After any merge, re-check every package you bumped against `git show origin/main:/package.json`. The reverse also happens: when review makes you *delete* code, the bump it justified can become dead — after acting on review, `git diff origin/main...HEAD -- /src` and drop the bump if it is empty. On #15001 two packages ended up byte-identical to `main` while still carrying a bump, which is noise at best and a version collision at worst. - **`@activepieces/shared` re-exports from `@activepieces/core-execution`, so a partial rebuild produces phantom "has no exported member" errors in unrelated files.** Rebuilding `core/shared` against a stale `core/execution` dist drops those re-exports, and the API typecheck then fails in `ee/agent/*` on symbols like `GetPersonalizationConfigRequest` — which live in `core/execution/src/lib/workers/worker-contract.ts`, not in shared at all. It reads exactly like a bad merge. The dependency order that actually works is `core/utils` → `core/piece-types` → `core/formula` → `core/execution` → `core/shared` → `server/utils` → `pieces/framework` → `core/ai-providers`; skipping a link silently poisons everything downstream of it. The same staleness makes an editor report missing enum members that exist in the source. - **To pull a file back out of a PR, restore it from the merge-base, never from `origin/main`.** A PR's diff is computed against the merge-base, so `git checkout origin/main -- ` does not "revert" the file — it imports every change `main` made to it since the fork and attributes them to you. Dropping one web file from [#15001](https://github.com/activepieces/activepieces/pull/15001) that way would have silently added 82 insertions / 44 deletions of somebody else's work. `git checkout $(git merge-base origin/main HEAD) -- ` makes it byte-identical to where the branch started, so it leaves the diff entirely and merges cleanly instead of conflicting. Verify with `git diff --quiet $(git merge-base origin/main HEAD) -- ` before committing, and read `git status` first — a `bun.lock` left dirty by an earlier `bun install` loves to ride along on a commit like this. diff --git a/brain/knowledge/engineering/server-module-anatomy.md b/brain/knowledge/engineering/server-module-anatomy.md index 5ad072027669..1793a0670e79 100644 --- a/brain/knowledge/engineering/server-module-anatomy.md +++ b/brain/knowledge/engineering/server-module-anatomy.md @@ -147,7 +147,7 @@ Verify with `npm run lint-dev` and `npm run test-api`. - **Duplicate migration timestamps are safe, not luck.** Several timestamps are shared by two or three files (1787, 1794, 1797, 1798, 1811, 1818, 1819, …). TypeORM sorts `getMigrations()` by parsed timestamp and `Array.prototype.sort` has been spec-stable since ES2019, so ties resolve to array order in `postgres-connection.ts` — the same code on every instance, hence the same order everywhere. Do not add a tie-break scheme. Do keep a real dependency (column → index on that column) on *distinct* timestamps rather than relying on array order to express it. - **`EntitySchema` supports partial-index `where`, but not expression columns.** For a partial index on a bare column (e.g. `ON file(platformId) WHERE projectId IS NULL`), pass `where: '"projectId" IS NULL'` alongside `columns: ['platformId']` — TypeORM 0.3.x's `EntitySchemaIndexOptions.where` is honored by the Postgres driver (`PostgresQueryRunner` line 2442: `${where ? "WHERE " + where : ""}`), so `synchronize` can stay on and `migration:generate` tracks the index correctly. Reserve `synchronize: false` for **expression indexes** — `columns` is `string[]` of bare column names with no expression syntax, so an index like `ON file(type, (metadata->>'flowId'))` (see `idx_file_sample_data_flow_id`) genuinely can't be expressed and needs the opt-out. Blindly using `synchronize: false` for every hand-written index (which I did once and got called on) leaves TypeORM blind to the index — future `migration:generate` won't drop it if you remove it from the entity, and drift can silently accumulate. - **`UpdateResult.affected` is `undefined` on PGlite — never branch on it.** TypeORM's Postgres driver sets `affected` from `raw.rowCount`, and `typeorm-pglite` returns PGlite's `Results` (`{ rows, fields, affectedRows }`) with no `rowCount`. So the compare-and-set idiom `if (result.affected === 0) return null` is *always false* on PGlite and every predicate in the `WHERE` becomes decorative — the guard silently passes. This is not test-only: `AP_DB_TYPE=PGLITE` is the documented one-line Docker install (`docs/install/options/docker.mdx`). It hit MCP OAuth (`mcpOAuthCodeService.consume`), where it made authorization codes replayable, unbound to their client and redirect_uri, and immune to expiry. Use `.returning('*')` and test `updateResult.raw` for emptiness instead — that works on both drivers. Confirmed against the pinned `@electric-sql/pglite` 0.3.14: a plain `UPDATE` answers `{ rows, fields, affectedRows }` with **`rowCount: undefined`**, while the same statement with `RETURNING *` fills `rows` correctly (0 on no match, 1 on match). Note PGlite *does* report `affectedRows` — it is only `rowCount`, the field TypeORM reads, that is missing, so "PGlite loses the count" is the wrong mental model. The remaining call sites were converted in 2026-08 (`ee/agent/agent-rpc-handlers.ts`, `ee/projects/platform-project-service.ts`); a `.affected` that only feeds a log line was left alone. **Integration tests here run on PGlite** (`.env.tests` sets `AP_DB_TYPE=PGLITE`), so they do exercise this class by default; it was still missed because nothing asserted on the guard. Earlier revisions of this page claimed the suite ran against a real Postgres server, which is wrong. Prefer `.returning('id')` over `.returning('*')`: on a table like `agent_conversation` the star form hauls the whole `messages` jsonb back on every write, and a row only has to be counted, not read. -- **No concurrency property can be tested in the api integration suite.** `.env.tests` runs PGlite, one in-process connection, so a second session cannot exist: `SELECT … FOR UPDATE` held from the test blocks nothing, two `Promise.all` requests serialise before either transaction opens, and a lost update is unobservable. A test written for a race there passes with the lock removed, which reads as proof and is the opposite. Measured Aug 2026 while adding a row lock to the agent draft-tools edit: deleting `setLock('pessimistic_write')` left all 13 tests green. So pin the user-visible invariant, mutation-test the parts that *are* observable (a name in a denylist, a guard's SQL), and say plainly in the commit that the lock rests on Postgres row semantics rather than a reproduced race. Row locks use `.createQueryBuilder().setLock('pessimistic_write')` inside `transaction(...)` — three of the four sites in the repo take that form. +- **No concurrency property can be tested in the api integration suite.** `.env.tests` runs PGlite, one in-process connection, so a second session cannot exist: `SELECT … FOR UPDATE` held from the test blocks nothing, two `Promise.all` requests serialise before either transaction opens, and a lost update is unobservable. A test written for a race there passes with the lock removed, which reads as proof and is the opposite. Measured Aug 2026 while adding a row lock to the agent draft-tools edit: deleting `setLock('pessimistic_write')` left all 13 tests green. So pin the user-visible invariant, mutation-test the parts that *are* observable (a name in a denylist, a guard's SQL), and say plainly in the commit that the lock rests on Postgres row semantics rather than a reproduced race. Row locks use `.createQueryBuilder().setLock('pessimistic_write')` inside `transaction(...)` — three of the four sites in the repo take that form. Better still, ask whether the coordination is needed at all: when two rows must agree, deriving one from the other at read time removes the class, while a lock only guards it and cannot be demonstrated here. That is how the agent-move race was settled in Sep 2026 — a run resolves its project from the agent row rather than from the conversation's copy of it, which is a pure function and therefore testable on any database. - **`breaking = true` is the rollback-safety flag, not the customer-facing one.** It marks destructive DDL (`DROP TABLE`/`DROP COLUMN`, `ADD ... NOT NULL` without a default) for `rollback-migrations.ts`. It does *not* by itself mean the PR needs the `⛓️‍💥 breaking-change` label — decide that from upgrade impact on self-hosters and API consumers. - **A new `AppSystemProp` needs three edits, not one.** Add the enum entry in `system-props.ts`, a default in `systemPropDefaultValues` (`system.ts`), *and* a validator in `systemPropValidators` (`system-validator.ts`). Miss the validator and `validateEnvPropsOnStartup` throws `systemPropValidators[prop] is not a function` at boot — every API test fails on setup, not just the new one. Document the var in `docs/install/reference/environment-variables.mdx` too. - **`permission: undefined` on `securityAccess.project(...)` silently allows any project member.** The argument is required in practice even though the type tolerates omitting it. diff --git a/brain/knowledge/execution-runtime/index.md b/brain/knowledge/execution-runtime/index.md index d3456d9df1a2..899ef70ff7c8 100644 --- a/brain/knowledge/execution-runtime/index.md +++ b/brain/knowledge/execution-runtime/index.md @@ -48,8 +48,6 @@ The four calls a run emits to the app during execution: `updateRunProgress`, `up - **The S3 piece-tarball cache shadows the CDN, so changing *what* gets cached means bumping `S3_PIECES_PREFIX`, not purging it.** `resolve()` (`piece-bundle.ts`) checks S3 before the CDN, so whatever `BUNDLE_PIECE` wrote wins for every later request. Until Aug 2026 that job cached the **npm** tarball, which for versions published before piece repackaging still declares its build-time deps — measured cost: 12 resident `@activepieces/shared` versions holding 388 MB of a 554 MB engine heap on cloud. The job now prefers the CDN artifact, but fixing the writer does not fix the objects already written, and *purging* them cannot work: a rolling deploy leaves old app instances writing npm tarballs back into the prefix for the rest of the rollout, and the purge has no way to know when the last one is gone. So the prefix is versioned (`pieces/` → `pieces/v2/`) — old code can only write the old prefix, so the new one is reachable only by a CDN-preferring writer. Same reflex as `LATEST_CACHE_VERSION` on the worker: when the meaning of a cached value changes, move the key; the abandoned prefix is dead storage to be swept later, never a correctness dependency. - **`extractConnectionIds` misses agent-tool connections.** It only reads step/trigger `settings.input.auth`, never `agentTools[].pieceMetadata.predefinedInput.auth`, so `flowVersion.connectionIds` under-reports and "which flows use this connection" lies. - **A code-sandbox `functions` entry must be a standalone declaration, never an object-method shorthand.** The v8 isolate re-injects each entry as source via `const ${key} = ${value.toString()}` (`v8-isolate-code-sandbox.ts`). A standalone `function flattenNestedKeys(...) {...}` (as exported from `script-evaluator.ts`) stringifies to a valid RHS and keeps recursion working by its inner name; an inline object-method shorthand stringifies to `flattenNestedKeys(...) {...}`, a syntax error as a `const` RHS. Keep it a standalone `function` export, never a method. For the same reason do **not** relocate a sandbox-injected function behind a separately-built package boundary (e.g. `@activepieces/core-utils`): its serialized `.toString()` would then depend on that package's build/minify config staying isolate-friendly. The trap: `no-op-code-sandbox.ts` passes the function by reference and tolerates either form, so a test run that skips the isolated-vm suite ships the bug green. Related: the `functions` **key** is also the global name users type in flow inputs (`{{flattenNestedKeys(...)}}`), so it is a public contract string, not an implementation detail. Keep it a hardcoded literal (matched by `FLATTEN_NESTED_KEYS_PATTERN` in `props-resolver.ts`); never derive it from the function's `.name`, which mangles under minification and would wrongly couple the token to the JS identifier. -- **The piece context is lazier and more mutable than it reads.** Three traps when assembling it anywhere new (they all surfaced when context assembly moved into the piece child process, `core/piece/piece-context-builder.ts`): `project.externalId` is a **function the piece calls**, not a value — resolving it while building the context fires a `/v1/worker/project` request on *every step*; the backward-compatibility wrapper (`backwardCompatabilityContextUtils.makeActionContextBackwardCompatible`) must wrap the finished context or pieces on older context versions die with `ctx.run.pause is not a function`; and the legacy pause shim calls `createWaitpoint()` **without awaiting it**, so whoever owns the context has to drain in-flight hook work before the process ends or the waitpoint POST never lands and the run hangs until timeout. -- **An error loses its friendly HTTP details the moment it crosses a process boundary.** `formatPieceError` (`friendly-piece-error.ts`) reads `error.response.{status,body}`, `error.status`, and falls back to `error.constructor.name` for `errorName` — but on `HttpError` (`pieces-common`) `response` is a **prototype getter** and `name` is plain `'Error'`. Structured clone, `{...e}`, and `JSON.stringify` all copy own enumerable props only, so a child-process runner that ships an error back verbatim silently drops `status`, `apiMessage`, and the error name, and the step renders as an opaque JSON blob. Serialize errors explicitly: read the getter keys by name (`response`, `request`, `status`, `headers`, `body`, `error`) plus own props, and carry `constructor.name` as `name`. Same trap applies to the run **result**: it must be JSON round-tripped, or an unresolved promise/function anywhere in the returned object throws `could not be cloned` from `process.send` and fails the step. - **A props-resolver script session is per-`resolve()`, never shared or hoisted.** `getPropsResolver(...).resolve(...)` builds a fresh `PropsResolver` per call, creates the script session via `scriptEvaluator.initSession()`, and disposes it in `resolve`'s `finally`, so an instance is single-use. Freshness is load-bearing: `setGlobal` is no-overwrite (`v8-isolate-code-sandbox.ts`) and injects each referenced step view once per resolve, so a session reused across resolves serves **stale step views** as flow state advances, and a reused instance would run on an already-disposed session. When refactoring props-resolver, capture `getStepView` and `scriptSession` inside `resolve` (they depend on the per-call `executionState`), not at instance scope, and never behind a shared mutable variable. --- diff --git a/brain/knowledge/flows-execution/flows.md b/brain/knowledge/flows-execution/flows.md index 3a984bc120bf..69d93dcbd4ef 100644 --- a/brain/knowledge/flows-execution/flows.md +++ b/brain/knowledge/flows-execution/flows.md @@ -35,6 +35,7 @@ Flows are the core automation primitive: a versioned directed graph of trigger + - **`transaction()` (`core/db/transaction.ts`) is a bare `dataSource.transaction()`** — it acquires a *new* connection, not a savepoint. Nesting it deadlocks, so check every caller before wrapping a service method that others may already call inside a transaction. - Step settings split a piece's props into an always-visible **essential** set and a collapsed **Advanced** section: a prop is Advanced only when it sets `advanced: true` (everything else — incl. `MARKDOWN`, tab/section group members, and checkbox reveal targets — stays essential). `propertyGroups` render as tabs, sectioned cards, or the "Add filter" builder. - **Flows stuck in `DELETING` keep eating the active-flow limit.** Deletion is a durable BullMQ system job (`delete-flow-`), not synchronous: `delete()` sets `operationStatus=DELETING` and enqueues, and the row plus `status=ENABLED` only go away when the job finishes. That job runs `sampleDataService.deleteForFlow`, whose `DELETE FROM file … metadata->>'flowId'=?` had no index — on the large prod `file` table it seq-scans, blows `statement_timeout`, exhausts its 2 attempts and lands **permanently** in the failed set. The flow is then hidden from the UI list (which filters `!=DELETING`) but still counted by the active-flows quota (`getUsage` counts `status=ENABLED`), so Publish silently shows the "Purchase Extra Active Flows" dialog instead of publishing — this is what breaks the `webhook-should-return-response` e2e monitor. Stuck flows are functionally dead (`preDelete` disables the trigger before the failing delete), so forcing their rows away is safe. Fixes on `fix/flow-delete-sample-data-timeout`: a partial expression index `idx_file_sample_data_flow_id` on `file (type, (metadata->>'flowId'))`, plus `operationStatus != DELETING` in the active-flow counts so the quota stops depending on delete-job success. +- **Any query that reads `flow_version.trigger` across a whole platform is TOAST-bound, not index-bound.** Trigger blobs are jsonb and TOAST'd on rows past ~2 KB, so a report that walks piece steps across all published flows on a platform pays ~2–5 ms of TOAST fetch + JSON parse per flow no matter how tight the WHERE clause is — a 10k-published-flow platform lands at 30 s–1 min. Endpoints of this shape (`platform/pieces-report/pieces-report.controller.ts` is the current example) must page + stream (`Readable.from(async iterable)`) so memory is bounded even when wall clock isn't; the escape hatch for the biggest platforms is a background job with async delivery, unlocked when a real sync request times out. Postgres jsonpath in-DB is not a shortcut — it still reads the whole TOASTed blob and the branch schema (`nextAction`/`children.*`/`onSuccessAction`/`firstLoopAction`/router children) drifts every time the flow shape changes, which is what `flowStructureUtil.getAllSteps` is authoritative over. - **`transferFlow` already deep-clones the whole flow — a callback that clones `step` again is quadratic.** `flowStructureUtil.transferFlow` opens with `JSON.parse(JSON.stringify(flowVersion))`, so the callback is handed a private copy and can mutate in place. Cloning per step instead is O(N²), because a `step` carries `nextAction` (the entire rest of the chain) plus loop/router children: cloning step *i* copies the remaining `N-i` steps. Measured on prod app containers (CDP CPU profile, 2026-08-21): the callback at `flow-version.service.ts` was **42% of wall-clock / ~84% of non-idle CPU**, at `transferStep` recursion depth 255 ≈ 32k step serializations per call, plus the GC churn behind ~2.5 GB RSS. It ran on **every** `getFlowVersionOrThrow` — including the default `removeConnectionsName=false, removeSampleData=false`, where the callback does nothing but the cloning still happens. Symptom was containers pegged at their `cpus: 1` cap and the 5s healthcheck `curl` timing out, which reads as "app unhealthy" with nothing crashed (the leftover zombie `curl`s are those killed healthchecks). Same pattern at `ee/…/project-state/diff/flow-diff.service.ts` (colder path, untouched here). When you write a `transferFlow` callback, mutate and return `step` — don't re-clone it. ### Editions diff --git a/bun.lock b/bun.lock index 6e37ad474b4d..17e5a5103503 100644 --- a/bun.lock +++ b/bun.lock @@ -105,6 +105,7 @@ "@ai-sdk/anthropic": "4.0.25", "@ai-sdk/azure": "4.0.26", "@ai-sdk/google": "4.0.29", + "@ai-sdk/google-vertex": "5.0.36", "@ai-sdk/openai": "4.0.25", "@ai-sdk/openai-compatible": "3.0.18", "@openrouter/ai-sdk-provider": "3.0.0", @@ -162,7 +163,7 @@ }, "packages/core/shared": { "name": "@activepieces/shared", - "version": "0.153.0", + "version": "0.155.0", "dependencies": { "@activepieces/core-execution": "workspace:*", "@activepieces/core-formula": "workspace:*", @@ -345,6 +346,7 @@ "@ai-sdk/anthropic": "3.0.72", "@ai-sdk/azure": "3.0.55", "@ai-sdk/google": "3.0.65", + "@ai-sdk/google-vertex": "4.0.113", "@ai-sdk/mcp": "1.0.11", "@ai-sdk/openai": "3.0.54", "@ai-sdk/openai-compatible": "2.0.42", @@ -1885,6 +1887,16 @@ "tslib": "2.6.2", }, }, + "packages/pieces/community/clay": { + "name": "@activepieces/piece-clay", + "version": "0.0.1", + "dependencies": { + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + "@activepieces/shared": "workspace:*", + "tslib": "2.6.2", + }, + }, "packages/pieces/community/clearout": { "name": "@activepieces/piece-clearout", "version": "0.1.8", @@ -2316,7 +2328,7 @@ }, "packages/pieces/community/cryptolens": { "name": "@activepieces/piece-cryptolens", - "version": "0.0.8", + "version": "0.1.0", "dependencies": { "@activepieces/core-piece-types": "workspace:*", "@activepieces/core-utils": "workspace:*", @@ -3314,6 +3326,20 @@ "tslib": "2.6.2", }, }, + "packages/pieces/community/formio": { + "name": "@activepieces/piece-formio", + "version": "0.0.1", + "dependencies": { + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*", + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + }, + "devDependencies": { + "tslib": "2.6.2", + "vitest": "3.2.6", + }, + }, "packages/pieces/community/formitable": { "name": "@activepieces/piece-formitable", "version": "0.0.8", @@ -4323,7 +4349,7 @@ }, "packages/pieces/community/hugging-face": { "name": "@activepieces/piece-hugging-face", - "version": "0.1.7", + "version": "0.1.8", "dependencies": { "@activepieces/core-piece-types": "workspace:*", "@activepieces/core-utils": "workspace:*", @@ -10228,7 +10254,7 @@ }, "packages/pieces/core/crypto": { "name": "@activepieces/piece-crypto", - "version": "0.0.27", + "version": "0.0.28", "dependencies": { "@activepieces/core-piece-types": "workspace:*", "@activepieces/core-utils": "workspace:*", @@ -10729,6 +10755,7 @@ "fastify-type-provider-zod": "6.1.0", "firebase-scrypt": "2.2.0", "fuse.js": "7.0.0", + "google-auth-library": "10.6.1", "http-status-codes": "2.2.0", "ioredis": "5.4.1", "jsonwebtoken": "9.0.1", @@ -11338,6 +11365,8 @@ "@activepieces/piece-claude": ["@activepieces/piece-claude@workspace:packages/pieces/community/claude"], + "@activepieces/piece-clay": ["@activepieces/piece-clay@workspace:packages/pieces/community/clay"], + "@activepieces/piece-clearout": ["@activepieces/piece-clearout@workspace:packages/pieces/community/clearout"], "@activepieces/piece-clearoutphone": ["@activepieces/piece-clearoutphone@workspace:packages/pieces/community/clearoutphone"], @@ -11564,6 +11593,8 @@ "@activepieces/piece-formbricks": ["@activepieces/piece-formbricks@workspace:packages/pieces/community/formbricks"], + "@activepieces/piece-formio": ["@activepieces/piece-formio@workspace:packages/pieces/community/formio"], + "@activepieces/piece-formitable": ["@activepieces/piece-formitable@workspace:packages/pieces/community/formitable"], "@activepieces/piece-forms": ["@activepieces/piece-forms@workspace:packages/pieces/core/forms"], @@ -17968,6 +17999,8 @@ "@activepieces/piece-ai/@ai-sdk/google": ["@ai-sdk/google@3.0.65", "", { "dependencies": { "@ai-sdk/provider": "3.0.9", "@ai-sdk/provider-utils": "4.0.24" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-SwdaJ6IqguyiVuDRgiRM4sHj7uUO4AETlQFFLF3jcEvu/3yrgIHfw2aM6bBNKSdalw0j25Pedx6qyHc2DWJwrg=="], + "@activepieces/piece-ai/@ai-sdk/google-vertex": ["@ai-sdk/google-vertex@4.0.113", "", { "dependencies": { "@ai-sdk/anthropic": "3.0.72", "@ai-sdk/google": "3.0.65", "@ai-sdk/openai-compatible": "2.0.42", "@ai-sdk/provider": "3.0.9", "@ai-sdk/provider-utils": "4.0.24", "google-auth-library": "^10.5.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-eG7dOZEt3umUWxHHlseYTXLDXdbsVkQ//99liHB38+vLat8SDYcp6+skvguev7OKbl66/8JHdgqRk5Wg7lcd3A=="], + "@activepieces/piece-ai/@ai-sdk/openai": ["@ai-sdk/openai@3.0.54", "", { "dependencies": { "@ai-sdk/provider": "3.0.9", "@ai-sdk/provider-utils": "4.0.24" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-j1qrNe/ebUKuE+fETzS+CVnczs11jQBR9y9M6aoKtJZAosg6SZnPC1Bb92e2u6yaSK+88TZoFhiY67uYphPitw=="], "@activepieces/piece-ai/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.42", "", { "dependencies": { "@ai-sdk/provider": "3.0.9", "@ai-sdk/provider-utils": "4.0.24" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-hjq485U/dpi6Hvjzw5+F1vohCrB1kibGHlUFknYGa4nOoCnSvFM1lTXEIyTAkjK1uXgTbNk8vw66lbEyWT12jg=="], @@ -19732,6 +19765,8 @@ "api/fast-xml-parser": ["fast-xml-parser@5.7.2", "", { "dependencies": { "@nodable/entities": "^2.1.0", "fast-xml-builder": "^1.1.5", "path-expression-matcher": "^1.5.0", "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w=="], + "api/google-auth-library": ["google-auth-library@10.6.1", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "7.1.3", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-5awwuLrzNol+pFDmKJd0dKtZ0fPLAtoA5p7YO4ODsDu6ONJUVqbYwvv8y2ZBO5MBNp9TJXigB19710kYpBPdtA=="], + "api/socket.io": ["socket.io@4.8.1", "", { "dependencies": { "accepts": "~1.3.4", "base64id": "~2.0.0", "cors": "~2.8.5", "debug": "~4.3.2", "engine.io": "~6.6.0", "socket.io-adapter": "~2.5.2", "socket.io-parser": "~4.2.4" } }, "sha512-oZ7iUCxph8WYRHHcjBEc9unw3adt5CmSNlppj/5Q4k2RIrhl8Z5yY2Xr4j9zj0+wzVZ0bxmYoGSzKJnRl6A4yg=="], "apify-client/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], @@ -20740,6 +20775,10 @@ "@activepieces/piece-ai/@ai-sdk/google/@ai-sdk/provider": ["@ai-sdk/provider@3.0.9", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-/ngMKqKdL9dSlY/eQ3NFDzzFyw0Hix+cbFFlyuKEKcOgpHdBt/spKUvX/i0wGrDLFPYJeVvv3N0j92LxWRL7yQ=="], + "@activepieces/piece-ai/@ai-sdk/google-vertex/@ai-sdk/provider": ["@ai-sdk/provider@3.0.9", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-/ngMKqKdL9dSlY/eQ3NFDzzFyw0Hix+cbFFlyuKEKcOgpHdBt/spKUvX/i0wGrDLFPYJeVvv3N0j92LxWRL7yQ=="], + + "@activepieces/piece-ai/@ai-sdk/google-vertex/google-auth-library": ["google-auth-library@10.6.1", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "7.1.3", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-5awwuLrzNol+pFDmKJd0dKtZ0fPLAtoA5p7YO4ODsDu6ONJUVqbYwvv8y2ZBO5MBNp9TJXigB19710kYpBPdtA=="], + "@activepieces/piece-ai/@ai-sdk/openai/@ai-sdk/provider": ["@ai-sdk/provider@3.0.9", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-/ngMKqKdL9dSlY/eQ3NFDzzFyw0Hix+cbFFlyuKEKcOgpHdBt/spKUvX/i0wGrDLFPYJeVvv3N0j92LxWRL7yQ=="], "@activepieces/piece-ai/@ai-sdk/openai-compatible/@ai-sdk/provider": ["@ai-sdk/provider@3.0.9", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-/ngMKqKdL9dSlY/eQ3NFDzzFyw0Hix+cbFFlyuKEKcOgpHdBt/spKUvX/i0wGrDLFPYJeVvv3N0j92LxWRL7yQ=="], @@ -21206,6 +21245,12 @@ "api/ai-gateway-provider/@openrouter/ai-sdk-provider": ["@openrouter/ai-sdk-provider@2.10.0", "", { "peerDependencies": { "ai": "^6.0.0", "zod": "^3.25.0 || ^4.0.0" } }, "sha512-FMsAEjLUt5pWuRE2LDC/LCvVrFjLlrEzUITH5+5SZtfq7KZ2wrOHjQVxzz92sju8S9ltpzW87CLW8/b0oBXVCw=="], + "api/google-auth-library/gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], + + "api/google-auth-library/gcp-metadata": ["gcp-metadata@8.1.2", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="], + + "api/google-auth-library/jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], + "api/socket.io/debug": ["debug@4.3.7", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ=="], "api/socket.io/engine.io": ["engine.io@6.6.7", "", { "dependencies": { "@types/cors": "^2.8.12", "@types/node": ">=10.0.0", "@types/ws": "^8.5.12", "accepts": "~1.3.4", "base64id": "2.0.0", "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.4.1", "engine.io-parser": "~5.2.1", "ws": "~8.18.3" } }, "sha512-DgOngfDKM2EviOH3Mr9m7ks1q8roetLy/IMmYthAYzbpInMbYc/GS+fWFA3rl1gvwKVsQrVV61fo5emD1y3OJQ=="], @@ -21978,6 +22023,12 @@ "@activepieces/piece-ai/@ai-sdk/amazon-bedrock/@smithy/util-utf8/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "@activepieces/piece-ai/@ai-sdk/google-vertex/google-auth-library/gaxios": ["gaxios@7.1.3", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2", "rimraf": "^5.0.1" } }, "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ=="], + + "@activepieces/piece-ai/@ai-sdk/google-vertex/google-auth-library/gcp-metadata": ["gcp-metadata@8.1.2", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="], + + "@activepieces/piece-ai/@ai-sdk/google-vertex/google-auth-library/jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], + "@activepieces/piece-gmail/google-auth-library/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], "@activepieces/piece-gmail/google-auth-library/jws/jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], @@ -22206,6 +22257,10 @@ "api/ai-gateway-provider/@openrouter/ai-sdk-provider/ai": ["ai@6.0.170", "", { "dependencies": { "@ai-sdk/gateway": "3.0.105", "@ai-sdk/provider": "3.0.9", "@ai-sdk/provider-utils": "4.0.24", "@opentelemetry/api": "1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-FWTKeGGDRcYJtPWIrdZDSuvOW5LCjI2NZUJmaml8OTOaPEsXnFdFvmawCXbT+wTGxyWKJTgZ9sZtCjbJsmjM2Q=="], + "api/google-auth-library/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + + "api/google-auth-library/jws/jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], + "api/socket.io/engine.io/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "api/socket.io/engine.io/ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="], @@ -22410,6 +22465,10 @@ "worker/@ai-sdk/mcp/@ai-sdk/provider-utils/undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="], + "@activepieces/piece-ai/@ai-sdk/google-vertex/google-auth-library/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + + "@activepieces/piece-ai/@ai-sdk/google-vertex/google-auth-library/jws/jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], + "@ai-sdk/google-vertex/google-auth-library/gcp-metadata/gaxios/node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], "@atlaskit/editor-json-transformer/@atlaskit/editor-prosemirror/prosemirror-commands/prosemirror-state/prosemirror-view": ["prosemirror-view@1.41.8", "", { "dependencies": { "prosemirror-model": "^1.20.0", "prosemirror-state": "^1.0.0", "prosemirror-transform": "^1.1.0" } }, "sha512-TnKDdohEatgyZNGCDWIdccOHXhYloJwbwU+phw/a23KBvJIR9lWQWW7WHHK3vBdOLDNuF7TaX98GObUZOWkOnA=="], diff --git a/docs/about/changelog.mdx b/docs/about/changelog.mdx index 4a9a753b6e7e..6dbd52ce8b27 100755 --- a/docs/about/changelog.mdx +++ b/docs/about/changelog.mdx @@ -4,6 +4,45 @@ description: "A log of all notable changes to Activepieces" icon: "code-commit" --- + + +### Agents you can keep, talk to, and put where they belong + +An agent used to be a bag of settings inside one flow step. Now it is something you +name, brief, talk to, and reuse. + +**Build one from a sentence** + +- **Describe the job**: the Agents page opens with a prompt box. Write *"summarise my + unread emails every morning"* and you get a drafted agent — a name, instructions, and + the tools it needs from the apps you have already connected. See + [Create an agent](/agents/create). +- **It only reaches for what you asked for**: the draft picks actions that read your data, + and adds one that sends, posts or files only when the sentence asks for it. +- **Edit with AI**: describe a change in the editor — *"only reply to paying customers"* — + and the instructions and tools are rewritten for you. Edits stay a draft until you press + **Save and go live**, and going live updates every flow already using the agent on its + next run, with nothing to republish. + +**Where an agent lives** + +- **Choose the project up front**: new agents no longer land wherever you happened to be. + The prompt box names the project and lets you change it, because the project decides + which connections, flows and files the agent can reach. +- **Move one later**: with a preview of what it costs first — the apps that have no + connection in the new project, the people who lose a share, and the fact that tools which + keep working now use the new project's accounts. See + [Manage an agent](/agents/manage). +- **Deleting is honest about consequences**: an agent still running inside a published flow + cannot be deleted or moved, and the flows are named so you know where to look. + +**Finding them** + +- The Agents page spans every project you can read, with search, sorting and a project + filter answered by the server, so a long list stays complete and fast. + + + ### AI-ready pieces: tool search, AI metadata, and audience diff --git a/docs/agents/create.mdx b/docs/agents/create.mdx index 83d3489e4f2c..70fa861cd71b 100644 --- a/docs/agents/create.mdx +++ b/docs/agents/create.mdx @@ -5,25 +5,34 @@ description: "The three routes in, and what each setting is for" icon: "pen" --- -## Pick a starting point +## Describe the job - +The **Agents** page opens with a prompt box. Write the job in a sentence and the agent is written for you: a name, instructions, and the tools it needs from the apps you have already connected. + + - Say what you need in a sentence. - - - Start from one already written. + *"Summarise my unread emails every morning."* You get a drafted agent to review. - - Write the instructions yourself. + + The chips under the box — triage support tickets, research a company, enrich a lead — are the same route with the sentence written. -The prompt box sits at the top of the Agents page. Templates and blank are behind **New agent**. All three land in the same editor, and all three need an [AI provider connected](/admin-guide/guides/setup-ai-providers). +Drafting needs an [AI provider connected](/admin-guide/guides/setup-ai-providers) and turned on for chat. Without one the page says so and offers to write the agent by hand instead. + + +Ask for what you want done, not for the tools. The draft only picks an action that reads your data unless the sentence asks for something to be sent, posted or filed, so *"tell me when something changes"* gets an agent that reports, not one that emails. + + +## Choose where it lives + +Under the prompt box, **New agents go to** names the project the agent will belong to, and lets you change it before you create it. That matters because the project is the boundary: an agent reaches only its own project's connections, flows, tables and knowledge. + +If you have one project, there is nothing to choose and the control stays out of the way. See [managing an agent](/agents/manage) for moving one later. ## Set it up -The **Configure** tab holds everything that changes what the agent does. +The **Configure** panel beside the conversation holds everything that changes what the agent does. @@ -46,9 +55,20 @@ The **Configure** tab holds everything that changes what the agent does. -## Try it +## Try it, then put it live + +The editor has two tabs beside the Configure panel. + + + + Describe a change — *"only reply to paying customers"* — and it rewrites the instructions and tools for you. + + + Talk to the agent as it is now, and watch which tools it reaches for. + + -The editor has a conversation panel beside the configuration. Talk to the agent there, watch which tools it reaches for, and adjust the instructions until it behaves. Changes take effect as soon as you save. +Edits are a draft until you press **Save and go live**. The header says which state you are in: *Changes not live yet* while you are editing, *Live* once saved, and *Needs a model to run* if no model is picked. Going live updates every flow already using the agent on its next run, with nothing to republish. Brief it like a capable new colleague. Say the goal and the edges, not every keystroke. The two people forget most: what "done" looks like, and which calls it should hand back to a human. diff --git a/docs/agents/manage.mdx b/docs/agents/manage.mdx new file mode 100644 index 000000000000..073936ba0cea --- /dev/null +++ b/docs/agents/manage.mdx @@ -0,0 +1,52 @@ +--- +title: "Manage an agent" +sidebarTitle: "Manage an agent" +description: "Where an agent lives, how to move it, and how to delete it safely" +icon: "folder-tree" +--- + +## The project is the boundary + +An agent belongs to one project, and that decides what it can reach: the connections it authenticates with, the flows it can call, the tables and files it looks things up in. Two projects with the same Gmail connection name are two different mailboxes. + +The **Agents** page spans every project you can see, so the card tells you which project each agent belongs to. The project filter beside the search box narrows the list, and it also moves the destination under the prompt box to that project. Whatever you pick in **New agents go to** wins, so check that line before you create anything. + +## Move it to another project + +From the `...` menu on a card, or from **Project** in the agent's own Advanced section, pick **Move to another project**. + +The agent takes its instructions, tools and conversations with it. Before you confirm, the dialog says what the move costs: + + + + A pinned connection is matched by name in the new project. Anything without a counterpart there stops working until you connect it, and the same is true for a flow or a file the new project does not have. + + + Tools that keep working now use the new project's accounts. Moving an agent between two clients points it at the second client's data. + + + An agent shared with someone who is not in the new project loses that share, and moving it back does not restore it. + + + +Two things refuse the move outright: a **published flow still running the agent**, because the reference is live and that flow would break, and a project you cannot create agents in. + + +Only the person who created the agent, or a project admin, can move or delete it. Everyone else in the project can use and edit it. + + +## Delete it + +**Delete** sits in the same `...` menu, and in a danger zone at the bottom of the agent's Advanced section. + +Deleting is permanent: the instructions, the tools and every conversation held with the agent go with it. A draft flow step pointing at the agent will break, and the dialog says so. + +If a **published flow** still runs the agent, deleting is refused and the flows are named. Take the agent out of those flows first. + +## Who can see it + +An agent is visible to everyone in its project. A restricted agent, shared with you and named colleagues instead, shows a lock beside its name in the list. + + +There is no screen for restricting an agent yet. Visibility is set through the API, with `visibility` and `sharedWithUserIds` on the agent create and update calls, and only the creator or a project admin can change who can see one. Everyone you share it with has to be a member of its project already, and a [move](#move-it-to-another-project) drops the shares that do not hold in the new project. + diff --git a/docs/agents/overview.mdx b/docs/agents/overview.mdx index 78a05761a784..1d50984c9acc 100644 --- a/docs/agents/overview.mdx +++ b/docs/agents/overview.mdx @@ -41,3 +41,7 @@ Each time it runs, it reads the situation, decides which tools to use, and keeps Say what you need and the agent gets written for you. + +## Where an agent lives + +An agent belongs to one project, and that decides which connections, flows and files it can reach. You choose the project when you create it, and you can [move it later](/agents/manage) — the dialog tells you what a move would break before you confirm. diff --git a/docs/docs.json b/docs/docs.json index 2ef74cda325d..0cadb571bff5 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -109,6 +109,7 @@ "pages": [ "agents/overview", "agents/create", + "agents/manage", "agents/tools", "agents/knowledge", "agents/in-flows" diff --git a/packages/core/ai-providers/package.json b/packages/core/ai-providers/package.json index f1da5a1a32b9..299b00757ee0 100644 --- a/packages/core/ai-providers/package.json +++ b/packages/core/ai-providers/package.json @@ -17,6 +17,7 @@ "@ai-sdk/anthropic": "4.0.25", "@ai-sdk/azure": "4.0.26", "@ai-sdk/google": "4.0.29", + "@ai-sdk/google-vertex": "5.0.36", "@ai-sdk/openai": "4.0.25", "@ai-sdk/openai-compatible": "3.0.18", "@openrouter/ai-sdk-provider": "3.0.0", diff --git a/packages/core/ai-providers/src/lib/create-language-model.test.ts b/packages/core/ai-providers/src/lib/create-language-model.test.ts index dc22fa0a1cf7..efc9c99da17f 100644 --- a/packages/core/ai-providers/src/lib/create-language-model.test.ts +++ b/packages/core/ai-providers/src/lib/create-language-model.test.ts @@ -1,15 +1,21 @@ import { AIProviderName } from '@activepieces/core-utils' +import { AIProviderConfig, AIProviderModelType, VertexProviderConfig } from '@activepieces/core-piece-types' import { describe, expect, it } from 'vitest' import { buildOpenAICompatibleHeaders, createLanguageModel } from './create-language-model' type ModelIdentity = { provider: string, modelId: string, settings?: { plugins?: unknown[] } } +type VertexModelIdentity = { config: { baseURL: string | (() => string) } } type CustomModelIdentity = { config: { headers: () => Record, fetch?: typeof globalThis.fetch } } function identify(model: unknown): ModelIdentity { return model as ModelIdentity } +function identifyVertex(model: unknown): VertexModelIdentity { + return model as VertexModelIdentity +} + function identifyCustom(model: unknown): CustomModelIdentity { return model as CustomModelIdentity } @@ -36,12 +42,19 @@ async function captureHeaders({ patchedFetch, headers }: { const authFor: Partial> = { [AIProviderName.BEDROCK]: { accessKeyId: 'a', secretAccessKey: 'b' }, + [AIProviderName.VERTEX]: { serviceAccountJson: JSON.stringify({ + type: 'service_account', + project_id: 'gcp-project', + client_email: 'sa@gcp-project.iam.gserviceaccount.com', + private_key: '-----BEGIN PRIVATE KEY-----\\nnot-a-real-key\\n-----END PRIVATE KEY-----\\n', + }) }, } const configFor: Partial> = { [AIProviderName.AZURE]: { resourceName: 'res', apiVersion: '2024-01-01' }, [AIProviderName.BEDROCK]: { region: 'us-east-1' }, [AIProviderName.CUSTOM]: { apiKeyHeader: 'x-api-key', baseUrl: 'https://example.test/v1', models: [] }, + [AIProviderName.VERTEX]: { project: 'gcp-project', region: 'europe-west4', models: [] }, } const buildFor = (provider: AIProviderName, options?: Record) => createLanguageModel({ @@ -54,6 +67,28 @@ const buildFor = (provider: AIProviderName, options?: Record) = const supportedProviders = Object.values(AIProviderName).filter((p) => p !== AIProviderName.CLOUDFLARE_GATEWAY) +describe('AIProviderConfig union', () => { + it('keeps every Vertex field instead of losing them to a looser member', () => { + const config = { + project: 'gcp-project', + region: 'europe-west4', + models: [{ modelId: 'gemini-2.5-pro', modelName: 'Gemini 2.5 Pro', modelType: AIProviderModelType.TEXT }], + } + + expect(AIProviderConfig.parse(config)).toEqual(config) + }) + + it('rejects a region that would escape the Vertex hostname', () => { + const withRegion = (region: string) => VertexProviderConfig.safeParse({ project: 'gcp-project', region, models: [] }).success + + expect(withRegion('europe-west4')).toBe(true) + expect(withRegion('global')).toBe(true) + expect(withRegion('evil.test/')).toBe(false) + expect(withRegion('foo.attacker.test')).toBe(false) + expect(withRegion('a/../../b')).toBe(false) + }) +}) + describe('createLanguageModel', () => { it.each(supportedProviders)('passes the model id straight through for %s', (provider) => { expect(identify(buildFor(provider)).modelId).toBe('some-model-id') @@ -73,6 +108,41 @@ describe('createLanguageModel', () => { expect(identify(buildFor(AIProviderName.OPENAI, { openaiResponsesModel: true })).provider).toBe('openai.responses') }) + it('routes Vertex straight at the configured GCP project and region', () => { + const model = buildFor(AIProviderName.VERTEX) + const { config } = identifyVertex(model) + const baseUrl = typeof config.baseURL === 'function' ? config.baseURL() : config.baseURL + + expect(identify(model).provider).toBe('google.vertex.chat') + expect(baseUrl).toBe('https://europe-west4-aiplatform.googleapis.com/v1beta1/projects/gcp-project/locations/europe-west4/publishers/google') + }) + + it('sends Model Garden Claude ids to the Vertex Anthropic client, not the Gemini one', () => { + const anthropicOnVertex = createLanguageModel({ + provider: AIProviderName.VERTEX, + auth: authFor[AIProviderName.VERTEX], + config: configFor[AIProviderName.VERTEX], + modelId: 'claude-sonnet-4-6', + }) + + expect(identify(buildFor(AIProviderName.VERTEX)).provider).toBe('google.vertex.chat') + expect(identify(anthropicOnVertex).provider).toBe('googleVertex.anthropic.messages') + }) + + it('sends Model Garden MaaS ids to the Vertex MaaS client', () => { + const build = (modelId: string) => identify(createLanguageModel({ + provider: AIProviderName.VERTEX, + auth: authFor[AIProviderName.VERTEX], + config: configFor[AIProviderName.VERTEX], + modelId, + })).provider + + expect(build('meta/llama-4-scout-17b-16e-instruct-maas')).toBe('vertex.maas.chat') + expect(build('mistral-large-2411-maas')).toBe('vertex.maas.chat') + expect(build('gemini-2.5-pro')).toBe('google.vertex.chat') + expect(build('claude-3-5-sonnet@20241022')).toBe('googleVertex.anthropic.messages') + }) + it('keeps the custom provider on chat completions unless apiStyle asks for responses', () => { const responsesConfig = { ...(configFor[AIProviderName.CUSTOM] as Record), apiStyle: 'responses' } const model = createLanguageModel({ diff --git a/packages/core/ai-providers/src/lib/create-language-model.ts b/packages/core/ai-providers/src/lib/create-language-model.ts index 9c18039bd55d..287db8519c45 100644 --- a/packages/core/ai-providers/src/lib/create-language-model.ts +++ b/packages/core/ai-providers/src/lib/create-language-model.ts @@ -1,6 +1,9 @@ import { AIProviderName, observedProviderFetch, ProviderOutcomeReporter, spreadIfDefined } from '@activepieces/core-utils' -import { AzureProviderConfig, BaseAIProviderAuthConfig, BedrockProviderAuthConfig, BedrockProviderConfig, OPENAI_COMPATIBLE_VENDOR_BASE_URLS, OpenAICompatibleProviderConfig } from '@activepieces/core-piece-types' +import { AzureProviderConfig, BaseAIProviderAuthConfig, BedrockProviderAuthConfig, BedrockProviderConfig, OPENAI_COMPATIBLE_VENDOR_BASE_URLS, OpenAICompatibleProviderConfig, VertexProviderAuthConfig, VertexProviderConfig } from '@activepieces/core-piece-types' import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock' +import { createVertex } from '@ai-sdk/google-vertex' +import { createVertexAnthropic } from '@ai-sdk/google-vertex/anthropic' +import { createVertexMaas } from '@ai-sdk/google-vertex/maas' import { createAnthropic } from '@ai-sdk/anthropic' import { createAzure } from '@ai-sdk/azure' import { createGoogleGenerativeAI } from '@ai-sdk/google' @@ -9,6 +12,8 @@ import { createOpenAICompatible } from '@ai-sdk/openai-compatible' import { createOpenRouter, OpenRouterChatSettings } from '@openrouter/ai-sdk-provider' import { LanguageModel } from 'ai' +const VERTEX_MAAS_SUFFIX = '-maas' +const VERTEX_ANTHROPIC_PREFIX = 'claude' const MISTRAL_BASE_URL = 'https://api.mistral.ai/v1' const AUTHORIZATION_HEADER = 'authorization' @@ -38,6 +43,12 @@ export function createLanguageModel({ provider, auth, config, modelId, options = const { region } = config as BedrockProviderConfig return createAmazonBedrock({ region, accessKeyId, secretAccessKey, ...observed })(modelId) } + case AIProviderName.VERTEX: { + const { serviceAccountJson } = auth as VertexProviderAuthConfig + const { project, region } = config as VertexProviderConfig + const vertexSettings = { project, location: region, googleAuthOptions: { credentials: parseServiceAccount(serviceAccountJson) }, ...observed } + return vertexClientFor({ modelId })(vertexSettings)(modelId) + } case AIProviderName.CUSTOM: { const { apiKey } = auth as BaseAIProviderAuthConfig const { apiKeyHeader, baseUrl, defaultHeaders, apiStyle } = config as OpenAICompatibleProviderConfig @@ -96,6 +107,27 @@ export function createLanguageModel({ provider, auth, config, modelId, options = } } +function vertexClientFor({ modelId }: { modelId: string }): typeof createVertex | typeof createVertexAnthropic | typeof createVertexMaas { + if (modelId.includes('/') || modelId.endsWith(VERTEX_MAAS_SUFFIX)) { + return createVertexMaas + } + if (modelId.toLowerCase().startsWith(VERTEX_ANTHROPIC_PREFIX)) { + return createVertexAnthropic + } + return createVertex +} + +function parseServiceAccount(serviceAccountJson: string): { client_email?: string, private_key?: string } { + const parsed: unknown = JSON.parse(serviceAccountJson) + const fields: Record = typeof parsed === 'object' && parsed !== null ? { ...parsed } : {} + const clientEmail = fields['client_email'] + const privateKey = fields['private_key'] + return { + client_email: typeof clientEmail === 'string' ? clientEmail : undefined, + private_key: typeof privateKey === 'string' ? privateKey.replace(/\\n/g, '\n') : undefined, + } +} + function stripDefaultAuthorization({ headers, delegate }: { headers: Record delegate?: typeof globalThis.fetch diff --git a/packages/core/execution/src/lib/engine/execution-errors.ts b/packages/core/execution/src/lib/engine/execution-errors.ts index 88fb2f21f01d..49680320d560 100644 --- a/packages/core/execution/src/lib/engine/execution-errors.ts +++ b/packages/core/execution/src/lib/engine/execution-errors.ts @@ -84,16 +84,6 @@ export class PausedFlowTimeoutError extends ExecutionError { } } -export class PieceMemoryLimitError extends ExecutionError { - constructor(heapLimitMb: string | undefined, standardError?: string, cause?: unknown) { - super('PieceMemoryLimitError', JSON.stringify({ - message: 'The piece ran out of memory', - heapLimitMb, - standardError, - }), ExecutionErrorType.USER, cause) - } -} - export class FileSizeError extends ExecutionError { constructor(currentFileSize: number, maximumSupportSize: number, cause?: unknown) { super('FileSizeError', JSON.stringify({ diff --git a/packages/core/piece-types/src/lib/ai-providers.ts b/packages/core/piece-types/src/lib/ai-providers.ts index ca5248ffc940..53cde753be74 100644 --- a/packages/core/piece-types/src/lib/ai-providers.ts +++ b/packages/core/piece-types/src/lib/ai-providers.ts @@ -24,6 +24,11 @@ const OpenAIProviderAuthConfig = BaseAIProviderAuthConfig const OpenRouterProviderAuthConfig = BaseAIProviderAuthConfig const MistralProviderAuthConfig = BaseAIProviderAuthConfig +export const VertexProviderAuthConfig = z.object({ + serviceAccountJson: z.string().check(z.minLength(1)), +}) +export type VertexProviderAuthConfig = z.infer + export const BedrockProviderAuthConfig = z.object({ accessKeyId: z.string().check(z.minLength(1)), secretAccessKey: z.string().check(z.minLength(1)), @@ -76,6 +81,13 @@ export const BedrockProviderConfig = z.object({ }) export type BedrockProviderConfig = z.infer +export const VertexProviderConfig = z.object({ + project: z.string().check(z.regex(/^[a-z0-9][a-z0-9-]{0,62}$/)), + region: z.string().check(z.regex(/^[a-z0-9][a-z0-9-]{0,62}$/)), + models: z.array(ProviderModelConfig), +}) +export type VertexProviderConfig = z.infer + export const OpenAiCompatibleVendorConfig = z.object({}) export type OpenAiCompatibleVendorConfig = z.infer @@ -89,6 +101,7 @@ export const AIProviderAuthConfig = z.union([ OpenAICompatibleProviderAuthConfig, ActivePiecesProviderAuthConfig, BedrockProviderAuthConfig, + VertexProviderAuthConfig, MistralProviderAuthConfig, ]) export type AIProviderAuthConfig = z.infer @@ -98,6 +111,7 @@ export const AIProviderConfig = z.union([ OpenAICompatibleProviderConfig, CloudflareGatewayProviderConfig, AzureProviderConfig, + VertexProviderConfig, BedrockProviderConfig, AnthropicProviderConfig, GoogleProviderConfig, @@ -228,6 +242,7 @@ export const ALLOWED_CHAT_MODELS_BY_PROVIDER: Partial `${AIProviderName.ANTHROPIC}/${m}`), ...OPENAI_CHAT_MODELS.map((m) => `${AIProviderName.OPENAI}/${m}`), @@ -274,6 +289,7 @@ const PROVIDER_MAX_CONTEXT_TOKENS: Partial> = { [AIProviderName.ANTHROPIC]: 200_000, [AIProviderName.GOOGLE]: 1_048_576, [AIProviderName.BEDROCK]: 200_000, + [AIProviderName.VERTEX]: 1_048_576, [AIProviderName.AZURE]: 128_000, [AIProviderName.OPENROUTER]: 128_000, [AIProviderName.ACTIVEPIECES]: 200_000, @@ -350,6 +366,7 @@ export const AI_PROVIDER_CAPABILITIES: Record export type DraftAgentRequest = z.infer export type AgentDraftFields = z.infer export type DraftAgentResponse = z.infer +export type AgentMoveLoss = z.infer +export type AgentMovePreview = z.infer export type ListAgentsRequest = z.infer +export type MoveAgentRequest = z.infer export type UpdateAgentRequest = z.infer diff --git a/packages/core/shared/src/lib/form-errors.ts b/packages/core/shared/src/lib/form-errors.ts index 3169be2b40e6..a99fa4e5932d 100644 --- a/packages/core/shared/src/lib/form-errors.ts +++ b/packages/core/shared/src/lib/form-errors.ts @@ -7,4 +7,5 @@ export const formErrors = { invalidFileName: 'invalidFileName', messageRequiresContentOrFiles: 'messageRequiresContentOrFiles', agentConfigTooLarge: 'agentConfigTooLarge', + invalidGcpResourceId: 'invalidGcpResourceId', } as const diff --git a/packages/core/shared/src/lib/management/ai-providers/index.ts b/packages/core/shared/src/lib/management/ai-providers/index.ts index 8f8b6b5e1ee4..2bdd7c0e9c51 100644 --- a/packages/core/shared/src/lib/management/ai-providers/index.ts +++ b/packages/core/shared/src/lib/management/ai-providers/index.ts @@ -1,5 +1,6 @@ import { AiProviderKeyStatus, AIProviderName, BaseModelSchema } from '@activepieces/core-utils' import { z } from 'zod' +import { formErrors } from '../../form-errors' export enum AIProviderModelType { IMAGE = 'image', @@ -11,6 +12,11 @@ export const BaseAIProviderAuthConfig = z.object({ }) export type BaseAIProviderAuthConfig = z.infer +export const VertexProviderAuthConfig = z.object({ + serviceAccountJson: z.string().min(1), +}) +export type VertexProviderAuthConfig = z.infer + export const AnthropicProviderAuthConfig = BaseAIProviderAuthConfig export type AnthropicProviderAuthConfig = z.infer @@ -101,6 +107,13 @@ export const BedrockProviderConfig = z.object({ }) export type BedrockProviderConfig = z.infer +export const VertexProviderConfig = z.object({ + project: z.string().regex(/^[a-z0-9][a-z0-9-]{0,62}$/, formErrors.invalidGcpResourceId), + region: z.string().regex(/^[a-z0-9][a-z0-9-]{0,62}$/, formErrors.invalidGcpResourceId), + models: z.array(ProviderModelConfig), +}) +export type VertexProviderConfig = z.infer + export const MistralProviderConfig = z.object({}) export type MistralProviderConfig = z.infer @@ -117,6 +130,7 @@ export const AIProviderAuthConfig = z.union([ OpenAICompatibleProviderAuthConfig, ActivePiecesProviderAuthConfig, BedrockProviderAuthConfig, + VertexProviderAuthConfig, MistralProviderAuthConfig, ]) export type AIProviderAuthConfig = z.infer @@ -125,6 +139,7 @@ export const AIProviderConfig = z.union([ OpenAICompatibleProviderConfig, CloudflareGatewayProviderConfig, AzureProviderConfig, + VertexProviderConfig, BedrockProviderConfig, AnthropicProviderConfig, GoogleProviderConfig, @@ -191,6 +206,12 @@ const ProviderConfigUnion = z.discriminatedUnion('provider', [ config: BedrockProviderConfig, auth: BedrockProviderAuthConfig, }), + z.object({ + displayName: z.string().min(1), + provider: z.literal(AIProviderName.VERTEX), + config: VertexProviderConfig, + auth: VertexProviderAuthConfig, + }), z.object({ displayName: z.string().min(1), provider: z.literal(AIProviderName.MISTRAL), @@ -322,6 +343,8 @@ export const GetProviderConfigResponse = z.object({ config: AIProviderConfig, auth: AIProviderAuthConfig, platformId: z.string(), + modelScope: AiProviderModelScope, + modelIds: z.array(z.string()), }) export type GetProviderConfigResponse = z.infer diff --git a/packages/core/utils/src/lib/permission.ts b/packages/core/utils/src/lib/permission.ts index 683e59283e6a..331d764dc7fa 100644 --- a/packages/core/utils/src/lib/permission.ts +++ b/packages/core/utils/src/lib/permission.ts @@ -55,6 +55,7 @@ export enum AIProviderName { CLOUDFLARE_GATEWAY = 'cloudflare-gateway', CUSTOM = 'custom', BEDROCK = 'bedrock', + VERTEX = 'vertex', MISTRAL = 'mistral', XAI = 'xai', DEEPSEEK = 'deepseek', diff --git a/packages/pieces/community/ai/package.json b/packages/pieces/community/ai/package.json index 67af54ccbf46..1be4ddf111ce 100644 --- a/packages/pieces/community/ai/package.json +++ b/packages/pieces/community/ai/package.json @@ -11,6 +11,7 @@ "@ai-sdk/anthropic": "3.0.72", "@ai-sdk/azure": "3.0.55", "@ai-sdk/google": "3.0.65", + "@ai-sdk/google-vertex": "4.0.113", "@ai-sdk/mcp": "1.0.11", "@ai-sdk/openai": "3.0.54", "@ai-sdk/openai-compatible": "2.0.42", diff --git a/packages/pieces/community/ai/src/lib/common/ai-sdk.ts b/packages/pieces/community/ai/src/lib/common/ai-sdk.ts index e7ba6d119080..77f63564d1a0 100644 --- a/packages/pieces/community/ai/src/lib/common/ai-sdk.ts +++ b/packages/pieces/community/ai/src/lib/common/ai-sdk.ts @@ -2,18 +2,23 @@ import { anthropic, createAnthropic } from '@ai-sdk/anthropic' import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock' import { createOpenAI, openai } from '@ai-sdk/openai' import { createGoogleGenerativeAI, google } from '@ai-sdk/google' +import { createVertex } from '@ai-sdk/google-vertex' +import { createVertexAnthropic } from '@ai-sdk/google-vertex/anthropic' +import { createVertexMaas } from '@ai-sdk/google-vertex/maas' import { createOpenAICompatible } from '@ai-sdk/openai-compatible' import { createAzure } from '@ai-sdk/azure' import { createOpenRouter } from '@openrouter/ai-sdk-provider' import { EmbeddingModel, ImageModel, LanguageModel } from 'ai' import { ProviderOptions } from '@ai-sdk/provider-utils' import { httpClient, HttpMethod } from '@activepieces/pieces-common' -import { AI_PROVIDER_CAPABILITIES, AIProviderName, AzureProviderConfig, BaseAIProviderAuthConfig, BedrockProviderAuthConfig, BedrockProviderConfig, CloudflareGatewayProviderConfig, GetProviderConfigResponse, OPENAI_COMPATIBLE_VENDOR_BASE_URLS, OpenAICompatibleProviderConfig, splitCloudflareGatewayModelId, spreadIfDefined } from '@activepieces/pieces-framework' +import { AI_PROVIDER_CAPABILITIES, AIProviderName, AzureProviderConfig, BaseAIProviderAuthConfig, BedrockProviderAuthConfig, BedrockProviderConfig, CloudflareGatewayProviderConfig, GetProviderConfigResponse, OPENAI_COMPATIBLE_VENDOR_BASE_URLS, OpenAICompatibleProviderConfig, splitCloudflareGatewayModelId, spreadIfDefined, VertexProviderAuthConfig, VertexProviderConfig } from '@activepieces/pieces-framework' import { createAiGateway } from 'ai-gateway-provider'; import { createAnthropic as createAnthropicGateway } from 'ai-gateway-provider/providers/anthropic'; import { createGoogleGenerativeAI as createGoogleGateway } from 'ai-gateway-provider/providers/google'; const AUTHORIZATION_HEADER = 'authorization' +const VERTEX_MAAS_SUFFIX = '-maas' +const VERTEX_ANTHROPIC_PREFIX = 'claude' async function fetchProviderConfig(params: { provider: AIProviderName, engineToken: string, apiUrl: string, configId?: string }) { const { body } = await httpClient.sendRequest({ @@ -147,6 +152,9 @@ function buildLanguageModel({ provider, auth, config, modelId, openaiResponsesMo const { region } = config as BedrockProviderConfig return createAmazonBedrock({ region, accessKeyId, secretAccessKey })(modelId) } + case AIProviderName.VERTEX: { + return vertexClientFor({ modelId })(buildVertexSettings({ auth, config }))(modelId) + } case AIProviderName.CUSTOM: { const { apiKey } = auth as BaseAIProviderAuthConfig const { apiKeyHeader, baseUrl, defaultHeaders, apiStyle } = config as OpenAICompatibleProviderConfig @@ -199,6 +207,33 @@ function buildLanguageModel({ provider, auth, config, modelId, openaiResponsesMo } } +function vertexClientFor({ modelId }: { modelId: string }): typeof createVertex | typeof createVertexAnthropic | typeof createVertexMaas { + if (modelId.includes('/') || modelId.endsWith(VERTEX_MAAS_SUFFIX)) { + return createVertexMaas + } + if (modelId.toLowerCase().startsWith(VERTEX_ANTHROPIC_PREFIX)) { + return createVertexAnthropic + } + return createVertex +} + +function buildVertexSettings({ auth, config }: { auth: unknown, config: unknown }): { project: string, location: string, googleAuthOptions: { credentials: { client_email?: string, private_key?: string } } } { + const { serviceAccountJson } = auth as VertexProviderAuthConfig + const { project, region } = config as VertexProviderConfig + return { project, location: region, googleAuthOptions: { credentials: parseServiceAccount(serviceAccountJson) } } +} + +function parseServiceAccount(serviceAccountJson: string): { client_email?: string, private_key?: string } { + const parsed: unknown = JSON.parse(serviceAccountJson) + const fields: Record = typeof parsed === 'object' && parsed !== null ? { ...parsed } : {} + const clientEmail = fields['client_email'] + const privateKey = fields['private_key'] + return { + client_email: typeof clientEmail === 'string' ? clientEmail : undefined, + private_key: typeof privateKey === 'string' ? privateKey.replace(/\\n/g, '\n') : undefined, + } +} + function stripDefaultAuthorization(headers: Record): typeof globalThis.fetch | undefined { const carriesAuthorization = Object.keys(headers).some((key) => key.trim().toLowerCase() === AUTHORIZATION_HEADER) if (carriesAuthorization) { @@ -247,6 +282,9 @@ function buildNativeImageModel({ provider, auth, config, modelId, metadataHeader const { region } = config as BedrockProviderConfig return createAmazonBedrock({ region, accessKeyId, secretAccessKey }).imageModel(modelId) } + case AIProviderName.VERTEX: { + return createVertex(buildVertexSettings({ auth, config })).imageModel(modelId) + } case AIProviderName.CUSTOM: { const { apiKey } = auth as BaseAIProviderAuthConfig const { apiKeyHeader, baseUrl, defaultHeaders } = config as OpenAICompatibleProviderConfig diff --git a/packages/pieces/community/formio/.eslintrc.json b/packages/pieces/community/formio/.eslintrc.json new file mode 100644 index 000000000000..6f1536634f91 --- /dev/null +++ b/packages/pieces/community/formio/.eslintrc.json @@ -0,0 +1,47 @@ +{ + "extends": [ + "../../../../.eslintrc.json" + ], + "ignorePatterns": [ + "!**/*" + ], + "overrides": [ + { + "files": [ + "*.ts", + "*.tsx", + "*.js", + "*.jsx" + ], + "rules": {} + }, + { + "files": [ + "*.ts", + "*.tsx" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + "lodash", + "lodash/*", + "@activepieces/core-*", + "@activepieces/server*", + "@activepieces/engine", + "@activepieces/shared" + ] + } + ] + } + }, + { + "files": [ + "*.js", + "*.jsx" + ], + "rules": {} + } + ] +} diff --git a/packages/pieces/community/formio/package.json b/packages/pieces/community/formio/package.json new file mode 100644 index 000000000000..00ab76de9247 --- /dev/null +++ b/packages/pieces/community/formio/package.json @@ -0,0 +1,23 @@ +{ + "name": "@activepieces/piece-formio", + "version": "0.0.1", + "type": "commonjs", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*" + }, + "devDependencies": { + "tslib": "2.6.2", + "vitest": "3.2.6" + }, + "scripts": { + "build": "tsc -p tsconfig.lib.json && cp package.json dist/", + "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", + "lint": "eslint 'src/**/*.ts'", + "test": "vitest run" + } +} diff --git a/packages/pieces/community/formio/src/i18n/translation.json b/packages/pieces/community/formio/src/i18n/translation.json new file mode 100644 index 000000000000..0e56ae6e1956 --- /dev/null +++ b/packages/pieces/community/formio/src/i18n/translation.json @@ -0,0 +1,30 @@ +{ + "Form": "Form", + "Submission ID": "Submission ID", + "Submission Data": "Submission Data", + "Filters": "Filters", + "Field": "Field", + "Operator": "Operator", + "Value": "Value", + "Limit": "Limit", + "Skip": "Skip", + "Sort By": "Sort By", + "Sort Direction": "Sort Direction", + "Type": "Type", + "Project URL": "Project URL", + "API Key": "API Key", + "Equals": "Equals", + "Not equal": "Not equal", + "Greater than": "Greater than", + "Greater than or equal": "Greater than or equal", + "Less than": "Less than", + "Less than or equal": "Less than or equal", + "In (comma separated)": "In (comma separated)", + "Matches regular expression": "Matches regular expression", + "Exists (true or false)": "Exists (true or false)", + "Newest first": "Newest first", + "Oldest first": "Oldest first", + "Forms": "Forms", + "Resources": "Resources", + "Merge With Existing Data": "Merge With Existing Data" +} diff --git a/packages/pieces/community/formio/src/index.ts b/packages/pieces/community/formio/src/index.ts new file mode 100644 index 000000000000..0786e6987010 --- /dev/null +++ b/packages/pieces/community/formio/src/index.ts @@ -0,0 +1,42 @@ +import { createCustomApiCallAction } from '@activepieces/pieces-common'; +import { createPiece, PieceCategory } from '@activepieces/pieces-framework'; +import { formioAuth } from './lib/auth'; +import { createSubmission } from './lib/actions/create-submission'; +import { deleteSubmission } from './lib/actions/delete-submission'; +import { findSubmissions } from './lib/actions/find-submissions'; +import { getForm } from './lib/actions/get-form'; +import { getSubmission } from './lib/actions/get-submission'; +import { listForms } from './lib/actions/list-forms'; +import { updateSubmission } from './lib/actions/update-submission'; +import { formioCommon } from './lib/common/client'; +import { newSubmission } from './lib/triggers/new-submission'; +import { updatedSubmission } from './lib/triggers/updated-submission'; + +export const formio = createPiece({ + displayName: 'Form.io', + description: + 'Build and manage forms and their submissions on Form.io, hosted or self-hosted', + auth: formioAuth, + minimumSupportedRelease: '0.36.1', + logoUrl: 'https://cdn.activepieces.com/pieces/formio.png', + categories: [PieceCategory.FORMS_AND_SURVEYS], + authors: ['odaithalji'], + actions: [ + createSubmission, + getSubmission, + findSubmissions, + updateSubmission, + deleteSubmission, + listForms, + getForm, + createCustomApiCallAction({ + auth: formioAuth, + baseUrl: (auth) => + auth ? formioCommon.normalizeProjectUrl(auth.props.projectUrl) : '', + authMapping: async (auth) => ({ + 'x-token': auth.props.apiKey, + }), + }), + ], + triggers: [newSubmission, updatedSubmission], +}); diff --git a/packages/pieces/community/formio/src/lib/actions/create-submission.ts b/packages/pieces/community/formio/src/lib/actions/create-submission.ts new file mode 100644 index 000000000000..6ce5c4db7be5 --- /dev/null +++ b/packages/pieces/community/formio/src/lib/actions/create-submission.ts @@ -0,0 +1,31 @@ +import { createAction } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from '../common/client'; +import { formioProps } from '../common/props'; +import { createSubmissionOutputSchema } from '../common/output-schemas'; + +export const createSubmission = createAction({ + auth: formioAuth, + name: 'create_submission', + displayName: 'Create Submission', + description: 'Submit data to a Form.io form', + classification: 'WRITE', + audience: 'both', + aiMetadata: { + description: + 'Creates a new submission on a Form.io form, with the field values keyed by the form component keys. Use it to file a record into Form.io from a flow, such as a citizen intake or a case created elsewhere. Requires the form and the submission data; not idempotent, since each call files a separate submission.', + idempotent: false, + }, + outputSchema: createSubmissionOutputSchema, + props: { + formPath: formioProps.formPath, + data: formioProps.submissionData, + }, + async run({ auth, propsValue }) { + return await formioCommon.createSubmission({ + auth: auth.props, + formPath: propsValue.formPath, + data: propsValue.data as Record, + }); + }, +}); diff --git a/packages/pieces/community/formio/src/lib/actions/delete-submission.ts b/packages/pieces/community/formio/src/lib/actions/delete-submission.ts new file mode 100644 index 000000000000..5ba28300875e --- /dev/null +++ b/packages/pieces/community/formio/src/lib/actions/delete-submission.ts @@ -0,0 +1,31 @@ +import { createAction } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from '../common/client'; +import { formioProps } from '../common/props'; +import { deleteSubmissionOutputSchema } from '../common/output-schemas'; + +export const deleteSubmission = createAction({ + auth: formioAuth, + name: 'delete_submission', + displayName: 'Delete Submission', + description: 'Delete a submission by its id', + classification: 'DESTRUCTIVE', + audience: 'both', + aiMetadata: { + description: + 'Deletes a Form.io submission by its id. Destructive and not recoverable through this piece, so confirm the id before calling it. Deleting an already-deleted submission has no further effect.', + idempotent: true, + }, + outputSchema: deleteSubmissionOutputSchema, + props: { + formPath: formioProps.formPath, + submissionId: formioProps.submissionId, + }, + async run({ auth, propsValue }) { + return await formioCommon.deleteSubmission({ + auth: auth.props, + formPath: propsValue.formPath, + submissionId: propsValue.submissionId, + }); + }, +}); diff --git a/packages/pieces/community/formio/src/lib/actions/find-submissions.ts b/packages/pieces/community/formio/src/lib/actions/find-submissions.ts new file mode 100644 index 000000000000..be42aa7796b3 --- /dev/null +++ b/packages/pieces/community/formio/src/lib/actions/find-submissions.ts @@ -0,0 +1,152 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from '../common/client'; +import { formioProps } from '../common/props'; +import { findSubmissionsOutputSchema } from '../common/output-schemas'; + +function buildQueryParams({ + filters, + limit, + skip, + sortField, + sortDirection, +}: { + filters: FilterRow[]; + limit: number | undefined; + skip: number | undefined; + sortField: string | undefined; + sortDirection: string | undefined; +}): Record { + const params: Record = {}; + + for (const filter of filters) { + const field = filter.field?.trim(); + if (!field) { + continue; + } + const key = + filter.operator && filter.operator !== 'equals' + ? `${field}__${filter.operator}` + : field; + params[key] = String(filter.value ?? ''); + } + + if (limit !== undefined && limit !== null) { + params['limit'] = String(limit); + } + if (skip !== undefined && skip !== null) { + params['skip'] = String(skip); + } + if (sortField) { + params['sort'] = sortDirection === 'asc' ? sortField : `-${sortField}`; + } + + return params; +} + +export const findSubmissions = createAction({ + auth: formioAuth, + name: 'find_submissions', + displayName: 'Find Submissions', + description: 'Search a form for submissions matching field filters', + classification: 'SEARCH', + audience: 'both', + aiMetadata: { + description: + 'Searches the submissions of a Form.io form, filtering on submitted field values or on the created and modified timestamps, with paging and sorting. Field paths are prefixed with data, for example data.email. Choose it to look records up by their content; use Get Submission when the id is already known. Read-only and idempotent.', + idempotent: true, + }, + outputSchema: findSubmissionsOutputSchema, + props: { + formPath: formioProps.formPath, + filters: Property.Array({ + displayName: 'Filters', + description: + 'Each row narrows the search. Use a data-prefixed path for a form field, for example data.email, or created / modified for the timestamps.', + required: false, + properties: { + field: Property.ShortText({ + displayName: 'Field', + description: 'For example data.category', + required: true, + }), + operator: Property.StaticDropdown({ + displayName: 'Operator', + required: true, + defaultValue: 'equals', + options: { + options: [ + { label: 'Equals', value: 'equals' }, + { label: 'Not equal', value: 'ne' }, + { label: 'Greater than', value: 'gt' }, + { label: 'Greater than or equal', value: 'gte' }, + { label: 'Less than', value: 'lt' }, + { label: 'Less than or equal', value: 'lte' }, + { label: 'In (comma separated)', value: 'in' }, + { label: 'Matches regular expression', value: 'regex' }, + { label: 'Exists (true or false)', value: 'exists' }, + ], + }, + }), + value: Property.ShortText({ + displayName: 'Value', + required: true, + }), + }, + }), + limit: Property.Number({ + displayName: 'Limit', + description: 'How many submissions to return. Form.io defaults to 10.', + required: false, + }), + skip: Property.Number({ + displayName: 'Skip', + description: 'How many submissions to skip, for paging', + required: false, + }), + sortField: Property.ShortText({ + displayName: 'Sort By', + description: 'For example created, modified, or data.refNumber', + required: false, + defaultValue: 'created', + }), + sortDirection: Property.StaticDropdown({ + displayName: 'Sort Direction', + required: false, + defaultValue: 'desc', + options: { + options: [ + { label: 'Newest first', value: 'desc' }, + { label: 'Oldest first', value: 'asc' }, + ], + }, + }), + }, + async run({ auth, propsValue }) { + const filters = (propsValue.filters ?? []) as FilterRow[]; + + const { submissions, total } = await formioCommon.findSubmissions({ + auth: auth.props, + formPath: propsValue.formPath, + queryParams: buildQueryParams({ + filters, + limit: propsValue.limit, + skip: propsValue.skip, + sortField: propsValue.sortField, + sortDirection: propsValue.sortDirection, + }), + }); + + return { + submissions, + count: submissions.length, + total: total ?? submissions.length, + }; + }, +}); + +export type FilterRow = { + field: string; + operator: string; + value: unknown; +}; diff --git a/packages/pieces/community/formio/src/lib/actions/get-form.ts b/packages/pieces/community/formio/src/lib/actions/get-form.ts new file mode 100644 index 000000000000..1b55a5293132 --- /dev/null +++ b/packages/pieces/community/formio/src/lib/actions/get-form.ts @@ -0,0 +1,29 @@ +import { createAction } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from '../common/client'; +import { formioProps } from '../common/props'; +import { getFormOutputSchema } from '../common/output-schemas'; + +export const getForm = createAction({ + auth: formioAuth, + name: 'get_form', + displayName: 'Get Form', + description: 'Read one form definition, including its components', + classification: 'READ', + audience: 'both', + aiMetadata: { + description: + 'Reads a single Form.io form definition, including the component tree that describes its fields. Use it to discover which field keys a form expects before creating or updating a submission. Read-only and idempotent.', + idempotent: true, + }, + outputSchema: getFormOutputSchema, + props: { + formPath: formioProps.formPath, + }, + async run({ auth, propsValue }) { + return await formioCommon.getForm({ + auth: auth.props, + formPath: propsValue.formPath, + }); + }, +}); diff --git a/packages/pieces/community/formio/src/lib/actions/get-submission.ts b/packages/pieces/community/formio/src/lib/actions/get-submission.ts new file mode 100644 index 000000000000..baba4642c680 --- /dev/null +++ b/packages/pieces/community/formio/src/lib/actions/get-submission.ts @@ -0,0 +1,31 @@ +import { createAction } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from '../common/client'; +import { formioProps } from '../common/props'; +import { getSubmissionOutputSchema } from '../common/output-schemas'; + +export const getSubmission = createAction({ + auth: formioAuth, + name: 'get_submission', + displayName: 'Get Submission', + description: 'Read one submission by its id', + classification: 'READ', + audience: 'both', + aiMetadata: { + description: + 'Reads a single Form.io submission by its id, returning the submitted data along with its owner, timestamps and metadata. Use it to look up a record whose id you already have; prefer Find Submissions to search by field values. Read-only and idempotent.', + idempotent: true, + }, + outputSchema: getSubmissionOutputSchema, + props: { + formPath: formioProps.formPath, + submissionId: formioProps.submissionId, + }, + async run({ auth, propsValue }) { + return await formioCommon.getSubmission({ + auth: auth.props, + formPath: propsValue.formPath, + submissionId: propsValue.submissionId, + }); + }, +}); diff --git a/packages/pieces/community/formio/src/lib/actions/list-forms.ts b/packages/pieces/community/formio/src/lib/actions/list-forms.ts new file mode 100644 index 000000000000..00f3f7f442fb --- /dev/null +++ b/packages/pieces/community/formio/src/lib/actions/list-forms.ts @@ -0,0 +1,45 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from '../common/client'; +import { listFormsOutputSchema } from '../common/output-schemas'; + +export const listForms = createAction({ + auth: formioAuth, + name: 'list_forms', + displayName: 'List Forms', + description: 'List the forms in the Form.io project', + classification: 'READ', + audience: 'both', + aiMetadata: { + description: + 'Lists the forms in a Form.io project, each with its title, path and id. Use it to discover which forms exist, or to resolve a form title to the path the submission actions need. Set the type to resource to list resources instead of forms. Read-only and idempotent.', + idempotent: true, + }, + outputSchema: listFormsOutputSchema, + props: { + type: Property.StaticDropdown({ + displayName: 'Type', + required: false, + defaultValue: 'form', + options: { + options: [ + { label: 'Forms', value: 'form' }, + { label: 'Resources', value: 'resource' }, + ], + }, + }), + limit: Property.Number({ + displayName: 'Limit', + description: 'How many to return. Defaults to 100.', + required: false, + }), + }, + async run({ auth, propsValue }) { + const forms = await formioCommon.listForms({ + auth: auth.props, + type: propsValue.type === 'resource' ? 'resource' : 'form', + limit: propsValue.limit ?? 100, + }); + return { forms, count: forms.length }; + }, +}); diff --git a/packages/pieces/community/formio/src/lib/actions/update-submission.ts b/packages/pieces/community/formio/src/lib/actions/update-submission.ts new file mode 100644 index 000000000000..4bc0ade24a79 --- /dev/null +++ b/packages/pieces/community/formio/src/lib/actions/update-submission.ts @@ -0,0 +1,41 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from '../common/client'; +import { formioProps } from '../common/props'; +import { updateSubmissionOutputSchema } from '../common/output-schemas'; + +export const updateSubmission = createAction({ + auth: formioAuth, + name: 'update_submission', + displayName: 'Update Submission', + description: 'Replace the data on an existing submission', + classification: 'WRITE', + audience: 'both', + aiMetadata: { + description: + 'Updates an existing Form.io submission, identified by its id. By default the fields supplied are merged into the submission, leaving every other field as it was. Turn Merge off to replace the submission data outright, which is what the Form.io API does natively and which clears any field not supplied. Merging reads the submission and writes it back, and Form.io cannot reject a write based on the version read, so two flows updating the same submission at the same time will lose one set of changes. Use it to correct or progress a record already in Form.io.', + idempotent: true, + }, + outputSchema: updateSubmissionOutputSchema, + props: { + formPath: formioProps.formPath, + submissionId: formioProps.submissionId, + data: formioProps.submissionData, + merge: Property.Checkbox({ + displayName: 'Merge With Existing Data', + description: + 'On by default: the fields above are merged into the submission and everything else is left alone. Turn it off to replace the submission data outright, which clears any field you do not supply.\n\nMerging reads the submission and writes it back, and Form.io offers no way to reject a write based on the version that was read. So if another flow or person changes the same submission during that moment, one of the two sets of changes is lost. Where several flows update one submission at once, have each write only the fields it owns with merge off, or serialise them.', + required: false, + defaultValue: true, + }), + }, + async run({ auth, propsValue }) { + return await formioCommon.updateSubmission({ + auth: auth.props, + formPath: propsValue.formPath, + submissionId: propsValue.submissionId, + data: propsValue.data as Record, + merge: propsValue.merge !== false, + }); + }, +}); diff --git a/packages/pieces/community/formio/src/lib/auth.ts b/packages/pieces/community/formio/src/lib/auth.ts new file mode 100644 index 000000000000..f8c15b1588da --- /dev/null +++ b/packages/pieces/community/formio/src/lib/auth.ts @@ -0,0 +1,35 @@ +import { PieceAuth, Property } from '@activepieces/pieces-framework'; +import { formioCommon } from './common/client'; + +export const formioAuth = PieceAuth.CustomAuth({ + description: `Connect to a Form.io project, hosted or self-hosted. + +**Project URL** — the URL your forms live under. On a self-hosted server that is usually the host itself, for example \`https://forms.example.gov\`, or the host plus the project name if your deployment uses projects, for example \`https://forms.example.gov/intake\`. On Form.io's cloud it looks like \`https://xyzabc.form.io\`. + +**API Key** — a project API key, sent as the \`x-token\` header. On a self-hosted server these come from the \`API_KEYS\` environment variable; on cloud, from Project Settings → API Keys. A user login (JWT) is not supported.`, + required: true, + props: { + projectUrl: Property.ShortText({ + displayName: 'Project URL', + description: 'For example https://forms.example.gov/intake', + required: true, + }), + apiKey: PieceAuth.SecretText({ + displayName: 'API Key', + description: 'Sent as the x-token header', + required: true, + }), + }, + validate: async ({ auth }) => { + try { + await formioCommon.validateAuth(auth); + return { valid: true }; + } catch (error) { + return { + valid: false, + error: + 'Could not reach this Form.io project with that API key. Check the Project URL and that the key is listed in the project (or in API_KEYS on a self-hosted server).', + }; + } + }, +}); diff --git a/packages/pieces/community/formio/src/lib/common/client.ts b/packages/pieces/community/formio/src/lib/common/client.ts new file mode 100644 index 000000000000..86c077f5dddd --- /dev/null +++ b/packages/pieces/community/formio/src/lib/common/client.ts @@ -0,0 +1,304 @@ +import { + HttpMethod, + HttpRequest, + httpClient, +} from '@activepieces/pieces-common'; + +function normalizeProjectUrl(projectUrl: string): string { + const trimmed = projectUrl.trim().replace(/\/+$/, ''); + if (trimmed.length === 0) { + throw new Error('Project URL is required'); + } + if (!/^https?:\/\//i.test(trimmed)) { + throw new Error( + 'Project URL must start with http:// or https:// (for example https://forms.example.gov/intake)' + ); + } + return trimmed; +} + +async function sendRequest({ + auth, + method, + path, + queryParams, + body, +}: { + auth: FormioAuth; + method: HttpMethod; + path: string; + queryParams?: Record; + body?: unknown; +}): Promise> { + const request: HttpRequest = { + method, + url: `${normalizeProjectUrl(auth.projectUrl)}/${path.replace(/^\//, '')}`, + headers: { 'x-token': auth.apiKey }, + ...(queryParams ? { queryParams } : {}), + ...(body ? { body } : {}), + }; + + const response = await httpClient.sendRequest(request); + return { + body: response.body, + total: parseTotal(response.headers?.['content-range']), + }; +} + +function parseTotal(contentRange: unknown): number | undefined { + if (typeof contentRange !== 'string') { + return undefined; + } + const total = Number(contentRange.split('/')[1]); + return Number.isFinite(total) ? total : undefined; +} + +export const formioCommon = { + normalizeProjectUrl, + + async listForms({ + auth, + type = 'form', + limit = 100, + }: { + auth: FormioAuth; + type?: 'form' | 'resource'; + limit?: number; + }): Promise { + const { body } = await sendRequest({ + auth, + method: HttpMethod.GET, + path: 'form', + queryParams: { type, limit: String(limit) }, + }); + return Array.isArray(body) ? body : []; + }, + + async getForm({ auth, formPath }: { auth: FormioAuth; formPath: string }) { + const { body } = await sendRequest({ + auth, + method: HttpMethod.GET, + path: formPath, + }); + return body; + }, + + async findFormId({ + auth, + formPath, + }: { + auth: FormioAuth; + formPath: string; + }): Promise { + const form = await formioCommon.getForm({ auth, formPath }); + if (!form?._id) { + throw new Error(`Form "${formPath}" was not found on this Form.io project`); + } + return form._id; + }, + + async createSubmission({ + auth, + formPath, + data, + }: { + auth: FormioAuth; + formPath: string; + data: Record; + }) { + const { body } = await sendRequest({ + auth, + method: HttpMethod.POST, + path: `${formPath}/submission`, + body: { data }, + }); + return body; + }, + + async getSubmission({ + auth, + formPath, + submissionId, + }: { + auth: FormioAuth; + formPath: string; + submissionId: string; + }) { + const { body } = await sendRequest({ + auth, + method: HttpMethod.GET, + path: `${formPath}/submission/${submissionId}`, + }); + return body; + }, + + async findSubmissions({ + auth, + formPath, + queryParams, + }: { + auth: FormioAuth; + formPath: string; + queryParams: Record; + }): Promise<{ submissions: FormioSubmission[]; total: number | undefined }> { + const { body, total } = await sendRequest({ + auth, + method: HttpMethod.GET, + path: `${formPath}/submission`, + queryParams, + }); + return { submissions: Array.isArray(body) ? body : [], total }; + }, + + async updateSubmission({ + auth, + formPath, + submissionId, + data, + merge, + }: { + auth: FormioAuth; + formPath: string; + submissionId: string; + data: Record; + merge: boolean; + }) { + const existing = merge + ? await formioCommon.getSubmission({ auth, formPath, submissionId }) + : undefined; + + const { body } = await sendRequest({ + auth, + method: HttpMethod.PUT, + path: `${formPath}/submission/${submissionId}`, + body: { + _id: submissionId, + data: existing ? { ...existing.data, ...data } : data, + }, + }); + return body; + }, + + async deleteSubmission({ + auth, + formPath, + submissionId, + }: { + auth: FormioAuth; + formPath: string; + submissionId: string; + }) { + await sendRequest({ + auth, + method: HttpMethod.DELETE, + path: `${formPath}/submission/${submissionId}`, + }); + return { deleted: true, submissionId }; + }, + + async createWebhookAction({ + auth, + formId, + webhookUrl, + events, + }: { + auth: FormioAuth; + formId: string; + webhookUrl: string; + events: FormioActionMethod[]; + }): Promise { + const { body } = await sendRequest({ + auth, + method: HttpMethod.POST, + path: `form/${formId}/action`, + body: { + title: 'Webhook', + name: 'webhook', + priority: 0, + handler: ['after'], + method: events, + settings: { url: webhookUrl, method: 'post' }, + }, + }); + if (!body?._id) { + throw new Error( + 'Form.io did not return an id for the webhook action it created' + ); + } + return body._id; + }, + + async deleteWebhookAction({ + auth, + formId, + actionId, + }: { + auth: FormioAuth; + formId: string; + actionId: string; + }) { + await sendRequest({ + auth, + method: HttpMethod.DELETE, + path: `form/${formId}/action/${actionId}`, + }); + }, + + async validateAuth(auth: FormioAuth): Promise { + await sendRequest({ + auth, + method: HttpMethod.GET, + path: 'role', + }); + }, +}; + +export const FORMIO_AUTH_HEADER = 'x-token'; + +export type FormioAuth = { + projectUrl: string; + apiKey: string; +}; + +export type FormioActionMethod = 'create' | 'update' | 'delete'; + +export type FormioForm = { + _id: string; + title: string; + name: string; + path: string; + type: string; + created?: string; + modified?: string; +}; + +export type FormioSubmission = { + _id: string; + form: string; + data: Record; + owner?: string | null; + roles?: unknown[]; + access?: unknown[]; + metadata?: Record; + externalIds?: unknown[]; + created?: string; + modified?: string; +}; + +export type FormioAction = { + _id: string; + name: string; + form: string; + settings?: Record; +}; + +export type FormioResponse = { + body: T; + total: number | undefined; +}; + +export type FormioWebhookPayload = { + request?: Record; + submission?: FormioSubmission; + params?: { formId?: string; submissionId?: string }; +}; diff --git a/packages/pieces/community/formio/src/lib/common/output-schemas.ts b/packages/pieces/community/formio/src/lib/common/output-schemas.ts new file mode 100644 index 000000000000..4f771fa4cf0f --- /dev/null +++ b/packages/pieces/community/formio/src/lib/common/output-schemas.ts @@ -0,0 +1,145 @@ +import { OutputSchema, OutputSchemaField } from '@activepieces/pieces-framework'; + +const submissionDataField: OutputSchemaField = { + key: 'data', + label: 'Submitted Data', + description: + 'The submitted values, keyed by the form component keys. The shape depends on the form, so expand it after a test run to see this form fields.', +}; + +const submissionMetadataField: OutputSchemaField = { + key: 'metadata', + label: 'Metadata', + description: + 'What Form.io recorded about the request that created the submission. Form.io keeps only a safe subset of the headers.', + children: [ + { + key: 'headers', + label: 'Request Headers', + children: [ + { key: 'host', label: 'Host' }, + { key: 'user-agent', label: 'User Agent' }, + { key: 'content-type', label: 'Content Type' }, + { key: 'content-length', label: 'Content Length' }, + ], + }, + ], +}; + +const submissionFields: OutputSchemaField[] = [ + { key: '_id', label: 'Submission ID' }, + { key: 'form', label: 'Form ID' }, + submissionDataField, + { key: 'owner', label: 'Owner' }, + { key: 'roles', label: 'Roles' }, + { key: 'access', label: 'Access' }, + { key: 'externalIds', label: 'External IDs' }, + submissionMetadataField, + { key: 'created', label: 'Created', format: 'datetime' }, + { key: 'modified', label: 'Modified', format: 'datetime' }, +]; + +const formFields: OutputSchemaField[] = [ + { key: '_id', label: 'Form ID' }, + { key: 'title', label: 'Title' }, + { key: 'name', label: 'Name' }, + { key: 'path', label: 'Path' }, + { key: 'type', label: 'Type' }, + { key: 'display', label: 'Display' }, + { key: 'machineName', label: 'Machine Name' }, + { key: 'owner', label: 'Owner' }, + { key: 'tags', label: 'Tags' }, + { + key: 'components', + label: 'Components', + labelKey: 'label', + description: 'The fields the form is built from.', + listItems: [ + { key: 'key', label: 'Key' }, + { key: 'label', label: 'Label' }, + { key: 'type', label: 'Type' }, + { key: 'input', label: 'Is Input', format: 'boolean' }, + ], + }, + { + key: 'access', + label: 'Access', + labelKey: 'type', + listItems: [ + { key: 'type', label: 'Type' }, + { key: 'roles', label: 'Roles' }, + ], + }, + { key: 'submissionAccess', label: 'Submission Access' }, + { key: 'created', label: 'Created', format: 'datetime' }, + { key: 'modified', label: 'Modified', format: 'datetime' }, +]; + +export const createSubmissionOutputSchema: OutputSchema = { + fields: submissionFields, +}; + +export const getSubmissionOutputSchema: OutputSchema = { + fields: submissionFields, +}; + +export const updateSubmissionOutputSchema: OutputSchema = { + fields: submissionFields, +}; + +export const findSubmissionsOutputSchema: OutputSchema = { + fields: [ + { + key: 'submissions', + label: 'Submissions', + listItems: submissionFields, + }, + { + key: 'count', + label: 'Returned Count', + format: 'number', + description: 'How many submissions this step returned.', + }, + { + key: 'total', + label: 'Total Matching', + format: 'number', + description: + 'How many submissions match in the whole form, which can exceed the returned count when a limit is set.', + }, + ], +}; + +export const deleteSubmissionOutputSchema: OutputSchema = { + fields: [ + { key: 'deleted', label: 'Deleted', format: 'boolean' }, + { key: 'submissionId', label: 'Submission ID' }, + ], +}; + +export const listFormsOutputSchema: OutputSchema = { + fields: [ + { + key: 'forms', + label: 'Forms', + labelKey: 'title', + listItems: formFields, + }, + { key: 'count', label: 'Count', format: 'number' }, + ], +}; + +export const getFormOutputSchema: OutputSchema = { + fields: formFields, +}; + +export const submissionTriggerOutputSchema: OutputSchema = { + fields: [ + ...submissionFields, + { + key: 'deleted', + label: 'Deleted At', + description: 'Set only once the submission has been deleted.', + }, + ], +}; diff --git a/packages/pieces/community/formio/src/lib/common/props.ts b/packages/pieces/community/formio/src/lib/common/props.ts new file mode 100644 index 000000000000..def9cfd532ea --- /dev/null +++ b/packages/pieces/community/formio/src/lib/common/props.ts @@ -0,0 +1,59 @@ +import { Property } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { formioCommon } from './client'; + +export const formioProps = { + formPath: Property.Dropdown({ + auth: formioAuth, + displayName: 'Form', + description: 'The form whose submissions this step works with', + required: true, + refreshers: [], + options: async ({ auth }) => { + if (!auth) { + return { + disabled: true, + options: [], + placeholder: 'Connect a Form.io project first', + }; + } + + try { + const forms = await formioCommon.listForms({ auth: auth.props }); + if (forms.length === 0) { + return { + disabled: true, + options: [], + placeholder: 'This project has no forms yet', + }; + } + return { + disabled: false, + options: forms.map((form) => ({ + label: form.title ?? form.name ?? form.path, + value: form.path, + })), + }; + } catch (error) { + return { + disabled: true, + options: [], + placeholder: 'Could not load forms from this Form.io project', + }; + } + }, + }), + + submissionId: Property.ShortText({ + displayName: 'Submission ID', + description: "The submission's `_id`", + required: true, + }), + + submissionData: Property.Json({ + displayName: 'Submission Data', + description: + 'The submission fields, keyed by the form component keys, for example {"fullName": "Amina Haddad", "email": "amina@example.gov"}', + required: true, + }), +}; diff --git a/packages/pieces/community/formio/src/lib/triggers/new-submission.ts b/packages/pieces/community/formio/src/lib/triggers/new-submission.ts new file mode 100644 index 000000000000..c5d1eb00c561 --- /dev/null +++ b/packages/pieces/community/formio/src/lib/triggers/new-submission.ts @@ -0,0 +1,11 @@ +import { registerSubmissionTrigger } from './register-submission-trigger'; + +export const newSubmission = registerSubmissionTrigger({ + name: 'new_submission', + displayName: 'New Submission', + description: 'Fires when a form receives a new submission', + aiDescription: + 'Fires when the chosen Form.io form receives a new submission. The payload is the saved submission: its id, the submitted data keyed by the form component keys, the owner, and the created and modified timestamps. Use it to start a flow on citizen or customer form intake.', + events: ['create'], + timestampField: 'created', +}); diff --git a/packages/pieces/community/formio/src/lib/triggers/register-submission-trigger.ts b/packages/pieces/community/formio/src/lib/triggers/register-submission-trigger.ts new file mode 100644 index 000000000000..1e9b842269ee --- /dev/null +++ b/packages/pieces/community/formio/src/lib/triggers/register-submission-trigger.ts @@ -0,0 +1,146 @@ +import { pollingHelper } from '@activepieces/pieces-common'; +import { createTrigger, TriggerStrategy } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { + FormioActionMethod, + FormioSubmission, + formioCommon, +} from '../common/client'; +import { submissionTriggerOutputSchema } from '../common/output-schemas'; +import { formioProps } from '../common/props'; +import { submissionSample, SubmissionTimestamp } from './submission-sample'; + +function submissionFromWebhook(body: unknown): FormioSubmission | undefined { + if (typeof body !== 'object' || body === null || !('submission' in body)) { + return undefined; + } + const { submission } = body; + if (typeof submission !== 'object' || submission === null) { + return undefined; + } + if (!('_id' in submission) || !('data' in submission)) { + return undefined; + } + return submission as FormioSubmission; +} + +export function registerSubmissionTrigger({ + name, + displayName, + description, + aiDescription, + events, + timestampField, +}: { + name: string; + displayName: string; + description: string; + aiDescription: string; + events: FormioActionMethod[]; + timestampField: SubmissionTimestamp; +}) { + const sample = submissionSample(timestampField); + const storeKey = `formio_${name}`; + + return createTrigger({ + auth: formioAuth, + name, + displayName, + description, + aiMetadata: { description: aiDescription }, + props: { formPath: formioProps.formPath }, + type: TriggerStrategy.WEBHOOK, + outputSchema: submissionTriggerOutputSchema, + sampleData: SAMPLE_SUBMISSION, + + async onEnable(context) { + const stale = await context.store.get(storeKey); + if (stale) { + try { + await formioCommon.deleteWebhookAction({ + auth: context.auth.props, + formId: stale.formId, + actionId: stale.actionId, + }); + } catch (error) { + await context.store.delete(storeKey); + } + await context.store.delete(storeKey); + } + + const formId = await formioCommon.findFormId({ + auth: context.auth.props, + formPath: context.propsValue.formPath, + }); + + const actionId = await formioCommon.createWebhookAction({ + auth: context.auth.props, + formId, + webhookUrl: context.webhookUrl, + events, + }); + + await context.store.put(storeKey, { + formId, + actionId, + }); + }, + + async onDisable(context) { + const registration = await context.store.get( + storeKey + ); + if (!registration) { + return; + } + await formioCommon.deleteWebhookAction({ + auth: context.auth.props, + formId: registration.formId, + actionId: registration.actionId, + }); + await context.store.delete(storeKey); + }, + + async run(context) { + const submission = submissionFromWebhook(context.payload.body); + return submission ? [submission] : []; + }, + + async test(context) { + return await pollingHelper.test(sample, { + auth: context.auth, + propsValue: context.propsValue, + store: context.store, + files: context.files, + }); + }, + }); +} + +export const SAMPLE_SUBMISSION = { + _id: '6a97e02d2a5c0ca5c20b1ab9', + form: '6a97df802a5c0ca5c20b1a2d', + data: { + fullName: 'Amina Haddad', + email: 'amina@example.gov', + category: 'permit', + }, + owner: null, + roles: [], + access: [], + externalIds: [], + metadata: { + headers: { + host: 'forms.example.gov', + 'user-agent': 'Mozilla/5.0', + 'content-type': 'application/json', + }, + }, + created: '2026-09-02T08:37:01.278Z', + modified: '2026-09-02T08:37:01.279Z', +}; + +export type WebhookRegistration = { + formId: string; + actionId: string; +}; diff --git a/packages/pieces/community/formio/src/lib/triggers/submission-sample.ts b/packages/pieces/community/formio/src/lib/triggers/submission-sample.ts new file mode 100644 index 000000000000..f47a1902385f --- /dev/null +++ b/packages/pieces/community/formio/src/lib/triggers/submission-sample.ts @@ -0,0 +1,62 @@ +import { DedupeStrategy, Polling } from '@activepieces/pieces-common'; +import { AppConnectionValueForAuthProperty } from '@activepieces/pieces-framework'; +import { formioAuth } from '../auth'; +import { FormioSubmission, formioCommon } from '../common/client'; + +const SAMPLE_PAGE_SIZE = 25; + +export function submissionSample( + timestampField: SubmissionTimestamp +): Polling { + return { + strategy: DedupeStrategy.TIMEBASED, + items: async ({ auth, propsValue }) => { + const { submissions } = await formioCommon.findSubmissions({ + auth: auth.props, + formPath: propsValue.formPath, + queryParams: { + limit: String(SAMPLE_PAGE_SIZE), + sort: `-${timestampField}`, + }, + }); + + return submissions + .filter((submission) => wasEdited({ submission, timestampField })) + .map((submission) => ({ + epochMilliSeconds: timestampOf({ submission, timestampField }), + data: submission, + })); + }, + }; +} + +function wasEdited({ + submission, + timestampField, +}: { + submission: FormioSubmission; + timestampField: SubmissionTimestamp; +}): boolean { + if (timestampField !== 'modified') { + return true; + } + return submission.modified !== submission.created; +} + +function timestampOf({ + submission, + timestampField, +}: { + submission: FormioSubmission; + timestampField: SubmissionTimestamp; +}): number { + const value = submission[timestampField] ?? submission.created; + const parsed = value ? Date.parse(value) : Number.NaN; + return Number.isNaN(parsed) ? 0 : parsed; +} + +export type SubmissionTimestamp = 'created' | 'modified'; + +export type FormioAuthValue = AppConnectionValueForAuthProperty< + typeof formioAuth +>; diff --git a/packages/pieces/community/formio/src/lib/triggers/updated-submission.ts b/packages/pieces/community/formio/src/lib/triggers/updated-submission.ts new file mode 100644 index 000000000000..211d3a1fa04d --- /dev/null +++ b/packages/pieces/community/formio/src/lib/triggers/updated-submission.ts @@ -0,0 +1,11 @@ +import { registerSubmissionTrigger } from './register-submission-trigger'; + +export const updatedSubmission = registerSubmissionTrigger({ + name: 'updated_submission', + displayName: 'Updated Submission', + description: 'Fires when an existing submission is changed', + aiDescription: + 'Fires when an existing submission on the chosen Form.io form is updated. The payload is the submission as it stands after the change, so compare the created and modified timestamps to tell an edit from an original intake. Use it to react to a record being corrected or progressed.', + events: ['update'], + timestampField: 'modified', +}); diff --git a/packages/pieces/community/formio/test/client.test.ts b/packages/pieces/community/formio/test/client.test.ts new file mode 100644 index 000000000000..39c1ff2babaa --- /dev/null +++ b/packages/pieces/community/formio/test/client.test.ts @@ -0,0 +1,268 @@ +/// + +const sendRequest = vi.fn(); + +vi.mock('@activepieces/pieces-common', () => ({ + HttpMethod: { + GET: 'GET', + POST: 'POST', + PUT: 'PUT', + DELETE: 'DELETE', + }, + httpClient: { + sendRequest: (...args: unknown[]) => sendRequest(...args), + }, +})); + +import { formioCommon } from '../src/lib/common/client'; + +const auth = { projectUrl: 'https://forms.example.gov/intake', apiKey: 'a-key' }; + +const lastRequest = () => sendRequest.mock.calls.at(-1)?.[0]; + +function reply(body: unknown, headers: Record = {}) { + sendRequest.mockResolvedValueOnce({ body, headers }); +} + +describe('project URL normalisation', () => { + test('a trailing slash is dropped so paths are never joined twice', () => { + expect(formioCommon.normalizeProjectUrl('https://forms.example.gov/intake/')).toBe( + 'https://forms.example.gov/intake' + ); + }); + + test('several trailing slashes are dropped', () => { + expect(formioCommon.normalizeProjectUrl('https://forms.example.gov///')).toBe( + 'https://forms.example.gov' + ); + }); + + test('surrounding whitespace is ignored', () => { + expect(formioCommon.normalizeProjectUrl(' https://forms.example.gov ')).toBe( + 'https://forms.example.gov' + ); + }); + + test('an empty URL is refused', () => { + expect(() => formioCommon.normalizeProjectUrl(' ')).toThrow(/required/i); + }); + + test('a URL without a scheme is refused rather than guessed at', () => { + expect(() => formioCommon.normalizeProjectUrl('forms.example.gov')).toThrow(/http/i); + }); + + test('http is accepted, since a self-hosted server may not have TLS', () => { + expect(formioCommon.normalizeProjectUrl('http://localhost:3001')).toBe('http://localhost:3001'); + }); +}); + +describe('requests', () => { + beforeEach(() => sendRequest.mockReset()); + + test('the API key travels in the x-token header', async () => { + reply([]); + await formioCommon.listForms({ auth }); + + expect(lastRequest().headers).toEqual({ 'x-token': 'a-key' }); + }); + + test('a path is appended to the project URL exactly once', async () => { + reply([]); + await formioCommon.listForms({ auth }); + + expect(lastRequest().url).toBe('https://forms.example.gov/intake/form'); + }); + + test('submission URLs are built from the form path', async () => { + reply({ _id: 's1' }); + await formioCommon.getSubmission({ auth, formPath: 'citizen-intake', submissionId: 's1' }); + + expect(lastRequest().url).toBe( + 'https://forms.example.gov/intake/citizen-intake/submission/s1' + ); + }); + + test('listForms asks for forms rather than resources by default', async () => { + reply([]); + await formioCommon.listForms({ auth }); + + expect(lastRequest().queryParams).toMatchObject({ type: 'form' }); + }); + + test('a non-array forms response degrades to an empty list', async () => { + reply({ message: 'nope' }); + + await expect(formioCommon.listForms({ auth })).resolves.toEqual([]); + }); +}); + +describe('Content-Range', () => { + beforeEach(() => sendRequest.mockReset()); + + test('the total comes from the header, not the row count', async () => { + reply([{ _id: 'a' }, { _id: 'b' }], { 'content-range': '0-1/57' }); + + const { submissions, total } = await formioCommon.findSubmissions({ + auth, + formPath: 'citizen-intake', + queryParams: { limit: '2' }, + }); + + expect(submissions).toHaveLength(2); + expect(total).toBe(57); + }); + + test('a missing header leaves the total undefined rather than wrong', async () => { + reply([{ _id: 'a' }]); + + const { total } = await formioCommon.findSubmissions({ + auth, + formPath: 'citizen-intake', + queryParams: {}, + }); + + expect(total).toBeUndefined(); + }); + + test('an unparseable header leaves the total undefined', async () => { + reply([], { 'content-range': 'nonsense' }); + + const { total } = await formioCommon.findSubmissions({ + auth, + formPath: 'citizen-intake', + queryParams: {}, + }); + + expect(total).toBeUndefined(); + }); +}); + +describe('updateSubmission', () => { + beforeEach(() => sendRequest.mockReset()); + + test('merging reads the submission first and keeps the fields not supplied', async () => { + reply({ _id: 's1', data: { fullName: 'Amina', email: 'amina@example.gov', refNumber: 7 } }); + reply({ _id: 's1', data: {} }); + + await formioCommon.updateSubmission({ + auth, + formPath: 'citizen-intake', + submissionId: 's1', + data: { category: 'complaint' }, + merge: true, + }); + + expect(sendRequest).toHaveBeenCalledTimes(2); + expect(lastRequest().body.data).toEqual({ + fullName: 'Amina', + email: 'amina@example.gov', + refNumber: 7, + category: 'complaint', + }); + }); + + test('a supplied field wins over the stored one', async () => { + reply({ _id: 's1', data: { category: 'permit' } }); + reply({ _id: 's1', data: {} }); + + await formioCommon.updateSubmission({ + auth, + formPath: 'citizen-intake', + submissionId: 's1', + data: { category: 'complaint' }, + merge: true, + }); + + expect(lastRequest().body.data.category).toBe('complaint'); + }); + + test('replacing sends only the supplied fields and never reads first', async () => { + reply({ _id: 's1', data: {} }); + + await formioCommon.updateSubmission({ + auth, + formPath: 'citizen-intake', + submissionId: 's1', + data: { fullName: 'Only This' }, + merge: false, + }); + + expect(sendRequest).toHaveBeenCalledTimes(1); + expect(lastRequest().body.data).toEqual({ fullName: 'Only This' }); + }); + + test('the update is a PUT carrying the submission id', async () => { + reply({ _id: 's1', data: {} }); + + await formioCommon.updateSubmission({ + auth, + formPath: 'citizen-intake', + submissionId: 's1', + data: {}, + merge: false, + }); + + expect(lastRequest().method).toBe('PUT'); + expect(lastRequest().body._id).toBe('s1'); + }); +}); + +describe('webhook action registration', () => { + beforeEach(() => sendRequest.mockReset()); + + test('the action is created on the form, scoped to the events asked for', async () => { + reply({ _id: 'action-1' }); + + const actionId = await formioCommon.createWebhookAction({ + auth, + formId: 'form-1', + webhookUrl: 'https://ap.example/api/v1/webhooks/abc', + events: ['create'], + }); + + expect(actionId).toBe('action-1'); + expect(lastRequest().url).toBe('https://forms.example.gov/intake/form/form-1/action'); + expect(lastRequest().body).toMatchObject({ + name: 'webhook', + handler: ['after'], + method: ['create'], + settings: { url: 'https://ap.example/api/v1/webhooks/abc' }, + }); + }); + + test('an action created without an id is an error rather than a silent no-op', async () => { + reply({}); + + await expect( + formioCommon.createWebhookAction({ + auth, + formId: 'form-1', + webhookUrl: 'https://ap.example/hook', + events: ['update'], + }) + ).rejects.toThrow(/id/i); + }); + + test('deleting an action addresses it by id', async () => { + reply({}); + + await formioCommon.deleteWebhookAction({ auth, formId: 'form-1', actionId: 'action-1' }); + + expect(lastRequest().method).toBe('DELETE'); + expect(lastRequest().url).toBe( + 'https://forms.example.gov/intake/form/form-1/action/action-1' + ); + }); +}); + +describe('deleteSubmission', () => { + beforeEach(() => sendRequest.mockReset()); + + test('it reports what it deleted, so a flow can log the id', async () => { + reply({}); + + await expect( + formioCommon.deleteSubmission({ auth, formPath: 'citizen-intake', submissionId: 's9' }) + ).resolves.toEqual({ deleted: true, submissionId: 's9' }); + }); +}); diff --git a/packages/pieces/community/formio/test/submission-sample.test.ts b/packages/pieces/community/formio/test/submission-sample.test.ts new file mode 100644 index 000000000000..356fa25f38b8 --- /dev/null +++ b/packages/pieces/community/formio/test/submission-sample.test.ts @@ -0,0 +1,144 @@ +/// + +import { submissionSample } from '../src/lib/triggers/submission-sample'; + +const findSubmissions = vi.fn(); + +vi.mock('../src/lib/common/client', () => ({ + formioCommon: { + findSubmissions: (...args: unknown[]) => findSubmissions(...args), + }, +})); + +const auth = { props: { projectUrl: 'https://forms.example.gov', apiKey: 'k' } }; +const propsValue = { formPath: 'citizen-intake' }; + +function submission({ + id, + created, + modified, +}: { + id: string; + created: string; + modified?: string; +}) { + return { + _id: id, + form: 'f1', + data: { fullName: id }, + created, + modified: modified ?? created, + }; +} + +async function fetchSample({ + timestampField, + rows, + formPath = propsValue.formPath, +}: { + timestampField: 'created' | 'modified'; + rows: ReturnType[]; + formPath?: string; +}) { + findSubmissions.mockResolvedValueOnce({ submissions: rows, total: rows.length }); + const sample = submissionSample(timestampField); + const items = await sample.items({ auth, propsValue: { formPath } } as never); + const call = findSubmissions.mock.calls.at(-1)?.[0]; + return { items, query: call?.queryParams, formPath: call?.formPath }; +} + +describe('submission sample data', () => { + beforeEach(() => findSubmissions.mockReset()); + + test('the newest submissions are asked for, so a sample is recognisable to whoever is building the flow', async () => { + const { query } = await fetchSample({ timestampField: 'created', rows: [] }); + + expect(query.sort).toBe('-created'); + }); + + test('the sample sorts on the field its trigger fires for', async () => { + const { query } = await fetchSample({ timestampField: 'modified', rows: [] }); + + expect(query.sort).toBe('-modified'); + }); + + test('a page size is always requested, so one sample cannot pull an unbounded list', async () => { + const { query } = await fetchSample({ timestampField: 'created', rows: [] }); + + expect(Number(query.limit)).toBeGreaterThan(0); + }); + + test('no timestamp filter is sent, since a sample is not resuming from anywhere', async () => { + const { query } = await fetchSample({ timestampField: 'created', rows: [] }); + + expect(query['created__gt']).toBeUndefined(); + expect(query['modified__gt']).toBeUndefined(); + }); + + test('the updated sample skips submissions that have never been edited', async () => { + const { items } = await fetchSample({ + timestampField: 'modified', + rows: [ + submission({ id: 'untouched', created: '2026-09-02T10:00:00.000Z' }), + submission({ + id: 'edited', + created: '2026-09-01T09:00:00.000Z', + modified: '2026-09-02T11:00:00.000Z', + }), + ], + }); + + expect(items.map((item) => (item.data as { _id: string })._id)).toEqual(['edited']); + }); + + test('the new-submission sample keeps every row, edited or not', async () => { + const { items } = await fetchSample({ + timestampField: 'created', + rows: [ + submission({ id: 'a', created: '2026-09-02T10:00:00.000Z' }), + submission({ + id: 'b', + created: '2026-09-02T10:00:01.000Z', + modified: '2026-09-02T12:00:00.000Z', + }), + ], + }); + + expect(items).toHaveLength(2); + }); + + test('the reported timestamp comes from the field being sampled', async () => { + const { items } = await fetchSample({ + timestampField: 'modified', + rows: [ + submission({ + id: 's1', + created: '2026-09-01T00:00:00.000Z', + modified: '2026-09-02T10:00:00.000Z', + }), + ], + }); + + expect(items[0].epochMilliSeconds).toBe(Date.parse('2026-09-02T10:00:00.000Z')); + }); + + test('an unparseable timestamp becomes 0 rather than NaN', async () => { + const { items } = await fetchSample({ + timestampField: 'created', + rows: [{ _id: 'x', form: 'f1', data: {}, created: 'not a date' } as never], + }); + + expect(items[0].epochMilliSeconds).toBe(0); + expect(Number.isNaN(items[0].epochMilliSeconds)).toBe(false); + }); + + test('the form chosen in the step is the form queried', async () => { + const { formPath } = await fetchSample({ + timestampField: 'created', + rows: [], + formPath: 'permit-renewal', + }); + + expect(formPath).toBe('permit-renewal'); + }); +}); diff --git a/packages/pieces/community/formio/tsconfig.json b/packages/pieces/community/formio/tsconfig.json new file mode 100644 index 000000000000..059cd8166183 --- /dev/null +++ b/packages/pieces/community/formio/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} diff --git a/packages/pieces/community/formio/tsconfig.lib.json b/packages/pieces/community/formio/tsconfig.lib.json new file mode 100644 index 000000000000..1d249c8f4125 --- /dev/null +++ b/packages/pieces/community/formio/tsconfig.lib.json @@ -0,0 +1,22 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "baseUrl": ".", + "paths": {}, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "jest.config.ts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/packages/pieces/community/formio/vitest.config.ts b/packages/pieces/community/formio/vitest.config.ts new file mode 100644 index 000000000000..f520fc141133 --- /dev/null +++ b/packages/pieces/community/formio/vitest.config.ts @@ -0,0 +1,18 @@ +import path from 'path' +import { defineConfig } from 'vitest/config' + +const repoRoot = path.resolve(__dirname, '../../../..') + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + }, + resolve: { + alias: { + '@activepieces/shared': path.resolve(repoRoot, 'packages/core/shared/src/index.ts'), + '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'), + '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'), + }, + }, +}) diff --git a/packages/pieces/framework/src/index.ts b/packages/pieces/framework/src/index.ts index 9ecce7c5622b..31993de3caf8 100644 --- a/packages/pieces/framework/src/index.ts +++ b/packages/pieces/framework/src/index.ts @@ -64,6 +64,8 @@ export { GetProviderConfigResponse, OpenAICompatibleProviderConfig, OpenAiCompatibleVendorConfig, + VertexProviderAuthConfig, + VertexProviderConfig, OPENAI_COMPATIBLE_VENDOR_BASE_URLS, getEffectiveProviderAndModel, splitCloudflareGatewayModelId, diff --git a/packages/server/api/package.json b/packages/server/api/package.json index 547c00b251e0..a198d6bab878 100644 --- a/packages/server/api/package.json +++ b/packages/server/api/package.json @@ -19,6 +19,7 @@ "@ai-sdk/azure": "4.0.26", "@ai-sdk/google": "4.0.29", "@ai-sdk/google-vertex": "5.0.36", + "google-auth-library": "10.6.1", "@ai-sdk/mcp": "2.0.20", "@ai-sdk/openai": "4.0.25", "@ai-sdk/openai-compatible": "3.0.18", diff --git a/packages/server/api/src/app/ai/ai-provider-service.ts b/packages/server/api/src/app/ai/ai-provider-service.ts index 088ab5d77297..2416f586dd4e 100644 --- a/packages/server/api/src/app/ai/ai-provider-service.ts +++ b/packages/server/api/src/app/ai/ai-provider-service.ts @@ -162,7 +162,7 @@ export const aiProviderService = (log: FastifyBaseLogger) => ({ return null } const auth = await decryptRowAuth({ aiProvider: chatProvider, platformId }) - return { provider: chatProvider.provider, configId: chatProvider.id, auth, config: chatProvider.config, platformId } + return { provider: chatProvider.provider, configId: chatProvider.id, auth, config: chatProvider.config, platformId, modelScope: chatProvider.modelScope, modelIds: chatProvider.modelIds } }, async keyServesScope({ platformId, provider, configId, resolvedFor, target }: { platformId: PlatformId, provider?: AIProviderName, configId?: string, resolvedFor: ProviderScope, target: ProviderScope }): Promise { @@ -240,7 +240,7 @@ export const aiProviderService = (log: FastifyBaseLogger) => ({ async getConfigOrThrow({ platformId, provider, scope, configId }: { platformId: PlatformId, provider: AIProviderName, scope: ProviderScope, configId?: string }): Promise { const aiProvider = await resolveRowForScope({ platformId, provider, scope, configId }) const auth = await decryptRowAuth({ aiProvider, platformId }) - return { provider: aiProvider.provider, configId: aiProvider.id, auth, config: aiProvider.config, platformId } + return { provider: aiProvider.provider, configId: aiProvider.id, auth, config: aiProvider.config, platformId, modelScope: aiProvider.modelScope, modelIds: aiProvider.modelIds } }, async getOrCreateActivePiecesProviderAuthConfig(platformId: PlatformId): Promise { await ensureManagedProviderRow({ platformId }) @@ -490,7 +490,7 @@ async function enrichWithKeysIfNeeded(aiProvider: AIProviderSchema, platformId: config: {}, auth: await encryptUtils.encryptObject(rawAuth), }) - return { provider: savedAiProvider.provider, configId: savedAiProvider.id, auth: rawAuth, config: savedAiProvider.config, platformId } + return { provider: savedAiProvider.provider, configId: savedAiProvider.id, auth: rawAuth, config: savedAiProvider.config, platformId, modelScope: savedAiProvider.modelScope, modelIds: savedAiProvider.modelIds } } diff --git a/packages/server/api/src/app/ai/providers/index.ts b/packages/server/api/src/app/ai/providers/index.ts index cb40ead4326a..27027e01f67b 100644 --- a/packages/server/api/src/app/ai/providers/index.ts +++ b/packages/server/api/src/app/ai/providers/index.ts @@ -11,6 +11,7 @@ import { openAICompatibleProvider } from './openai-compatible-gateway-provider' import { openAiCompatibleVendor } from './openai-compatible-vendor' import { openaiProvider } from './openai-provider' import { openRouterProvider } from './openrouter-provider' +import { vertexProvider } from './vertex-provider' export const aiProviders: Record> = { [AIProviderName.OPENAI]: openaiProvider, @@ -21,6 +22,7 @@ export const aiProviders: Record = { + name: 'Google Vertex AI', + + async validateConnection( + authConfig: VertexProviderAuthConfig, + config: VertexProviderConfig, + _log: FastifyBaseLogger, + ): Promise { + const credentials = parseServiceAccount(authConfig.serviceAccountJson) + + const { data: token, error } = await tryCatch(() => new GoogleAuth({ + credentials, + projectId: config.project, + scopes: [CLOUD_PLATFORM_SCOPE], + }).getAccessToken()) + + if (error) { + throw error + } + if (!token) { + throw new Error('Vertex AI service account returned no access token') + } + }, + + async listModels(_authConfig: VertexProviderAuthConfig, config: VertexProviderConfig): Promise { + return config.models.map((model) => ({ + id: model.modelId, + name: model.modelName, + type: model.modelType, + })) + }, +} + +function parseServiceAccount(serviceAccountJson: string): ServiceAccountCredentials { + const { data: parsed, error } = tryCatchSync(() => JSON.parse(serviceAccountJson)) + if (error) { + throw new Error('Service account JSON is not valid JSON — paste the whole key file') + } + if (!isServiceAccount(parsed)) { + throw new Error('Service account JSON must have type "service_account" with project_id, client_email and private_key') + } + return { + type: parsed.type, + project_id: parsed.project_id, + client_email: parsed.client_email, + private_key: parsed.private_key.replace(/\\n/g, '\n'), + } +} + +function isServiceAccount(value: unknown): value is ServiceAccountCredentials { + if (typeof value !== 'object' || value === null) { + return false + } + const candidate: Record = { ...value } + return candidate['type'] === 'service_account' + && typeof candidate['project_id'] === 'string' + && typeof candidate['client_email'] === 'string' + && typeof candidate['private_key'] === 'string' +} + +const CLOUD_PLATFORM_SCOPE = 'https://www.googleapis.com/auth/cloud-platform' + +type ServiceAccountCredentials = { + type: string + project_id: string + client_email: string + private_key: string +} diff --git a/packages/server/api/src/app/app.ts b/packages/server/api/src/app/app.ts index 7568bbd446a6..91cdd9b2c78d 100644 --- a/packages/server/api/src/app/app.ts +++ b/packages/server/api/src/app/app.ts @@ -29,6 +29,7 @@ import { authorizationMiddleware } from './core/security/v2/authz/authorization- import { distributedLock, redisConnections } from './database/redis-connections' import { agentEvalModule } from './ee/agent/agent-eval-controller' import { agentHelpers } from './ee/agent/agent-helpers' +import { assertAgentsResolveInProject } from './ee/agent/agent-service' import { agentModule } from './ee/agent/agent.module' import { alertsModule } from './ee/alerts/alerts-module' import { apiKeyModule } from './ee/api-keys/api-key-module' @@ -71,6 +72,7 @@ import { userModule } from './ee/users/user.module' import { fileModule } from './file/file.module' import { flagModule } from './flags/flag.module' import { flagHooks } from './flags/flags.hooks' +import { flowPublishHooks } from './flows/flow/flow-publish-hooks' import { flowBackgroundJobs } from './flows/flow/flow.jobs' import { humanInputModule } from './flows/flow/human-input/human-input.module' import { flowRunModule } from './flows/flow-run/flow-run-module' @@ -100,6 +102,7 @@ import { pieceModule } from './pieces/metadata/piece-metadata-controller' import { pieceMetadataService } from './pieces/metadata/piece-metadata-service' import { pieceSyncService } from './pieces/piece-sync-service' import { billingProvider } from './platform/billing-provider' +import { piecesReportModule } from './platform/pieces-report/pieces-report.module' import { platformModule } from './platform/platform.module' import { projectHooks } from './project/project-hooks' import { storeEntryModule } from './store-entry/store-entry.module' @@ -240,6 +243,7 @@ export const setupApp = async (app: FastifyInstance): Promise = await app.register(authenticationModule) await app.register(triggerModule) await app.register(platformModule) + await app.register(piecesReportModule) await app.register(humanInputModule) await app.register(mcpServerModule) await app.register(mcpOAuthApproveController) @@ -349,6 +353,7 @@ export const setupApp = async (app: FastifyInstance): Promise = flagHooks.set(enterpriseFlagsHooks) billingProvider.set(autumnBillingProvider) resumePageHooks.set((log) => ({ getTheme: (params) => appearanceHelper.getTheme({ ...params, log }) })) + flowPublishHooks.set(() => ({ assertReferencesResolve: assertAgentsResolveInProject })) exceptionHandler.initializeSentry(system.get(AppSystemProp.SENTRY_DSN)) systemJobHandlers.registerJobHandler(SystemJobName.HARD_DELETE_PLATFORM, (data) => platformTeardownJobs(app.log).hardDeletePlatformHandler(data)) break @@ -385,6 +390,7 @@ export const setupApp = async (app: FastifyInstance): Promise = flagHooks.set(enterpriseFlagsHooks) billingProvider.set(autumnBillingProvider) resumePageHooks.set((log) => ({ getTheme: (params) => appearanceHelper.getTheme({ ...params, log }) })) + flowPublishHooks.set(() => ({ assertReferencesResolve: assertAgentsResolveInProject })) break case ApEdition.COMMUNITY: await app.register(platformProjectModule) diff --git a/packages/server/api/src/app/ee/agent/agent-controller.ts b/packages/server/api/src/app/ee/agent/agent-controller.ts index 9cdc4929d2e5..acfaf887240e 100644 --- a/packages/server/api/src/app/ee/agent/agent-controller.ts +++ b/packages/server/api/src/app/ee/agent/agent-controller.ts @@ -1,5 +1,5 @@ import { ActivepiecesError, ApId, assertNotNullOrUndefined, ErrorCode, Permission, SeekPage, UserId } from '@activepieces/core-utils' -import { Agent, AgentSummary, AgentWithUsage, ApplicationEventName, CreateAgentRequest, DraftAgentRequest, DraftAgentResponse, GetAgentRequest, ListAgentsRequest, PrincipalType, SERVICE_KEY_SECURITY_OPENAPI, UpdateAgentRequest } from '@activepieces/shared' +import { Agent, AgentMovePreview, AgentSummary, AgentWithUsage, ApplicationEventName, CreateAgentRequest, DraftAgentRequest, DraftAgentResponse, GetAgentRequest, ListAgentsRequest, MoveAgentRequest, PrincipalType, SERVICE_KEY_SECURITY_OPENAPI, UpdateAgentRequest } from '@activepieces/shared' import { FastifyRequest } from 'fastify' import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod' import { StatusCodes } from 'http-status-codes' @@ -117,6 +117,31 @@ export const agentController: FastifyPluginAsyncZod = async (app) => { return agentRedaction.withoutToolSecrets(agent) }) + app.get('/:id/move-preview', MovePreviewRoute, async (request): Promise => { + return agentService(request.log).movePreview({ + id: request.params.id, + projectId: request.projectId, + userId: await resolveUserId(request), + targetProjectId: request.query.projectId, + platformId: request.principal.platform.id, + }) + }) + + app.post('/:id/move', MoveAgentRoute, async (request): Promise => { + const agent = await agentService(request.log).move({ + id: request.params.id, + projectId: request.projectId, + userId: await resolveUserId(request), + targetProjectId: request.body.projectId, + platformId: request.principal.platform.id, + }) + applicationEvents(request.log).sendUserEvent(request, { + action: ApplicationEventName.AGENT_UPDATED, + data: { agent: { id: agent.id, displayName: agent.displayName } }, + }) + return agentRedaction.withoutToolSecrets(agent) + }) + app.delete('/:id', DeleteAgentRoute, async (request, reply): Promise => { const agent = await agentService(request.log).delete({ id: request.params.id, @@ -268,6 +293,38 @@ const UnpublishAgentRoute = { }, } +const moveSecurity = { + security: securityAccess.project( + [PrincipalType.USER, PrincipalType.SERVICE], + Permission.WRITE_AGENT, + { type: ProjectResourceType.TABLE, tableName: AgentEntity }, + ), +} + +const MovePreviewRoute = { + config: moveSecurity, + schema: { + tags: ['agents'], + security: [SERVICE_KEY_SECURITY_OPENAPI], + description: 'Report what moving an agent into another project would cost', + params: z.object({ id: ApId }), + querystring: MoveAgentRequest, + response: { [StatusCodes.OK]: AgentMovePreview }, + }, +} + +const MoveAgentRoute = { + config: moveSecurity, + schema: { + tags: ['agents'], + security: [SERVICE_KEY_SECURITY_OPENAPI], + description: 'Move an agent to another project', + params: z.object({ id: ApId }), + body: MoveAgentRequest, + response: { [StatusCodes.OK]: Agent }, + }, +} + const DeleteAgentRoute = { config: { security: securityAccess.project( diff --git a/packages/server/api/src/app/ee/agent/agent-conversation-service.ts b/packages/server/api/src/app/ee/agent/agent-conversation-service.ts index b34aa8afe59d..29d267bf9853 100644 --- a/packages/server/api/src/app/ee/agent/agent-conversation-service.ts +++ b/packages/server/api/src/app/ee/agent/agent-conversation-service.ts @@ -2,15 +2,34 @@ import { ActivepiecesError, apId, ErrorCode, isNil, sanitizeObjectForPostgresql, import { Agent, AgentConversation, AgentConversationStatus, AgentHistoryMessage, AgentRunSource, CreateAgentConversationRequest, PersistedAgentMessage, PersistedAgentRole, SetAgentMessageFeedbackRequest, UpdateAgentConversationRequest } from '@activepieces/shared' import { ModelMessage } from 'ai' import { FastifyBaseLogger } from 'fastify' +import { EntityManager } from 'typeorm' +import { transaction } from '../../core/db/transaction' import { buildPaginator } from '../../helper/pagination/build-paginator' import { paginationHelper } from '../../helper/pagination/pagination-utils' import { Order } from '../../helper/pagination/paginator' import { agentApprovalGate } from './agent-approval-gate' import { AgentConversationEntity } from './agent-conversation-entity' +import { AgentEntity } from './agent-entity' import { agentHelpers, EVAL_CONVERSATION_ID_PREFIX, isEvalConversationId } from './agent-helpers' import { agentService } from './agent-service' import { agentHistory } from './history/agent-history' +async function projectStillHoldingAgent({ agentId, authorisedProjectId, entityManager }: { agentId: string, authorisedProjectId: string, entityManager: EntityManager }): Promise { + const locked = await entityManager.getRepository(AgentEntity) + .createQueryBuilder('agent') + .select(['agent.projectId']) + .setLock('pessimistic_write') + .where('agent.id = :agentId', { agentId }) + .getOne() + if (isNil(locked) || locked.projectId !== authorisedProjectId) { + throw new ActivepiecesError({ + code: ErrorCode.VALIDATION, + params: { message: 'That agent has just moved to another project. Open it again to start a new chat.' }, + }) + } + return locked.projectId +} + export const agentConversationService = (log: FastifyBaseLogger) => ({ async createConversation({ platformId, userId, request, id }: CreateConversationParams): Promise { const agent = isNil(request.agentId) @@ -20,17 +39,19 @@ export const agentConversationService = (log: FastifyBaseLogger) => ({ const builderProjectId = builder ? await resolveBuilderProject({ agent, requestedProjectId: request.projectId, platformId, userId, log }) : null - const conversation = await agentHelpers.conversationRepo().save({ + const conversation = await transaction(async (entityManager) => entityManager.getRepository(AgentConversationEntity).save({ id: id ?? apId(), platformId, - projectId: agent?.projectId ?? builderProjectId, + projectId: isNil(agent) + ? builderProjectId + : await projectStillHoldingAgent({ agentId: agent.id, authorisedProjectId: agent.projectId, entityManager }), userId, agentId: agent?.id ?? null, source: builder ? AgentRunSource.AGENT_BUILDER : isNil(agent) ? AgentRunSource.CHAT : AgentRunSource.AGENT, title: request.title ?? null, modelName: request.modelName ?? null, messages: [], - }) + })) log.info({ conversation: { id: conversation.id }, platform: { id: platformId }, user: { id: userId } }, '[agentConversationService] Conversation created') return conversation }, diff --git a/packages/server/api/src/app/ee/agent/agent-helpers.ts b/packages/server/api/src/app/ee/agent/agent-helpers.ts index 0f1b0847a546..fecdb4f75557 100644 --- a/packages/server/api/src/app/ee/agent/agent-helpers.ts +++ b/packages/server/api/src/app/ee/agent/agent-helpers.ts @@ -1,6 +1,6 @@ import { ActivepiecesError, AIProviderName, apId, ErrorCode, isNil, ProviderOutcomeReporter, spreadIfDefined, tryCatch, unique } from '@activepieces/core-utils' import { agentAiUtils } from '@activepieces/server-utils' -import { ACTIVEPIECES_CHAT_TIERS, AgentConversation, AgentConversationStatus, AI_PROVIDER_ENTITY_TYPES, aiProviderUtils, DEFAULT_CHAT_TIER_ID, GetAgentMemoryResponse, GetProviderConfigResponse, Project, ProjectType, UserMemory } from '@activepieces/shared' +import { ACTIVEPIECES_CHAT_TIERS, AgentConversation, AgentConversationStatus, AI_PROVIDER_ENTITY_TYPES, AIProviderConfig, AiProviderModelScope, AIProviderModelType, aiProviderUtils, DEFAULT_CHAT_TIER_ID, GetAgentMemoryResponse, GetProviderConfigResponse, Project, ProjectType, UserMemory } from '@activepieces/shared' import { SharedV3ProviderOptions } from '@ai-sdk/provider' import { EmbeddingModel, LanguageModel } from 'ai' import { FastifyBaseLogger } from 'fastify' @@ -154,20 +154,42 @@ function resolveTier({ tierId }: { tierId: string | null }) { return findTier({ tierId }) ?? findTier({ tierId: DEFAULT_CHAT_TIER_ID }) ?? ACTIVEPIECES_CHAT_TIERS[0] } -function resolveModelIdForProvider({ provider, selectedModel }: { provider: AIProviderName, selectedModel: string | null }): string { - const curatedModels = aiProviderUtils.getCuratedChatModels({ provider }) - if (selectedModel && curatedModels?.some((model) => model.id === selectedModel)) { - return selectedModel +// An admin-listed catalog is the whole truth about what a key exposes, so an empty one means the key +// serves no text model - not that we may fall back to a curated id it was never configured for. +function manualTextModelCatalog({ config }: { config?: AIProviderConfig }): string[] | undefined { + if (isNil(config) || !('models' in config)) { + return undefined + } + return config.models.filter((model) => model.modelType === AIProviderModelType.TEXT).map((model) => model.modelId) +} + +function pickAllowedModel({ provider, selectedModel, candidates, modelScope, modelIds }: { provider: AIProviderName, selectedModel: string | null, candidates: string[], modelScope?: AiProviderModelScope, modelIds?: string[] }): string { + const allowed = modelScope === 'selected' && !isNil(modelIds) + ? candidates.filter((candidate) => modelIds.includes(candidate)) + : candidates + if (allowed.length === 0) { + throw new ActivepiecesError({ + code: ErrorCode.ENTITY_NOT_FOUND, + params: { entityId: provider, entityType: AI_PROVIDER_ENTITY_TYPES.provider }, + }, 'this AI provider key allows no text model a chat turn can run on') + } + return selectedModel && allowed.includes(selectedModel) ? selectedModel : allowed[0] +} + +function resolveModelIdForProvider({ provider, selectedModel, config, modelScope, modelIds }: { provider: AIProviderName, selectedModel: string | null, config?: AIProviderConfig, modelScope?: AiProviderModelScope, modelIds?: string[] }): string { + const catalog = manualTextModelCatalog({ config }) + if (!isNil(catalog)) { + return pickAllowedModel({ provider, selectedModel, candidates: catalog, modelScope, modelIds }) } + const curatedModels = aiProviderUtils.getCuratedChatModels({ provider }) const tierModelId = resolveTier({ tierId: selectedModel }).modelId if (provider === AIProviderName.ACTIVEPIECES || provider === AIProviderName.OPENROUTER) { return tierModelId } const nativeModelId = tierModelId.replace(/^[^/]+\//, '').replace(/\./g, '-') - if (isNil(curatedModels)) { - return nativeModelId - } - return curatedModels.some((model) => model.id === nativeModelId) ? nativeModelId : curatedModels[0].id + const candidates = isNil(curatedModels) ? [nativeModelId] : curatedModels.map((model) => model.id) + const preferred = selectedModel && candidates.includes(selectedModel) ? selectedModel : nativeModelId + return pickAllowedModel({ provider, selectedModel: preferred, candidates, modelScope, modelIds }) } // Analytics and billing report the model a turn ran on. The provider is unknown when a platform's @@ -198,7 +220,7 @@ function reportKeyOutcome({ platformId, providerId, log }: { platformId: string, async function resolveTierModel({ platformId, tierId, provider, providerConfigId, scope, log }: { platformId: string, tierId: string, provider?: AIProviderName, providerConfigId?: string, scope: ProviderScope, log: FastifyBaseLogger }): Promise<{ model: LanguageModel, modelId: string, provider: AIProviderName }> { const providerConfig = await resolveRunProvider({ platformId, scope, log, ...spreadIfDefined('provider', provider), ...spreadIfDefined('providerConfigId', providerConfigId) }) - const modelId = resolveModelIdForProvider({ provider: providerConfig.provider, selectedModel: tierId }) + const modelId = resolveModelIdForProvider({ provider: providerConfig.provider, selectedModel: tierId, config: providerConfig.config, modelScope: providerConfig.modelScope, modelIds: providerConfig.modelIds }) return { model: agentAiUtils.createChatModel({ provider: providerConfig.provider, @@ -216,8 +238,8 @@ async function resolveFastModel({ platformId, provider, providerConfigId, scope, return (await resolveTierModel({ platformId, tierId: FAST_TIER_ID, scope, log, ...spreadIfDefined('provider', provider), ...spreadIfDefined('providerConfigId', providerConfigId) })).model } -function resolveFastModelId({ provider }: { provider: AIProviderName }): string { - return resolveModelIdForProvider({ provider, selectedModel: FAST_TIER_ID }) +function resolveFastModelId({ provider, config, modelScope, modelIds }: { provider: AIProviderName, config?: AIProviderConfig, modelScope?: AiProviderModelScope, modelIds?: string[] }): string { + return resolveModelIdForProvider({ provider, selectedModel: FAST_TIER_ID, config, modelScope, modelIds }) } async function resolveEmbeddingModel({ platformId, provider, providerConfigId, scope, log }: { platformId: string, provider?: AIProviderName, providerConfigId?: string, scope: ProviderScope, log: FastifyBaseLogger }): Promise<{ model: EmbeddingModel, providerOptions: SharedV3ProviderOptions }> { diff --git a/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts b/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts index 5828dbc768c8..ccf3ce20f39c 100644 --- a/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts +++ b/packages/server/api/src/app/ee/agent/agent-rpc-handlers.ts @@ -181,7 +181,7 @@ export const agentRpcHandlers = (log: FastifyBaseLogger) => ({ const tier = agentHelpers.resolveTier({ tierId: namesItsOwnModel ? null : selectedModel }) const resolvedModelId = namesItsOwnModel && !isNil(modelName) ? modelName - : agentHelpers.resolveModelIdForProvider({ provider: providerConfig.provider, selectedModel }) + : agentHelpers.resolveModelIdForProvider({ provider: providerConfig.provider, selectedModel, config: providerConfig.config, modelScope: providerConfig.modelScope, modelIds: providerConfig.modelIds }) // Inject an inventory of the project's existing connections into context so the agent // never has to *guess* an app name to find out what's connected. Without this, discovery @@ -299,7 +299,7 @@ export const agentRpcHandlers = (log: FastifyBaseLogger) => ({ auth: providerConfig.auth as Record, providerConfig: providerConfig.config as Record, modelId: resolvedModelId, - fastModelId: agentHelpers.resolveFastModelId({ provider: providerConfig.provider }), + fastModelId: agentHelpers.resolveFastModelId({ provider: providerConfig.provider, config: providerConfig.config, modelScope: providerConfig.modelScope, modelIds: providerConfig.modelIds }), systemPrompt: systemPromptText, messages: messagesForLlm, allMessages, diff --git a/packages/server/api/src/app/ee/agent/agent-service.ts b/packages/server/api/src/app/ee/agent/agent-service.ts index 7e2ecf235259..075a70bab710 100644 --- a/packages/server/api/src/app/ee/agent/agent-service.ts +++ b/packages/server/api/src/app/ee/agent/agent-service.ts @@ -1,19 +1,23 @@ import { createHash } from 'node:crypto' import { AgentToolType, McpAuthType } from '@activepieces/core-piece-types' -import { ActivepiecesError, ApId, apId, Cursor, ErrorCode, isNil, omit, Permission, PlatformId, ProjectId, sanitizeObjectForPostgresql, SeekPage, UserId } from '@activepieces/core-utils' -import { Agent, AgentConfig, AgentListSort, AgentSummary, agentUtils, AgentVisibility, CreateAgentRequest, DEFAULT_CHAT_TIER_ID, DefaultProjectRole, Project, ProjectType, UpdateAgentRequest } from '@activepieces/shared' +import { ActivepiecesError, apId, ApId, connectionTemplate, Cursor, ErrorCode, isNil, omit, Permission, PlatformId, ProjectId, sanitizeObjectForPostgresql, SeekPage, unique, UserId } from '@activepieces/core-utils' +import { Agent, AgentConfig, AgentFlowTool, AgentKnowledgeBaseTool, AgentListSort, AgentMoveLoss, AgentMoveLossKind, AgentMovePreview, AgentRunSource, AgentSummary, agentUtils, AgentVisibility, CreateAgentRequest, DEFAULT_CHAT_TIER_ID, DefaultProjectRole, Project, ProjectType, UpdateAgentRequest } from '@activepieces/shared' import { FastifyBaseLogger } from 'fastify' -import { Brackets, In, SelectQueryBuilder } from 'typeorm' +import { Brackets, EntityManager, In, SelectQueryBuilder } from 'typeorm' +import { appConnectionService } from '../../app-connection/app-connection-service/app-connection-service' import { repoFactory } from '../../core/db/repo-factory' import { transaction } from '../../core/db/transaction' -import { publishedFlowsUsingAgent, PublishedFlowsUsingAgent } from '../../flows/flow-version/flow-version.service' +import { flowService } from '../../flows/flow/flow.service' +import { PublishedFlowsUsingAgent, publishedFlowsUsingAgent, publishedFlowVersionsUsingAgent } from '../../flows/flow-version/flow-version.service' import { buildPaginator } from '../../helper/pagination/build-paginator' import { paginationHelper } from '../../helper/pagination/pagination-utils' import { Order, OrderByConfig } from '../../helper/pagination/paginator' +import { knowledgeBaseService } from '../../knowledge-base/knowledge-base.service' import { resolvePermissionChecker } from '../../mcp/mcp-permissions' import { projectService } from '../../project/project-service' import { userService } from '../../user/user-service' import { projectMemberService } from '../projects/project-members/project-member.service' +import { AgentConversationEntity } from './agent-conversation-entity' import { AgentEntity, AgentWithRelations } from './agent-entity' import { agentHelpers } from './agent-helpers' @@ -25,6 +29,8 @@ export const agentAudit = { describePublished } export const agentRedaction = { withoutToolSecrets } +const AGENT_MOVED_AWAY = 'That agent has just been moved somewhere else. Reload the page and try again.' + export const agentService = (log: FastifyBaseLogger) => ({ async create({ platformId, projectId, ownerId, request }: CreateParams): Promise { const visibility = request.visibility ?? AgentVisibility.PROJECT @@ -113,7 +119,10 @@ export const agentService = (log: FastifyBaseLogger) => ({ }) const draft = isNil(request.draft) ? agent.draft : sanitizeObjectForPostgresql(request.draft) const published = goLive && agentUtils.isPublishable(draft) ? draft : agent.published - await agentRepo().save({ ...omit(agent, ['published']), ...omit(request, ['goLive']), draft, published, visibility, sharedWithUserIds }) + await transaction(async (entityManager) => { + await lockedAgentInProjectOrThrow({ entityManager, id, projectId }) + await entityManager.getRepository(AgentEntity).save({ id, ...omit(request, ['goLive', 'draft', 'visibility', 'sharedWithUserIds']), draft, published, visibility, sharedWithUserIds }) + }) return this.getOneOrThrow({ id, projectId, userId }) }, @@ -171,7 +180,7 @@ export const agentService = (log: FastifyBaseLogger) => ({ if (isNil(tools)) { return null } - await repo.save({ ...omit(agent, ['published']), draft: sanitizeObjectForPostgresql({ ...agent.draft, tools }) }) + await repo.save({ id, draft: sanitizeObjectForPostgresql({ ...agent.draft, tools }) }) return this.getOneOrThrow({ id, projectId, userId }) }) }, @@ -183,16 +192,75 @@ export const agentService = (log: FastifyBaseLogger) => ({ return { total: usage.total, names: usage.names } }, - async delete({ id, projectId, userId }: GetParams): Promise { + async movePreview({ id, projectId, userId, targetProjectId, platformId }: MoveParams & { id: string }): Promise { const agent = await this.getOneOrThrow({ id, projectId, userId }) await assertMayDestroy({ agent, projectId, userId, log }) - const flowsInUse = await this.publishedFlowsUsing({ agent, projectId, userId }) - if (flowsInUse.total > 0) { - throw new ActivepiecesError({ - code: ErrorCode.VALIDATION, - params: { message: describeFlowsInUse(flowsInUse) }, - }) + const target = await readableProjectOrThrow({ platformId, userId, targetProjectId, log }) + const [flowsInUse, mayCreateAgentsThere] = await Promise.all([ + this.publishedFlowsUsing({ agent, projectId, userId }), + mayWriteAgentsIn({ projectId: target.id, userId, log }), + ]) + if (!mayCreateAgentsThere) { + return { blockedByPublishedFlows: flowsInUse, mayCreateAgentsThere, toolsThatStopWorking: [], membersLosingAccess: 0 } + } + const [toolsThatStopWorking, sharedWithUserIds] = await Promise.all([ + toolsBrokenBy({ agent, targetProjectId: target.id, log }), + resolveShare({ visibility: agent.visibility, requested: undefined, stored: agent.sharedWithUserIds, projectId: target.id, log }), + ]) + return { + blockedByPublishedFlows: flowsInUse, + mayCreateAgentsThere, + toolsThatStopWorking, + membersLosingAccess: agent.sharedWithUserIds.length - sharedWithUserIds.length, } + }, + + async move({ id, projectId, userId, targetProjectId, platformId }: MoveParams & { id: string }): Promise { + const agent = await this.getOneOrThrow({ id, projectId, userId }) + if (agent.projectId === targetProjectId) { + return agent + } + await assertMayRemoveFromProject({ agent, projectId, userId, log }) + const target = await readableProjectOrThrow({ platformId, userId, targetProjectId, log }) + await assertMayWriteAgentsIn({ projectId: target.id, userId, log }) + await transaction(async (entityManager) => { + const repo = entityManager.getRepository(AgentEntity) + const locked = await lockedAgentInProjectOrThrow({ entityManager, id, projectId }) + const sharedWithUserIds = await resolveShare({ visibility: locked.visibility, requested: undefined, stored: locked.sharedWithUserIds, projectId: target.id, log }) + const clash = await repo.findOneBy({ projectId: target.id, externalId: agent.externalId }) + if (!isNil(clash)) { + throw new ActivepiecesError({ + code: ErrorCode.VALIDATION, + params: { message: `"${target.displayName}" already holds an agent with the same external id, so this one cannot move there.` }, + }) + } + const blocking = publishedFlowVersionsUsingAgent({ projectId, agentExternalId: agent.externalId, alias: 'blocking_version' }) + const moved = await repo.createQueryBuilder() + .update() + .set({ projectId: target.id, sharedWithUserIds }) + .where('"id" = :id AND "projectId" = :projectId', { id, projectId }) + .andWhere(`NOT EXISTS (${blocking.getQuery()})`) + .setParameters(blocking.getParameters()) + .returning('id') + .execute() + const movedRows: unknown[] = moved.raw ?? [] + if (movedRows.length === 0) { + throw new ActivepiecesError({ + code: ErrorCode.VALIDATION, + params: { message: describeFlowsInUse(await agentService(log).publishedFlowsUsing({ agent, projectId, userId })) }, + }) + } + await entityManager.getRepository(AgentConversationEntity).update( + { agentId: id, source: AgentRunSource.AGENT }, + { projectId: target.id }, + ) + }) + return this.getOneOrThrow({ id, projectId: target.id, userId }) + }, + + async delete({ id, projectId, userId }: GetParams): Promise { + const agent = await this.getOneOrThrow({ id, projectId, userId }) + await assertMayRemoveFromProject({ agent, projectId, userId, log }) await agentRepo().delete({ id, projectId }) return agent }, @@ -294,6 +362,117 @@ async function assertMayChangeWhoCanSee({ agent, request, projectId, userId, log }) } +export async function assertAgentsResolveInProject({ projectId, agentExternalIds, entityManager }: { projectId: ProjectId, agentExternalIds: string[], entityManager: EntityManager }): Promise { + if (agentExternalIds.length === 0) { + return + } + const resolved = await entityManager.getRepository(AgentEntity) + .createQueryBuilder('agent') + .select(['agent.externalId']) + .setLock('pessimistic_read') + .where('agent."projectId" = :projectId', { projectId }) + .andWhere('agent."externalId" IN (:...agentExternalIds)', { agentExternalIds }) + .getMany() + const missing = agentExternalIds.filter((externalId) => !resolved.some((agent) => agent.externalId === externalId)) + if (missing.length > 0) { + throw new ActivepiecesError({ + code: ErrorCode.VALIDATION, + params: { message: 'This flow runs an agent that is not in this project any more. Point the step at an agent here, then publish.' }, + }) + } +} + +async function readableProjectOrThrow({ platformId, userId, targetProjectId, log }: { platformId: PlatformId, userId: UserId, targetProjectId: ProjectId, log: FastifyBaseLogger }): Promise { + const [target] = await resolveReadableProjects({ platformId, userId, projectId: targetProjectId, log }) + if (isNil(target)) { + throw new ActivepiecesError({ + code: ErrorCode.AUTHORIZATION, + params: { message: 'That project is not one you can move an agent into' }, + }) + } + return target +} + +async function assertMayWriteAgentsIn({ projectId, userId, log }: { projectId: ProjectId, userId: UserId, log: FastifyBaseLogger }): Promise { + if (await mayWriteAgentsIn({ projectId, userId, log })) { + return + } + throw new ActivepiecesError({ + code: ErrorCode.AUTHORIZATION, + params: { message: 'Your role in that project cannot create or change agents there' }, + }) +} + +async function assertMayRemoveFromProject({ agent, projectId, userId, log }: AssertDestroyParams): Promise { + await assertMayDestroy({ agent, projectId, userId, log }) + const flowsInUse = await agentService(log).publishedFlowsUsing({ agent, projectId, userId }) + if (flowsInUse.total > 0) { + throw new ActivepiecesError({ + code: ErrorCode.VALIDATION, + params: { message: describeFlowsInUse(flowsInUse) }, + }) + } +} + +async function lockedAgentInProjectOrThrow({ entityManager, id, projectId }: { entityManager: EntityManager, id: string, projectId: ProjectId }): Promise { + const locked = await entityManager.getRepository(AgentEntity) + .createQueryBuilder('agent') + .setLock('pessimistic_write') + .where('agent.id = :id', { id }) + .getOne() + if (isNil(locked) || locked.projectId !== projectId) { + throw new ActivepiecesError({ + code: ErrorCode.VALIDATION, + params: { message: AGENT_MOVED_AWAY }, + }) + } + return locked +} + +async function mayWriteAgentsIn({ projectId, userId, log }: { projectId: ProjectId, userId: UserId, log: FastifyBaseLogger }): Promise { + const checker = await resolvePermissionChecker({ userId, projectId, log }) + return isNil(checker.check(Permission.WRITE_AGENT, '__move_agent_into_project')) +} + +async function toolsBrokenBy({ agent, targetProjectId, log }: { agent: Agent, targetProjectId: ProjectId, log: FastifyBaseLogger }): Promise { + const pinned = agent.draft.tools.flatMap((tool) => { + if (tool.type !== AgentToolType.PIECE) { + return [] + } + const externalId = connectionTemplate.unwrapExternalId(tool.pieceMetadata.predefinedInput?.auth) + return isNil(externalId) ? [] : [{ pieceName: tool.pieceMetadata.pieceName, externalId }] + }) + const flowTools = agent.draft.tools.filter((tool): tool is AgentFlowTool => tool.type === AgentToolType.FLOW) + const knowledgeTools = agent.draft.tools.filter((tool): tool is AgentKnowledgeBaseTool => tool.type === AgentToolType.KNOWLEDGE_BASE) + + const [connectionsThere, flowsThere, knowledgeThere] = await Promise.all([ + pinned.length === 0 ? Promise.resolve([]) : appConnectionService(log).getManyConnectionStates({ projectId: targetProjectId }), + flowTools.length === 0 ? Promise.resolve({ data: [] }) : flowService(log).list({ + projectIds: [targetProjectId], + externalIds: unique(flowTools.map((tool) => tool.externalFlowId)), + cursorRequest: null, + includeTriggerSource: false, + }), + knowledgeTools.length === 0 ? Promise.resolve([]) : knowledgeBaseService(log).getFilesByIds({ + projectId: targetProjectId, + ids: unique(knowledgeTools.map((tool) => tool.sourceId)), + }), + ]) + + const connectionIdsThere = new Set(connectionsThere.map((connection) => connection.externalId)) + const flowIdsThere = new Set(flowsThere.data.map((flow) => flow.externalId)) + const knowledgeIdsThere = new Set(knowledgeThere.map((file) => file.id)) + + return [ + ...unique(pinned.filter((pin) => !connectionIdsThere.has(pin.externalId)).map((pin) => pin.pieceName)) + .map((label) => ({ kind: AgentMoveLossKind.CONNECTION, label })), + ...unique(flowTools.filter((tool) => !flowIdsThere.has(tool.externalFlowId)).map((tool) => tool.flowDisplayName ?? tool.toolName)) + .map((label) => ({ kind: AgentMoveLossKind.FLOW, label })), + ...unique(knowledgeTools.filter((tool) => !knowledgeIdsThere.has(tool.sourceId)).map((tool) => tool.sourceName)) + .map((label) => ({ kind: AgentMoveLossKind.KNOWLEDGE, label })), + ] +} + async function assertMayDestroy({ agent, projectId, userId, log }: AssertDestroyParams): Promise { if (agent.ownerId === userId || await isProjectAdministrator({ projectId, userId, log })) { return @@ -385,6 +564,13 @@ type ListParams = { limit?: number } +type MoveParams = { + projectId: ProjectId + userId: UserId + targetProjectId: ProjectId + platformId: PlatformId +} + type EditDraftToolsParams = GetParams & { edit: (tools: AgentConfig['tools']) => AgentConfig['tools'] | null } diff --git a/packages/server/api/src/app/ee/agent/personalization/chat-personalization-service.ts b/packages/server/api/src/app/ee/agent/personalization/chat-personalization-service.ts index f3638b2e0138..87e16e389b17 100644 --- a/packages/server/api/src/app/ee/agent/personalization/chat-personalization-service.ts +++ b/packages/server/api/src/app/ee/agent/personalization/chat-personalization-service.ts @@ -279,8 +279,8 @@ export const chatPersonalizationService = (log: FastifyBaseLogger) => ({ provider: provider.provider, auth: provider.auth, providerConfig: provider.config ?? {}, - modelId: agentHelpers.resolveModelIdForProvider({ provider: providerName, selectedModel: null }), - fastModelId: agentHelpers.resolveFastModelId({ provider: providerName }), + modelId: agentHelpers.resolveModelIdForProvider({ provider: providerName, selectedModel: null, config: provider.config, modelScope: provider.modelScope, modelIds: provider.modelIds }), + fastModelId: agentHelpers.resolveFastModelId({ provider: providerName, config: provider.config, modelScope: provider.modelScope, modelIds: provider.modelIds }), user: { firstName: user.firstName, lastName: user.lastName, email: user.email }, platformName: platform.name, website: companyRow?.domain ?? null, diff --git a/packages/server/api/src/app/flows/flow-version/flow-version.service.ts b/packages/server/api/src/app/flows/flow-version/flow-version.service.ts index 094abc3b58e5..ea22f13d5cfd 100644 --- a/packages/server/api/src/app/flows/flow-version/flow-version.service.ts +++ b/packages/server/api/src/app/flows/flow-version/flow-version.service.ts @@ -2,7 +2,7 @@ import { ActivepiecesError, apId, Cursor, ErrorCode, FlowId, FlowVersionId, isNi import { FlowOperationRequest, flowOperations, FlowOperationType, flowStructureUtil, FlowTriggerType, FlowVersion, FlowVersionState, LATEST_FLOW_SCHEMA_VERSION, Note } from '@activepieces/shared' import dayjs from 'dayjs' import { FastifyBaseLogger } from 'fastify' -import { EntityManager, FindOneOptions } from 'typeorm' +import { EntityManager, FindOneOptions, SelectQueryBuilder } from 'typeorm' import { repoFactory } from '../../core/db/repo-factory' import { buildPaginator } from '../../helper/pagination/build-paginator' import { paginationHelper } from '../../helper/pagination/pagination-utils' @@ -16,13 +16,15 @@ import { flowVersionValidationUtil } from './flow-version-validator-util' export const flowVersionRepo = repoFactory(FlowVersionEntity) +export const publishedFlowVersionsUsingAgent = ({ projectId, agentExternalId, alias = 'flow_version' }: { projectId: ProjectId, agentExternalId: string, alias?: string }): SelectQueryBuilder => flowVersionRepo() + .createQueryBuilder(alias) + .innerJoin('flow', `${alias}_flow`, `${alias}_flow.id = ${alias}."flowId"`) + .where(`${alias}_flow."projectId" = :projectId`, { projectId }) + .andWhere(`${alias}.id = ${alias}_flow."publishedVersionId"`) + .andWhere(`${alias}."agentIds" && :agentExternalIds`, { agentExternalIds: [agentExternalId] }) + export const publishedFlowsUsingAgent = async ({ projectId, agentExternalId, nameLimit }: { projectId: ProjectId, agentExternalId: string, nameLimit: number }): Promise => { - const referencing = () => flowVersionRepo() - .createQueryBuilder('flow_version') - .innerJoin('flow', 'flow', 'flow.id = flow_version."flowId"') - .where('flow."projectId" = :projectId', { projectId }) - .andWhere('flow_version.id = flow."publishedVersionId"') - .andWhere('flow_version."agentIds" && :agentExternalIds', { agentExternalIds: [agentExternalId] }) + const referencing = () => publishedFlowVersionsUsingAgent({ projectId, agentExternalId }) const [total, named] = await Promise.all([ referencing().getCount(), referencing() diff --git a/packages/server/api/src/app/flows/flow/flow-publish-hooks.ts b/packages/server/api/src/app/flows/flow/flow-publish-hooks.ts new file mode 100644 index 000000000000..3a4631a7c2fb --- /dev/null +++ b/packages/server/api/src/app/flows/flow/flow-publish-hooks.ts @@ -0,0 +1,13 @@ +import { ProjectId } from '@activepieces/core-utils' +import { EntityManager } from 'typeorm' +import { hooksFactory } from '../../helper/hooks-factory' + +export const flowPublishHooks = hooksFactory.create(() => ({ + async assertReferencesResolve(): Promise { + return + }, +})) + +export type FlowPublishHooks = { + assertReferencesResolve(params: { projectId: ProjectId, agentExternalIds: string[], entityManager: EntityManager }): Promise +} diff --git a/packages/server/api/src/app/flows/flow/flow.service.ts b/packages/server/api/src/app/flows/flow/flow.service.ts index 8adfb78ae930..a11093228de2 100644 --- a/packages/server/api/src/app/flows/flow/flow.service.ts +++ b/packages/server/api/src/app/flows/flow/flow.service.ts @@ -21,6 +21,7 @@ import { flowVersionMigrationService } from '../flow-version/flow-version-migrat import { flowVersionRepo, flowVersionService } from '../flow-version/flow-version.service' import { flowFolderService } from '../folder/folder.service' import { flowExecutionCache } from './flow-execution-cache' +import { flowPublishHooks } from './flow-publish-hooks' import { flowPublishUtils } from './flow-publish-utils' import { flowSideEffects } from './flow-service-side-effects' import { FlowEntity } from './flow.entity' @@ -489,6 +490,11 @@ export const flowService = (log: FastifyBaseLogger) => ({ } const publishedFlow = await transaction(async (entityManager) => { + await flowPublishHooks.get(log).assertReferencesResolve({ + projectId, + agentExternalIds: flowVersionToPublish.agentIds ?? [], + entityManager, + }) const lockedFlowVersion = await lockFlowVersionIfNotLocked({ flowVersion: flowVersionToPublish, userId, diff --git a/packages/server/api/src/app/platform/pieces-report/pieces-report.controller.ts b/packages/server/api/src/app/platform/pieces-report/pieces-report.controller.ts new file mode 100644 index 000000000000..62eda72eead3 --- /dev/null +++ b/packages/server/api/src/app/platform/pieces-report/pieces-report.controller.ts @@ -0,0 +1,133 @@ +import { Readable } from 'node:stream' +import { FlowActionType, FlowStatus, flowStructureUtil, FlowTrigger, FlowTriggerType, PrincipalType } from '@activepieces/shared' +import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod' +import { StatusCodes } from 'http-status-codes' +import { securityAccess } from '../../core/security/authorization/fastify-security' +import { flowRepo } from '../../flows/flow/flow.repo' + +const BATCH_SIZE = 200 + +const CSV_HEADER = [ + 'projectId', + 'projectName', + 'flowId', + 'flowName', + 'flowVersionId', + 'versionCreatedAt', + 'stepName', + 'stepType', + 'pieceName', + 'pieceVersion', +] + +export const piecesReportController: FastifyPluginAsyncZod = async (app) => { + app.get('/pieces-report.csv', PiecesReportRequest, async (req, reply) => { + const platformId = req.principal.platform.id + const today = new Date().toISOString().slice(0, 10) + const filename = `pieces-report-${platformId}-${today}.csv` + + return reply + .header('Content-Disposition', `attachment; filename="${filename}"`) + .header('Cache-Control', 'no-store') + .type('text/csv; charset=utf-8') + .status(StatusCodes.OK) + .send(Readable.from(csvStream(platformId))) + }) +} + +async function* csvStream(platformId: string): AsyncIterable { + yield csvRow(CSV_HEADER) + let cursor: string | undefined + while (true) { + const batch = await fetchBatch({ platformId, cursor, limit: BATCH_SIZE }) + if (batch.length === 0) { + break + } + for (const row of batch) { + for (const step of flowStructureUtil.getAllSteps(row.trigger)) { + if (step.type !== FlowActionType.PIECE && step.type !== FlowTriggerType.PIECE) { + continue + } + yield csvRow([ + row.projectId, + row.projectName, + row.flowId, + row.flowName, + row.flowVersionId, + toIso(row.versionCreatedAt), + step.name, + step.type, + step.settings.pieceName, + step.settings.pieceVersion, + ]) + } + } + cursor = batch[batch.length - 1].flowVersionId + } +} + +async function fetchBatch({ platformId, cursor, limit }: FetchBatchParams): Promise { + const qb = flowRepo().createQueryBuilder('flow') + .innerJoin('flow_version', 'fv', 'fv.id = flow."publishedVersionId"') + .innerJoin('project', 'p', 'p.id = flow."projectId"') + .where('p."platformId" = :platformId', { platformId }) + .andWhere('flow."publishedVersionId" IS NOT NULL') + .andWhere('flow.status = :status', { status: FlowStatus.ENABLED }) + .select([ + 'flow."projectId" AS "projectId"', + 'p."displayName" AS "projectName"', + 'flow.id AS "flowId"', + 'fv."displayName" AS "flowName"', + 'fv.id AS "flowVersionId"', + 'fv.created AS "versionCreatedAt"', + 'fv.trigger AS "trigger"', + ]) + .orderBy('fv.id', 'ASC') + .limit(limit) + if (cursor !== undefined) { + qb.andWhere('fv.id > :cursor', { cursor }) + } + return qb.getRawMany() +} + +function csvRow(fields: (string | null | undefined)[]): string { + return fields.map(csvField).join(',') + '\n' +} + +function csvField(value: string | null | undefined): string { + if (value === null || value === undefined) { + return '' + } + const disarmed = CSV_FORMULA_LEADS.has(value[0]) ? `'${value}` : value + const needsQuoting = /[",\r\n]/.test(disarmed) + const escaped = disarmed.replace(/"/g, '""') + return needsQuoting ? `"${escaped}"` : escaped +} + +const CSV_FORMULA_LEADS = new Set(['=', '+', '-', '@', '\t', '\r']) + +function toIso(value: Date | string): string { + return value instanceof Date ? value.toISOString() : new Date(value).toISOString() +} + +type FlowRow = { + projectId: string + projectName: string + flowId: string + flowName: string + flowVersionId: string + versionCreatedAt: Date | string + trigger: FlowTrigger +} + +type FetchBatchParams = { + platformId: string + cursor: string | undefined + limit: number +} + +const PiecesReportRequest = { + config: { + security: securityAccess.platformAdminOnly([PrincipalType.USER]), + }, +} diff --git a/packages/server/api/src/app/platform/pieces-report/pieces-report.module.ts b/packages/server/api/src/app/platform/pieces-report/pieces-report.module.ts new file mode 100644 index 000000000000..e8426c726175 --- /dev/null +++ b/packages/server/api/src/app/platform/pieces-report/pieces-report.module.ts @@ -0,0 +1,6 @@ +import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod' +import { piecesReportController } from './pieces-report.controller' + +export const piecesReportModule: FastifyPluginAsyncZod = async (app) => { + await app.register(piecesReportController, { prefix: '/v1/platform' }) +} diff --git a/packages/server/api/test/integration/ce/platform/pieces-report.test.ts b/packages/server/api/test/integration/ce/platform/pieces-report.test.ts new file mode 100644 index 000000000000..8e39e1ffa614 --- /dev/null +++ b/packages/server/api/test/integration/ce/platform/pieces-report.test.ts @@ -0,0 +1,177 @@ +import { FlowAction, FlowActionType, FlowStatus, FlowTrigger, FlowTriggerType, FlowVersionState } from '@activepieces/shared' +import { FastifyInstance } from 'fastify' +import { StatusCodes } from 'http-status-codes' +import { db } from '../../../helpers/db' +import { createMockFlow, createMockFlowVersion, mockAndSaveBasicSetup } from '../../../helpers/mocks' +import { createTestContext, TestContext } from '../../../helpers/test-context' +import { setupTestEnvironment, teardownTestEnvironment } from '../../../helpers/test-setup' + +let app: FastifyInstance +let ctx: TestContext + +beforeAll(async () => { + app = await setupTestEnvironment() +}) + +afterAll(async () => { + await teardownTestEnvironment() +}) + +beforeEach(async () => { + ctx = await createTestContext(app) +}) + +describe('GET /v1/platform/pieces-report.csv', () => { + it('streams a CSV of PIECE actions and triggers on ENABLED published flows across the platform, excluding disabled, drafts and other platforms', async () => { + const enabledFlow = createMockFlow({ projectId: ctx.project.id, status: FlowStatus.ENABLED }) + await db.save('flow', enabledFlow) + const enabledVersion = createMockFlowVersion({ + flowId: enabledFlow.id, + state: FlowVersionState.LOCKED, + trigger: pieceTrigger({ name: 'trigger', pieceName: 'gmail', pieceVersion: '0.7.0', triggerName: 'new_email' }), + displayName: 'Enabled Flow', + }) + enabledVersion.trigger.nextAction = pieceAction({ name: 'step_1', pieceName: 'slack', pieceVersion: '0.3.4', actionName: 'send_message' }) + await db.save('flow_version', enabledVersion) + enabledFlow.publishedVersionId = enabledVersion.id + await db.save('flow', enabledFlow) + + const disabledFlow = createMockFlow({ projectId: ctx.project.id, status: FlowStatus.DISABLED }) + await db.save('flow', disabledFlow) + const disabledVersion = createMockFlowVersion({ + flowId: disabledFlow.id, + state: FlowVersionState.LOCKED, + trigger: pieceTrigger({ name: 'trigger', pieceName: 'notion', pieceVersion: '1.2.0', triggerName: 'new_page' }), + displayName: 'Disabled Flow', + }) + await db.save('flow_version', disabledVersion) + disabledFlow.publishedVersionId = disabledVersion.id + await db.save('flow', disabledFlow) + + const draftOnlyFlow = createMockFlow({ projectId: ctx.project.id, status: FlowStatus.DISABLED }) + await db.save('flow', draftOnlyFlow) + await db.save('flow_version', createMockFlowVersion({ + flowId: draftOnlyFlow.id, + state: FlowVersionState.DRAFT, + trigger: pieceTrigger({ name: 'trigger', pieceName: 'openai', pieceVersion: '0.1.0', triggerName: 'ask' }), + displayName: 'Draft Only', + })) + + const other = await mockAndSaveBasicSetup() + const otherFlow = createMockFlow({ projectId: other.mockProject.id, status: FlowStatus.ENABLED }) + await db.save('flow', otherFlow) + const otherVersion = createMockFlowVersion({ + flowId: otherFlow.id, + state: FlowVersionState.LOCKED, + trigger: pieceTrigger({ name: 'trigger', pieceName: 'other-piece', pieceVersion: '9.9.9', triggerName: 'other' }), + displayName: 'Other Platform Flow', + }) + await db.save('flow_version', otherVersion) + otherFlow.publishedVersionId = otherVersion.id + await db.save('flow', otherFlow) + + const response = await ctx.get('/v1/platform/pieces-report.csv') + + expect(response.statusCode).toBe(StatusCodes.OK) + expect(response.headers['content-type']).toContain('text/csv') + expect(response.headers['content-disposition']).toContain(`pieces-report-${ctx.platform.id}-`) + + const rows = response.body.trim().split('\n') + expect(rows[0]).toBe('projectId,projectName,flowId,flowName,flowVersionId,versionCreatedAt,stepName,stepType,pieceName,pieceVersion') + + const body = rows.slice(1).map((line) => line.split(',')) + const cells = body.map((cols) => ({ + flowId: cols[2], + stepName: cols[6], + stepType: cols[7], + pieceName: cols[8], + pieceVersion: cols[9], + })) + + expect(cells).toHaveLength(2) + expect(cells).toContainEqual({ + flowId: enabledFlow.id, + stepName: 'trigger', + stepType: FlowTriggerType.PIECE, + pieceName: 'gmail', + pieceVersion: '0.7.0', + }) + expect(cells).toContainEqual({ + flowId: enabledFlow.id, + stepName: 'step_1', + stepType: FlowActionType.PIECE, + pieceName: 'slack', + pieceVersion: '0.3.4', + }) + for (const c of cells) { + expect(c.flowId).not.toBe(disabledFlow.id) + expect(c.pieceName).not.toBe('notion') + expect(c.pieceName).not.toBe('openai') + expect(c.pieceName).not.toBe('other-piece') + } + }) + + it('disarms CSV formula prefixes in flow and project names', async () => { + const project = await db.findOneByOrFail<{ id: string, displayName: string }>('project', { id: ctx.project.id }) + project.displayName = '=cmd|"/c calc"!A1' + await db.save('project', project) + + const flow = createMockFlow({ projectId: ctx.project.id, status: FlowStatus.ENABLED }) + await db.save('flow', flow) + const version = createMockFlowVersion({ + flowId: flow.id, + state: FlowVersionState.LOCKED, + trigger: pieceTrigger({ name: 'trigger', pieceName: 'gmail', pieceVersion: '0.7.0', triggerName: 'new_email' }), + displayName: '@SUM(1+1)', + }) + await db.save('flow_version', version) + flow.publishedVersionId = version.id + await db.save('flow', flow) + + const response = await ctx.get('/v1/platform/pieces-report.csv') + + expect(response.statusCode).toBe(StatusCodes.OK) + expect(response.body).toContain('"\'=cmd|""/c calc""!A1"') + expect(response.body).toContain('\'@SUM(1+1)') + expect(response.body).not.toMatch(/(^|,)=cmd/) + expect(response.body).not.toMatch(/(^|,)@SUM/) + }) + +}) + +type PieceTriggerParams = { name: string, pieceName: string, pieceVersion: string, triggerName: string } +type PieceActionParams = { name: string, pieceName: string, pieceVersion: string, actionName: string } + +function pieceTrigger({ name, pieceName, pieceVersion, triggerName }: PieceTriggerParams): FlowTrigger { + return { + type: FlowTriggerType.PIECE, + name, + displayName: pieceName, + valid: true, + lastUpdatedDate: new Date().toISOString(), + settings: { + pieceName, + pieceVersion, + triggerName, + input: {}, + propertySettings: {}, + }, + } +} + +function pieceAction({ name, pieceName, pieceVersion, actionName }: PieceActionParams): FlowAction { + return { + type: FlowActionType.PIECE, + name, + displayName: pieceName, + valid: true, + lastUpdatedDate: new Date().toISOString(), + settings: { + pieceName, + pieceVersion, + actionName, + input: {}, + propertySettings: {}, + }, + } +} diff --git a/packages/server/api/test/integration/ee/agent/agent-controller.test.ts b/packages/server/api/test/integration/ee/agent/agent-controller.test.ts index ab0f8023fc48..375603e29451 100644 --- a/packages/server/api/test/integration/ee/agent/agent-controller.test.ts +++ b/packages/server/api/test/integration/ee/agent/agent-controller.test.ts @@ -1,11 +1,12 @@ import { AIProviderName, apId, Permission, RoleType } from '@activepieces/core-utils' -import { AgentIcon, AgentRunSource, AgentVisibility, ColorName, DEFAULT_AGENT_MAX_STEPS, DefaultProjectRole, FlowStatus, FlowVersionState, MAX_DRAFT_PROMPT_LENGTH } from '@activepieces/shared' +import { AgentIcon, AgentRunSource, AgentToolType, KnowledgeBaseSourceType, AgentVisibility, ColorName, DefaultProjectRole, FlowStatus, FlowVersionState } from '@activepieces/shared' import { FastifyInstance } from 'fastify' import { StatusCodes } from 'http-status-codes' +import { agentConversationService } from '../../../../src/app/ee/agent/agent-conversation-service' +import { agentService } from '../../../../src/app/ee/agent/agent-service' import { db } from '../../../helpers/db' -import { createMockFlow, createMockFlowVersion, createMockProjectRole, mockAndSaveAIProvider } from '../../../helpers/mocks' +import { createMockFlow, createMockFlowVersion, createMockProject, createMockProjectRole, mockAndSaveAIProvider } from '../../../helpers/mocks' import { createMemberContext, createTestContext, TestContext } from '../../../helpers/test-context' -import { DRAFTS_PER_MINUTE } from '../../../../src/app/ee/agent/agent-controller' import { setupTestEnvironment, teardownTestEnvironment } from '../../../helpers/test-setup' let app: FastifyInstance @@ -685,6 +686,194 @@ describe('agent list across projects', () => { }) }) +describe('moving an agent to another project', () => { + const secondProjectOf = async (ctx: TestContext) => { + const project = createMockProject({ + ownerId: ctx.user.id, + platformId: ctx.platform.id, + displayName: 'Second project', + }) + await db.save('project', project) + return project + } + + it('moves the agent, its conversations, and nothing else', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const target = await secondProjectOf(ctx) + const conversationId = apId() + await db.save('agent_conversation', { + id: conversationId, + created: new Date().toISOString(), + updated: new Date().toISOString(), + platformId: ctx.platform.id, + projectId: ctx.project.id, + userId: ctx.user.id, + agentId: agent.id, + source: AgentRunSource.AGENT, + messages: [], + status: 'IDLE', + }) + + const moved = await ctx.post(`/v1/agents/${agent.id}/move`, { projectId: target.id }) + + expect(moved.statusCode).toBe(StatusCodes.OK) + expect(moved.json().projectId).toBe(target.id) + const inTarget = (await ctx.get('/v1/agents', { projectId: target.id })).json().data + expect(inTarget.map((row: { id: string }) => row.id)).toStrictEqual([agent.id]) + const inSource = (await ctx.get('/v1/agents', { projectId: ctx.project.id })).json().data + expect(inSource).toStrictEqual([]) + const row = await db.findOneByOrFail('agent_conversation', { id: conversationId }) + expect((row as { projectId: string }).projectId).toBe(target.id) + }) + + it('refuses a project the caller cannot reach', async () => { + const ctx = await context() + const stranger = await context() + const agent = await createAgent(ctx) + + const response = await ctx.post(`/v1/agents/${agent.id}/move`, { projectId: stranger.project.id }) + + expect(response.statusCode).toBe(StatusCodes.FORBIDDEN) + expect((await db.findOneByOrFail('agent', { id: agent.id }) as { projectId: string }).projectId).toBe(ctx.project.id) + }) + + it('refuses while a published flow still runs it, and says which', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const target = await secondProjectOf(ctx) + await publishFlowRunningAgent({ projectId: ctx.project.id, externalId: agent.externalId, displayName: 'Nightly sweep' }) + + const response = await ctx.post(`/v1/agents/${agent.id}/move`, { projectId: target.id }) + + expect(response.statusCode).toBe(StatusCodes.CONFLICT) + expect(JSON.stringify(response.json())).toContain('Nightly sweep') + expect((await db.findOneByOrFail('agent', { id: agent.id }) as { projectId: string }).projectId).toBe(ctx.project.id) + }) + + it('says up front what the move would cost', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const target = await secondProjectOf(ctx) + await publishFlowRunningAgent({ projectId: ctx.project.id, externalId: agent.externalId, displayName: 'Nightly sweep' }) + + const preview = await ctx.get(`/v1/agents/${agent.id}/move-preview`, { projectId: target.id }) + + expect(preview.statusCode).toBe(StatusCodes.OK) + expect(preview.json().blockedByPublishedFlows.total).toBe(1) + expect(preview.json().blockedByPublishedFlows.names).toStrictEqual(['Nightly sweep']) + expect(preview.json().mayCreateAgentsThere).toBe(true) + expect(preview.json().toolsThatStopWorking).toStrictEqual([]) + expect(preview.json().membersLosingAccess).toBe(0) + }) + + it('leaves the project alone when the agent is edited after moving', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const target = await secondProjectOf(ctx) + expect((await ctx.post(`/v1/agents/${agent.id}/move`, { projectId: target.id })).statusCode).toBe(StatusCodes.OK) + const before = await db.findOneByOrFail('agent', { id: agent.id }) as { updated: string } + + const renamed = await ctx.post(`/v1/agents/${agent.id}`, { displayName: 'Renamed after the move' }) + + expect(renamed.statusCode).toBe(StatusCodes.OK) + expect(renamed.json().displayName).toBe('Renamed after the move') + expect(renamed.json().projectId).toBe(target.id) + const after = await db.findOneByOrFail('agent', { id: agent.id }) as { updated: string, projectId: string } + expect(after.projectId).toBe(target.id) + // The sort on the list page reads this, so an update has to keep moving it. + expect(new Date(after.updated).getTime()).toBeGreaterThanOrEqual(new Date(before.updated).getTime()) + }) + + it('still opens a chat with the agent once it has moved', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const target = await secondProjectOf(ctx) + expect((await ctx.post(`/v1/agents/${agent.id}/move`, { projectId: target.id })).statusCode).toBe(StatusCodes.OK) + + const conversation = await agentConversationService(app.log).createConversation({ + platformId: ctx.platform.id, + userId: ctx.user.id, + request: { agentId: agent.id }, + }) + + expect(conversation.projectId).toBe(target.id) + }) + + it('refuses to publish a flow that points at an agent from another project', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const target = await secondProjectOf(ctx) + const flow = createMockFlow({ projectId: ctx.project.id, status: FlowStatus.DISABLED }) + await db.save('flow', flow) + const version = createMockFlowVersion({ flowId: flow.id, updatedBy: ctx.user.id, state: FlowVersionState.DRAFT }) + await db.save('flow_version', { ...version, agentIds: [agent.externalId] }) + expect((await ctx.post(`/v1/agents/${agent.id}/move`, { projectId: target.id })).statusCode).toBe(StatusCodes.OK) + + const published = await ctx.post(`/v1/flows/${flow.id}`, { + type: 'LOCK_AND_PUBLISH', + request: {}, + }) + + expect(published.statusCode).toBe(StatusCodes.CONFLICT) + const row = await db.findOneByOrFail('flow', { id: flow.id }) as { publishedVersionId: string | null } + expect(row.publishedVersionId).toBeNull() + }) + + it('names every kind of tool that would stop working, not only connections', async () => { + const ctx = await context() + const target = await secondProjectOf(ctx) + const agent = await createAgent(ctx, { + draft: { + ...agentBody(ctx.project.id).draft, + tools: [ + { + type: AgentToolType.FLOW, + toolName: 'run_the_intake_flow', + externalFlowId: apId(), + flowDisplayName: 'Client intake', + }, + { + type: AgentToolType.KNOWLEDGE_BASE, + toolName: 'search_the_handbook', + sourceType: KnowledgeBaseSourceType.FILE, + sourceId: apId(), + sourceName: 'Handbook.pdf', + }, + ], + }, + }) + + const preview = (await ctx.get(`/v1/agents/${agent.id}/move-preview`, { projectId: target.id })).json() + + expect(preview.toolsThatStopWorking).toStrictEqual([ + { kind: 'flow', label: 'Client intake' }, + { kind: 'knowledge', label: 'Handbook.pdf' }, + ]) + }) + + it('tells someone who cannot create agents there, and reveals nothing about that project', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const stranger = await context() + + const preview = await ctx.get(`/v1/agents/${agent.id}/move-preview`, { projectId: stranger.project.id }) + + expect(preview.statusCode).toBe(StatusCodes.FORBIDDEN) + }) + + it('is not something a member can do to someone else\'s agent', async () => { + const ctx = await context() + const agent = await createAgent(ctx) + const target = await secondProjectOf(ctx) + const member = await createMemberContext(app, ctx, { projectRole: DefaultProjectRole.EDITOR }) + + const response = await member.post(`/v1/agents/${agent.id}/move`, { projectId: target.id }) + + expect([StatusCodes.FORBIDDEN, StatusCodes.NOT_FOUND]).toContain(response.statusCode) + }) +}) + describe('agent permissions', () => { it('lets a viewer read an agent but never create or change one', async () => { const owner = await context() diff --git a/packages/server/api/test/unit/app/ee/agent/agent-model-resolution.test.ts b/packages/server/api/test/unit/app/ee/agent/agent-model-resolution.test.ts index dc9ca9818c7a..4047962be876 100644 --- a/packages/server/api/test/unit/app/ee/agent/agent-model-resolution.test.ts +++ b/packages/server/api/test/unit/app/ee/agent/agent-model-resolution.test.ts @@ -1,4 +1,5 @@ import { AIProviderName } from '@activepieces/core-utils' +import { AIProviderModelType } from '@activepieces/shared' import { describe, expect, it, vi } from 'vitest' import { agentHelpers } from '../../../../../src/app/ee/agent/agent-helpers' @@ -12,6 +13,45 @@ const resolve = ({ provider, selectedModel }: { provider: AIProviderName, select agentHelpers.resolveModelIdForProvider({ provider, selectedModel }) describe('resolveModelIdForProvider', () => { + const vertexConfig = (models: { modelId: string, modelType: AIProviderModelType }[]) => ({ + project: 'gcp-project', + region: 'europe-west4', + models: models.map((model) => ({ ...model, modelName: model.modelId })), + }) + + it('picks from the models an admin listed on the key, not the curated list', () => { + const config = vertexConfig([ + { modelId: 'claude-3-5-sonnet@20241022', modelType: AIProviderModelType.TEXT }, + { modelId: 'gemini-2.5-flash', modelType: AIProviderModelType.TEXT }, + ]) + + expect(agentHelpers.resolveModelIdForProvider({ provider: AIProviderName.VERTEX, selectedModel: 'smart', config })).toBe('claude-3-5-sonnet@20241022') + expect(agentHelpers.resolveModelIdForProvider({ provider: AIProviderName.VERTEX, selectedModel: 'gemini-2.5-flash', config })).toBe('gemini-2.5-flash') + }) + + it('honours the key model allow-list over the curated default', () => { + const scoped = { modelScope: 'selected' as const, modelIds: ['gemini-2.5-flash'] } + + expect(agentHelpers.resolveModelIdForProvider({ provider: AIProviderName.GOOGLE, selectedModel: 'smart', ...scoped })).toBe('gemini-2.5-flash') + expect(agentHelpers.resolveModelIdForProvider({ provider: AIProviderName.GOOGLE, selectedModel: 'gemini-2.5-pro', ...scoped })).toBe('gemini-2.5-flash') + }) + + it('refuses when the allow-list excludes every candidate', () => { + expect(() => agentHelpers.resolveModelIdForProvider({ + provider: AIProviderName.GOOGLE, + selectedModel: 'smart', + modelScope: 'selected', + modelIds: ['a-model-this-provider-does-not-offer'], + })).toThrow() + }) + + it('refuses a key that lists no text model rather than falling back to one it never offered', () => { + const imageOnly = vertexConfig([{ modelId: 'imagen-4.0-generate-001', modelType: AIProviderModelType.IMAGE }]) + + expect(() => agentHelpers.resolveModelIdForProvider({ provider: AIProviderName.VERTEX, selectedModel: 'smart', config: imageOnly })).toThrow() + expect(() => agentHelpers.resolveModelIdForProvider({ provider: AIProviderName.VERTEX, selectedModel: 'smart', config: vertexConfig([]) })).toThrow() + }) + it('keeps the tier model id for the activepieces provider', () => { expect(resolve({ provider: AIProviderName.ACTIVEPIECES, selectedModel: 'smart' })).toBe('anthropic/claude-sonnet-4.6') expect(resolve({ provider: AIProviderName.ACTIVEPIECES, selectedModel: 'fast' })).toBe('anthropic/claude-haiku-4.5') diff --git a/packages/server/engine/esbuild.config.mjs b/packages/server/engine/esbuild.config.mjs index 663bc9c729c0..e43451fbacf5 100644 --- a/packages/server/engine/esbuild.config.mjs +++ b/packages/server/engine/esbuild.config.mjs @@ -6,7 +6,6 @@ import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const outdir = path.resolve(__dirname, '../../../dist/packages/engine'); const proxyOutfile = path.join(outdir, 'main.js'); -const pieceChildOutfile = path.join(outdir, 'piece-child.js'); const watch = process.argv.includes('--watch'); @@ -56,9 +55,9 @@ function rebuildLogger(outfile) { }; } -function buildOptions({ outfile, entry = 'src/main.ts' }) { +function buildOptions({ outfile }) { return { - entryPoints: [path.resolve(__dirname, entry)], + entryPoints: [path.resolve(__dirname, 'src/main.ts')], bundle: true, platform: 'node', target: 'node20', @@ -82,17 +81,14 @@ function buildOptions({ outfile, entry = 'src/main.ts' }) { }; } -const targets = [ - buildOptions({ outfile: proxyOutfile }), - buildOptions({ outfile: pieceChildOutfile, entry: 'src/piece-child.ts' }), -]; - if (watch) { - for (const target of targets) { - const ctx = await esbuild.context(target); - await ctx.rebuild(); - await ctx.watch(); - } + const ctx = await esbuild.context( + buildOptions({ outfile: proxyOutfile }) + ); + await ctx.rebuild(); + await ctx.watch(); } else { - await Promise.all(targets.map((target) => esbuild.build(target))); + await esbuild.build( + buildOptions({ outfile: proxyOutfile }) + ); } diff --git a/packages/server/engine/src/lib/core/piece/piece-auth.ts b/packages/server/engine/src/lib/core/piece/piece-auth.ts deleted file mode 100644 index 33b9619c65df..000000000000 --- a/packages/server/engine/src/lib/core/piece/piece-auth.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { isNil } from '@activepieces/core-utils' -import { getAuthPropertyForValue, PieceAuthProperty, PropertyType } from '@activepieces/pieces-framework' -import { AppConnectionType, AppConnectionValue, PiecePackage } from '@activepieces/shared' -import { EngineConstants } from '../../handler/context/engine-constants' -import { PieceDescription } from './piece-protocol' -import { PieceRef, pieceRunner } from './piece-runner' - -export const pieceAuth = { - callMethod: async ({ operation, authValueType, methodPath }: CallMethodParams): Promise => { - const piece: PieceRef = { - pieceName: operation.piece.pieceName, - pieceVersion: operation.piece.pieceVersion, - devPieces: EngineConstants.DEV_PIECES, - } - const description = await pieceRunner.describe(piece) - const selected = select({ description, authValueType }) - if (isNil(selected)) { - return { called: false } - } - const path = [...selected.path, ...methodPath] - if (!description.hasPath(path)) { - return { called: false, property: selected.property } - } - const argument = argumentFor({ property: selected.property, value: operation.auth }) - if (isNil(argument)) { - return { called: false, property: selected.property, mismatch: true } - } - const server = { - apiUrl: operation.internalApiUrl.endsWith('/') ? operation.internalApiUrl : `${operation.internalApiUrl}/`, - publicUrl: operation.publicApiUrl, - } - return { - called: true, - property: selected.property, - result: (await pieceRunner.call({ piece, path, args: [{ auth: argument.argument, server }] })).result, - } - }, -} - -function select({ description, authValueType }: SelectParams): SelectedAuth | undefined { - const auth = description.metadata.auth - if (isNil(auth)) { - return undefined - } - const property = getAuthPropertyForValue({ authValueType, pieceAuth: auth }) - if (isNil(property)) { - return undefined - } - const index = Array.isArray(auth) ? auth.indexOf(property) : -1 - return { - property, - path: index === -1 ? ['auth'] : ['auth', String(index)], - } -} - -function argumentFor({ property, value }: ArgumentParams): { argument: unknown } | undefined { - switch (property.type) { - case PropertyType.OAUTH2: - return [AppConnectionType.OAUTH2, AppConnectionType.CLOUD_OAUTH2, AppConnectionType.PLATFORM_OAUTH2].includes(value.type) ? { argument: value } : undefined - case PropertyType.BASIC_AUTH: - return value.type === AppConnectionType.BASIC_AUTH ? { argument: value } : undefined - case PropertyType.SECRET_TEXT: - return value.type === AppConnectionType.SECRET_TEXT ? { argument: value.secret_text } : undefined - case PropertyType.CUSTOM_AUTH: - return value.type === AppConnectionType.CUSTOM_AUTH ? { argument: value.props } : undefined - case PropertyType.OIDC: - return value.type === AppConnectionType.OIDC ? { argument: value.props } : undefined - default: - return undefined - } -} - -type SelectParams = { - description: PieceDescription - authValueType: AppConnectionType -} - -type ArgumentParams = { - property: PieceAuthProperty - value: AppConnectionValue -} - -type SelectedAuth = { - property: PieceAuthProperty - path: string[] -} - -type CallMethodParams = { - operation: { - piece: PiecePackage - auth: AppConnectionValue - internalApiUrl: string - publicApiUrl: string - } - authValueType: AppConnectionType - methodPath: string[] -} - -export type AuthCallResult = - | { called: false, property?: PieceAuthProperty, mismatch?: boolean } - | { called: true, property: PieceAuthProperty, result: unknown } diff --git a/packages/server/engine/src/lib/core/piece/piece-child.ts b/packages/server/engine/src/lib/core/piece/piece-child.ts deleted file mode 100644 index 9fe447e159fe..000000000000 --- a/packages/server/engine/src/lib/core/piece/piece-child.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { isNil, isObject } from '@activepieces/core-utils' -import { Piece } from '@activepieces/pieces-framework' -import { extractPieceFromModule } from '@activepieces/shared' -import { buildContext } from './piece-context-builder' -import { ChildMessage, ParentMessage, pieceProtocol } from './piece-protocol' - -export const pieceChild = { - listen: (): void => { - process.on('message', (message: ParentMessage) => void handleParentMessage(message)) - process.on('disconnect', () => process.exit(0)) - process.on('unhandledRejection', (reason) => report({ type: 'done', success: false, error: pieceProtocol.serializeError(reason) })) - process.on('uncaughtException', (error) => report({ type: 'done', success: false, error: pieceProtocol.serializeError(error) })) - }, -} - -async function handleParentMessage(message: ParentMessage): Promise { - try { - const piece = await loadPiece(message) - if (message.type === 'describe') { - report({ type: 'done', success: true, result: describe(piece) }) - return - } - const built = isNil(message.context) ? undefined : await buildContext({ piece, request: message.context }) - const method = resolveMethod({ piece, path: message.path }) - const result = await method.call(...[...message.args, ...built?.args ?? []]) - await Promise.allSettled(built?.pending ?? []) - report({ - type: 'done', - success: true, - result: pieceProtocol.toTransferable(result), - hooks: built?.hooks, - }) - } - catch (error) { - report({ type: 'done', success: false, error: pieceProtocol.serializeError(error) }) - } -} - -async function loadPiece({ piecePath, pieceName, pieceVersion }: { piecePath: string, pieceName: string, pieceVersion: string }): Promise { - const pieceModule = await import(piecePath) - return extractPieceFromModule({ module: pieceModule, pieceName, pieceVersion }) -} - -function describe(piece: Piece): unknown { - return { - metadata: pieceProtocol.toTransferable(piece.metadata()), - functionPaths: collectFunctionPaths({ value: callableRoot(piece), path: [], depth: 0, seen: new Set() }), - } -} - -function callableRoot(piece: Piece): Record { - return { - actions: piece.actions(), - triggers: piece.triggers(), - auth: piece.auth, - events: piece.events, - } -} - -function resolveMethod({ piece, path }: { piece: Piece, path: string[] }): BoundMethod { - let owner: unknown = undefined - let current: unknown = callableRoot(piece) - for (const segment of path) { - if (!isObject(current)) { - throw new Error(`Path not found in piece: ${path.join('.')}`) - } - owner = current - current = Reflect.get(current, segment) - } - if (typeof current !== 'function') { - throw new Error(`Path is not callable in piece: ${path.join('.')}`) - } - const method = current - return { call: async (...args: unknown[]) => method.apply(owner, args) } -} - -function collectFunctionPaths({ value, path, depth, seen }: CollectParams): string[] { - if (depth > MAX_FUNCTION_PATH_DEPTH || !isObject(value) || seen.has(value)) { - return [] - } - return Object.entries(value).flatMap(([key, item]) => { - const itemPath = [...path, key] - return typeof item === 'function' ? [itemPath.join('.')] : collectFunctionPaths({ value: item, path: itemPath, depth: depth + 1, seen: new Set([...seen, value]) }) - }) -} - -function report(message: ChildMessage): void { - if (settled) { - return - } - settled = true - process.send?.(message, () => process.exit(0)) -} - -const MAX_FUNCTION_PATH_DEPTH = 6 -let settled = false - -type BoundMethod = { - call: (...args: unknown[]) => Promise -} - -type CollectParams = { - value: unknown - path: string[] - depth: number - seen: Set -} diff --git a/packages/server/engine/src/lib/core/piece/piece-context-builder.ts b/packages/server/engine/src/lib/core/piece/piece-context-builder.ts deleted file mode 100644 index 69561c05fec9..000000000000 --- a/packages/server/engine/src/lib/core/piece/piece-context-builder.ts +++ /dev/null @@ -1,326 +0,0 @@ -import { isNil, isObject } from '@activepieces/core-utils' -import { ActionContext, backwardCompatabilityContextUtils, CreateWaitpointHook, CreateWaitpointParams, CreateWaitpointResult, InputPropertyMap, Piece, PieceAuthProperty, PiecePropertyMap, SetScheduleRequest, StaticPropsValue, StopHookParams, TagsManager } from '@activepieces/pieces-framework' -import { AUTHENTICATION_PROPERTY_NAME, EngineGenericError, InvalidCronExpressionError, InvalidScheduleIntervalError, PausedFlowTimeoutError, ScheduleOptions, TriggerSourceScheduleType } from '@activepieces/shared' -import { isValidCron } from 'cron-validator' -import dayjs from 'dayjs' -import { retryFetch } from '../../api/retry-fetch' -import { flowRunProgressReporter } from '../../helper/flow-run-progress-reporter' -import { createFileUploader } from '../../piece-context/file-uploader' -import { createFlowsContext } from '../../piece-context/flows' -import { createContextStore } from '../../piece-context/store' -import { waitpointClient } from '../../piece-context/waitpoint-client' -import { utils } from '../../utils' -import { propsProcessor } from '../../variables/props-processor' -import { ActionContextRequest, CollectedHooks, ContextRequest, PieceRuntime, PropsContextRequest, TriggerContextRequest } from './piece-protocol' - -export async function buildContext({ piece, request }: BuildContextParams): Promise { - const hooks: CollectedHooks = { - hookResponse: { type: 'none', tags: [] }, - listeners: [], - } - const pending: Promise[] = [] - switch (request.kind) { - case 'action': - return { args: [await buildActionContext({ piece, request, hooks, pending })], hooks, pending } - case 'trigger': - return { args: [await buildTriggerContext({ piece, request, hooks })], hooks, pending } - case 'props': - return { args: [request.resolvedInput, buildPropsContext(request)], hooks, pending } - } -} - -async function buildActionContext({ piece, request, hooks, pending }: ActionParams): Promise { - const { runtime, stepName, actionName } = request - const action = piece.getAction(actionName) - if (isNil(action)) { - throw new EngineGenericError('ActionNotFoundError', `Action not found, actionName=${actionName}`) - } - const propsValue = await processProps({ request, props: action.props, requireAuth: action.requireAuth, piece }) - - const context: ActionContext = { - executionType: request.executionType, - resumePayload: request.resumePayload!, - store: createContextStore({ - apiUrl: runtime.internalApiUrl, - prefix: '', - flowId: runtime.flowId, - engineToken: runtime.engineToken, - }), - output: runtime.actionRunMode - ? { update: async (): Promise => Promise.resolve() } - : flowRunProgressReporter.createOutputContext(runtime), - flows: createFlowsContext({ - engineToken: runtime.engineToken, - internalApiUrl: runtime.internalApiUrl, - flowId: runtime.flowId, - flowVersionId: runtime.flowVersionId, - }), - step: { name: stepName }, - auth: propsValue[AUTHENTICATION_PROPERTY_NAME], - files: createFileUploader({ apiUrl: runtime.internalApiUrl, engineToken: runtime.engineToken }), - server: { - token: runtime.engineToken, - apiUrl: runtime.internalApiUrl, - publicUrl: runtime.publicApiUrl, - }, - propsValue, - tags: createTagsManager(hooks), - connections: createConnections({ runtime, target: 'actions', hooks }), - run: { - id: runtime.flowRunId, - stop: (request?: StopHookParams) => { - hooks.hookResponse = { ...hooks.hookResponse, type: 'stopped', response: request ?? { response: {} } } - }, - respond: (request?: StopHookParams) => { - hooks.hookResponse = { ...hooks.hookResponse, type: 'respond', response: request ?? { response: {} } } - }, - createWaitpoint: createWaitpointHook({ runtime, stepName, hooks, pending }), - waitForWaitpoint: () => { - assertCanSuspend(runtime) - hooks.hookResponse = { ...hooks.hookResponse, type: 'paused' } - }, - }, - project: createProjectContext(runtime), - } - - return backwardCompatabilityContextUtils.makeActionContextBackwardCompatible({ - contextVersion: runtime.contextVersion, - context, - }) -} - -async function buildTriggerContext({ piece, request, hooks }: TriggerParams): Promise { - const { runtime, stepName } = request - const trigger = piece.getTrigger(stepName) - if (isNil(trigger)) { - throw new EngineGenericError('TriggerNotFoundError', `Trigger not found, stepName=${stepName}`) - } - const propsValue = await processProps({ request, props: trigger.props, requireAuth: trigger.requireAuth, piece }) - - return { - store: createContextStore({ - apiUrl: runtime.internalApiUrl, - prefix: request.storePrefix, - flowId: runtime.flowId, - engineToken: runtime.engineToken, - }), - step: { name: stepName }, - app: { - createListeners: ({ events, identifierKey, identifierValue }: { events: string[], identifierKey: string, identifierValue: string }): void => { - hooks.listeners.push({ events, identifierValue, identifierKey }) - }, - }, - setSchedule: (scheduleRequest: SetScheduleRequest) => { - hooks.scheduleOptions = parseSchedule(scheduleRequest) - }, - flows: createFlowsContext({ - engineToken: runtime.engineToken, - internalApiUrl: runtime.internalApiUrl, - flowId: runtime.flowId, - flowVersionId: runtime.flowVersionId, - }), - webhookUrl: request.webhookUrl, - isRepublish: request.isRepublish, - auth: propsValue[AUTHENTICATION_PROPERTY_NAME], - propsValue, - payload: request.payload ?? {}, - run: { id: runtime.flowRunId }, - project: createProjectContext(runtime), - server: { - token: runtime.engineToken, - apiUrl: runtime.internalApiUrl, - publicUrl: runtime.publicApiUrl, - }, - connections: createConnections({ runtime, target: 'triggers', hooks }), - ...(request.includeFiles ? { files: createFileUploader({ apiUrl: runtime.internalApiUrl, engineToken: runtime.engineToken }) } : {}), - } -} - -function buildPropsContext({ runtime, stepName, searchValue }: PropsContextRequest): unknown { - return { - searchValue, - server: { - token: runtime.engineToken, - apiUrl: runtime.internalApiUrl, - publicUrl: runtime.publicApiUrl, - }, - project: createProjectContext(runtime), - flows: createFlowsContext({ - engineToken: runtime.engineToken, - internalApiUrl: runtime.internalApiUrl, - flowId: runtime.flowId, - flowVersionId: runtime.flowVersionId, - }), - step: { name: stepName }, - connections: createConnections({ - runtime, - target: 'properties', - hooks: { hookResponse: { type: 'none', tags: [] }, listeners: [] }, - }), - } -} - -async function processProps({ request, props, requireAuth, piece }: ProcessPropsParams): Promise> { - const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators( - request.resolvedInput, - props, - piece.auth, - requireAuth, - request.propertySettings, - ) - if (Object.keys(errors).length > 0) { - throw new Error(JSON.stringify(errors, null, 2)) - } - return processedInput -} - -function createConnections({ runtime, target, hooks }: { runtime: PieceRuntime, target: 'actions' | 'triggers' | 'properties', hooks: CollectedHooks }): ReturnType { - return utils.createConnectionManager({ - apiUrl: runtime.internalApiUrl, - projectId: runtime.projectId, - engineToken: runtime.engineToken, - target, - hookResponse: hooks.hookResponse, - contextVersion: runtime.contextVersion, - pieceName: runtime.pieceName, - }) -} - -function createProjectContext(runtime: PieceRuntime): { id: string, externalId: () => Promise } { - return { - id: runtime.projectId, - externalId: async () => { - const response = await retryFetch(`${runtime.internalApiUrl}v1/worker/project`, { - headers: { Authorization: `Bearer ${runtime.engineToken}` }, - }) - const project = await response.json() - return isObject(project) && typeof project.externalId === 'string' ? project.externalId : undefined - }, - } -} - -function createTagsManager(hooks: CollectedHooks): TagsManager { - return { - add: async ({ name }: { name: string }): Promise => { - hooks.hookResponse.tags.push(name) - }, - } -} - -function createWaitpointHook({ runtime, stepName, hooks, pending }: WaitpointHookParams): CreateWaitpointHook { - return (params: CreateWaitpointParams) => { - const created = createWaitpoint({ runtime, stepName, hooks, params }) - pending.push(created) - return created - } -} - -async function createWaitpoint({ runtime, stepName, hooks, params }: SubmitWaitpointParams): Promise { - assertCanSuspend(runtime) - assertDelayWithinTimeout(params.resumeDateTime) - if (!isNil(params.responseToSend)) { - hooks.hookResponse = { ...hooks.hookResponse, responseToSend: params.responseToSend } - } - const result = await waitpointClient.create({ - apiUrl: runtime.internalApiUrl, - engineToken: runtime.engineToken, - flowRunId: runtime.flowRunId, - projectId: runtime.projectId, - stepName, - type: params.type, - version: params.version ?? 'V1', - resumeDateTime: params.resumeDateTime, - responseToSend: params.responseToSend, - workerHandlerId: runtime.workerHandlerId, - httpRequestId: runtime.httpRequestId, - }) - return { - ...result, - buildResumeUrl: ({ queryParams, sync }) => { - const url = new URL(`${result.resumeUrl}${sync ? '/sync' : ''}`) - url.search = new URLSearchParams(queryParams).toString() - return url.toString() - }, - } -} - -function parseSchedule(request: SetScheduleRequest): ScheduleOptions { - if ('intervalMs' in request) { - const parsed = ScheduleOptions.safeParse({ type: TriggerSourceScheduleType.INTERVAL, intervalMs: request.intervalMs }) - if (!parsed.success) { - throw new InvalidScheduleIntervalError(request.intervalMs) - } - return parsed.data - } - if (!isValidCron(request.cronExpression)) { - throw new InvalidCronExpressionError(request.cronExpression) - } - return { - type: TriggerSourceScheduleType.CRON_EXPRESSION, - cronExpression: request.cronExpression, - timezone: request.timezone ?? 'UTC', - } -} - -function assertCanSuspend(runtime: PieceRuntime): void { - if (runtime.actionRunMode) { - throw new Error('This action pauses the run (waitpoint) and can only run inside a flow, not as a action run.') - } -} - -function assertDelayWithinTimeout(resumeDateTime?: string): void { - if (isNil(resumeDateTime)) { - return - } - if (dayjs(resumeDateTime).diff(dayjs(), 'days') > AP_PAUSED_FLOW_TIMEOUT_DAYS) { - throw new PausedFlowTimeoutError(undefined, AP_PAUSED_FLOW_TIMEOUT_DAYS) - } -} - -const AP_PAUSED_FLOW_TIMEOUT_DAYS = Number(process.env.AP_PAUSED_FLOW_TIMEOUT_DAYS) - - -type BuildContextParams = { - piece: Piece - request: ContextRequest -} - -type ActionParams = { - piece: Piece - request: ActionContextRequest - hooks: CollectedHooks - pending: Promise[] -} - -type TriggerParams = { - piece: Piece - request: TriggerContextRequest - hooks: CollectedHooks -} - -type ProcessPropsParams = { - request: ActionContextRequest | TriggerContextRequest - props: Parameters[1] - requireAuth: boolean - piece: Piece -} - -type WaitpointHookParams = { - runtime: PieceRuntime - stepName: string - hooks: CollectedHooks - pending: Promise[] -} - -type SubmitWaitpointParams = { - runtime: PieceRuntime - stepName: string - hooks: CollectedHooks - params: CreateWaitpointParams -} - -export type BuiltContext = { - args: unknown[] - hooks: CollectedHooks - pending: Promise[] -} diff --git a/packages/server/engine/src/lib/core/piece/piece-path.ts b/packages/server/engine/src/lib/core/piece/piece-path.ts deleted file mode 100644 index 043d0593fac3..000000000000 --- a/packages/server/engine/src/lib/core/piece/piece-path.ts +++ /dev/null @@ -1,142 +0,0 @@ -import fs from 'fs/promises' -import path from 'path' -import { isNil } from '@activepieces/core-utils' -import { EngineGenericError, getPackageAliasForPiece, getPieceNameFromAlias, trimVersionFromAlias } from '@activepieces/shared' -import { utils } from '../../utils' -import { PieceRef } from './piece-runner' - -export const piecePath = { - resolve: async ({ pieceName, pieceVersion, devPieces }: PieceRef): Promise => { - const packageName = getPackageAlias({ pieceName, pieceVersion, devPieces }) - const piecePath = devPieces.includes(getPieceNameFromAlias(packageName)) - ? await findInDistFolder(packageName) - : await traverseAllParentFoldersToFindPiece(packageName) - if (isNil(piecePath)) { - throw new EngineGenericError('PieceNotFoundError', `Piece not found for package: ${packageName}`) - } - return piecePath - }, - -} - -function getPackageAlias({ pieceName, pieceVersion, devPieces }: PieceRef): string { - if (devPieces.includes(getPieceNameFromAlias(pieceName))) { - return pieceName - } - - return getPackageAliasForPiece({ - pieceName, - pieceVersion, - }) -} - -async function findInDistFolder(packageName: string): Promise { - const sourcePiecesPath = path.resolve('packages/pieces') - if (!await utils.folderExists(sourcePiecesPath)) { - return null - } - const distPackageJsonPaths = await findDistPackageJsonFiles(sourcePiecesPath) - for (const packageJsonPath of distPackageJsonPaths) { - const { data: result } = await utils.tryCatchAndThrowOnEngineError(async () => { - const content = await fs.readFile(packageJsonPath, 'utf-8') - const packageJson = JSON.parse(content) - if (packageJson.name === packageName) { - return path.join(path.dirname(packageJsonPath), 'src', 'index.js') - } - return null - }) - if (result) { - return result - } - } - return null -} - -async function findDistPackageJsonFiles(dirPath: string): Promise { - const results: string[] = [] - const ignoredDirs = ['node_modules', '.turbo', 'framework', 'common'] - - async function scanDir(currentPath: string): Promise { - const items = await fs.readdir(currentPath, { withFileTypes: true }) - for (const item of items) { - if (!item.isDirectory() || ignoredDirs.includes(item.name)) { - continue - } - const fullPath = path.join(currentPath, item.name) - if (item.name === 'dist') { - const pkgJson = path.join(fullPath, 'package.json') - if (await utils.folderExists(pkgJson)) { - results.push(pkgJson) - } - } - else { - await scanDir(fullPath) - } - } - } - - await scanDir(dirPath) - return results -} - - -async function traverseAllParentFoldersToFindPiece(packageName: string): Promise { - const trimmedName = trimVersionFromAlias(packageName) - const customPaths = (process.env.AP_CUSTOM_PIECES_PATHS ?? '').split(':').filter(Boolean) - for (const customPath of customPaths) { - const entry = await resolveInstalledPieceEntry(path.resolve(customPath, 'pieces', packageName), trimmedName) - if (!isNil(entry)) { - return entry - } - } - - const rootDir = path.parse(__dirname).root - let currentDir = __dirname - const maxIterations = currentDir.split(path.sep).length - for (let i = 0; i < maxIterations; i++) { - const entry = await resolveInstalledPieceEntry(path.resolve(currentDir, 'pieces', packageName), trimmedName) - if (!isNil(entry)) { - return entry - } - - const parentDir = path.dirname(currentDir) - if (parentDir === currentDir || currentDir === rootDir) { - break - } - currentDir = parentDir - } - return null -} - -// A piece entry is resolved from its package.json "main" (defaulting to src/index.js). -// Registry/dev installs keep the package nested in node_modules; a packed-archive bundle is -// extracted straight to the install-folder root. Try the nested package first, then the root. -async function resolveInstalledPieceEntry(pieceFolder: string, trimmedName: string): Promise { - const packageDir = path.join(pieceFolder, 'node_modules', trimmedName) - if (await utils.folderExists(packageDir)) { - return resolveEntryFromPackageDir(packageDir) - } - // Only return an entry that actually exists: a half-installed registry folder also has a - // stub package.json (no "main") at this point, for which resolveEntryFromPackageDir would - // otherwise return a non-existent src/index.js — fall through to a clean PieceNotFoundError. - const rootManifest = path.join(pieceFolder, 'package.json') - if (await utils.folderExists(rootManifest)) { - const rootEntry = await resolveEntryFromPackageDir(pieceFolder) - if (await utils.folderExists(rootEntry)) { - return rootEntry - } - } - return null -} - -async function resolveEntryFromPackageDir(packageDir: string): Promise { - const { data: mainEntry } = await utils.tryCatchAndThrowOnEngineError(async () => { - const packageJson = JSON.parse(await fs.readFile(path.join(packageDir, 'package.json'), 'utf-8')) - if (isNil(packageJson.main)) { - return null - } - const resolved = path.join(packageDir, packageJson.main) - return await utils.folderExists(resolved) ? resolved : null - }) - return mainEntry ?? path.join(packageDir, 'src', 'index.js') -} diff --git a/packages/server/engine/src/lib/core/piece/piece-protocol.ts b/packages/server/engine/src/lib/core/piece/piece-protocol.ts deleted file mode 100644 index 24705a29d175..000000000000 --- a/packages/server/engine/src/lib/core/piece/piece-protocol.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { inspect } from 'node:util' -import { isNil, isObject } from '@activepieces/core-utils' -import { ContextVersion, PieceMetadata } from '@activepieces/pieces-framework' -import { ExecutionError, ExecutionErrorType, ExecutionType, PropertySettings, ResumePayload, ScheduleOptions } from '@activepieces/shared' -import { HookResponse } from '../../utils' - -export const pieceProtocol = { - toTransferable: (value: unknown): unknown => { - if (typeof value === 'function' || isThenable(value)) { - return undefined - } - if (Array.isArray(value)) { - return value.map((item) => pieceProtocol.toTransferable(item)) - } - if (!isObject(value) || Buffer.isBuffer(value) || value instanceof Date) { - return value - } - const entries = Object.entries(value) - .map(([key, item]) => [key, pieceProtocol.toTransferable(item)]) - .filter(([, item]) => item !== undefined) - return Object.fromEntries(entries) - }, - - serializeError: (error: unknown): SerializedError => { - if (!(error instanceof Error)) { - return { message: String(error) } - } - const details = Object.fromEntries( - [...Object.keys(error), ...ERROR_DETAIL_KEYS] - .map((key) => [key, readJsonSafe(() => Reflect.get(error, key)).data] as const) - .filter(([, data]) => data !== undefined), - ) - return { - ...details, - message: error.message, - name: error.name === 'Error' ? error.constructor.name : error.name, - stack: error.stack, - type: error instanceof ExecutionError ? error.type : undefined, - cause: isNil(error.cause) ? undefined : inspect(error.cause), - } - }, - - deserializeError: ({ message, name, stack, type, ...details }: SerializedError): Error => { - if (!isNil(type)) { - return new ExecutionError(name ?? 'ExecutionError', message, type) - } - const error = Object.assign(new Error(message), details) - error.name = name ?? error.name - error.stack = stack ?? error.stack - return error - }, -} - -function isThenable(value: unknown): boolean { - return isObject(value) && typeof value.then === 'function' -} - -function readJsonSafe(read: () => unknown): { data: unknown } { - try { - return { data: JSON.parse(JSON.stringify(read())) } - } - catch { - return { data: undefined } - } -} - -const ERROR_DETAIL_KEYS = ['response', 'request', 'status', 'headers', 'body', 'error'] - -type PieceIdentity = { - piecePath: string - pieceName: string - pieceVersion: string -} - -export type PieceRuntime = { - internalApiUrl: string - publicApiUrl: string - engineToken: string - projectId: string - flowId: string - flowVersionId: string - flowRunId: string - pieceName: string - contextVersion?: ContextVersion - actionRunMode: boolean - workerHandlerId?: string - httpRequestId?: string -} - -export type ActionContextRequest = { - kind: 'action' - runtime: PieceRuntime - actionName: string - stepName: string - resolvedInput: Record - propertySettings: Record - executionType: ExecutionType - resumePayload?: ResumePayload -} - -export type TriggerContextRequest = { - kind: 'trigger' - runtime: PieceRuntime - stepName: string - resolvedInput: Record - propertySettings: Record - payload: unknown - storePrefix: string - includeFiles: boolean - webhookUrl?: string - isRepublish?: boolean -} - -export type PropsContextRequest = { - kind: 'props' - runtime: PieceRuntime - stepName: string - resolvedInput: Record - searchValue?: string -} - -export type ContextRequest = ActionContextRequest | TriggerContextRequest | PropsContextRequest - -export type PieceDescription = { - metadata: DescribedMetadata - functionPaths: string[] - hasPath: (path: string[]) => boolean -} - -type DescribedMetadata = Omit & { - i18n?: PieceMetadata['i18n'] -} - -type AppListener = { - events: string[] - identifierValue: string - identifierKey: string -} - -export type CollectedHooks = { - hookResponse: HookResponse - listeners: AppListener[] - scheduleOptions?: ScheduleOptions -} - -export type SerializedError = { - message: string - name?: string - stack?: string - type?: ExecutionErrorType - [key: string]: unknown -} - -export type ParentMessage = - | (PieceIdentity & { type: 'describe' }) - | (PieceIdentity & { type: 'call', path: string[], args: unknown[], context?: ContextRequest }) - -export type ChildMessage = - | { type: 'done', success: true, result: unknown, hooks?: CollectedHooks } - | { type: 'done', success: false, error: SerializedError } diff --git a/packages/server/engine/src/lib/core/piece/piece-runner.ts b/packages/server/engine/src/lib/core/piece/piece-runner.ts deleted file mode 100644 index 3e8d412cb992..000000000000 --- a/packages/server/engine/src/lib/core/piece/piece-runner.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { spawn } from 'node:child_process' -import path from 'node:path' -import { isNil, tryCatchSync } from '@activepieces/core-utils' -import { EngineGenericError, PieceMemoryLimitError } from '@activepieces/shared' -import { piecePath } from './piece-path' -import { ChildMessage, CollectedHooks, ContextRequest, ParentMessage, PieceDescription, pieceProtocol } from './piece-protocol' - -export const pieceRunner = { - describe: async (piece: PieceRef): Promise => { - const cacheKey = `${piece.pieceName}@${piece.pieceVersion}` - const cached = descriptions.get(cacheKey) - if (!isNil(cached)) { - return cached - } - const description = runInChildProcess({ piece, request: { type: 'describe' } }).then(({ result }) => toPieceDescription(result)) - descriptions.set(cacheKey, description) - description.catch(() => descriptions.delete(cacheKey)) - return description - }, - - call: async ({ piece, path: methodPath, args = [], context }: CallParams): Promise => { - return runInChildProcess({ piece, request: { type: 'call', path: methodPath, args, context } }) - }, -} - -async function runInChildProcess({ piece, request }: RunInChildProcessParams): Promise { - const entryPath = await piecePath.resolve(piece) - - return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [...process.execArgv, childEntryPath()], { - stdio: ['pipe', 'pipe', 'pipe', 'ipc'], - serialization: 'advanced', - }) - let settled = false - let output = '' - - const settle = (apply: () => void): void => { - if (settled) { - return - } - settled = true - child.kill() - apply() - } - - child.stdout?.on('data', (data: Buffer) => { - output += data.toString() - console.log(data.toString().trimEnd()) - }) - child.stderr?.on('data', (data: Buffer) => { - output += data.toString() - console.error(data.toString().trimEnd()) - }) - - child.on('message', (message: ChildMessage) => { - settle(() => message.success - ? resolve({ result: message.result, hooks: message.hooks }) - : reject(pieceProtocol.deserializeError(message.error))) - }) - - child.on('close', (code, signal) => { - settle(() => reject(toExitError({ code, signal, output }))) - }) - - child.on('error', (error) => { - settle(() => reject(new EngineGenericError('PieceProcessError', withOutput(error.message, output)))) - }) - - const identity = { piecePath: entryPath, pieceName: piece.pieceName, pieceVersion: piece.pieceVersion } - const { error: sendError } = tryCatchSync(() => { - const message: ParentMessage = request.type === 'describe' - ? { ...identity, type: 'describe' } - : { ...identity, type: 'call', path: request.path, args: request.args, context: request.context } - child.send(message) - }) - if (sendError) { - settle(() => reject(new EngineGenericError('PieceArgumentsNotSerializableError', sendError.message))) - } - }) -} - -function toPieceDescription(value: unknown): PieceDescription { - const metadata = Reflect.get(Object(value), 'metadata') - const functionPaths = Reflect.get(Object(value), 'functionPaths') - if (isNil(metadata) || !Array.isArray(functionPaths)) { - throw new EngineGenericError('PieceDescriptionInvalidError', 'Piece process returned an unexpected description') - } - return { - metadata, - functionPaths, - hasPath: (path: string[]) => functionPaths.includes(path.join('.')), - } -} - -export function toExitError({ code, signal, output }: ExitParams): Error { - if (isOutOfMemory({ code, signal, output })) { - return new PieceMemoryLimitError(heapLimitMb(), output.trim()) - } - return new EngineGenericError('PieceProcessExitedError', withOutput(`Piece process exited with code ${code} and signal ${signal}`, output)) -} - -// Mirrors the engine-side signatures in sandbox.ts: V8 saying it ran out of heap, or aborting, -// is unambiguous. A SIGKILL is ambiguous there because shutdown kills the engine the same way — -// here it is not, since the only kill we issue happens after the call has already settled. -function isOutOfMemory({ code, signal, output }: ExitParams): boolean { - return output.includes('JavaScript heap out of memory') - || code === 134 - || signal === 'SIGABRT' - || signal === 'SIGKILL' -} - -function heapLimitMb(): string | undefined { - return process.execArgv.find((arg) => arg.startsWith('--max-old-space-size='))?.split('=')[1] -} - -function withOutput(message: string, output: string): string { - return output.trim().length === 0 ? message : `${message}\n${output.trim()}` -} - -function childEntryPath(): string { - return process.env.AP_PIECE_CHILD_ENTRY ?? path.join(__dirname, 'piece-child.js') -} - -const descriptions = new Map>() - -type RunInChildProcessParams = { - piece: PieceRef - request: { type: 'describe' } | { type: 'call', path: string[], args: unknown[], context?: ContextRequest } -} - -type ExitParams = { - code: number | null - signal: NodeJS.Signals | null - output: string -} - -export type PieceRef = { - pieceName: string - pieceVersion: string - devPieces: string[] -} - -export type CallParams = { - piece: PieceRef - path: string[] - args?: unknown[] - context?: ContextRequest -} - -export type CallResult = { - result: unknown - hooks?: CollectedHooks -} diff --git a/packages/server/engine/src/lib/core/piece/trigger-runner.ts b/packages/server/engine/src/lib/core/piece/trigger-runner.ts deleted file mode 100644 index 19e737c5c6fe..000000000000 --- a/packages/server/engine/src/lib/core/piece/trigger-runner.ts +++ /dev/null @@ -1,206 +0,0 @@ -import { assertEqual, isNil, isObject } from '@activepieces/core-utils' -import { PiecePropertyMap, StaticPropsValue, TriggerStrategy } from '@activepieces/pieces-framework' -import { EngineGenericError, EngineHttpResponse, ExecuteTriggerResponse, FlowTrigger, PieceTrigger, PropertySettings, TriggerHookType } from '@activepieces/shared' -import { EngineConstants, ResolvedExecuteTriggerOperation } from '../../handler/context/engine-constants' -import { FlowExecutorContext } from '../../handler/context/flow-execution-context' -import { buildRuntime } from '../../handler/piece-executor' -import { createPropsResolver } from '../../variables/props-resolver' -import { CollectedHooks, TriggerContextRequest } from './piece-protocol' -import { PieceRef, pieceRunner } from './piece-runner' - -export const triggerRunner = { - async executeOnStart({ trigger, constants, payload }: ExecuteOnStartParams): Promise { - const { pieceName, pieceVersion, triggerName, input, propertySettings } = (trigger as PieceTrigger).settings - assertTriggerName(triggerName) - - const piece: PieceRef = { pieceName, pieceVersion, devPieces: constants.devPieces } - const description = await pieceRunner.describe(piece) - if (!description.hasPath(['triggers', triggerName, 'onStart'])) { - return - } - await pieceRunner.call({ - piece, - path: ['triggers', triggerName, 'onStart'], - context: await buildTriggerContext({ - piece, - constants, - triggerName, - input, - propertySettings, - contextVersion: description.metadata.contextInfo?.version, - payload, - storePrefix: '', - includeFiles: false, - }), - }) - }, - - async executeTrigger({ params, constants }: ExecuteTriggerParams): Promise> { - const { pieceName, pieceVersion, triggerName, input, propertySettings } = (params.flowVersion.trigger as PieceTrigger).settings - assertTriggerName(triggerName) - - const piece: PieceRef = { pieceName, pieceVersion, devPieces: constants.devPieces } - const description = await pieceRunner.describe(piece) - const pieceTrigger = description.metadata.triggers[triggerName] - if (isNil(pieceTrigger)) { - throw new EngineGenericError('TriggerNotFoundError', `Trigger not found, pieceName=${pieceName}, triggerName=${triggerName}`) - } - - const context = await buildTriggerContext({ - piece, - constants, - triggerName, - input, - propertySettings, - contextVersion: description.metadata.contextInfo?.version, - payload: params.triggerPayload, - storePrefix: params.test ? 'test' : '', - includeFiles: params.hookType === TriggerHookType.TEST || params.hookType === TriggerHookType.RUN, - webhookUrl: params.webhookUrl, - isRepublish: params.isRepublish, - }) - const runHook = async (methodName: string): Promise<{ result: unknown, hooks?: CollectedHooks }> => - pieceRunner.call({ piece, path: ['triggers', triggerName, methodName], context }) - - switch (params.hookType) { - case TriggerHookType.ON_DISABLE: { - await runHook('onDisable') - return {} - } - case TriggerHookType.ON_ENABLE: { - const { hooks } = await runHook('onEnable') - return { - listeners: hooks?.listeners ?? [], - scheduleOptions: pieceTrigger.type === TriggerStrategy.POLLING ? hooks?.scheduleOptions : undefined, - } - } - case TriggerHookType.RENEW: { - assertEqual(pieceTrigger.type, TriggerStrategy.WEBHOOK, 'triggerType', 'WEBHOOK') - await runHook('onRenew') - return {} - } - case TriggerHookType.HANDSHAKE: { - const { result } = await runHook('onHandshake') - return { response: toWebhookResponse(result) } - } - case TriggerHookType.TEST: { - const { result } = await runHook('test') - return { output: toItems(result) } - } - case TriggerHookType.RUN: { - if (pieceTrigger.type === TriggerStrategy.APP_WEBHOOK) { - await verifyAppWebhook({ piece, description, params, pieceName }) - } - const { result } = await runHook('run') - return { output: toItems(result) } - } - } - }, -} - -async function buildTriggerContext({ piece, constants, triggerName, input, propertySettings, contextVersion, payload, storePrefix, includeFiles, webhookUrl, isRepublish }: BuildTriggerContextParams): Promise { - const { resolvedInput } = await createPropsResolver({ - apiUrl: constants.internalApiUrl, - projectId: constants.projectId, - engineToken: constants.engineToken, - contextVersion, - stepNames: constants.stepNames, - pieceName: piece.pieceName, - }).resolve>({ - unresolvedInput: input, - executionState: FlowExecutorContext.empty(), - }) - - return { - kind: 'trigger', - runtime: buildRuntime({ constants, pieceName: piece.pieceName, contextVersion }), - stepName: triggerName, - resolvedInput, - propertySettings, - payload, - storePrefix, - includeFiles, - webhookUrl, - isRepublish, - } -} - -async function verifyAppWebhook({ piece, description, params, pieceName }: VerifyAppWebhookParams): Promise { - if (!params.appWebhookUrl) { - throw new EngineGenericError('AppWebhookUrlNotAvailableError', `App webhook url is not available for piece name ${pieceName}`) - } - if (!params.webhookSecret) { - throw new EngineGenericError('WebhookSecretNotAvailableError', `Webhook secret is not available for piece name ${pieceName}`) - } - if (!description.hasPath(['events', 'verify'])) { - throw new Error('Webhook is not verified') - } - const { result } = await pieceRunner.call({ - piece, - path: ['events', 'verify'], - args: [{ - appWebhookUrl: params.appWebhookUrl, - payload: params.triggerPayload, - webhookSecret: params.webhookSecret, - }], - }) - if (result !== true) { - throw new Error('Webhook is not verified') - } -} - -function assertTriggerName(triggerName: string | undefined): asserts triggerName is string { - if (isNil(triggerName)) { - throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set') - } -} - -function toItems(value: unknown): unknown[] { - if (!Array.isArray(value)) { - throw new EngineGenericError('TriggerOutputNotArrayError', `Trigger returned ${typeof value} instead of an array of items`) - } - return value -} - -function toWebhookResponse(value: unknown): { status: number, body?: unknown, headers?: Record } | undefined { - if (!isObject(value)) { - return undefined - } - return { - status: typeof value.status === 'number' ? value.status : 200, - body: value.body, - headers: EngineHttpResponse.shape.headers.safeParse(value.headers).data ?? {}, - } -} - -type ExecuteOnStartParams = { - trigger: FlowTrigger - constants: EngineConstants - payload: unknown -} - -type ExecuteTriggerParams = { - params: ResolvedExecuteTriggerOperation - constants: EngineConstants -} - -type BuildTriggerContextParams = { - piece: PieceRef - constants: EngineConstants - triggerName: string - input: unknown - propertySettings: Record - contextVersion: TriggerContextRequest['runtime']['contextVersion'] - payload: unknown - storePrefix: string - includeFiles: boolean - webhookUrl?: string - isRepublish?: boolean -} - -type VerifyAppWebhookParams = { - piece: PieceRef - description: Awaited> - params: ResolvedExecuteTriggerOperation - pieceName: string -} diff --git a/packages/server/engine/src/lib/handler/flow-executor.ts b/packages/server/engine/src/lib/handler/flow-executor.ts index 5525a464944a..550839abc4b5 100644 --- a/packages/server/engine/src/lib/handler/flow-executor.ts +++ b/packages/server/engine/src/lib/handler/flow-executor.ts @@ -2,9 +2,9 @@ import { performance } from 'node:perf_hooks' import { isNil } from '@activepieces/core-utils' import { EngineGenericError, ExecutionType, FlowAction, FlowActionType, FlowRunStatus, FlowTrigger, GenericStepOutput, StepOutputStatus } from '@activepieces/shared' import dayjs from 'dayjs' -import { triggerRunner } from '../core/piece/trigger-runner' import { flowRunProgressReporter } from '../helper/flow-run-progress-reporter' import { loggingUtils } from '../helper/logging-utils' +import { triggerHelper } from '../helper/trigger-helper' import { BaseExecutor } from './base-executor' import { codeExecutor } from './code-executor' import { EngineConstants, ResolvedExecuteFlowOperation } from './context/engine-constants' @@ -49,7 +49,7 @@ export const flowExecutor = { void flowRunProgressReporter.backup().catch((err) => { console.error('[Progress] Initial payload upload failed', err) }) - await triggerRunner.executeOnStart({ trigger, constants, payload: input.triggerPayload }) + await triggerHelper.executeOnStart(trigger, constants, input.triggerPayload) await flowRunProgressReporter.sendUpdate({ engineConstants: constants, flowExecutorContext: executionState, diff --git a/packages/server/engine/src/lib/handler/piece-executor.ts b/packages/server/engine/src/lib/handler/piece-executor.ts index 3b8d95ea308a..7f3bf13bd759 100644 --- a/packages/server/engine/src/lib/handler/piece-executor.ts +++ b/packages/server/engine/src/lib/handler/piece-executor.ts @@ -1,15 +1,22 @@ -import { ActivepiecesError, ErrorCode, isNil, isObject } from '@activepieces/core-utils' -import { PiecePropertyMap, StaticPropsValue } from '@activepieces/pieces-framework' -import { EngineGenericError, ExecutionType, FlowActionType, FlowRunStatus, GenericStepOutput, PieceAction, RespondResponse, StepOutputStatus } from '@activepieces/shared' +import { isNil, isObject } from '@activepieces/core-utils' +import { ActionContext, backwardCompatabilityContextUtils, CreateWaitpointHook, CreateWaitpointParams, CreateWaitpointResult, InputPropertyMap, PieceAuthProperty, PiecePropertyMap, RespondHook, RespondHookParams, StaticPropsValue, StopHook, StopHookParams, TagsManager, WaitForWaitpointHook } from '@activepieces/pieces-framework' +import { AUTHENTICATION_PROPERTY_NAME, EngineGenericError, ExecutionType, FlowActionType, FlowRunStatus, GenericStepOutput, PausedFlowTimeoutError, PieceAction, RespondResponse, StepOutputStatus } from '@activepieces/shared' +import dayjs from 'dayjs' import { engineRunApi } from '../api/engine-run-api' -import { PieceRuntime } from '../core/piece/piece-protocol' -import { pieceRunner } from '../core/piece/piece-runner' import { continueIfFailureHandler, runWithExponentialBackoff } from '../helper/error-handling' import { flowRunProgressReporter } from '../helper/flow-run-progress-reporter' +import { pieceLoader } from '../helper/piece-loader' +import { createFileUploader } from '../piece-context/file-uploader' +import { createFlowsContext } from '../piece-context/flows' +import { createContextStore } from '../piece-context/store' +import { waitpointClient } from '../piece-context/waitpoint-client' import { HookResponse, utils } from '../utils' +import { propsProcessor } from '../variables/props-processor' import { ActionHandler, BaseExecutor, failStep } from './base-executor' import { EngineConstants } from './context/engine-constants' +const AP_PAUSED_FLOW_TIMEOUT_DAYS = Number(process.env.AP_PAUSED_FLOW_TIMEOUT_DAYS) + export const pieceExecutor: BaseExecutor = { async handle({ action, @@ -33,32 +40,44 @@ const executeAction: ActionHandler = async ({ action, executionStat }) const { data: executionStateResult, error: executionStateError } = await utils.tryCatchAndThrowOnEngineError((async () => { - const { actionName, pieceName, pieceVersion, propertySettings } = action.settings - if (isNil(actionName)) { + if (isNil(action.settings.actionName)) { throw new EngineGenericError('ActionNameNotSetError', 'Action name is not set') } - const piece = { pieceName, pieceVersion, devPieces: constants.devPieces } - const description = await pieceRunner.describe(piece) - if (isNil(description.metadata.actions[actionName])) { - throw new ActivepiecesError({ - code: ErrorCode.ENTITY_NOT_FOUND, - params: { - entityType: 'step', - entityId: actionName, - message: `Action not found for piece ${pieceName}@${pieceVersion}`, - extra: { pieceName, pieceVersion }, - }, - }) - } - const contextVersion = description.metadata.contextInfo?.version + const { pieceAction, piece } = await pieceLoader.getPieceAndActionOrThrow({ + pieceName: action.settings.pieceName, + pieceVersion: action.settings.pieceVersion, + actionName: action.settings.actionName, + devPieces: constants.devPieces, + }) - const { resolvedInput, censoredInput } = await constants.getPropsResolver({ contextVersion, pieceName }).resolve>({ + const { resolvedInput, censoredInput } = await constants.getPropsResolver({ contextVersion: piece.getContextInfo?.().version, pieceName: action.settings.pieceName }).resolve>({ unresolvedInput: action.settings.input, executionState, }) + stepOutput.input = censoredInput + const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators(resolvedInput, pieceAction.props, piece.auth, pieceAction.requireAuth, action.settings.propertySettings) + if (Object.keys(errors).length > 0) { + throw new Error(JSON.stringify(errors, null, 2)) + } + + + const params: { + hookResponse: HookResponse + } = { + hookResponse: { + type: 'none', + tags: [], + }, + } + const outputContext = constants.actionRunMode + ? { update: async (): Promise => { /* no-op: action runs have no live progress channel */ } } + : flowRunProgressReporter.createOutputContext({ + engineConstants: constants, + }) + const isPaused = executionState.isPaused({ stepName: action.name }) if (!isPaused) { await flowRunProgressReporter.sendUpdate({ @@ -67,29 +86,69 @@ const executeAction: ActionHandler = async ({ action, executionStat stepNameToUpdate: action.name, }) } - - const testSingleStepMode = !isNil(constants.stepNameToTest) - const useTestMethod = testSingleStepMode && description.hasPath(['actions', actionName, 'test']) - const { result: output, hooks } = await pieceRunner.call({ - piece, - path: ['actions', actionName, useTestMethod ? 'test' : 'run'], - context: { - kind: 'action', - runtime: buildRuntime({ constants, pieceName, contextVersion }), - actionName, - stepName: action.name, - resolvedInput, - propertySettings, - executionType: isPaused ? ExecutionType.RESUME : ExecutionType.BEGIN, - resumePayload: constants.resumePayload, + const context: ActionContext = { + executionType: isPaused ? ExecutionType.RESUME : ExecutionType.BEGIN, + resumePayload: constants.resumePayload!, + store: createContextStore({ + apiUrl: constants.internalApiUrl, + prefix: '', + flowId: constants.flowId, + engineToken: constants.engineToken, + }), + output: outputContext, + flows: createFlowsContext({ + engineToken: constants.engineToken, + internalApiUrl: constants.internalApiUrl, + flowId: constants.flowId, + flowVersionId: constants.flowVersionId, + }), + step: { + name: action.name, + }, + auth: processedInput[AUTHENTICATION_PROPERTY_NAME], + files: createFileUploader({ + apiUrl: constants.internalApiUrl, + engineToken: constants.engineToken, + }), + server: { + token: constants.engineToken, + apiUrl: constants.internalApiUrl, + publicUrl: constants.publicApiUrl, + }, + propsValue: processedInput, + tags: createTagsManager(params), + connections: utils.createConnectionManager({ + apiUrl: constants.internalApiUrl, + projectId: constants.projectId, + engineToken: constants.engineToken, + target: 'actions', + hookResponse: params.hookResponse, + contextVersion: piece.getContextInfo?.().version, + pieceName: action.settings.pieceName, + }), + run: { + id: constants.flowRunId, + stop: createStopHook(params), + respond: createRespondHook(params), + createWaitpoint: createWaitpointHook({ constants, stepName: action.name, hookParams: params }), + waitForWaitpoint: createWaitForWaitpointHook({ constants, hookParams: params }), }, + project: { + id: constants.projectId, + externalId: constants.externalProjectId, + }, + } + const backwardCompatibleContext = backwardCompatabilityContextUtils.makeActionContextBackwardCompatible({ + contextVersion: piece.getContextInfo?.().version, + context, }) + const testSingleStepMode = !isNil(constants.stepNameToTest) + const runMethodToExecute = (testSingleStepMode && !isNil(pieceAction.test)) ? pieceAction.test : pieceAction.run + const output = await runMethodToExecute(backwardCompatibleContext) + const newExecutionContext = executionState.addTags(params.hookResponse.tags) - const hookResponse: HookResponse = hooks?.hookResponse ?? { type: 'none', tags: [] } - const newExecutionContext = executionState.addTags(hookResponse.tags) - - const webhookResponse = getResponse(hookResponse) - const isSamePiece = constants.triggerPieceName === pieceName + const webhookResponse = getResponse(params.hookResponse) + const isSamePiece = constants.triggerPieceName === action.settings.pieceName if (!isNil(webhookResponse) && !isNil(constants.workerHandlerId) && !isNil(constants.httpRequestId) && isSamePiece) { await engineRunApi.sendFlowResponse({ apiUrl: constants.internalApiUrl, @@ -107,17 +166,17 @@ const executeAction: ActionHandler = async ({ action, executionStat } const stepEndTime = performance.now() - if (hookResponse.type === 'stopped') { - if (isNil(hookResponse.response)) { + if (params.hookResponse.type === 'stopped') { + if (isNil(params.hookResponse.response)) { throw new EngineGenericError('StopResponseNotSetError', 'Stop response is not set') } const succeeded = stepOutput.setOutput(output).setStatus(StepOutputStatus.SUCCEEDED).setDuration(stepEndTime - stepStartTime) return (await newExecutionContext.upsertStep(action.name, succeeded)).incrementStepsExecuted().setVerdict({ status: FlowRunStatus.SUCCEEDED, - stopResponse: hookResponse.response.response, + stopResponse: (params.hookResponse.response as StopHookParams).response, }) } - if (hookResponse.type === 'paused') { + if (params.hookResponse.type === 'paused') { const paused = stepOutput.setOutput(output).setStatus(StepOutputStatus.PAUSED).setDuration(stepEndTime - stepStartTime) return (await newExecutionContext.upsertStep(action.name, paused)) .incrementStepsExecuted() @@ -153,25 +212,108 @@ function getResponse(hookResponse: HookResponse): RespondResponse | undefined { } } -export function buildRuntime({ constants, pieceName, contextVersion }: BuildRuntimeParams): PieceRuntime { +const createTagsManager = (hkParams: createTagsManagerParams): TagsManager => { return { - internalApiUrl: constants.internalApiUrl, - publicApiUrl: constants.publicApiUrl, + add: async (params: addTagsParams): Promise => { + hkParams.hookResponse.tags.push(params.name) + }, + + } +} + +type addTagsParams = { + name: string +} + +type createTagsManagerParams = { + hookResponse: HookResponse +} + + +function createStopHook(params: CreateStopHookParams): StopHook { + return (req?: StopHookParams) => { + params.hookResponse = { + ...params.hookResponse, + type: 'stopped', + response: req ?? { response: {} }, + } + } +} +type CreateStopHookParams = { + hookResponse: HookResponse +} + +function createRespondHook(params: CreateRespondHookParams): RespondHook { + return (req?: RespondHookParams) => { + params.hookResponse = { + ...params.hookResponse, + type: 'respond', + response: req ?? { response: {} }, + } + } +} + +type CreateRespondHookParams = { + hookResponse: HookResponse +} + +function createWaitpointHook({ constants, stepName, hookParams }: { constants: EngineConstants, stepName: string, hookParams: { hookResponse: HookResponse } }): CreateWaitpointHook { + return (req: CreateWaitpointParams): Promise => { + assertActionRunCannotSuspend(constants) + return submitWaitpoint({ constants, stepName, hookParams, req }) + } +} + +async function submitWaitpoint({ constants, stepName, hookParams, req }: { constants: EngineConstants, stepName: string, hookParams: { hookResponse: HookResponse }, req: CreateWaitpointParams }): Promise { + assertDelayWithinTimeout(req.resumeDateTime) + if (!isNil(req.responseToSend)) { + hookParams.hookResponse = { ...hookParams.hookResponse, responseToSend: req.responseToSend } + } + const result = await waitpointClient.create({ + apiUrl: constants.internalApiUrl, engineToken: constants.engineToken, - projectId: constants.projectId, - flowId: constants.flowId, - flowVersionId: constants.flowVersionId, flowRunId: constants.flowRunId, - pieceName, - contextVersion, - actionRunMode: constants.actionRunMode, + projectId: constants.projectId, + stepName, + type: req.type, + version: req.version ?? 'V1', + resumeDateTime: req.resumeDateTime, + responseToSend: req.responseToSend, workerHandlerId: constants.workerHandlerId ?? undefined, httpRequestId: constants.httpRequestId ?? undefined, + }) + return { + ...result, + buildResumeUrl: (params: { queryParams: Record, sync?: boolean }): string => { + const url = new URL(`${result.resumeUrl}${params.sync ? '/sync' : ''}`) + url.search = new URLSearchParams(params.queryParams).toString() + return url.toString() + }, + } +} + +function createWaitForWaitpointHook({ constants, hookParams }: { constants: EngineConstants, hookParams: { hookResponse: HookResponse } }): WaitForWaitpointHook { + return (_waitpointId: string) => { + assertActionRunCannotSuspend(constants) + hookParams.hookResponse = { + ...hookParams.hookResponse, + type: 'paused', + } + } +} + +function assertActionRunCannotSuspend(constants: EngineConstants): void { + if (constants.actionRunMode) { + throw new Error('This action pauses the run (waitpoint) and can only run inside a flow, not as a action run.') } } -type BuildRuntimeParams = { - constants: EngineConstants - pieceName: string - contextVersion?: PieceRuntime['contextVersion'] +function assertDelayWithinTimeout(resumeDateTime?: string): void { + if (isNil(resumeDateTime)) { + return + } + const diffInDays = dayjs(resumeDateTime).diff(dayjs(), 'days') + if (diffInDays > AP_PAUSED_FLOW_TIMEOUT_DAYS) { + throw new PausedFlowTimeoutError(undefined, AP_PAUSED_FLOW_TIMEOUT_DAYS) + } } diff --git a/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts b/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts index 181d112fe854..618c1920ee5c 100644 --- a/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts +++ b/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts @@ -70,16 +70,17 @@ export const flowRunProgressReporter = { }) }) }, - createOutputContext: ({ internalApiUrl, engineToken, projectId, flowRunId }: CreateOutputContextParams): OutputContext => { + createOutputContext: (params: CreateOutputContextParams): OutputContext => { + const { engineConstants } = params return { update: async (params: { data: unknown }) => { // Streaming output is best-effort — a failed push must never fail the run. const { error } = await tryCatch(() => engineRunApi.updateStepProgress({ - apiUrl: internalApiUrl, - engineToken, + apiUrl: engineConstants.internalApiUrl, + engineToken: engineConstants.engineToken, request: { - projectId, - runId: flowRunId, + projectId: engineConstants.projectId, + runId: engineConstants.flowRunId, output: params.data, }, })) @@ -243,10 +244,7 @@ type UpdateStepProgressParams = { } type CreateOutputContextParams = { - internalApiUrl: string - engineToken: string - projectId: string - flowRunId: string + engineConstants: EngineConstants } type ExtractStepResponse = { diff --git a/packages/server/engine/src/lib/helper/piece-helper.ts b/packages/server/engine/src/lib/helper/piece-helper.ts new file mode 100644 index 000000000000..30df4dfb5841 --- /dev/null +++ b/packages/server/engine/src/lib/helper/piece-helper.ts @@ -0,0 +1,366 @@ +import path from 'path' +import { isNil } from '@activepieces/core-utils' +import { + DropdownProperty, + DynamicProperties, + ExecutePropsResult, + getAuthPropertyForValue, + MultiSelectDropdownProperty, + PieceAuthProperty, + PieceMetadata, + PiecePropertyMap, + pieceTranslation, + PropertyType, + StaticPropsValue } from '@activepieces/pieces-framework' +import { AppConnectionType, AppConnectionValue, EngineGenericError, ExecuteExtractPieceMetadata, ExecutePropsOptions, ExecuteRefreshTokenAuthOperation, ExecuteRefreshTokenAuthResponse, ExecuteResolveConnectionIdentifierOperation, ExecuteResolveConnectionIdentifierResponse, ExecuteValidateAuthOperation, ExecuteValidateAuthResponse } from '@activepieces/shared' +import { EngineConstants } from '../handler/context/engine-constants' + +const DEFAULT_REFRESH_EXPIRES_IN_SECONDS = 3300 +import { testExecutionContext } from '../handler/context/test-execution-context' +import { createFlowsContext } from '../piece-context/flows' +import { utils } from '../utils' +import { createPropsResolver } from '../variables/props-resolver' +import { dynamicPropKeys } from './dynamic-prop-keys' +import { pieceLoader } from './piece-loader' + +export const pieceHelper = { + async executeProps( operation: ExecutePropsParams): Promise> { + const constants = EngineConstants.fromExecutePropertyInput(operation) + const executionState = await testExecutionContext.stateFromFlowVersion({ + apiUrl: operation.internalApiUrl, + flowVersion: operation.flowVersion, + projectId: operation.projectId, + engineToken: operation.engineToken, + sampleData: operation.sampleData, + engineConstants: constants, + }) + const { property, piece } = await pieceLoader.getPropOrThrow({ pieceName: operation.pieceName, pieceVersion: operation.pieceVersion, actionOrTriggerName: operation.actionOrTriggerName, propertyName: operation.propertyName, devPieces: EngineConstants.DEV_PIECES }) + + if (property.type !== PropertyType.DROPDOWN && property.type !== PropertyType.MULTI_SELECT_DROPDOWN && property.type !== PropertyType.DYNAMIC) { + throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property.type} for ${property.displayName}`) + } + const { data: executePropsResult, error: executePropsError } = await utils.tryCatchAndThrowOnEngineError((async (): Promise> => { + const { resolvedInput } = await createPropsResolver({ + apiUrl: constants.internalApiUrl, + projectId: constants.projectId, + engineToken: constants.engineToken, + contextVersion: piece.getContextInfo?.().version, + stepNames: constants.stepNames, + pieceName: operation.pieceName, + }).resolve< + StaticPropsValue + >({ + unresolvedInput: operation.input, + executionState, + }) + const ctx = { + searchValue: operation.searchValue, + server: { + token: constants.engineToken, + apiUrl: constants.internalApiUrl, + publicUrl: operation.publicApiUrl, + }, + project: { + id: constants.projectId, + externalId: constants.externalProjectId, + }, + flows: createFlowsContext(constants), + step: { + name: operation.actionOrTriggerName, + }, + connections: utils.createConnectionManager({ + projectId: constants.projectId, + engineToken: constants.engineToken, + apiUrl: constants.internalApiUrl, + target: 'properties', + contextVersion: piece.getContextInfo?.().version, + pieceName: operation.pieceName, + }), + } + + switch (property.type) { + case PropertyType.DYNAMIC: { + const dynamicProperty = property as DynamicProperties + const props = await dynamicProperty.props(resolvedInput, ctx) + return { + type: PropertyType.DYNAMIC, + options: dynamicPropKeys.escapePropsKeys(props), + } + } + case PropertyType.MULTI_SELECT_DROPDOWN: { + const multiSelectProperty = property as MultiSelectDropdownProperty< + unknown, + boolean + > + const options = await multiSelectProperty.options(resolvedInput, ctx) + return { + type: PropertyType.MULTI_SELECT_DROPDOWN, + options, + } + } + case PropertyType.DROPDOWN: { + const dropdownProperty = property as DropdownProperty + const options = await dropdownProperty.options(resolvedInput, ctx) + return { + type: PropertyType.DROPDOWN, + options, + } + } + default: { + throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property}`) + } + } + })) + + if (executePropsError) { + console.error(executePropsError) + return { + type: property.type, + options: { + disabled: true, + options: [], + placeholder: 'Throws an error, reconnect or refresh the page', + }, + } + } + + return executePropsResult + }, + + async executeValidateAuth( + { params, devPieces }: { params: ExecuteValidateAuthOperation, devPieces: string[] }, + ): Promise { + const { piece: piecePackage } = params + + const piece = await pieceLoader.loadPieceOrThrow({ pieceName: piecePackage.pieceName, pieceVersion: piecePackage.pieceVersion, devPieces }) + const server = buildServerContext(params) + return validateAuth({ + authValue: params.auth, + pieceAuth: piece.auth, + server, + }) + + }, + + async executeResolveConnectionIdentifier( + { params, devPieces }: { params: ExecuteResolveConnectionIdentifierOperation, devPieces: string[] }, + ): Promise { + const { piece: piecePackage } = params + + const piece = await pieceLoader.loadPieceOrThrow({ pieceName: piecePackage.pieceName, pieceVersion: piecePackage.pieceVersion, devPieces }) + const server = buildServerContext(params) + return resolveConnectionIdentifier({ + authValue: params.auth, + connectionType: params.connectionType, + pieceAuth: piece.auth, + server, + }) + }, + + async executeRefreshTokenAuth( + { params, devPieces }: { params: ExecuteRefreshTokenAuthOperation, devPieces: string[] }, + ): Promise { + const { piece: piecePackage } = params + + const piece = await pieceLoader.loadPieceOrThrow({ pieceName: piecePackage.pieceName, pieceVersion: piecePackage.pieceVersion, devPieces }) + + if (params.auth.type !== AppConnectionType.CUSTOM_AUTH) { + return { skipped: true } + } + + const pieceAuth = getAuthPropertyForValue({ authValueType: params.auth.type, pieceAuth: piece.auth }) + + if (isNil(pieceAuth) || pieceAuth.type !== PropertyType.CUSTOM_AUTH || isNil(pieceAuth.refresh)) { + return { skipped: true } + } + + const server = buildServerContext(params) + const result = await pieceAuth.refresh.generate({ + auth: params.auth.props, + server, + }) + + const expiresIn = result.expires_in ?? pieceAuth.refresh.defaultExpiresIn ?? DEFAULT_REFRESH_EXPIRES_IN_SECONDS + + return { + skipped: false, + access_token: result.access_token, + expires_in: expiresIn, + } + }, + + async extractPieceMetadata({ devPieces, params }: { devPieces: string[], params: ExecuteExtractPieceMetadata }): Promise { + const { pieceName, pieceVersion } = params + const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) + const pieceAlias = pieceLoader.getPackageAlias({ pieceName, pieceVersion, devPieces }) + const pieceIndexPath = await pieceLoader.getPiecePath({ packageName: pieceAlias, devPieces }) + const pieceDistRoot = path.dirname(path.dirname(pieceIndexPath)) + const i18n = await pieceTranslation.initializeI18n(pieceDistRoot) + const fullMetadata = piece.metadata() + return { + ...fullMetadata, + name: pieceName, + version: pieceVersion, + authors: piece.authors, + i18n, + } + }, +} + +type ExecutePropsParams = Omit & { pieceName: string, pieceVersion: string } + + +function mismatchAuthTypeErrorMessage(pieceAuthType: PropertyType, connectionType: AppConnectionType): ExecuteValidateAuthResponse { + return { + valid: false, + error: `Connection value type does not match piece auth type: ${pieceAuthType} !== ${connectionType}`, + } +} + +const validateAuth = async ({ + server, + authValue, + pieceAuth, +}: ValidateAuthParams): Promise => { + if (isNil(pieceAuth)) { + return { + valid: true, + } + } + const usedPieceAuth = getAuthPropertyForValue({ + authValueType: authValue.type, + pieceAuth, + }) + + if (isNil(usedPieceAuth)) { + return { + valid: false, + error: 'No piece auth found for auth value', + } + } + if (isNil(usedPieceAuth.validate)) { + return { + valid: true, + } + } + + + switch (usedPieceAuth.type) { + case PropertyType.OAUTH2:{ + if (authValue.type !== AppConnectionType.OAUTH2 && authValue.type !== AppConnectionType.CLOUD_OAUTH2 && authValue.type !== AppConnectionType.PLATFORM_OAUTH2) { + return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) + } + return usedPieceAuth.validate({ + auth: authValue, + server, + }) + } + case PropertyType.BASIC_AUTH:{ + if (authValue.type !== AppConnectionType.BASIC_AUTH) { + return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) + } + return usedPieceAuth.validate({ + auth: authValue, + server, + }) + } + case PropertyType.SECRET_TEXT:{ + if (authValue.type !== AppConnectionType.SECRET_TEXT) { + return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) + } + return usedPieceAuth.validate({ + auth: authValue.secret_text, + server, + }) + } + case PropertyType.CUSTOM_AUTH:{ + if (authValue.type !== AppConnectionType.CUSTOM_AUTH) { + return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) + } + return usedPieceAuth.validate({ + auth: authValue.props, + server, + }) + } + case PropertyType.OIDC:{ + if (authValue.type !== AppConnectionType.OIDC) { + return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) + } + return usedPieceAuth.validate({ + auth: authValue.props, + server, + }) + } + default: { + throw new EngineGenericError('InvalidAuthTypeError', 'Invalid auth type') + } + } +} + +const resolveConnectionIdentifier = async ({ + server, + authValue, + connectionType, + pieceAuth, +}: ResolveConnectionIdentifierParams): Promise => { + if (isNil(pieceAuth)) { + return { identifier: undefined } + } + const usedPieceAuth = getAuthPropertyForValue({ + authValueType: connectionType, + pieceAuth, + }) + if (isNil(usedPieceAuth)) { + return { identifier: undefined } + } + switch (usedPieceAuth.type) { + case PropertyType.OAUTH2: { + if (!('access_token' in authValue)) { + return { identifier: undefined } + } + return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue, server }) } + } + case PropertyType.BASIC_AUTH: { + if (!('username' in authValue)) { + return { identifier: undefined } + } + return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue, server }) } + } + case PropertyType.SECRET_TEXT: { + if (!('secret_text' in authValue)) { + return { identifier: undefined } + } + return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue.secret_text, server }) } + } + case PropertyType.CUSTOM_AUTH: + case PropertyType.OIDC: { + if (!('props' in authValue)) { + return { identifier: undefined } + } + return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue.props, server }) } + } + default: { + return { identifier: undefined } + } + } +} + +type ValidateAuthParams = { + server: { + apiUrl: string + publicUrl: string + } + authValue: AppConnectionValue + pieceAuth: PieceAuthProperty | PieceAuthProperty[] | undefined +} + +type ResolveConnectionIdentifierParams = ValidateAuthParams & { + connectionType: AppConnectionType +} + +function buildServerContext({ internalApiUrl, publicApiUrl }: { internalApiUrl: string, publicApiUrl: string }) { + return { + apiUrl: internalApiUrl.endsWith('/') ? internalApiUrl : internalApiUrl + '/', + publicUrl: publicApiUrl, + } +} \ No newline at end of file diff --git a/packages/server/engine/src/lib/helper/piece-loader.ts b/packages/server/engine/src/lib/helper/piece-loader.ts new file mode 100644 index 000000000000..8e8211c2a0d5 --- /dev/null +++ b/packages/server/engine/src/lib/helper/piece-loader.ts @@ -0,0 +1,342 @@ +import fs from 'fs/promises' +import { createRequire } from 'node:module' +import path from 'path' +import { ActivepiecesError, ErrorCode, isNil } from '@activepieces/core-utils' +import { Action, Piece, PiecePropertyMap, Trigger } from '@activepieces/pieces-framework' +import { EngineGenericError, extractPieceFromModule, getPackageAliasForPiece, getPieceNameFromAlias, trimVersionFromAlias } from '@activepieces/shared' +import { utils } from '../utils' + +export const pieceLoader = { + loadPieceOrThrow: async ( + { pieceName, pieceVersion, devPieces }: LoadPieceParams, + ): Promise => { + const { data: piece, error: pieceError } = await utils.tryCatchAndThrowOnEngineError(async () => { + const packageName = pieceLoader.getPackageAlias({ + pieceName, + pieceVersion, + devPieces, + }) + const piecePath = await pieceLoader.getPiecePath({ packageName, devPieces }) + const module = loadAndCapPieces(piecePath) + + const piece = extractPieceFromModule({ + module, + pieceName, + pieceVersion, + }) + + if (isNil(piece)) { + throw new EngineGenericError('PieceNotFoundError', `Piece not found for piece: ${pieceName}, pieceVersion: ${pieceVersion}`) + } + return piece + }) + if (pieceError) { + throw pieceError + } + return piece + }, + + getPieceAndTriggerOrThrow: async (params: GetPieceAndTriggerParams): Promise<{ piece: Piece, pieceTrigger: Trigger }> => { + const { pieceName, pieceVersion, triggerName, devPieces } = params + const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) + const trigger = piece.getTrigger(triggerName) + + if (trigger === undefined) { + throw new EngineGenericError('TriggerNotFoundError', `Trigger not found, pieceName=${pieceName}, triggerName=${triggerName}`) + } + + return { + piece, + pieceTrigger: trigger, + } + }, + + getPieceAndActionOrThrow: async (params: GetPieceAndActionParams): Promise<{ piece: Piece, pieceAction: Action }> => { + const { pieceName, pieceVersion, actionName, devPieces } = params + + const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) + const pieceAction = piece.getAction(actionName) + + if (isNil(pieceAction)) { + throw new ActivepiecesError({ + code: ErrorCode.ENTITY_NOT_FOUND, + params: { + entityType: 'step', + entityId: actionName, + message: `Action not found for piece ${pieceName}@${pieceVersion}`, + extra: { pieceName, pieceVersion }, + }, + }) + } + + return { + piece, + pieceAction, + } + }, + + getPropOrThrow: async ({ pieceName, pieceVersion, actionOrTriggerName, propertyName, devPieces }: GetPropParams) => { + const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) + + const actionOrTrigger = piece.getAction(actionOrTriggerName) ?? piece.getTrigger(actionOrTriggerName) + + if (isNil(actionOrTrigger)) { + throw new ActivepiecesError({ + code: ErrorCode.ENTITY_NOT_FOUND, + params: { + entityType: 'step', + entityId: actionOrTriggerName, + message: `Step not found for piece ${pieceName}@${pieceVersion}`, + extra: { pieceName, pieceVersion }, + }, + }) + } + + const property = (actionOrTrigger.props as PiecePropertyMap)[propertyName] + + if (isNil(property)) { + throw new ActivepiecesError({ + code: ErrorCode.ENTITY_NOT_FOUND, + params: { + entityType: 'config', + entityId: propertyName, + message: `Config not found for step ${actionOrTriggerName} in piece ${pieceName}@${pieceVersion}`, + extra: { pieceName, pieceVersion, stepName: actionOrTriggerName }, + }, + }) + } + + return { property, piece } + }, + + getPackageAlias: ({ pieceName, pieceVersion, devPieces }: GetPackageAliasParams) => { + if (devPieces.includes(getPieceNameFromAlias(pieceName))) { + return pieceName + } + + return getPackageAliasForPiece({ + pieceName, + pieceVersion, + }) + }, + + getPiecePath: async ({ packageName, devPieces }: GetPiecePathParams): Promise => { + const piecePath = devPieces.includes(getPieceNameFromAlias(packageName)) + ? await findInDistFolder(packageName) + : await traverseAllParentFoldersToFindPiece(packageName) + if (isNil(piecePath)) { + throw new EngineGenericError('PieceNotFoundError', `Piece not found for package: ${packageName}`) + } + return piecePath + }, +} + +const MAX_LOADED_PIECES = 5 +const engineRequire = createRequire(__filename) +const loadedPieceEntries = new Map() +const baselineModuleIds = new Set(Object.keys(engineRequire.cache)) + +function loadAndCapPieces(piecePath: string): Record { + const disposableRequire = createRequire(__filename) + const resolvedEntry = disposableRequire.resolve(piecePath) + const loadedModule: Record = disposableRequire(resolvedEntry) + loadedPieceEntries.delete(resolvedEntry) + loadedPieceEntries.set(resolvedEntry, true) + if (loadedPieceEntries.size > MAX_LOADED_PIECES) { + const leastRecentEntry = loadedPieceEntries.keys().next().value + if (!isNil(leastRecentEntry)) { + loadedPieceEntries.delete(leastRecentEntry) + evictPieceSubtree(leastRecentEntry) + global.gc?.() + } + } + return loadedModule +} + +function evictPieceSubtree(evictedEntry: string): void { + const survivorReachable = collectReachableModules([...loadedPieceEntries.keys()]) + const evictedReachable = collectReachableModules([evictedEntry]) + const doomed = new Set([...evictedReachable].filter((id) => !survivorReachable.has(id) && !baselineModuleIds.has(id))) + if (doomed.size === 0) { + return + } + for (const id of Object.keys(engineRequire.cache)) { + const mod = engineRequire.cache[id] + if (!isNil(mod) && !doomed.has(id)) { + mod.children = mod.children.filter((child) => !doomed.has(child.id)) + } + } + if (!isNil(engineRequire.main)) { + engineRequire.main.children = engineRequire.main.children.filter((child) => !doomed.has(child.id)) + } + for (const id of doomed) { + Reflect.deleteProperty(engineRequire.cache, id) + } +} + +function collectReachableModules(rootIds: string[]): Set { + const reachable = new Set() + const stack = rootIds.filter((id) => !isNil(engineRequire.cache[id])) + while (stack.length > 0) { + const id = stack.pop() + if (isNil(id) || reachable.has(id)) { + continue + } + reachable.add(id) + for (const child of engineRequire.cache[id]?.children ?? []) { + stack.push(child.id) + } + } + return reachable +} + +async function findInDistFolder(packageName: string): Promise { + const sourcePiecesPath = path.resolve('packages/pieces') + if (!await utils.folderExists(sourcePiecesPath)) { + return null + } + const distPackageJsonPaths = await findDistPackageJsonFiles(sourcePiecesPath) + for (const packageJsonPath of distPackageJsonPaths) { + const { data: result } = await utils.tryCatchAndThrowOnEngineError(async () => { + const content = await fs.readFile(packageJsonPath, 'utf-8') + const packageJson = JSON.parse(content) + if (packageJson.name === packageName) { + return path.join(path.dirname(packageJsonPath), 'src', 'index.js') + } + return null + }) + if (result) { + return result + } + } + return null +} + +async function findDistPackageJsonFiles(dirPath: string): Promise { + const results: string[] = [] + const ignoredDirs = ['node_modules', '.turbo', 'framework', 'common'] + + async function scanDir(currentPath: string): Promise { + const items = await fs.readdir(currentPath, { withFileTypes: true }) + for (const item of items) { + if (!item.isDirectory() || ignoredDirs.includes(item.name)) { + continue + } + const fullPath = path.join(currentPath, item.name) + if (item.name === 'dist') { + const pkgJson = path.join(fullPath, 'package.json') + if (await utils.folderExists(pkgJson)) { + results.push(pkgJson) + } + } + else { + await scanDir(fullPath) + } + } + } + + await scanDir(dirPath) + return results +} + + +async function traverseAllParentFoldersToFindPiece(packageName: string): Promise { + const trimmedName = trimVersionFromAlias(packageName) + const customPaths = (process.env.AP_CUSTOM_PIECES_PATHS ?? '').split(':').filter(Boolean) + for (const customPath of customPaths) { + const entry = await resolveInstalledPieceEntry(path.resolve(customPath, 'pieces', packageName), trimmedName) + if (!isNil(entry)) { + return entry + } + } + + const rootDir = path.parse(__dirname).root + let currentDir = __dirname + const maxIterations = currentDir.split(path.sep).length + for (let i = 0; i < maxIterations; i++) { + const entry = await resolveInstalledPieceEntry(path.resolve(currentDir, 'pieces', packageName), trimmedName) + if (!isNil(entry)) { + return entry + } + + const parentDir = path.dirname(currentDir) + if (parentDir === currentDir || currentDir === rootDir) { + break + } + currentDir = parentDir + } + return null +} + +// A piece entry is resolved from its package.json "main" (defaulting to src/index.js). +// Registry/dev installs keep the package nested in node_modules; a packed-archive bundle is +// extracted straight to the install-folder root. Try the nested package first, then the root. +async function resolveInstalledPieceEntry(pieceFolder: string, trimmedName: string): Promise { + const packageDir = path.join(pieceFolder, 'node_modules', trimmedName) + if (await utils.folderExists(packageDir)) { + return resolveEntryFromPackageDir(packageDir) + } + // Only return an entry that actually exists: a half-installed registry folder also has a + // stub package.json (no "main") at this point, for which resolveEntryFromPackageDir would + // otherwise return a non-existent src/index.js — fall through to a clean PieceNotFoundError. + const rootManifest = path.join(pieceFolder, 'package.json') + if (await utils.folderExists(rootManifest)) { + const rootEntry = await resolveEntryFromPackageDir(pieceFolder) + if (await utils.folderExists(rootEntry)) { + return rootEntry + } + } + return null +} + +async function resolveEntryFromPackageDir(packageDir: string): Promise { + const { data: mainEntry } = await utils.tryCatchAndThrowOnEngineError(async () => { + const packageJson = JSON.parse(await fs.readFile(path.join(packageDir, 'package.json'), 'utf-8')) + if (isNil(packageJson.main)) { + return null + } + const resolved = path.join(packageDir, packageJson.main) + return await utils.folderExists(resolved) ? resolved : null + }) + return mainEntry ?? path.join(packageDir, 'src', 'index.js') +} + +type GetPiecePathParams = { + packageName: string + devPieces: string[] +} + +type LoadPieceParams = { + pieceName: string + pieceVersion: string + devPieces: string[] +} + +type GetPieceAndTriggerParams = { + pieceName: string + pieceVersion: string + triggerName: string + devPieces: string[] +} + +type GetPieceAndActionParams = { + pieceName: string + pieceVersion: string + actionName: string + devPieces: string[] +} + +type GetPropParams = { + pieceName: string + pieceVersion: string + actionOrTriggerName: string + propertyName: string + devPieces: string[] +} + +type GetPackageAliasParams = { + pieceName: string + devPieces: string[] + pieceVersion: string +} + diff --git a/packages/server/engine/src/lib/helper/trigger-helper.ts b/packages/server/engine/src/lib/helper/trigger-helper.ts new file mode 100644 index 000000000000..0d6417fd80a9 --- /dev/null +++ b/packages/server/engine/src/lib/helper/trigger-helper.ts @@ -0,0 +1,301 @@ +import { assertEqual, isNil } from '@activepieces/core-utils' +import { PiecePropertyMap, SetScheduleRequest, StaticPropsValue, TriggerStrategy } from '@activepieces/pieces-framework' +import { AUTHENTICATION_PROPERTY_NAME, EngineGenericError, EventPayload, ExecuteTriggerResponse, FlowTrigger, InvalidCronExpressionError, InvalidScheduleIntervalError, PieceTrigger, PropertySettings, ScheduleOptions, TriggerHookType, TriggerSourceScheduleType } from '@activepieces/shared' +import { isValidCron } from 'cron-validator' +import { EngineConstants, ResolvedExecuteTriggerOperation } from '../handler/context/engine-constants' +import { FlowExecutorContext } from '../handler/context/flow-execution-context' +import { createFileUploader } from '../piece-context/file-uploader' +import { createFlowsContext } from '../piece-context/flows' +import { createContextStore } from '../piece-context/store' +import { utils } from '../utils' +import { propsProcessor } from '../variables/props-processor' +import { createPropsResolver } from '../variables/props-resolver' +import { pieceLoader } from './piece-loader' + +type Listener = { + events: string[] + identifierValue: string + identifierKey: string +} + +export const triggerHelper = { + async executeOnStart(trigger: FlowTrigger, constants: EngineConstants, payload: unknown) { + const { pieceName, pieceVersion, triggerName, input, propertySettings } = (trigger as PieceTrigger).settings + + if (isNil(triggerName)) { + throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set') + } + + const { pieceTrigger, processedInput, piece } = await prepareTriggerExecution({ + pieceName, + pieceVersion, + triggerName, + input, + projectId: constants.projectId, + apiUrl: constants.internalApiUrl, + engineToken: constants.engineToken, + devPieces: constants.devPieces, + propertySettings, + stepNames: constants.stepNames, + }) + const isOldVersionOrNotSupported = isNil(pieceTrigger.onStart) + if (isOldVersionOrNotSupported) { + return + } + const context = { + store: createContextStore({ + apiUrl: constants.internalApiUrl, + prefix: '', + flowId: constants.flowId, + engineToken: constants.engineToken, + }), + auth: processedInput[AUTHENTICATION_PROPERTY_NAME], + propsValue: processedInput, + payload, + run: { + id: constants.flowRunId, + }, + step: { + name: triggerName, + }, + project: { + id: constants.projectId, + externalId: constants.externalProjectId, + }, + connections: utils.createConnectionManager({ + apiUrl: constants.internalApiUrl, + projectId: constants.projectId, + engineToken: constants.engineToken, + target: 'triggers', + contextVersion: piece.getContextInfo?.().version, + pieceName, + }), + } + await pieceTrigger.onStart(context) + }, + + async executeTrigger({ params, constants }: ExecuteTriggerParams): Promise> { + const { pieceName, pieceVersion, triggerName, input, propertySettings } = (params.flowVersion.trigger as PieceTrigger).settings + + if (isNil(triggerName)) { + throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set') + } + + const { piece, pieceTrigger, processedInput } = await prepareTriggerExecution({ + pieceName, + pieceVersion, + triggerName, + input, + projectId: params.projectId, + apiUrl: constants.internalApiUrl, + engineToken: params.engineToken, + devPieces: constants.devPieces, + propertySettings, + stepNames: constants.stepNames, + }) + + const appListeners: Listener[] = [] + const prefix = params.test ? 'test' : '' + let scheduleOptions: ScheduleOptions | undefined = undefined + const context = { + store: createContextStore({ + apiUrl: constants.internalApiUrl, + prefix, + flowId: params.flowVersion.flowId, + engineToken: params.engineToken, + }), + step: { + name: triggerName, + }, + app: { + createListeners({ events, identifierKey, identifierValue }: Listener): void { + appListeners.push({ events, identifierValue, identifierKey }) + }, + }, + setSchedule(request: SetScheduleRequest) { + if ('intervalMs' in request) { + const parsed = ScheduleOptions.safeParse({ + type: TriggerSourceScheduleType.INTERVAL, + intervalMs: request.intervalMs, + }) + if (!parsed.success) { + throw new InvalidScheduleIntervalError(request.intervalMs) + } + scheduleOptions = parsed.data + return + } + if (!isValidCron(request.cronExpression)) { + throw new InvalidCronExpressionError(request.cronExpression) + } + scheduleOptions = { + type: TriggerSourceScheduleType.CRON_EXPRESSION, + cronExpression: request.cronExpression, + timezone: request.timezone ?? 'UTC', + } + }, + flows: createFlowsContext({ + engineToken: params.engineToken, + internalApiUrl: constants.internalApiUrl, + flowId: params.flowVersion.flowId, + flowVersionId: params.flowVersion.id, + }), + webhookUrl: params.webhookUrl, + isRepublish: params.isRepublish, + auth: processedInput[AUTHENTICATION_PROPERTY_NAME], + propsValue: processedInput, + payload: params.triggerPayload ?? {}, + project: { + id: params.projectId, + externalId: constants.externalProjectId, + }, + server: { + token: params.engineToken, + apiUrl: constants.internalApiUrl, + publicUrl: params.publicApiUrl, + }, + connections: utils.createConnectionManager({ + apiUrl: constants.internalApiUrl, + projectId: constants.projectId, + engineToken: constants.engineToken, + target: 'triggers', + contextVersion: piece.getContextInfo?.().version, + pieceName, + }), + } + switch (params.hookType) { + case TriggerHookType.ON_DISABLE: { + await pieceTrigger.onDisable(context) + return {} + } + case TriggerHookType.ON_ENABLE: { + await pieceTrigger.onEnable(context) + return { + listeners: appListeners, + scheduleOptions: pieceTrigger.type === TriggerStrategy.POLLING ? scheduleOptions : undefined, + } + } + case TriggerHookType.RENEW: { + assertEqual(pieceTrigger.type, TriggerStrategy.WEBHOOK, 'triggerType', 'WEBHOOK') + await pieceTrigger.onRenew(context) + return {} + } + case TriggerHookType.HANDSHAKE: { + const { data: handshakeResponse, error: handshakeResponseError } = await utils.tryCatchAndThrowOnEngineError(() => pieceTrigger.onHandshake(context)) + + if (handshakeResponseError) { + throw handshakeResponseError + } + return { + response: handshakeResponse, + } + } + case TriggerHookType.TEST: { + const { data: testResponse, error: testResponseError } = await utils.tryCatchAndThrowOnEngineError(() => pieceTrigger.test({ + ...context, + files: createFileUploader({ + apiUrl: constants.internalApiUrl, + engineToken: params.engineToken!, + }), + })) + + if (testResponseError) { + throw testResponseError + } + return { + output: testResponse, + } + } + case TriggerHookType.RUN: { + if (pieceTrigger.type === TriggerStrategy.APP_WEBHOOK) { + + const { data: verified, error: verifiedError } = await utils.tryCatchAndThrowOnEngineError(async () => { + if (!params.appWebhookUrl) { + throw new EngineGenericError('AppWebhookUrlNotAvailableError', `App webhook url is not available for piece name ${pieceName}`) + } + if (!params.webhookSecret) { + throw new EngineGenericError('WebhookSecretNotAvailableError', `Webhook secret is not available for piece name ${pieceName}`) + } + + return piece.events?.verify({ + appWebhookUrl: params.appWebhookUrl, + payload: params.triggerPayload as EventPayload, + webhookSecret: params.webhookSecret, + }) + }) + + if (verifiedError) { + throw verifiedError + } + if (isNil(verified)) { + throw new Error('Webhook is not verified') + } + } + + const { data: triggerRunResult, error: triggerRunError } = await utils.tryCatchAndThrowOnEngineError(async () => { + const items = await pieceTrigger.run({ + ...context, + files: createFileUploader({ + apiUrl: constants.internalApiUrl, + engineToken: params.engineToken!, + }), + }) + return { + output: items, + } + }) + + if (triggerRunError) { + throw triggerRunError + } + return triggerRunResult + } + } + }, +} + +type ExecuteTriggerParams = { + params: ResolvedExecuteTriggerOperation + constants: EngineConstants +} + +async function prepareTriggerExecution({ pieceName, pieceVersion, triggerName, input, propertySettings, projectId, apiUrl, engineToken, devPieces, stepNames }: PrepareTriggerExecutionParams) { + const { piece, pieceTrigger } = await pieceLoader.getPieceAndTriggerOrThrow({ + pieceName, + pieceVersion, + triggerName, + devPieces, + }) + + const { resolvedInput } = await createPropsResolver({ + apiUrl, + projectId, + engineToken, + contextVersion: piece.getContextInfo?.().version, + stepNames, + pieceName, + }).resolve>({ + unresolvedInput: input, + executionState: FlowExecutorContext.empty(), + }) + + const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators(resolvedInput, pieceTrigger.props, piece.auth, pieceTrigger.requireAuth, propertySettings) + + if (Object.keys(errors).length > 0) { + throw new Error(JSON.stringify(errors, null, 2)) + } + + return { piece, pieceTrigger, processedInput } +} + +type PrepareTriggerExecutionParams = { + pieceName: string + pieceVersion: string + triggerName: string + input: unknown + propertySettings: Record + projectId: string + apiUrl: string + engineToken: string + devPieces: string[] + stepNames: string[] +} diff --git a/packages/server/engine/src/lib/operations/auth-refresh.operation.ts b/packages/server/engine/src/lib/operations/auth-refresh.operation.ts index d780e54e19d2..fc6919767b34 100644 --- a/packages/server/engine/src/lib/operations/auth-refresh.operation.ts +++ b/packages/server/engine/src/lib/operations/auth-refresh.operation.ts @@ -1,36 +1,21 @@ -import { isObject } from '@activepieces/core-utils' -import { PropertyType } from '@activepieces/pieces-framework' import { - AppConnectionType, EngineResponse, EngineResponseStatus, ExecuteRefreshTokenAuthOperation, ExecuteRefreshTokenAuthResponse, } from '@activepieces/shared' -import { pieceAuth } from '../core/piece/piece-auth' +import { EngineConstants } from '../handler/context/engine-constants' +import { pieceHelper } from '../helper/piece-helper' export const authRefreshOperation = { execute: async (operation: ExecuteRefreshTokenAuthOperation): Promise> => { + const output = await pieceHelper.executeRefreshTokenAuth({ + params: operation, + devPieces: EngineConstants.DEV_PIECES, + }) return { status: EngineResponseStatus.OK, - response: await refreshAuth(operation), + response: output, } }, } - -async function refreshAuth(operation: ExecuteRefreshTokenAuthOperation): Promise { - if (operation.auth.type !== AppConnectionType.CUSTOM_AUTH) { - return { skipped: true } - } - const call = await pieceAuth.callMethod({ operation, authValueType: operation.auth.type, methodPath: ['refresh', 'generate'] }) - if (!call.called || call.property.type !== PropertyType.CUSTOM_AUTH || !isObject(call.result) || typeof call.result.access_token !== 'string') { - return { skipped: true } - } - return { - skipped: false, - access_token: call.result.access_token, - expires_in: typeof call.result.expires_in === 'number' ? call.result.expires_in : call.property.refresh?.defaultExpiresIn ?? DEFAULT_REFRESH_EXPIRES_IN_SECONDS, - } -} - -const DEFAULT_REFRESH_EXPIRES_IN_SECONDS = 3300 diff --git a/packages/server/engine/src/lib/operations/auth-validation.operation.ts b/packages/server/engine/src/lib/operations/auth-validation.operation.ts index 9c338a86d1af..8025ba237380 100644 --- a/packages/server/engine/src/lib/operations/auth-validation.operation.ts +++ b/packages/server/engine/src/lib/operations/auth-validation.operation.ts @@ -1,36 +1,23 @@ -import { isObject } from '@activepieces/core-utils' import { EngineResponse, EngineResponseStatus, ExecuteValidateAuthOperation, ExecuteValidateAuthResponse, } from '@activepieces/shared' -import { pieceAuth } from '../core/piece/piece-auth' +import { EngineConstants } from '../handler/context/engine-constants' +import { pieceHelper } from '../helper/piece-helper' export const authValidationOperation = { execute: async (operation: ExecuteValidateAuthOperation): Promise> => { - const call = await pieceAuth.callMethod({ operation, authValueType: operation.auth.type, methodPath: ['validate'] }) - if (!call.called) { - return { - status: EngineResponseStatus.OK, - response: call.mismatch - ? { valid: false, error: `Connection value type does not match piece auth type: ${call.property?.type} !== ${operation.auth.type}` } - : { valid: true }, - } - } + const input = operation as ExecuteValidateAuthOperation + const output = await pieceHelper.executeValidateAuth({ + params: input, + devPieces: EngineConstants.DEV_PIECES, + }) + return { status: EngineResponseStatus.OK, - response: toValidateAuthResponse(call.result), + response: output, } }, -} - -function toValidateAuthResponse(value: unknown): ExecuteValidateAuthResponse { - if (!isObject(value)) { - return { valid: false, error: 'Connection validation returned an unexpected result' } - } - if (value.valid === true) { - return { valid: true } - } - return { valid: false, error: typeof value.error === 'string' ? value.error : 'Connection validation failed' } -} +} \ No newline at end of file diff --git a/packages/server/engine/src/lib/operations/flow.operation.ts b/packages/server/engine/src/lib/operations/flow.operation.ts index c62023da068e..99cf6b385e33 100644 --- a/packages/server/engine/src/lib/operations/flow.operation.ts +++ b/packages/server/engine/src/lib/operations/flow.operation.ts @@ -1,12 +1,12 @@ import { isNil, tryCatch } from '@activepieces/core-utils' import { EngineGenericError, EngineResponse, EngineResponseStatus, ExecuteFlowOperation, ExecuteTriggerResponse, ExecutionError, ExecutionErrorType, ExecutionState, ExecutionType, FlowActionType, FlowRunStatus, flowStructureUtil, GenericStepOutput, LoopStepOutput, ResumePayload, ResumeReason, StepOutput, StepOutputStatus, TriggerHookType, TriggerPayload } from '@activepieces/shared' import { engineFileApi } from '../api/engine-file-api' -import { triggerRunner } from '../core/piece/trigger-runner' import { EngineConstants, ResolvedBeginExecuteFlowOperation, ResolvedExecuteFlowOperation } from '../handler/context/engine-constants' import { FlowExecutorContext } from '../handler/context/flow-execution-context' import { testExecutionContext } from '../handler/context/test-execution-context' import { flowExecutor } from '../handler/flow-executor' import { flowRunProgressReporter } from '../helper/flow-run-progress-reporter' +import { triggerHelper } from '../helper/trigger-helper' import { utils } from '../utils' import { resolveJobPayload } from './utils/resolve-job-payload' @@ -183,7 +183,7 @@ async function runOrReturnPayload(input: ResolvedBeginExecuteFlowOperation, cons if (!input.executeTrigger) { return input.triggerPayload as TriggerPayload } - const newPayload = await triggerRunner.executeTrigger({ + const newPayload = await triggerHelper.executeTrigger({ params: { ...input, hookType: TriggerHookType.RUN, diff --git a/packages/server/engine/src/lib/operations/piece-metadata.operation.ts b/packages/server/engine/src/lib/operations/piece-metadata.operation.ts index 1262acc01f2b..d47fcafc37d2 100644 --- a/packages/server/engine/src/lib/operations/piece-metadata.operation.ts +++ b/packages/server/engine/src/lib/operations/piece-metadata.operation.ts @@ -1,32 +1,23 @@ -import path from 'path' -import { PieceMetadata, pieceTranslation } from '@activepieces/pieces-framework' +import { PieceMetadata } from '@activepieces/pieces-framework' import { EngineResponse, EngineResponseStatus, ExecuteExtractPieceMetadataOperation, } from '@activepieces/shared' -import { piecePath } from '../core/piece/piece-path' -import { pieceRunner } from '../core/piece/piece-runner' import { EngineConstants } from '../handler/context/engine-constants' +import { pieceHelper } from '../helper/piece-helper' + export const pieceMetadataOperation = { extract: async (operation: ExecuteExtractPieceMetadataOperation): Promise> => { - const piece = { - pieceName: operation.pieceName, - pieceVersion: operation.pieceVersion, + const input = operation as ExecuteExtractPieceMetadataOperation + const output = await pieceHelper.extractPieceMetadata({ + params: input, devPieces: EngineConstants.DEV_PIECES, - } - const { metadata } = await pieceRunner.describe(piece) - const entryPath = await piecePath.resolve(piece) - const i18n = await pieceTranslation.initializeI18n(path.dirname(path.dirname(entryPath))) + }) return { status: EngineResponseStatus.OK, - response: { - ...metadata, - name: operation.pieceName, - version: operation.pieceVersion, - i18n, - }, + response: output, } }, -} +} \ No newline at end of file diff --git a/packages/server/engine/src/lib/operations/property.operation.ts b/packages/server/engine/src/lib/operations/property.operation.ts index e4db1b810c7e..82027a0f5be5 100644 --- a/packages/server/engine/src/lib/operations/property.operation.ts +++ b/packages/server/engine/src/lib/operations/property.operation.ts @@ -1,138 +1,22 @@ -import { isNil, isObject } from '@activepieces/core-utils' -import { DropdownState, ExecutePropsResult, InputPropertyMap, PiecePropertyMap, PropertyType, StaticPropsValue } from '@activepieces/pieces-framework' +import { ExecutePropsResult, PropertyType } from '@activepieces/pieces-framework' import { - EngineGenericError, EngineResponse, EngineResponseStatus, ExecutePropsOptions, } from '@activepieces/shared' -import * as z from 'zod/mini' -import { PieceDescription } from '../core/piece/piece-protocol' -import { pieceRunner } from '../core/piece/piece-runner' -import { EngineConstants } from '../handler/context/engine-constants' -import { testExecutionContext } from '../handler/context/test-execution-context' -import { buildRuntime } from '../handler/piece-executor' -import { dynamicPropKeys } from '../helper/dynamic-prop-keys' -import { utils } from '../utils' -import { createPropsResolver } from '../variables/props-resolver' +import { pieceHelper } from '../helper/piece-helper' + export const propertyOperation = { - execute: async (operation: ExecutePropsOptions): Promise>> => { + execute: async (operation: ExecutePropsOptions): Promise>> => { + const output = await pieceHelper.executeProps({ + ...operation, + pieceName: operation.piece.pieceName, + pieceVersion: operation.piece.pieceVersion, + }) return { status: EngineResponseStatus.OK, - response: await executeProps(operation), + response: output, } }, -} - -async function executeProps(operation: ExecutePropsOptions): Promise> { - const constants = EngineConstants.fromExecutePropertyInput({ - ...operation, - pieceName: operation.piece.pieceName, - pieceVersion: operation.piece.pieceVersion, - }) - const piece = { - pieceName: operation.piece.pieceName, - pieceVersion: operation.piece.pieceVersion, - devPieces: EngineConstants.DEV_PIECES, - } - const description = await pieceRunner.describe(piece) - const { propertyType, path } = resolvePropertyPath({ description, operation }) - - const { data: result, error } = await utils.tryCatchAndThrowOnEngineError(async () => { - const executionState = await testExecutionContext.stateFromFlowVersion({ - apiUrl: operation.internalApiUrl, - flowVersion: operation.flowVersion, - projectId: operation.projectId, - engineToken: operation.engineToken, - sampleData: operation.sampleData, - engineConstants: constants, - }) - const contextVersion = description.metadata.contextInfo?.version - const { resolvedInput } = await createPropsResolver({ - apiUrl: constants.internalApiUrl, - projectId: constants.projectId, - engineToken: constants.engineToken, - contextVersion, - stepNames: constants.stepNames, - pieceName: piece.pieceName, - }).resolve>({ - unresolvedInput: operation.input, - executionState, - }) - const { result } = await pieceRunner.call({ - piece, - path, - context: { - kind: 'props', - runtime: buildRuntime({ constants, pieceName: piece.pieceName, contextVersion }), - stepName: operation.actionOrTriggerName, - resolvedInput, - searchValue: operation.searchValue, - }, - }) - return result - - }) - - if (error) { - console.error(error) - return { - type: propertyType, - options: { - disabled: true, - options: [], - placeholder: 'Throws an error, reconnect or refresh the page', - }, - } - } - return toPropsResult({ propertyType, result }) -} - -function resolvePropertyPath({ description, operation }: ResolvePropertyPathParams): { propertyType: ExecutablePropertyType, path: string[] } { - const { actionOrTriggerName, propertyName } = operation - const root = isNil(description.metadata.actions[actionOrTriggerName]) ? 'triggers' : 'actions' - const step = description.metadata[root][actionOrTriggerName] - const property = step?.props[propertyName] - if (isNil(property)) { - throw new EngineGenericError('PropertyNotFoundError', `Property not found: ${actionOrTriggerName}.${propertyName}`) - } - if (property.type !== PropertyType.DROPDOWN && property.type !== PropertyType.MULTI_SELECT_DROPDOWN && property.type !== PropertyType.DYNAMIC) { - throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property.type} for ${property.displayName}`) - } - return { - propertyType: property.type, - path: [root, actionOrTriggerName, 'props', propertyName, property.type === PropertyType.DYNAMIC ? 'props' : 'options'], - } -} - -function toPropsResult({ propertyType, result }: { propertyType: ExecutablePropertyType, result: unknown }): ExecutePropsResult { - if (propertyType === PropertyType.DYNAMIC) { - return { - type: propertyType, - options: dynamicPropKeys.escapePropsKeys(toInputPropertyMap(result)), - } - } - return { - type: propertyType, - options: toDropdownState(result), - } -} - -function toInputPropertyMap(result: unknown): InputPropertyMap { - return DynamicProps.safeParse(result).data ?? {} -} - -function toDropdownState(result: unknown): DropdownState { - return DropdownResult.safeParse(result).data ?? { disabled: false, options: [] } -} - -const DynamicProps = z.custom((value) => isObject(value)) -const DropdownResult = z.custom>((value) => isObject(value) && Array.isArray(Reflect.get(value, 'options'))) - -type ExecutablePropertyType = PropertyType.DROPDOWN | PropertyType.MULTI_SELECT_DROPDOWN | PropertyType.DYNAMIC - -type ResolvePropertyPathParams = { - description: PieceDescription - operation: ExecutePropsOptions -} +} \ No newline at end of file diff --git a/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts b/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts index a6aef6194769..96b3f3e839ad 100644 --- a/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts +++ b/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts @@ -4,15 +4,19 @@ import { ExecuteResolveConnectionIdentifierOperation, ExecuteResolveConnectionIdentifierResponse, } from '@activepieces/shared' -import { pieceAuth } from '../core/piece/piece-auth' +import { EngineConstants } from '../handler/context/engine-constants' +import { pieceHelper } from '../helper/piece-helper' export const resolveConnectionIdentifierOperation = { execute: async (operation: ExecuteResolveConnectionIdentifierOperation): Promise> => { - const call = await pieceAuth.callMethod({ operation, authValueType: operation.connectionType, methodPath: ['getConnectionIdentifier'] }) - const identifier = call.called ? call.result : undefined + const output = await pieceHelper.executeResolveConnectionIdentifier({ + params: operation, + devPieces: EngineConstants.DEV_PIECES, + }) + return { status: EngineResponseStatus.OK, - response: { identifier: typeof identifier === 'string' ? identifier : undefined }, + response: output, } }, } diff --git a/packages/server/engine/src/lib/operations/trigger-hook.operation.ts b/packages/server/engine/src/lib/operations/trigger-hook.operation.ts index 90b778ead670..bd51a412448c 100644 --- a/packages/server/engine/src/lib/operations/trigger-hook.operation.ts +++ b/packages/server/engine/src/lib/operations/trigger-hook.operation.ts @@ -1,8 +1,8 @@ import { inspect } from 'util' import { formatPieceError } from '@activepieces/core-utils' import { EngineResponse, EngineResponseStatus, ExecuteTriggerOperation, ExecuteTriggerResponse, TriggerHookType } from '@activepieces/shared' -import { triggerRunner } from '../core/piece/trigger-runner' import { EngineConstants, ResolvedExecuteTriggerOperation } from '../handler/context/engine-constants' +import { triggerHelper } from '../helper/trigger-helper' import { utils } from '../utils' import { resolveJobPayload } from './utils/resolve-job-payload' @@ -18,7 +18,7 @@ export const triggerHookOperation = { }), } const { data: output, error } = await utils.tryCatchAndThrowOnEngineError(() => - triggerRunner.executeTrigger({ + triggerHelper.executeTrigger({ params: input, constants: EngineConstants.fromExecuteTriggerInput(input), }), diff --git a/packages/server/engine/src/piece-child.ts b/packages/server/engine/src/piece-child.ts deleted file mode 100644 index 8df5d36b2fcc..000000000000 --- a/packages/server/engine/src/piece-child.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { pieceChild } from './lib/core/piece/piece-child' - -pieceChild.listen() diff --git a/packages/server/engine/test/core/piece/piece-auth.test.ts b/packages/server/engine/test/core/piece/piece-auth.test.ts deleted file mode 100644 index c507ec78383d..000000000000 --- a/packages/server/engine/test/core/piece/piece-auth.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { PieceAuth, PieceAuthProperty, PropertyType } from '@activepieces/pieces-framework' -import { AppConnectionType, AppConnectionValue, PiecePackage } from '@activepieces/shared' -import { pieceAuth } from '../../../src/lib/core/piece/piece-auth' -import { CollectedHooks, PieceDescription } from '../../../src/lib/core/piece/piece-protocol' -import { pieceRunner } from '../../../src/lib/core/piece/piece-runner' - -const PIECE = { pieceName: '@activepieces/piece-test', pieceVersion: '1.0.0' } as unknown as PiecePackage - -const HOOKS: CollectedHooks = { hookResponse: {}, listeners: [] } as unknown as CollectedHooks - -const SECRET_TEXT_AUTH = PieceAuth.SecretText({ displayName: 'API Key', required: true }) -const CUSTOM_AUTH = PieceAuth.CustomAuth({ displayName: 'Custom', required: true, props: {} }) - -const SECRET_TEXT_VALUE: AppConnectionValue = { type: AppConnectionType.SECRET_TEXT, secret_text: 'my-secret' } -const CUSTOM_AUTH_VALUE: AppConnectionValue = { type: AppConnectionType.CUSTOM_AUTH, props: { apiKey: 'k' } } - -function makeDescription({ auth, paths }: MakeDescriptionParams): PieceDescription { - return { - metadata: { auth } as unknown as PieceDescription['metadata'], - functionPaths: paths, - hasPath: (path: string[]) => paths.includes(path.join('.')), - } -} - -function operationFor(auth: AppConnectionValue) { - return { - piece: PIECE, - auth, - internalApiUrl: 'http://internal', - publicApiUrl: 'http://public', - } -} - -describe('piece-auth callMethod', () => { - beforeEach(() => { - vi.restoreAllMocks() - }) - - it('unwraps the piece value from the { result, hooks } wrapper', async () => { - vi.spyOn(pieceRunner, 'describe').mockResolvedValue(makeDescription({ auth: SECRET_TEXT_AUTH, paths: ['auth.validate'] })) - vi.spyOn(pieceRunner, 'call').mockResolvedValue({ result: { valid: true }, hooks: HOOKS }) - - const result = await pieceAuth.callMethod({ operation: operationFor(SECRET_TEXT_VALUE), authValueType: AppConnectionType.SECRET_TEXT, methodPath: ['validate'] }) - - expect(result).toEqual({ called: true, property: SECRET_TEXT_AUTH, result: { valid: true } }) - }) - - it('unwraps the refresh result so access_token is reachable', async () => { - vi.spyOn(pieceRunner, 'describe').mockResolvedValue(makeDescription({ auth: CUSTOM_AUTH, paths: ['auth.refresh.generate'] })) - vi.spyOn(pieceRunner, 'call').mockResolvedValue({ result: { access_token: 'tok', expires_in: 60 }, hooks: HOOKS }) - - const result = await pieceAuth.callMethod({ operation: operationFor(CUSTOM_AUTH_VALUE), authValueType: AppConnectionType.CUSTOM_AUTH, methodPath: ['refresh', 'generate'] }) - - expect(result).toEqual({ called: true, property: CUSTOM_AUTH, result: { access_token: 'tok', expires_in: 60 } }) - }) - - it('passes the resolved path, argument and slash-normalized server url to the runner', async () => { - vi.spyOn(pieceRunner, 'describe').mockResolvedValue(makeDescription({ auth: SECRET_TEXT_AUTH, paths: ['auth.validate'] })) - const call = vi.spyOn(pieceRunner, 'call').mockResolvedValue({ result: { valid: true }, hooks: HOOKS }) - - await pieceAuth.callMethod({ operation: operationFor(SECRET_TEXT_VALUE), authValueType: AppConnectionType.SECRET_TEXT, methodPath: ['validate'] }) - - expect(call).toHaveBeenCalledWith({ - piece: expect.objectContaining({ pieceName: '@activepieces/piece-test', pieceVersion: '1.0.0' }), - path: ['auth', 'validate'], - args: [{ auth: 'my-secret', server: { apiUrl: 'http://internal/', publicUrl: 'http://public' } }], - }) - }) - - it('selects the indexed auth path when the piece exposes an auth array', async () => { - vi.spyOn(pieceRunner, 'describe').mockResolvedValue(makeDescription({ auth: [SECRET_TEXT_AUTH, CUSTOM_AUTH], paths: ['auth.1.validate'] })) - const call = vi.spyOn(pieceRunner, 'call').mockResolvedValue({ result: { valid: true }, hooks: HOOKS }) - - await pieceAuth.callMethod({ operation: operationFor(CUSTOM_AUTH_VALUE), authValueType: AppConnectionType.CUSTOM_AUTH, methodPath: ['validate'] }) - - expect(call).toHaveBeenCalledWith(expect.objectContaining({ path: ['auth', '1', 'validate'] })) - }) - - it('returns called:false when the piece declares no auth', async () => { - vi.spyOn(pieceRunner, 'describe').mockResolvedValue(makeDescription({ auth: undefined, paths: [] })) - const call = vi.spyOn(pieceRunner, 'call') - - const result = await pieceAuth.callMethod({ operation: operationFor(SECRET_TEXT_VALUE), authValueType: AppConnectionType.SECRET_TEXT, methodPath: ['validate'] }) - - expect(result).toEqual({ called: false }) - expect(call).not.toHaveBeenCalled() - }) - - it('returns called:false with the property when the method path is absent', async () => { - vi.spyOn(pieceRunner, 'describe').mockResolvedValue(makeDescription({ auth: SECRET_TEXT_AUTH, paths: [] })) - const call = vi.spyOn(pieceRunner, 'call') - - const result = await pieceAuth.callMethod({ operation: operationFor(SECRET_TEXT_VALUE), authValueType: AppConnectionType.SECRET_TEXT, methodPath: ['validate'] }) - - expect(result).toEqual({ called: false, property: SECRET_TEXT_AUTH }) - expect(call).not.toHaveBeenCalled() - }) - - it('returns called:false with mismatch when the connection value type does not fit the property', async () => { - vi.spyOn(pieceRunner, 'describe').mockResolvedValue(makeDescription({ auth: SECRET_TEXT_AUTH, paths: ['auth.validate'] })) - const call = vi.spyOn(pieceRunner, 'call') - - const result = await pieceAuth.callMethod({ operation: operationFor(CUSTOM_AUTH_VALUE), authValueType: AppConnectionType.SECRET_TEXT, methodPath: ['validate'] }) - - expect(result).toEqual({ called: false, property: SECRET_TEXT_AUTH, mismatch: true }) - expect(call).not.toHaveBeenCalled() - }) -}) - -type MakeDescriptionParams = { - auth: PieceAuthProperty | PieceAuthProperty[] | undefined - paths: string[] -} diff --git a/packages/server/engine/test/core/piece/piece-memory.test.ts b/packages/server/engine/test/core/piece/piece-memory.test.ts deleted file mode 100644 index 279379df8818..000000000000 --- a/packages/server/engine/test/core/piece/piece-memory.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { ExecutionError, ExecutionErrorType } from '@activepieces/shared' -import { toExitError } from '../../../src/lib/core/piece/piece-runner' - -const heapMessage = 'FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory' - -describe('piece child exit classification', () => { - it.each([ - ['the V8 heap message', { code: 1, signal: null, output: heapMessage }], - ['an abort exit code', { code: 134, signal: null, output: '' }], - ['SIGABRT', { code: null, signal: 'SIGABRT' as const, output: '' }], - ['a kernel OOM kill', { code: null, signal: 'SIGKILL' as const, output: '' }], - ])('reports %s as a user-level memory failure', (_name, exit) => { - const error = toExitError(exit) - - expect(error).toBeInstanceOf(ExecutionError) - expect(error).toMatchObject({ name: 'PieceMemoryLimitError', type: ExecutionErrorType.USER }) - expect(JSON.parse(error.message).message).toBe('The piece ran out of memory') - }) - - it('reports any other abnormal exit as an engine error carrying the child output', () => { - const error = toExitError({ code: 7, signal: null, output: 'some stack trace' }) - - expect(error).toMatchObject({ name: 'PieceProcessExitedError', type: ExecutionErrorType.ENGINE }) - expect(error.message).toContain('some stack trace') - }) -}) diff --git a/packages/server/engine/test/core/piece/piece-protocol-error-cause.test.ts b/packages/server/engine/test/core/piece/piece-protocol-error-cause.test.ts deleted file mode 100644 index ecf3360192e3..000000000000 --- a/packages/server/engine/test/core/piece/piece-protocol-error-cause.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { inspect } from 'node:util' -import { formatPieceError } from '@activepieces/core-utils' -import { describe, expect, it } from 'vitest' -import { pieceProtocol } from '../../../src/lib/core/piece/piece-protocol' - -function acrossBoundary(error: unknown): Error { - return pieceProtocol.deserializeError(JSON.parse(JSON.stringify(pieceProtocol.serializeError(error)))) -} - -describe('piece protocol error cause', () => { - it('carries an undici cause across the RPC boundary into the raw payload', async () => { - const original = await fetch('http://127.0.0.1:9/nope').catch((error: Error) => error) - expect(original.message).toBe('fetch failed') - - const revived = acrossBoundary(original) - const { raw, message } = formatPieceError(revived, { raw: inspect(revived) }) - - expect(message).toBe('fetch failed') - expect(raw).toContain((original.cause as Error).message) - }) - - it('leaves errors without a cause untouched', () => { - expect(acrossBoundary(new Error('plain')).cause).toBeUndefined() - }) -}) diff --git a/packages/server/engine/test/core/piece/piece-protocol.test.ts b/packages/server/engine/test/core/piece/piece-protocol.test.ts deleted file mode 100644 index c4a507825625..000000000000 --- a/packages/server/engine/test/core/piece/piece-protocol.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { ExecutionError, ExecutionErrorType } from '@activepieces/shared' -import { pieceProtocol } from '../../../src/lib/core/piece/piece-protocol' - -describe('piece protocol', () => { - it('keeps the execution error type across the boundary', () => { - for (const type of [ExecutionErrorType.ENGINE, ExecutionErrorType.USER]) { - const restored = pieceProtocol.deserializeError(pieceProtocol.serializeError(new ExecutionError('BoomError', 'boom', type))) - - expect(restored).toBeInstanceOf(ExecutionError) - expect(restored).toMatchObject({ name: 'BoomError', message: 'boom', type }) - } - }) - - it('keeps http details and the constructor name of a plain piece error', () => { - class HttpError extends Error { - constructor(readonly status: number) { - super('request failed') - } - } - - const restored = pieceProtocol.deserializeError(pieceProtocol.serializeError(new HttpError(404))) - - expect(restored).not.toBeInstanceOf(ExecutionError) - expect(restored).toMatchObject({ name: 'HttpError', message: 'request failed', status: 404 }) - }) - - it('falls back to a message for a thrown non-error', () => { - expect(pieceProtocol.serializeError('just a string')).toEqual({ message: 'just a string' }) - }) - - it('drops functions and promises from a result but keeps buffers and dates', () => { - const date = new Date('2020-01-01T00:00:00.000Z') - - const transferable = pieceProtocol.toTransferable({ - keep: Buffer.from('bytes'), - when: date, - nested: { fn: () => undefined, pending: Promise.resolve(1), value: 2 }, - list: [1, () => undefined, 'three'], - }) - - expect(transferable).toEqual({ - keep: Buffer.from('bytes'), - when: date, - nested: { value: 2 }, - list: [1, undefined, 'three'], - }) - }) -}) diff --git a/packages/server/engine/test/handler/flow-log-size.test.ts b/packages/server/engine/test/handler/flow-log-size.test.ts index 044b189201f9..49065fb938c3 100644 --- a/packages/server/engine/test/handler/flow-log-size.test.ts +++ b/packages/server/engine/test/handler/flow-log-size.test.ts @@ -13,8 +13,8 @@ vi.mock('../../src/lib/helper/flow-run-progress-reporter', () => ({ }, })) -vi.mock('../../src/lib/core/piece/trigger-runner', () => ({ - triggerRunner: { +vi.mock('../../src/lib/helper/trigger-helper', () => ({ + triggerHelper: { executeOnStart: vi.fn().mockResolvedValue(undefined), }, })) diff --git a/packages/server/engine/test/handler/flow-waitpoint-response.test.ts b/packages/server/engine/test/handler/flow-waitpoint-response.test.ts index 9684c7a0a50c..ceee9c5d7555 100644 --- a/packages/server/engine/test/handler/flow-waitpoint-response.test.ts +++ b/packages/server/engine/test/handler/flow-waitpoint-response.test.ts @@ -2,13 +2,17 @@ import { FlowRunStatus } from '@activepieces/shared' import { vi } from 'vitest' import { FlowExecutorContext } from '../../src/lib/handler/context/flow-execution-context' import { flowExecutor } from '../../src/lib/handler/flow-executor' -import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' import { buildPieceAction, generateMockEngineConstants } from './test-helper' const { mockSendFlowResponse } = vi.hoisted(() => ({ mockSendFlowResponse: vi.fn().mockResolvedValue(undefined), })) +vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ + waitpointClient: { + create: vi.fn().mockResolvedValue({ id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }), + }, +})) vi.mock('../../src/lib/api/engine-run-api', () => ({ engineRunApi: { @@ -17,17 +21,6 @@ vi.mock('../../src/lib/api/engine-run-api', () => ({ })) describe('flow waitpoint response propagation', () => { - let engineApi: EngineApiStub - - beforeEach(async () => { - engineApi = await startEngineApiStub({ - 'POST /v1/waitpoints': { id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }, - }) - }) - - afterEach(async () => { - await engineApi.close() - }) beforeEach(() => { vi.clearAllMocks() @@ -55,7 +48,6 @@ describe('flow waitpoint response propagation', () => { action, executionState: FlowExecutorContext.empty(), constants: generateMockEngineConstants({ - internalApiUrl: engineApi.url, triggerPieceName: '@activepieces/piece-webhook', workerHandlerId: 'test-handler-id', httpRequestId: 'test-request-id', @@ -79,9 +71,6 @@ describe('flow waitpoint response propagation', () => { }, }, }) - const sentHeaders = mockSendFlowResponse.mock.calls[0][0].request.runResponse.headers - expect(typeof sentHeaders['x-activepieces-resume-webhook-url']).toBe('string') - expect(sentHeaders['x-activepieces-resume-webhook-url']).toMatch(/^https?:\/\//) }) it('should not call sendFlowResponse when triggerPieceName does not match', async () => { @@ -103,7 +92,6 @@ describe('flow waitpoint response propagation', () => { action, executionState: FlowExecutorContext.empty(), constants: generateMockEngineConstants({ - internalApiUrl: engineApi.url, triggerPieceName: 'some-other-piece', workerHandlerId: 'test-handler-id', httpRequestId: 'test-request-id', diff --git a/packages/server/engine/test/handler/flow-with-delay.test.ts b/packages/server/engine/test/handler/flow-with-delay.test.ts index d299431ab776..b219cf2f424e 100644 --- a/packages/server/engine/test/handler/flow-with-delay.test.ts +++ b/packages/server/engine/test/handler/flow-with-delay.test.ts @@ -1,22 +1,20 @@ import { FlowRunStatus } from '@activepieces/shared' +import { vi } from 'vitest' import { FlowExecutorContext } from '../../src/lib/handler/context/flow-execution-context' import { flowExecutor } from '../../src/lib/handler/flow-executor' -import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' +import { waitpointClient } from '../../src/lib/piece-context/waitpoint-client' import { buildCodeAction, buildPieceAction, generateMockEngineConstants } from './test-helper' -const WAITPOINT_PATH = '/v1/waitpoints' +vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ + waitpointClient: { + create: vi.fn().mockResolvedValue({ id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }), + }, +})) describe('flow with delay', () => { - let engineApi: EngineApiStub - beforeEach(async () => { - engineApi = await startEngineApiStub({ - [`POST ${WAITPOINT_PATH}`]: { id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }, - }) - }) - - afterEach(async () => { - await engineApi.close() + beforeEach(() => { + vi.clearAllMocks() }) it('delay-for pauses flow and calls waitpointClient.create with DELAY type', async () => { @@ -37,13 +35,13 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: delayForFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) expect(result.verdict).toEqual({ status: FlowRunStatus.PAUSED, }) - expect(engineApi.requestsFor(WAITPOINT_PATH)[0].body).toEqual( + expect(waitpointClient.create).toHaveBeenCalledWith( expect.objectContaining({ type: 'DELAY', resumeDateTime: expect.any(String), @@ -69,7 +67,7 @@ describe('flow with delay', () => { const pauseResult = await flowExecutor.execute({ action: delayForFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) const resumeResult = await flowExecutor.execute({ @@ -78,7 +76,6 @@ describe('flow with delay', () => { status: FlowRunStatus.RUNNING, }), constants: generateMockEngineConstants({ - internalApiUrl: engineApi.url, resumePayload: { queryParams: {}, body: {}, @@ -109,13 +106,13 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: shortDelayFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) expect(result.verdict).toEqual({ status: FlowRunStatus.RUNNING, }) - expect(engineApi.requestsFor(WAITPOINT_PATH)).toHaveLength(0) + expect(waitpointClient.create).not.toHaveBeenCalled() }) it('delay-until pauses flow for future dates', async () => { @@ -136,13 +133,13 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: delayUntilFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) expect(result.verdict).toEqual({ status: FlowRunStatus.PAUSED, }) - expect(engineApi.requestsFor(WAITPOINT_PATH)[0].body).toEqual( + expect(waitpointClient.create).toHaveBeenCalledWith( expect.objectContaining({ type: 'DELAY', resumeDateTime: expect.any(String), @@ -164,12 +161,12 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: delayUntilFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) expect(result.verdict).toEqual({ status: FlowRunStatus.RUNNING, }) - expect(engineApi.requestsFor(WAITPOINT_PATH)).toHaveLength(0) + expect(waitpointClient.create).not.toHaveBeenCalled() }) }) diff --git a/packages/server/engine/test/handler/flow-with-pause.test.ts b/packages/server/engine/test/handler/flow-with-pause.test.ts index a047ad7be6ea..effea6b03974 100644 --- a/packages/server/engine/test/handler/flow-with-pause.test.ts +++ b/packages/server/engine/test/handler/flow-with-pause.test.ts @@ -3,9 +3,13 @@ import { vi } from 'vitest' import { FlowExecutorContext } from '../../src/lib/handler/context/flow-execution-context' import { StepExecutionPath } from '../../src/lib/handler/context/step-execution-path' import { flowExecutor } from '../../src/lib/handler/flow-executor' -import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' import { buildCodeAction, buildPieceAction, buildRouterWithOneCondition, buildSimpleLoopAction, generateMockEngineConstants } from './test-helper' +vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ + waitpointClient: { + create: vi.fn().mockResolvedValue({ id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }), + }, +})) const simplePauseFlow = buildPieceAction({ @@ -61,23 +65,12 @@ const pauseFlowWithLoopAndBranch = buildSimpleLoopAction({ }) describe('flow with pause', () => { - let engineApi: EngineApiStub - - beforeEach(async () => { - engineApi = await startEngineApiStub({ - 'POST /v1/waitpoints': { id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }, - }) - }) - - afterEach(async () => { - await engineApi.close() - }) it('should pause and resume successfully with loops and branch', async () => { const pauseResult = await flowExecutor.execute({ action: pauseFlowWithLoopAndBranch, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url, stepNames: ['loop'] }), + constants: generateMockEngineConstants({ stepNames: ['loop'] }), }) expect(pauseResult.verdict).toEqual({ status: FlowRunStatus.PAUSED, @@ -97,7 +90,6 @@ describe('flow with pause', () => { status: FlowRunStatus.RUNNING, }), constants: generateMockEngineConstants({ - internalApiUrl: engineApi.url, stepNames: ['loop'], resumePayload: { queryParams: { @@ -127,13 +119,12 @@ describe('flow with pause', () => { const pauseResult1 = await flowExecutor.execute({ action: flawWithTwoPause, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) const resumeResult1 = await flowExecutor.execute({ action: flawWithTwoPause, executionState: pauseResult1, constants: generateMockEngineConstants({ - internalApiUrl: engineApi.url, resumePayload: { queryParams: { action: 'approve', @@ -152,7 +143,6 @@ describe('flow with pause', () => { status: FlowRunStatus.RUNNING, }), constants: generateMockEngineConstants({ - internalApiUrl: engineApi.url, resumePayload: { queryParams: { action: 'approve', @@ -173,7 +163,7 @@ describe('flow with pause', () => { const pauseResult = await flowExecutor.execute({ action: simplePauseFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) expect(pauseResult.verdict).toStrictEqual({ status: FlowRunStatus.PAUSED, @@ -185,7 +175,6 @@ describe('flow with pause', () => { action: simplePauseFlow, executionState: pauseResult, constants: generateMockEngineConstants({ - internalApiUrl: engineApi.url, resumePayload: { queryParams: { action: 'approve', @@ -248,7 +237,7 @@ describe('flow with pause', () => { const result = await flowExecutor.execute({ action: routerWithTwoPauseActions, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), + constants: generateMockEngineConstants(), }) expect(result.verdict).toStrictEqual({ diff --git a/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts b/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts index 8ff996625f58..3a9282097f32 100644 --- a/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts +++ b/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts @@ -170,7 +170,7 @@ describe('flow-run-progress-reporter slicing in single-step test mode', () => { logsFileId: 'logs-1', }) - const outputContext = flowRunProgressReporter.createOutputContext(engineConstants) + const outputContext = flowRunProgressReporter.createOutputContext({ engineConstants }) const big = { big: 'x'.repeat(40_000) } await outputContext.update({ data: big }) @@ -195,7 +195,7 @@ describe('flow-run-progress-reporter slicing in single-step test mode', () => { }) updateStepProgressMock.mockRejectedValueOnce(new Error('Failed to POST step-progress: 400 Bad Request')) - const outputContext = flowRunProgressReporter.createOutputContext(engineConstants) + const outputContext = flowRunProgressReporter.createOutputContext({ engineConstants }) // Must resolve, not reject — a streaming failure must not fail the run. await expect(outputContext.update({ data: { partial: true } })).resolves.toBeUndefined() diff --git a/packages/server/engine/test/helpers/engine-api-stub.ts b/packages/server/engine/test/helpers/engine-api-stub.ts deleted file mode 100644 index 0727630331f3..000000000000 --- a/packages/server/engine/test/helpers/engine-api-stub.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { createServer, IncomingMessage, ServerResponse } from 'node:http' -import { AddressInfo } from 'node:net' - -export async function startEngineApiStub(routes: Routes = {}): Promise { - const requests: RecordedRequest[] = [] - - const server = createServer((req, res) => void handle({ req, res, routes, requests })) - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) - const { port } = server.address() as AddressInfo - - return { - url: `http://127.0.0.1:${port}/`, - requests, - requestsFor: (path: string) => requests.filter((request) => request.path === path), - close: async () => new Promise((resolve) => server.close(() => resolve())), - } -} - -async function handle({ req, res, routes, requests }: HandleParams): Promise { - const path = (req.url ?? '').split('?')[0] - const body = await readBody(req) - requests.push({ method: req.method ?? 'GET', path, body }) - - const route = routes[`${req.method} ${path}`] ?? routes[path] - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify(route ?? {})) -} - -async function readBody(req: IncomingMessage): Promise { - const chunks: Buffer[] = [] - for await (const chunk of req) { - chunks.push(Buffer.from(chunk)) - } - if (chunks.length === 0) { - return undefined - } - try { - return JSON.parse(Buffer.concat(chunks).toString()) - } - catch { - return Buffer.concat(chunks).toString() - } -} - -type Routes = Record - -type HandleParams = { - req: IncomingMessage - res: ServerResponse - routes: Routes - requests: RecordedRequest[] -} - -export type RecordedRequest = { - method: string - path: string - body: unknown -} - -export type EngineApiStub = { - url: string - requests: RecordedRequest[] - requestsFor: (path: string) => RecordedRequest[] - close: () => Promise -} diff --git a/packages/server/engine/test/operations/flow-operation-invariants.test.ts b/packages/server/engine/test/operations/flow-operation-invariants.test.ts index be94321c5fca..4043cedf2f69 100644 --- a/packages/server/engine/test/operations/flow-operation-invariants.test.ts +++ b/packages/server/engine/test/operations/flow-operation-invariants.test.ts @@ -31,8 +31,8 @@ vi.mock('../../src/lib/helper/flow-run-progress-reporter', () => ({ const { mockExecuteTrigger } = vi.hoisted(() => ({ mockExecuteTrigger: vi.fn(), })) -vi.mock('../../src/lib/core/piece/trigger-runner', () => ({ - triggerRunner: { +vi.mock('../../src/lib/helper/trigger-helper', () => ({ + triggerHelper: { executeTrigger: mockExecuteTrigger, executeOnStart: vi.fn().mockResolvedValue(undefined), }, @@ -49,9 +49,16 @@ vi.mock('../../src/lib/api/engine-file-api', () => ({ }, })) +const { mockCreateWaitpoint } = vi.hoisted(() => ({ + mockCreateWaitpoint: vi.fn(), +})) +vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ + waitpointClient: { + create: mockCreateWaitpoint, + }, +})) import { flowOperation } from '../../src/lib/operations/flow.operation' -import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' function makeFlowVersion(): FlowVersion { return { @@ -82,7 +89,7 @@ function makeBeginOperation(overrides?: Partial): Beg return { projectId: 'proj-1', engineToken: 'test-token', - internalApiUrl: engineApi.url, + internalApiUrl: 'http://localhost:3000/', publicApiUrl: 'http://localhost:4200/api/', timeoutInSeconds: 600, platformId: 'plat-1', @@ -151,7 +158,7 @@ function makeResumeOperation(overrides?: Partial): R return { projectId: 'proj-1', engineToken: 'test-token', - internalApiUrl: engineApi.url, + internalApiUrl: 'http://localhost:3000/', publicApiUrl: 'http://localhost:4200/api/', timeoutInSeconds: 600, platformId: 'plat-1', @@ -170,18 +177,7 @@ function makeResumeOperation(overrides?: Partial): R } } -let engineApi: EngineApiStub - describe('flow operation invariants', () => { - beforeEach(async () => { - engineApi = await startEngineApiStub({ - 'POST /v1/waitpoints': { id: 'wp-new', resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-new' }, - }) - }) - - afterEach(async () => { - await engineApi.close() - }) describe('RESUME execution state hydration', () => { it('throws EngineGenericError when RESUME has empty execution state in logs file', async () => { mockDownload.mockReset() @@ -272,6 +268,11 @@ describe('flow operation invariants', () => { // (waitpoint path). With the fix, step_1 stays FAILED in the restored state, // `isCompleted` short-circuits piece-executor, and no new waitpoint is created. mockDownload.mockReset() + mockCreateWaitpoint.mockReset() + mockCreateWaitpoint.mockResolvedValue({ + id: 'wp-new', + resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-new', + }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -312,7 +313,7 @@ describe('flow operation invariants', () => { await flowOperation.execute(operation) - expect(engineApi.requestsFor('/v1/waitpoints')).toHaveLength(0) + expect(mockCreateWaitpoint).not.toHaveBeenCalled() }) it('drops FAILED steps on a retry resume (resumeReason=RETRY — FlowRetryStrategy.FROM_FAILED_STEP)', async () => { @@ -321,6 +322,11 @@ describe('flow operation invariants', () => { // engine to replay the failed step. Preserving FAILED on this path would silently turn // retry into a no-op. The discriminator is the explicit `resumeReason` field. mockDownload.mockReset() + mockCreateWaitpoint.mockReset() + mockCreateWaitpoint.mockResolvedValue({ + id: 'wp-retry', + resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-retry', + }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -355,7 +361,7 @@ describe('flow operation invariants', () => { // step_1 (FAILED) was dropped because resumeReason=RETRY → engine replayed it from // BEGIN, which creates a waitpoint via the approval piece. - expect(engineApi.requestsFor('/v1/waitpoints').length).toBeGreaterThan(0) + expect(mockCreateWaitpoint).toHaveBeenCalled() }) it('drops non-terminal statuses (e.g. RUNNING from a mid-step crash) on any resume', async () => { @@ -364,6 +370,11 @@ describe('flow operation invariants', () => { // whether resumePayload is present. Only SUCCEEDED, PAUSED, and FAILED (the last // conditionally) survive restoration. mockDownload.mockReset() + mockCreateWaitpoint.mockReset() + mockCreateWaitpoint.mockResolvedValue({ + id: 'wp-replay', + resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-replay', + }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -403,7 +414,7 @@ describe('flow operation invariants', () => { await flowOperation.execute(operation) - expect(engineApi.requestsFor('/v1/waitpoints')).toHaveLength(1) + expect(mockCreateWaitpoint).toHaveBeenCalledTimes(1) }) it('preserves FAILED steps on a delay-piece waitpoint resume even though resumePayload is null', async () => { @@ -413,6 +424,11 @@ describe('flow operation invariants', () => { // engine would drop FAILED — replaying any `continueOnFailure` step that preceded // the delay. With `resumeReason: WAITPOINT`, FAILED is preserved correctly. mockDownload.mockReset() + mockCreateWaitpoint.mockReset() + mockCreateWaitpoint.mockResolvedValue({ + id: 'wp-delay', + resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-delay', + }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -451,7 +467,7 @@ describe('flow operation invariants', () => { await flowOperation.execute(operation) - expect(engineApi.requestsFor('/v1/waitpoints')).toHaveLength(0) + expect(mockCreateWaitpoint).not.toHaveBeenCalled() }) }) @@ -487,7 +503,7 @@ describe('flow operation invariants', () => { } expect(mockDownload).toHaveBeenCalledWith({ - apiUrl: engineApi.url, + apiUrl: 'http://localhost:3000/', engineToken: 'test-token', fileId: 'payload-file-1', }) @@ -617,6 +633,7 @@ describe('flow operation invariants', () => { describe('RESUME payload hydration', () => { it('resolves a ref resumePayload via the engine file client', async () => { mockDownload.mockReset() + mockCreateWaitpoint.mockReset() mockDownload.mockImplementation(({ fileId }: { fileId: string }) => { if (fileId === 'logs-file-1') { return Promise.resolve(new TextEncoder().encode(JSON.stringify({ @@ -648,7 +665,7 @@ describe('flow operation invariants', () => { } expect(mockDownload).toHaveBeenCalledWith({ - apiUrl: engineApi.url, + apiUrl: 'http://localhost:3000/', engineToken: 'test-token', fileId: 'resume-file-1', }) diff --git a/packages/server/engine/test/operations/trigger-hook-operation.test.ts b/packages/server/engine/test/operations/trigger-hook-operation.test.ts index caec7d824493..11ead75f56c1 100644 --- a/packages/server/engine/test/operations/trigger-hook-operation.test.ts +++ b/packages/server/engine/test/operations/trigger-hook-operation.test.ts @@ -19,8 +19,8 @@ vi.mock('../../src/lib/api/engine-file-api', () => ({ const { mockExecuteTrigger } = vi.hoisted(() => ({ mockExecuteTrigger: vi.fn(), })) -vi.mock('../../src/lib/core/piece/trigger-runner', () => ({ - triggerRunner: { +vi.mock('../../src/lib/helper/trigger-helper', () => ({ + triggerHelper: { executeTrigger: mockExecuteTrigger, }, })) diff --git a/packages/server/engine/vitest.config.ts b/packages/server/engine/vitest.config.ts index fd25c528cff3..8b3a73c79e8c 100644 --- a/packages/server/engine/vitest.config.ts +++ b/packages/server/engine/vitest.config.ts @@ -1,5 +1,4 @@ import path from 'path' -import { buildSync } from 'esbuild' import { defineConfig } from 'vitest/config' // Change CWD to repo root for compatibility with piece-loader path resolution @@ -11,29 +10,6 @@ process.env.AP_BASE_CODE_DIRECTORY = 'packages/server/engine/test/resources/code process.env.AP_TEST_MODE = 'true' process.env.AP_DEV_PIECES = 'http,data-mapper,approval,webhook,delay' -const alias = { - '@activepieces/shared': path.resolve(__dirname, '../../core/shared/src/index.ts'), - '@activepieces/pieces-framework': path.resolve(__dirname, '../../pieces/framework/src/index.ts'), - '@activepieces/pieces-common': path.resolve(__dirname, '../../pieces/common/src/index.ts'), - '@activepieces/core-formula': path.resolve(__dirname, '../../core/formula/src/index.ts'), - '@activepieces/core-piece-types': path.resolve(__dirname, '../../core/piece-types/src/index.ts'), - '@activepieces/core-utils': path.resolve(__dirname, '../../core/utils/src/index.ts'), - '@activepieces/core-execution': path.resolve(__dirname, '../../core/execution/src/index.ts'), -} - -const pieceChildEntry = path.resolve(__dirname, '../../../dist/packages/engine-test/piece-child.js') -buildSync({ - entryPoints: [path.resolve(__dirname, 'src/piece-child.ts')], - bundle: true, - platform: 'node', - target: 'node20', - outfile: pieceChildEntry, - format: 'cjs', - alias, - external: ['isolated-vm', 'utf-8-validate', 'bufferutil'], -}) -process.env.AP_PIECE_CHILD_ENTRY = pieceChildEntry - export default defineConfig({ // esbuild injects this at bundle time; vitest runs the source directly, so define it here too. // Tests exercise the proxy-included path (the no-proxy bundle's behaviour is the build-flag flip). @@ -47,6 +23,14 @@ export default defineConfig({ include: [path.resolve(__dirname, 'test/**/*.test.ts')], }, resolve: { - alias, + alias: { + '@activepieces/shared': path.resolve(__dirname, '../../../packages/core/shared/src/index.ts'), + '@activepieces/pieces-framework': path.resolve(__dirname, '../../../packages/pieces/framework/src/index.ts'), + '@activepieces/pieces-common': path.resolve(__dirname, '../../../packages/pieces/common/src/index.ts'), + '@activepieces/core-formula': path.resolve(__dirname, '../../../packages/core/formula/src/index.ts'), + '@activepieces/core-piece-types': path.resolve(__dirname, '../../../packages/core/piece-types/src/index.ts'), + '@activepieces/core-utils': path.resolve(__dirname, '../../../packages/core/utils/src/index.ts'), + '@activepieces/core-execution': path.resolve(__dirname, '../../../packages/core/execution/src/index.ts'), + }, }, }) diff --git a/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts b/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts index 975f0f863e6a..a788db11812b 100644 --- a/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts +++ b/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts @@ -8,19 +8,20 @@ import { ApEnvironment } from '@activepieces/shared' import { nanoid } from 'nanoid' import { SandboxSettings } from '../../types' -const engineDistPath = 'dist/packages/engine' -const engineBundles = ['main.js', 'piece-child.js'] +const engineExecutablePath = 'dist/packages/engine/main.js' const installedPaths = new Map>() export const engineInstaller = (_log: ApLogger, getSettings: () => SandboxSettings) => ({ async install({ path }: InstallParams): Promise { const isDev = getSettings().ENVIRONMENT === ApEnvironment.DEVELOPMENT + // The egress proxy was removed, so there is a single engine bundle (main.js). + const source = engineExecutablePath const inFlight = installedPaths.get(path) if (!isNil(inFlight) && !isDev) { await inFlight return { cacheHit: true } } - const install = copyEngine({ path }) + const install = copyEngine({ source, path }) installedPaths.set(path, install) const { error } = await tryCatch(() => install) if (error) { @@ -31,11 +32,9 @@ export const engineInstaller = (_log: ApLogger, getSettings: () => SandboxSettin }, }) -async function copyEngine({ path }: CopyEngineParams): Promise { - for (const bundle of engineBundles) { - await atomicCopy(`${engineDistPath}/${bundle}`, `${path}/${bundle}`) - await atomicCopy(`${engineDistPath}/${bundle}.map`, `${path}/${bundle}.map`) - } +async function copyEngine({ source, path }: CopyEngineParams): Promise { + await atomicCopy(source, `${path}/main.js`) + await atomicCopy(`${source}.map`, `${path}/main.js.map`) } async function atomicCopy(src: PathLike, dest: PathLike): Promise { @@ -47,6 +46,7 @@ async function atomicCopy(src: PathLike, dest: PathLike): Promise { } type CopyEngineParams = { + source: string path: string } diff --git a/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts b/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts index 8711a0b8ceb3..609bab38522f 100644 --- a/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts +++ b/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts @@ -16,8 +16,6 @@ async function makeSandboxRoot(): Promise { await mkdir(join(root, ENGINE_SOURCE_DIR), { recursive: true }) await writeFile(join(root, ENGINE_SOURCE_DIR, 'main.js'), 'engine-bundle', 'utf8') await writeFile(join(root, ENGINE_SOURCE_DIR, 'main.js.map'), '{}', 'utf8') - await writeFile(join(root, ENGINE_SOURCE_DIR, 'piece-child.js'), 'piece-child-bundle', 'utf8') - await writeFile(join(root, ENGINE_SOURCE_DIR, 'piece-child.js.map'), '{}', 'utf8') const target = join(root, 'cache', 'common') await mkdir(target, { recursive: true }) process.chdir(root) @@ -53,7 +51,6 @@ describe('engineInstaller', () => { expect(second.cacheHit).toBe(true) expect(third.cacheHit).toBe(true) expect(await readFile(join(target, 'main.js'), 'utf8')).toBe('engine-bundle') - expect(await readFile(join(target, 'piece-child.js'), 'utf8')).toBe('piece-child-bundle') }) it('is not invalidated by another container writing the shared cache.json', async () => { diff --git a/packages/web/public/locales/en/translation.json b/packages/web/public/locales/en/translation.json index 4f5465b6bf11..a02ce779091f 100644 --- a/packages/web/public/locales/en/translation.json +++ b/packages/web/public/locales/en/translation.json @@ -31,6 +31,25 @@ "Needs a model": "Needs a model", "New agent": "New agent", "New agents go to": "New agents go to", + "Pick where this agent should live.": "Pick where this agent should live.", + "Checking what this move affects…": "Checking what this move affects…", + "Couldn't check what this move affects. Try again.": "Couldn't check what this move affects. Try again.", + "Your role in {project} cannot create agents there.": "Your role in {project} cannot create agents there.", + "Move anyway": "Move anyway", + "Moved {name} to {project}": "Moved {name} to {project}", + "another project": "another project", + "agentMoveNothingBreaks": "Every tool it uses works in {project} too. From now on it works with that project's accounts, files and flows.", + "agentMoveUsesTargetAccounts": "Everything else keeps working, but with {project}'s accounts instead of this project's.", + "agentMoveAndMore": "{names} and {count, plural, =1 {1 more} other {# more}}", + "agentMoveLosesConnections": "{count, plural, =1 {{names} has no connection in {project}, so that tool stops working until you connect it there.} other {{names} have no connections in {project}, so those tools stop working until you connect them there.}}", + "agentMoveLosesFlows": "{count, plural, =1 {The flow {names} does not exist in {project}, so that tool stops working.} other {The flows {names} do not exist in {project}, so those tools stop working.}}", + "agentMoveLosesKnowledge": "{count, plural, =1 {The file {names} is not in {project}, so that tool stops working.} other {The files {names} are not in {project}, so those tools stop working.}}", + "Move to another project": "Move to another project", + "Moved {name}": "Moved {name}", + "That agent could not be moved.": "That agent could not be moved.", + "Nothing breaks: every tool it uses is connected there too.": "Nothing breaks: every tool it uses is connected there too.", + "agentMoveLosesConnections": "{count, plural, =1 {{names} has no connection in {project}, so that tool stops working until you connect it there.} other {{names} have no connections in {project}, so those tools stop working until you connect them there.}}", + "agentMoveLosesMembers": "{count, plural, =1 {1 person it is shared with is not in {project}, so they lose access. Moving it back will not give it back.} other {# people it is shared with are not in {project}, so they lose access. Moving it back will not give it back.}}", "Load more": "Load more", "Showing {count} so far": "Showing {count} so far", "No provider is turned on for chat": "No provider is turned on for chat", @@ -69,6 +88,8 @@ "Copied to clipboard": "Copied to clipboard", "Copy to clipboard": "Copy to clipboard", "Download JSON": "Download JSON", + "Download Report": "Download Report", + "Failed to download pieces report": "Failed to download pieces report", "Response from {pieceDisplayName}": "Response from {pieceDisplayName}", "What the service said": "What the service said", "Error message": "Error message", @@ -2716,6 +2737,12 @@ "Not live yet": "Not live yet", "Changes not live yet": "Changes not live yet", "These changes are not live. Save and go live to hand them to this agent and every flow using it.": "These changes are not live. Save and go live to hand them to this agent and every flow using it.", + "Service account JSON": "Service account JSON", + "Google Cloud project ID": "Google Cloud project ID", + "Region": "Region", + "The complete JSON key file for a service account with the Vertex AI User role.": "The complete JSON key file for a service account with the Vertex AI User role.", + "Connect a Google Cloud service account to call Vertex AI models inside your own GCP project.\n\n1. Open the [Google Cloud Console](https://console.cloud.google.com) and select the project you want to bill.\n2. Enable the **Vertex AI API** under *APIs & Services > Enabled APIs*.\n3. Go to **IAM & Admin > Service Accounts** and create a service account, granting it the **Vertex AI User** role. Avoid broader roles — follow least-privilege so a leaked key has limited blast radius.\n4. Open the service account, go to **Keys > Add Key > Create new key**, choose **JSON**, and download it.\n5. Paste the entire contents of that JSON file below, then set the project ID and the region your models are served from.\n6. Add the model ids you want to expose — Vertex model availability depends on your project, region and Model Garden access, so we cannot list them for you.": "Connect a Google Cloud service account to call Vertex AI models inside your own GCP project.\n\n1. Open the [Google Cloud Console](https://console.cloud.google.com) and select the project you want to bill.\n2. Enable the **Vertex AI API** under *APIs & Services > Enabled APIs*.\n3. Go to **IAM & Admin > Service Accounts** and create a service account, granting it the **Vertex AI User** role. Avoid broader roles — follow least-privilege so a leaked key has limited blast radius.\n4. Open the service account, go to **Keys > Add Key > Create new key**, choose **JSON**, and download it.\n5. Paste the entire contents of that JSON file below, then set the project ID and the region your models are served from.\n6. Add the model ids you want to expose — Vertex model availability depends on your project, region and Model Garden access, so we cannot list them for you.", + "invalidGcpResourceId": "Use lowercase letters, numbers and hyphens only", "API style": "API style", "Chat completions": "Chat completions", "Responses": "Responses", diff --git a/packages/web/src/app/routes/agents/id/index.tsx b/packages/web/src/app/routes/agents/id/index.tsx index b65eded53191..77cb3e5ca3f3 100644 --- a/packages/web/src/app/routes/agents/id/index.tsx +++ b/packages/web/src/app/routes/agents/id/index.tsx @@ -94,6 +94,12 @@ import { agentsMutations, agentsQueries, } from '@/features/agents/hooks/agents-hooks'; +import { MoveAgentDialog } from '@/features/agents/move-agent-dialog'; +import { + ApProjectDisplay, + getProjectName, + projectCollectionUtils, +} from '@/features/projects'; import { useAuthorization } from '@/hooks/authorization-hooks'; import { flagsHooks } from '@/hooks/flags-hooks'; import { api } from '@/lib/api'; @@ -193,6 +199,56 @@ const AgentEditorSkeleton = () => ( ); +const AgentProjectRow = ({ agent }: { agent: Agent }) => { + const [moving, setMoving] = useState(false); + const { checkAccess } = useAuthorization(agent.projectId); + const { data: allProjects } = projectCollectionUtils.useAll(); + const home = (allProjects ?? []).find( + (project) => project.id === agent.projectId, + ); + + if ( + !checkAccess(Permission.WRITE_AGENT) || + (allProjects ?? []).length < 2 || + home === undefined + ) { + return null; + } + + return ( + + +
+ + +
+ + projectCollectionUtils.setCurrentProject( + projectId, + `/projects/${agent.projectId}/agents/${agent.id}`, + ) + } + /> + + ); +}; + const AgentDangerZone = ({ agent, onDeleted, @@ -544,6 +600,7 @@ const ConfigureFields = ({ )} /> + diff --git a/packages/web/src/app/routes/platform/setup/ai/providers-tab/config-detail.tsx b/packages/web/src/app/routes/platform/setup/ai/providers-tab/config-detail.tsx index 6772224c51c0..ef688623e02e 100644 --- a/packages/web/src/app/routes/platform/setup/ai/providers-tab/config-detail.tsx +++ b/packages/web/src/app/routes/platform/setup/ai/providers-tab/config-detail.tsx @@ -8,6 +8,7 @@ import { OpenAICompatibleProviderConfig, Project, UpdateAIProviderRequest, + VertexProviderConfig, } from '@activepieces/shared'; import { useQuery } from '@tanstack/react-query'; import { t } from 'i18next'; @@ -426,6 +427,7 @@ function draftOf(config: AIProviderWithoutSensitiveData): ConfigDraft { } const ManualProviderConfig = z.union([ + VertexProviderConfig, OpenAICompatibleProviderConfig, CloudflareGatewayProviderConfig, ]); diff --git a/packages/web/src/app/routes/platform/setup/ai/providers-tab/connect-provider-dialog.tsx b/packages/web/src/app/routes/platform/setup/ai/providers-tab/connect-provider-dialog.tsx index 03e4f24d3403..f52fab14dab8 100644 --- a/packages/web/src/app/routes/platform/setup/ai/providers-tab/connect-provider-dialog.tsx +++ b/packages/web/src/app/routes/platform/setup/ai/providers-tab/connect-provider-dialog.tsx @@ -46,6 +46,7 @@ import { SelectTrigger, SelectValue, } from '@/components/ui/select'; +import { Textarea } from '@/components/ui/textarea'; import { AiProviderInfo, SUPPORTED_AI_PROVIDERS } from '@/features/agents'; import { aiProviderMutations } from '@/features/platform-admin'; import { cn } from '@/lib/utils'; @@ -69,7 +70,7 @@ export function ConnectProviderDialog({ }) { return ( - + + ) : field.type === 'textarea' ? ( + +