[pull] main from activepieces:main

.github/actions/setup/action.yml

@@ -0,0 +1,56 @@
+name: Setup
+description: Node, bun, workspace dependencies and the turbo remote cache. Shared by every CI job.
+
+inputs:
+  turbo-s3-bucket:
+    description: S3 bucket for the turbo remote cache
+    required: true
+  turbo-s3-endpoint:
+    description: S3 endpoint for the turbo remote cache
+    required: true
+  turbo-s3-access-key-id:
+    description: S3 access key id for the turbo remote cache
+    required: true
+  turbo-s3-secret-access-key:
+    description: S3 secret access key for the turbo remote cache
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-node@v6
+      with:
+        node-version: 24
+
+    - uses: oven-sh/setup-bun@v2
+      with:
+        bun-version-file: package.json
+
+    - uses: actions/cache@v4
+      with:
+        path: ~/.bun/install/cache
+        key: ${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
+        restore-keys: ${{ runner.os }}-bun-
+
+    # redis-memory-server compiles Redis from source in its postinstall (about 3 minutes) unless
+    # the binary for the version pinned in the root package.json is already in this folder.
+    - uses: actions/cache@v4
+      with:
+        path: node_modules/.cache/redis-memory-server
+        key: ${{ runner.os }}-redis-memory-server-${{ hashFiles('package.json') }}
+        restore-keys: ${{ runner.os }}-redis-memory-server-
+
+    - run: bun install --frozen-lockfile
+      shell: bash
+
+    - name: Turbo Cache
+      uses: rharkor/caching-for-turbo@v2.4.2
+      with:
+        provider: s3
+        s3-bucket: ${{ inputs.turbo-s3-bucket }}
+        s3-region: auto
+        s3-endpoint: ${{ inputs.turbo-s3-endpoint }}
+        s3-access-key-id: ${{ inputs.turbo-s3-access-key-id }}
+        s3-secret-access-key: ${{ inputs.turbo-s3-secret-access-key }}
+        max-age: 3w
+        max-size: 20gb

.github/workflows/ci-v2.yml

@@ -0,0 +1,232 @@
+name: CI v2
+on:
+  pull_request:
+
+permissions:
+  actions: read
+  contents: read
+
+concurrency:
+  group: ci-v2-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  # turbo --affected diffs against this ref. CI checkouts have origin/<base> but no local main.
+  TURBO_SCM_BASE: origin/${{ github.base_ref }}
+  TURBO_TELEMETRY_DISABLED: "1"
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    if: github.repository == 'activepieces/activepieces'
+    outputs:
+      packages: ${{ steps.affected.outputs.packages }}
+      all: ${{ steps.affected.outputs.all }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Find affected packages
+        id: affected
+        run: |
+          set -euo pipefail
+          ALL=false
+          # Files outside any package (workflows, root config, tools) are invisible to turbo's
+          # graph, so a change there runs everything. Docs and markdown never do.
+          OUTSIDE_PACKAGES=$(git diff --name-only "$TURBO_SCM_BASE...HEAD" | grep -vE '^(packages/|docs/|brain/|\.agents/|\.claude/)|\.md$' || true)
+          if [ -n "$OUTSIDE_PACKAGES" ]; then
+            ALL=true
+          fi
+          # Pieces follow the old ci.yml rule: a piece runs when its own files changed, and every
+          # piece runs when pieces/framework or pieces/common changed. A change to a core package
+          # (core-piece-types, core-utils, ...) does not pull the 700+ pieces in, even though they
+          # depend on it; a break there is caught when pieces are published, not in the PR.
+          if ! DIRECT=$(npx --yes turbo@2.9.14 ls --filter="[$TURBO_SCM_BASE...HEAD]" --output=json | jq -c '[.packages.items[].name]') \
+            || ! AFFECTED=$(npx --yes turbo@2.9.14 ls --affected --output=json | jq -c '[.packages.items[].name]'); then
+            ALL=true
+            PACKAGES='[]'
+          else
+            PACKAGES=$(jq -cn --argjson direct "$DIRECT" --argjson affected "$AFFECTED" '
+              ($direct | any(. == "@activepieces/pieces-framework" or . == "@activepieces/pieces-common")) as $allPieces
+              | [$affected[] | select((startswith("@activepieces/piece-") | not) or $allPieces or IN($direct[]))]')
+          fi
+          echo "all=$ALL" >> "$GITHUB_OUTPUT"
+          echo "packages=$PACKAGES" >> "$GITHUB_OUTPUT"
+          echo "all=$ALL"
+          echo "packages=$PACKAGES"
+
+  lint:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.all == 'true' || needs.changes.outputs.packages != '[]'
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/setup
+        with:
+          turbo-s3-bucket: ${{ secrets.TURBO_CACHE_S3_BUCKET }}
+          turbo-s3-endpoint: ${{ secrets.TURBO_CACHE_S3_ENDPOINT }}
+          turbo-s3-access-key-id: ${{ secrets.TURBO_CACHE_S3_ACCESS_KEY_ID }}
+          turbo-s3-secret-access-key: ${{ secrets.TURBO_CACHE_S3_SECRET_ACCESS_KEY }}
+
+      - name: Lint core packages
+        if: needs.changes.outputs.all == 'true'
+        run: npm run lint-core
+
+      # Also on full runs: a PR that edits a workflow and pieces/framework in one go still needs
+      # every piece linted, and the changes job's list is what knows that.
+      - name: Lint affected packages
+        env:
+          AFFECTED_PACKAGES: ${{ needs.changes.outputs.packages }}
+        run: |
+          set -euo pipefail
+          FILTERS=$(echo "$AFFECTED_PACKAGES" | jq -r '.[] | "--filter=" + .' | tr '\n' ' ')
+          if [ -n "$FILTERS" ]; then
+            npx turbo run lint $FILTERS
+          fi
+
+  build-test:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.all == 'true' || needs.changes.outputs.packages != '[]'
+    services:
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+    env:
+      AP_REDIS_HOST: localhost
+      AP_REDIS_PORT: "6379"
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/setup
+        with:
+          turbo-s3-bucket: ${{ secrets.TURBO_CACHE_S3_BUCKET }}
+          turbo-s3-endpoint: ${{ secrets.TURBO_CACHE_S3_ENDPOINT }}
+          turbo-s3-access-key-id: ${{ secrets.TURBO_CACHE_S3_ACCESS_KEY_ID }}
+          turbo-s3-secret-access-key: ${{ secrets.TURBO_CACHE_S3_SECRET_ACCESS_KEY }}
+
+      # One turbo invocation so every package builds once: the affected packages from the changes
+      # job, plus the core set on full runs. --affected and --filter intersect in turbo, so the
+      # union is built here as explicit filters. api is excluded because its `test` script runs
+      # the three integration suites serially; the api-tests matrix covers those.
+      - name: Typecheck, build and test
+        env:
+          AFFECTED_PACKAGES: ${{ needs.changes.outputs.packages }}
+          RUN_ALL: ${{ needs.changes.outputs.all }}
+        run: |
+          set -euo pipefail
+          FILTERS=$(echo "$AFFECTED_PACKAGES" | jq -r '.[] | select(. != "api") | "--filter=" + .' | tr '\n' ' ')
+          if [ "$RUN_ALL" = "true" ]; then
+            FILTERS="$FILTERS $(npx tsx tools/scripts/test-filters.ts)"
+          fi
+          npx turbo run typecheck build test $FILTERS
+
+      - name: Check new migrations are rollback-safe
+        run: npx tsx tools/scripts/check-migration-rollback.ts
+
+  # The api e2e tests import the worker's source, and the worker resolves @activepieces/sandbox
+  # through its built dist, so the api jobs build worker too and also run when worker is affected.
+  api-tests:
+    name: api (${{ matrix.name }})
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.all == 'true' || contains(needs.changes.outputs.packages, '"api"') || contains(needs.changes.outputs.packages, '"worker"')
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: ce
+            tasks: test-ce
+          - name: cloud
+            tasks: test-cloud
+          - name: ee, unit, migrations
+            tasks: test-ee test-unit check-migrations
+    services:
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+    env:
+      # .env.tests pins AP_REDIS_HOST=redis, the alias inside the job's docker network, which a
+      # non-container job is not on. dotenv does not override values already in the environment.
+      AP_REDIS_HOST: localhost
+      AP_REDIS_PORT: "6379"
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: ./.github/actions/setup
+        with:
+          turbo-s3-bucket: ${{ secrets.TURBO_CACHE_S3_BUCKET }}
+          turbo-s3-endpoint: ${{ secrets.TURBO_CACHE_S3_ENDPOINT }}
+          turbo-s3-access-key-id: ${{ secrets.TURBO_CACHE_S3_ACCESS_KEY_ID }}
+          turbo-s3-secret-access-key: ${{ secrets.TURBO_CACHE_S3_SECRET_ACCESS_KEY }}
+
+      - run: npx turbo run build ${{ matrix.tasks }} --filter=api --filter=worker
+
+  # Only the tool-search integration tests, against real PostgreSQL + pgvector. The api-tests
+  # matrix runs them on PGLITE, which papers over node-postgres driver behaviour.
+  tool-search-postgres:
+    name: tool-search (postgres)
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.all == 'true' || contains(needs.changes.outputs.packages, '"api"')
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: activepieces
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    env:
+      AP_DB_TYPE: POSTGRES
+      AP_POSTGRES_HOST: localhost
+      AP_POSTGRES_PORT: "5432"
+      AP_POSTGRES_DATABASE: activepieces
+      AP_POSTGRES_USERNAME: postgres
+      AP_POSTGRES_PASSWORD: postgres
+      AP_POSTGRES_USE_SSL: "false"
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: ./.github/actions/setup
+        with:
+          turbo-s3-bucket: ${{ secrets.TURBO_CACHE_S3_BUCKET }}
+          turbo-s3-endpoint: ${{ secrets.TURBO_CACHE_S3_ENDPOINT }}
+          turbo-s3-access-key-id: ${{ secrets.TURBO_CACHE_S3_ACCESS_KEY_ID }}
+          turbo-s3-secret-access-key: ${{ secrets.TURBO_CACHE_S3_SECRET_ACCESS_KEY }}
+
+      - name: Enable pgvector extension
+        run: |
+          PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d activepieces \
+            -c 'CREATE EXTENSION IF NOT EXISTS vector;'
+
+      - run: npx turbo run build --filter=api
+
+      # Serial: every file migrates the one shared service DB on DataSource.initialize().
+      - name: Run tool-search integration tests on Postgres
+        working-directory: packages/server/api
+        run: npx vitest run test/integration/ce/tool-search --bail 1 --passWithNoTests=false --no-file-parallelism

.github/workflows/warm-ci-cache.yml

@@ -0,0 +1,22 @@
+name: Warm CI cache
+# GitHub only lets a pull request restore caches created on the same PR or on the base branch.
+# CI v2 runs on pull requests only, so this seeds the base-branch copies (bun downloads and the
+# compiled redis-memory-server binary) that every new PR's first run restores.
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  warm:
+    runs-on: ubuntu-latest
+    if: github.repository == 'activepieces/activepieces'
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: ./.github/actions/setup
+        with:
+          turbo-s3-bucket: ${{ secrets.TURBO_CACHE_S3_BUCKET }}
+          turbo-s3-endpoint: ${{ secrets.TURBO_CACHE_S3_ENDPOINT }}
+          turbo-s3-access-key-id: ${{ secrets.TURBO_CACHE_S3_ACCESS_KEY_ID }}
+          turbo-s3-secret-access-key: ${{ secrets.TURBO_CACHE_S3_SECRET_ACCESS_KEY }}

brain/knowledge/data-storage-observability/tables.md

@@ -27,6 +27,7 @@ A built-in relational database inside Activepieces: users store structured data
 - `record.create()` bulk insert caps at 50 per batch, transactional.
 - When adding any new table/field/record route, the `permission` arg to `securityAccess.project(...)` is required — passing `undefined` silently allows any project member.
 - The per-create `field.validateCount()` check alone races on bulk paths: all concurrent creates read the same pre-save count and pass. Bulk/import paths (`table.create` with fields, project-state/project-replace apply) MUST call it once up front with the batch size **before** their `Promise.all`.
+- The web table store keys each cell by the column's position (`fieldIndex`), not by field id, so every field mutator in `ap-tables-client-state.tsx` has to renumber surviving cells (`deleteField` didn't, see ENG-537). A cell edit sends only the edited cell, and `record.update` upserts just those cells, so stale indexes can't blank other columns on the server.
 - Concurrent field reorders are last-write-wins, same as rename — no distributed lock. Reordering *existing* fields through project-release apply is not supported: `FieldState` carries no position, so array order only applies to newly created fields.
 - The web client stores positional `cell.fieldIndex` references, so it must remap every record's cells when fields move.
 

brain/knowledge/engineering/ci-pipeline.md

@@ -0,0 +1,36 @@
+---
+icon: ⏱️
+---
+
+# CI Pipeline
+
+The PR checks that decide whether a change builds and passes tests. Lives in `.github/workflows/ci-v2.yml` (`ci.yml` runs beside it until the side-by-side comparison ends). The gates that shape *how a PR is reviewed* are on *CI PR Review Hygiene*.
+
+**changes job** — the first job. Runs `turbo ls --affected` to list the packages the PR touches plus everything that depends on them, and sets `all=true` when a file outside any package changed (workflows, root config, `tools/`). Every other job keys off those two outputs.
+
+**affected** — turbo's own `--affected` filter: the dependency graph decides what runs, not path globs. A `shared` change runs api, web and worker; a piece change runs that piece and, if api depends on it, api. When turbo cannot diff, it runs everything, so the failure mode is slow, never skipped. _Avoid_: hand-written `git diff | grep pieces/` filters, the graph already knows.
+
+**api matrix** — the api suites split one per runner (`test-ce`, `test-cloud`, `test-ee test-unit check-migrations`) and only when `api` is affected. They were the whole critical path when they shared one 4-core runner.
+
+**setup action** — `.github/actions/setup`: node, bun, `bun install --frozen-lockfile`, turbo remote cache. Every job uses it; change install behaviour there once.
+
+## Gotchas
+
+- `redis-memory-server` downloads and compiles Redis from source in its postinstall (about 3 minutes, most of the old 3.5-minute `bun install`). Api tests, unit tests included, start an in-memory Redis from that binary at runtime, so skipping the postinstall (`REDISMS_DISABLE_POSTINSTALL=1`) only moves the compile into the first test that needs it and trips the 60 s and 120 s timeouts. Cache `node_modules/.cache/redis-memory-server` (keyed on the root `package.json`, which pins the version under `redisMemoryServer.version`) instead. Do not swap the in-memory Redis for the service container to avoid the compile: each vitest fork gets its own Redis today, a shared one lets parallel test files interfere.
+- Every `build` task is `cache: false` in `turbo.json`, so each separate `turbo run` invocation rebuilds its whole dependency chain. Tasks that share dependencies belong in one invocation.
+- Api integration tests boot the full server once per test file (`pool: forks`, isolated). The number of test files is the CI cost driver: 122 files in June 2026 → 310 in September moved the test step from 6 to 19 minutes on one runner.
+- The GitHub org is on the free plan: 20 concurrent jobs org-wide, all workflows included. More jobs per PR means queueing at busy hours. `cancel-in-progress` per PR number keeps superseded runs from holding slots.
+- api's `test` script runs ce+ee+cloud serially. Any `turbo run test` over many packages must exclude api (`--filter='!api'`; `tools/scripts/test-filters.ts` does the same).
+- The api e2e tests (`execute-flow-e2e`, `test-step-e2e`, `piece-options-e2e`) import the worker's source, and `worker.ts` imports `@activepieces/sandbox`, which resolves to `packages/server/sandbox/dist`. A job that builds only api fails those three files with "Failed to resolve entry for package @activepieces/sandbox". The api jobs build `worker` as well and also run when worker is affected.
+- A cache saved during a pull request run is visible only to that PR and to nothing else. Caches every PR should hit (bun downloads, the compiled Redis binary) must be created on `main`; `warm-ci-cache.yml` does that on every push to main.
+- Two turbo invocations running at the same time in one job race on `cache: false` builds: the old `ci.yml` failed `api#build` with "@activepieces/shared has no exported member" for a member that exists, because a second invocation was rewriting `shared/dist` while api's tsc read it. One invocation per job.
+- `--affected` and `--filter` intersect in turbo; there is no union flag. To run "the affected packages plus this fixed set" in one invocation, pass the affected names (the `changes` job outputs them as JSON) as explicit `--filter=<name>` arguments alongside the fixed ones. Two invocations rebuild the shared dependency chain twice because `build` is uncached.
+- Pieces are the one place the pipeline overrides the dependency graph, by decision (2026-09-24). A piece runs when its own files changed; every piece runs when `pieces/framework` or `pieces/common` changed. A change to `core-piece-types`, `core-utils`, `core-formula` or `core-execution` does not pull the 700+ pieces in, although they depend on it, because that cost about 20 minutes per such PR (5 of the last 60). The accepted blind spot: a core type change that breaks piece compilation is caught when pieces are published (`release-pieces.yml`), not in the PR. The `changes` job computes this with two turbo listings, `ls --filter='[base...HEAD]'` (changed directly) and `ls --affected` (plus dependents).
+- `TURBO_SCM_BASE` must be `origin/<base branch>` in CI. turbo's default base is `main`, which a CI checkout does not have, so without it every run degrades to "everything affected".
+
+## Key files
+
+- `.github/workflows/ci-v2.yml` — the PR pipeline
+- `.github/actions/setup` — shared install and cache steps
+- `tools/scripts/test-filters.ts` — every package with a `test` script, minus api
+- `tools/scripts/check-migration-rollback.ts` — new migrations must declare `breaking` and `down()`

brain/knowledge/engineering/index.md

@@ -34,6 +34,7 @@ The **Activepieces engineering brain**: how the system works, and *why* it was b
 - **Web Navigation Shell** — the rail, the three sidebars and the layouts that choose between them, which primitives are shared, and why platform admin renders without the rail
 - **Cloud Deployment Paths** — canary → prod, the `cloud-hotfix` override, and the breaking-migration gate that blocks both
 - **Helm Chart** — the Kubernetes install we ship to self-hosters, its two competing paths for an `AP_*` variable, and the secrets it never creates
+- **CI Pipeline** — the PR checks that build and test a change: the affected-packages gate, the api matrix, and why install and test time grew
 - **CI PR Review Hygiene** — draft-first Greptile review, the per-area PR size gate, and the workflow conventions reviewers keep re-litigating
 - **E2E Tests & Monitors** — the one Playwright suite behind CI, the Checkly monitors that run it against production Cloud, and the BetterStack script the repo pushes on merge
 - **Architecture Spine** — the load-bearing structure of the codebase, and the gotchas that come with it: request-body `.max()` as data loss, TypeORM soft-delete across a canary window, and canary not proxying websockets