diff --git a/.agents/skills/tinybird-cli-guidelines/SKILL.md b/.agents/skills/tinybird-cli-guidelines/SKILL.md new file mode 100644 index 00000000000..1ba9b32bdb3 --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/SKILL.md @@ -0,0 +1,45 @@ +--- +name: tinybird-cli-guidelines +description: Tinybird CLI commands, workflows, and operations. Use when running tb commands, managing local development, deploying, or working with data operations. +--- + +# Tinybird CLI Guidelines + +Guidance for using the Tinybird CLI (tb) for local development, deployments, data operations, and workspace management. + +## When to Apply + +- Running any `tb` command +- Choosing a development workflow (local, branch, or cloud) +- Local development with Tinybird Local +- Branch development with Tinybird Cloud branches +- Building and deploying projects +- Setting up CI/CD pipelines +- Appending, replacing, or deleting data +- Managing tokens and secrets via CLI +- Generating mock data +- Running tests + +## Rule Files + +- `rules/development-workflows.md` +- `rules/cli-commands.md` +- `rules/build-deploy.md` +- `rules/local-development.md` +- `rules/branch-development.md` +- `rules/ci-cd.md` +- `rules/data-operations.md` +- `rules/append-data.md` +- `rules/mock-data.md` +- `rules/tokens.md` +- `rules/secrets.md` + +## Quick Reference + +- CLI 4.0 workflow: configure `dev_mode` once, then use plain `tb build` and `tb deploy`. +- `tb build` targets your configured development environment (`branch` or `local`) in tinybird.config.json. +- `tb deploy` targets Tinybird Cloud production. +- Use `--cloud`/`--local`/`--branch` only as explicit manual overrides. +- Use `tb info` to check CLI context. +- Use `tb endpoint data ` to test endpoints (not `tb pipe data`). +- Never invent commands or flags; run `tb --help` to verify. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/append-data.md b/.agents/skills/tinybird-cli-guidelines/rules/append-data.md new file mode 100644 index 00000000000..c87ee4f134d --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/append-data.md @@ -0,0 +1,25 @@ +# Append Data + +Tinybird CLI supports three ways to append data to an existing datasource: local file, remote URL, or events payload. + +## CLI: tb datasource append + +``` +tb datasource append [datasource_name] --file /path/to/local/file +``` + +``` +tb datasource append [datasource_name] --url https://example.com/data.csv +``` + +``` +tb datasource append [datasource_name] --events '{"a":"b", "c":"d"}' +``` + +Notes: + +- The command appends to an existing datasource. +- Use `tb --cloud datasource append` to target Cloud; Local is the default. +- For ingesting data from Kafka, S3 or GCS, see: https://www.tinybird.co/docs/forward/get-data-in/connectors + +You can also send POST request to v0/events (streaming) and v0/datasources (batch) endpoints. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/branch-development.md b/.agents/skills/tinybird-cli-guidelines/rules/branch-development.md new file mode 100644 index 00000000000..1deb5ab18de --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/branch-development.md @@ -0,0 +1,108 @@ +# Branch Development + +## Overview + +Tinybird Cloud branches provide isolated environments for development and testing. Each branch gets its own copy of resources and can optionally include production data. Branches are the recommended workflow for teams collaborating on the same workspace. + +## When to Use Branches + +- Developing features that need real production data shapes for testing +- Collaborating with a team where multiple people work on the same workspace +- Testing schema changes or new endpoints before deploying to production +- CI/CD workflows that validate changes on pull requests + +For solo development or quick iteration, Tinybird Local (`dev_mode=local`) may be faster. See `rules/local-development.md`. + +## Branch Workflow + +1. Create a git branch for your feature +2. Run `tb dev` — Tinybird automatically creates a Cloud branch matching your git branch name +3. Develop and test against the branch (file changes are watched and auto-rebuilt) +4. Push changes and create a PR +5. Merge to deploy to production + +## Creating Branches + +Automatic (recommended): + +Check out a git branch and run `tb dev` or `tb build`. Tinybird automatically creates or uses a Cloud branch with the same name as your git branch. + +Manual: + +``` +tb branch create my_feature +``` + +Branch names must use underscores, not hyphens (e.g., `my_feature`, not `my-feature`). + +### The `--last-partition` Flag + +Use `--last-partition` to copy the latest partition of production data into the branch: + +``` +tb branch create my_feature --last-partition +``` + +This is useful when you need real data to test queries, validate endpoint behavior, or debug issues that depend on production data shapes. Without it, the branch starts empty. + +### The `--with-connections` Flag + +Use `--with-connections` to enable connectors (Kafka, S3, GCS) in the branch: + +``` +tb branch create my_feature --last-partition --with-connections +``` + +For S3/GCS, import sample data with `tb --branch=my_feature datasource sample --wait`. Kafka connections are stopped by default and need to be started explicitly with `tb --branch=my_feature datasource start `. + +## Working with Branch Tokens + +After creating a branch, you may need its token to connect client applications (dashboards, APIs, scripts) to the branch environment instead of production. + +List tokens for a branch: + +``` +tb --branch my_feature token ls +``` + +### Using Branch Tokens in Client Apps + +A common pattern is to set an environment variable that your application checks, falling back to the production token when no branch token is set: + +```env +# .env.local +TINYBIRD_API_URL=https://api.tinybird.co +TINYBIRD_API_TOKEN= +TINYBIRD_BRANCH_TOKEN= +``` + +In your application, prioritize the branch token when present: + +``` +token = TINYBIRD_BRANCH_TOKEN || TINYBIRD_API_TOKEN +``` + +This way, setting or unsetting the branch token switches between branch and production data without code changes. + +## Branch Commands Reference + +- `tb branch ls`: List all branches +- `tb branch create `: Create a new branch (empty) +- `tb branch create --last-partition`: Create a branch with latest production data +- `tb branch create --last-partition --with-connections`: Create a branch with data and connectors +- `tb branch rm `: Remove a branch +- `tb branch clear`: Clear branch state +- `tb dev`: Start development session (auto-creates branch from git branch name, watches files) +- `tb --branch open`: Open the branch in the Tinybird UI + +## Targeting a Branch Explicitly + +Most commands can target a specific branch with the `--branch` flag: + +``` +tb --branch my_feature endpoint data my_endpoint +tb --branch my_feature sql "SELECT count() FROM my_datasource" +tb --branch my_feature token ls +``` + +When `dev_mode=branch`, `tb build` targets the branch automatically without needing `--branch`. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/build-deploy.md b/.agents/skills/tinybird-cli-guidelines/rules/build-deploy.md new file mode 100644 index 00000000000..47e99826606 --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/build-deploy.md @@ -0,0 +1,56 @@ +# Build & Deploy + +Use this rule to keep local files, development environments, and production deployments aligned under the CLI 4.0 workflow. + +## Default Workflow (CLI 4.0) + +1. Configure `dev_mode` in `tinybird.config.json` (`branch`, `local`, or `manual`). +2. Run `tb build` to validate and sync to the configured development target. +3. Run `tb deploy` to deploy to Tinybird Cloud main (production). + +In CLI 4.0, build/deploy should usually be run without `--cloud`, `--local`, or `--branch`. + +## `tb build` Behavior + +- `dev_mode=local`: builds against Tinybird Local. +- `dev_mode=branch`: builds against a Cloud branch derived from the current git branch (created automatically if needed). +- `dev_mode=manual`: requires explicit flags (`--local`, `--cloud`, `--branch`) for environment selection. +- In branch mode, building from `main`/`master` is blocked to avoid accidental production changes. + +## `tb deploy` Behavior + +- `tb deploy` deploys current project files to Tinybird Cloud main. +- Use only when the user explicitly requests a production deployment. +- Ask for confirmation before deploying. + +## Deploy Check + +- Run `tb deploy --check` before real deploys to catch schema/dependency issues early. +- Use check mode whenever deployment intent is uncertain. + +## Destructive operations and flags + +- Deleting datasources, pipes, or connections locally requires an explicit destructive deploy. +- Use `tb deploy --allow-destructive-operations` only when the user confirms deletion or data loss is acceptable. +- If you see warnings about deletions, stop and ask for confirmation before re-running with the flag. + +Example: + +``` +tb deploy --allow-destructive-operations +``` + +## Manual Overrides + +- Explicit flags still work and override `dev_mode`. +- Use overrides only when the user explicitly asks for a specific environment target. + +## Validation intent (why) + +- Building keeps development environments aligned with local files for fast iteration. +- Deploy checks reduce failed deployments by validating changes before publishing. + +## What not to do + +- Do not deploy destructive changes without `--allow-destructive-operations` and explicit user confirmation. +- Do not assume production is updated after `tb build`; `build` and `deploy` are separate operations. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/ci-cd.md b/.agents/skills/tinybird-cli-guidelines/rules/ci-cd.md new file mode 100644 index 00000000000..79342b4eb58 --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/ci-cd.md @@ -0,0 +1,204 @@ +# CI/CD Integration + +## Recommended Pattern + +Use Tinybird Local in CI to build and test with `tb --local build` and `tb --local test run`, then `tb --cloud deploy --check` to validate against Cloud. In CD, use `tb --cloud deploy` to deploy on merge to the main branch. + +## CI: Pull Request Validation + +The recommended CI flow uses a Tinybird Local service container for building and testing, then validates the deployment against Cloud: + +1. `tb --local build` — build the project against Tinybird Local +2. `tb --local test run` — run tests against Tinybird Local +3. `tb --cloud deploy --check` — validate the deployment would succeed on Cloud (dry run) + +The `deploy --check` step catches schema compatibility, dependency resolution, and resource naming issues before they reach production. + +## CD: Production Deployment + +Run when changes are merged to the main branch: + +``` +tb --cloud deploy +``` + +This creates a staging deployment, migrates data, and promotes to live. + +For projects that prefer explicit confirmation, use a two-step process: + +``` +tb --cloud deployment create --wait +tb --cloud deployment promote +``` + +## Example: GitHub Actions + +```yaml +# .github/workflows/tinybird-ci.yml +name: Tinybird CI +on: + pull_request: + paths: + - 'tinybird/**' + +env: + TINYBIRD_HOST: https://api.tinybird.co + TINYBIRD_TOKEN: ${{ secrets.TB_ADMIN_TOKEN }} + +jobs: + validate: + runs-on: ubuntu-latest + services: + tinybird: + image: tinybirdco/tinybird-local:latest + ports: + - 7181:7181 + steps: + - uses: actions/checkout@v4 + - name: Install Tinybird CLI + run: curl https://tinybird.co | sh + - name: Build project + run: tb --local build + working-directory: tinybird + - name: Test project + run: tb --local test run + working-directory: tinybird + - name: Deployment check + run: tb --cloud --host ${{ env.TINYBIRD_HOST }} --token ${{ env.TINYBIRD_TOKEN }} deploy --check + working-directory: tinybird +``` + +```yaml +# .github/workflows/tinybird-cd.yml +name: Tinybird CD +on: + push: + branches: [main] + paths: + - 'tinybird/**' + +env: + TINYBIRD_HOST: https://api.tinybird.co + TINYBIRD_TOKEN: ${{ secrets.TB_ADMIN_TOKEN }} + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Tinybird CLI + run: curl https://tinybird.co | sh + - name: Deploy + run: tb --cloud --host ${{ env.TINYBIRD_HOST }} --token ${{ env.TINYBIRD_TOKEN }} deploy + working-directory: tinybird +``` + +## Example: GitLab CI + +```yaml +tinybird_ci: + image: ubuntu:latest + stage: test + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + - tinybird/** + services: + - name: tinybirdco/tinybird-local:latest + alias: tinybird-local + before_script: + - apt update && apt install -y curl + - curl https://tinybird.co | sh + - export PATH="$HOME/.local/bin:$PATH" + script: + - cd tinybird + - tb --local build + - tb --local test run + - tb --cloud --host $TINYBIRD_HOST --token $TINYBIRD_TOKEN deploy --check + +tinybird_cd: + image: ubuntu:latest + stage: deploy + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + - tinybird/** + before_script: + - apt update && apt install -y curl + - curl https://tinybird.co | sh + - export PATH="$HOME/.local/bin:$PATH" + script: + - cd tinybird + - tb --cloud --host $TINYBIRD_HOST --token $TINYBIRD_TOKEN deploy +``` + +## Preview Environments + +Preview environments create an ephemeral Tinybird branch per pull request, so you can test changes with production data before merging. + +### Using the TypeScript or Python SDK + +The `tinybird preview` command (available in `@tinybirdco/sdk` and `tinybird-sdk`, not the `tb` CLI) creates a branch named `tmp_ci_`, builds resources, and deploys them: + +```yaml +# GitHub Actions example +- run: npx tinybird preview + env: + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_TOKEN }} +``` + +The SDK auto-detects CI environments (GitHub Actions, GitLab CI, Vercel, CircleCI, Azure Pipelines, Bitbucket Pipelines) and resolves the correct branch token. The host is inferred from the token. + +If a branch with the same name already exists, it is deleted and recreated. + +### Using the tb CLI + +The `tb` CLI doesn't have a `preview` subcommand. Create preview branches manually: + +```yaml +- name: Create preview branch + run: tb --host ${{ env.TINYBIRD_HOST }} --token ${{ env.TINYBIRD_TOKEN }} branch create tmp_ci_${{ github.head_ref }} --last-partition +- name: Build on branch + run: tb --host ${{ env.TINYBIRD_HOST }} --token ${{ env.TINYBIRD_TOKEN }} --branch=tmp_ci_${{ github.head_ref }} build +``` + +### Cleanup + +Delete preview branches when the PR is closed: + +```yaml +# SDK +- run: npx tinybird branch delete tmp_ci_${{ github.head_ref }} + +# tb CLI +- run: tb --host ${{ env.TINYBIRD_HOST }} --token ${{ env.TINYBIRD_TOKEN }} branch rm tmp_ci_${{ github.head_ref }} +``` + +### Preview with connectors + +When your project uses Kafka, S3, or GCS connectors, the `tinybird preview` command doesn't ingest data from connectors in preview branches. To test with connector data, create the branch manually with `--with-connections`: + +``` +tb branch create tmp_ci_my_feature --last-partition --with-connections +``` + +For S3/GCS connectors, import sample data: + +``` +tb --branch=tmp_ci_my_feature datasource sample my_datasource --wait +``` + +Kafka connections are stopped by default in preview branches. Start them explicitly: + +``` +tb --branch=tmp_ci_my_feature datasource start my_kafka_datasource +``` + +## Key Principles + +- Production deploys should happen through CI/CD, not manually. +- Use Tinybird Local in CI for building and testing (`tb --local build`, `tb --local test run`), then `tb --cloud deploy --check` to validate against Cloud. +- Use `--wait` in CD pipelines so the job reflects the actual deployment result. +- Store the admin token as a CI/CD secret, never in code. +- Scope CI triggers to Tinybird project file paths to avoid unnecessary runs. +- Use preview environments when you need a full working branch per PR with production data. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/cli-commands.md b/.agents/skills/tinybird-cli-guidelines/rules/cli-commands.md new file mode 100644 index 00000000000..9dc124841fe --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/cli-commands.md @@ -0,0 +1,146 @@ +# Tinybird CLI Commands + +**⚠️ Never invent commands or flags.** If you are unsure whether a command or flag exists, run `tb --help` to verify before using it. Only use commands and flags documented here or confirmed via `--help`. + +## Build/Deploy Context (CLI 4.0) + +- Preferred flow: configure `dev_mode` once, then run plain `tb build` and `tb deploy`. +- Use `--cloud`, `--local`, and `--branch` only as explicit manual overrides. + +## Global Overrides + +- `tb --cloud `: Run command against Cloud +- `tb --local `: Run command against Local +- `tb --branch `: Run command against a specific branch +- `tb --debug `: Print debug information + +## Project & Development + +- `tb init`: Initialize a new project +- `tb create`: Deprecated alias for `tb init` +- `tb info`: Show project information and CLI context +- `tb build`: Validate and build the project +- `tb build --watch`: Build and watch for changes +- `tb dev`: Build and watch for changes +- `tb dev --ui`: Connect local project to Tinybird UI +- `tb preview`: Create/update preview environment for the current branch +- `tb open`: Open workspace in the browser +- `tb fmt `: Format a .datasource, .pipe, or .connection file +- `tb fmt --diff`: Show diff without modifying file + +## Deploy & Deployments + +- `tb deploy`: Deploy the project +- `tb deploy --check`: Validate deployment without actually creating +- `tb deploy --wait`: Wait for deployment to finish +- `tb deploy --allow-destructive-operations`: Allow destructive changes (requires explicit confirmation) +- `tb deployment ls`: List all deployments +- `tb deployment create`: Create a staging deployment and validate before promoting +- `tb deployment promote`: Promote a staging deployment to production +- `tb deployment discard`: Discard a pending deployment + +## Logs + +- `tb logs`: Show recent logs from common service datasources +- `tb logs --start -30m --source '*'`: Query all sources for a custom time range +- `tb logs --output json`: Emit logs as JSON for scripting + +## Data Sources + +- `tb datasource ls`: List all data sources +- `tb datasource append --file `: Append data from local file +- `tb datasource append --url `: Append data from URL +- `tb datasource append --events ''`: Append JSON events +- `tb datasource replace `: Full replace of data source +- `tb datasource replace --sql-condition ""`: Selective replace +- `tb datasource delete --sql-condition ""`: Delete matching rows +- `tb datasource delete --sql-condition "" --wait`: Delete and wait for completion +- `tb datasource truncate --yes`: Delete all rows +- `tb datasource truncate --cascade --yes`: Truncate including dependent MVs +- `tb datasource sync --yes`: Sync from S3/GCS connection +- `tb datasource export --format csv`: Export data to file + +## Pipes & Endpoints + +- `tb pipe ls`: List all pipes +- `tb endpoint ls`: List all endpoints +- `tb endpoint data `: Get data from endpoint (use this to test endpoints) +- `tb endpoint data --param_name value`: Get data with parameters +- `tb endpoint stats `: Show endpoint stats for last 7 days +- `tb endpoint url `: Print endpoint URL +- `tb endpoint token `: Get token to read endpoint + +Note: use `tb endpoint data` to test endpoints, not `tb pipe data`. The `endpoint data` command calls the endpoint as a consumer would, with parameter validation and output formatting. + +## SQL Queries + +- `tb sql ""`: Run SQL query +- `tb sql "" --stats`: Run query and show stats +- `tb sql --pipe --node `: Run SQL from a specific pipe node + +## Materializations & Copy Pipes + +- `tb materialization ls`: List all materializations +- `tb copy ls`: List all copy pipes +- `tb copy run `: Run a copy pipe manually +- `tb copy run --param key=value`: Run with parameters + +## Testing + +- `tb test run`: Run the full test suite +- `tb test run `: Run specific test file or test +- `tb test update `: Update test expectations + +## Mock Data + +- `tb mock` was removed in CLI 4.0 +- Use the `fixtures/` folder and agent skills to generate sample data, then append with `tb datasource append` + +## Tokens & Secrets + +- `tb token ls`: List all tokens +- `tb secret ls`: List all secrets +- `tb secret set `: Create or update a secret +- `tb secret rm `: Delete a secret + +## Connections & Sinks + +- `tb connection ls`: List all connections +- `tb sink ls`: List all sinks + +## Jobs + +- `tb job ls`: List all jobs +- `tb job cancel `: Cancel a running job + +## Branches + +- `tb branch ls`: List all branches +- `tb branch create `: Create a new branch (starts empty) +- `tb branch create --last-partition`: Create a branch with latest production data partition +- `tb branch rm `: Remove a branch +- `tb branch clear`: Clear branch state +- `tb --branch token ls`: List tokens for a specific branch +- `tb --branch endpoint data `: Test endpoint on a specific branch + +## Tinybird Local + +- `tb local start`: Start Tinybird Local container +- `tb local stop`: Stop Tinybird Local +- `tb local restart --yes`: Restart Tinybird Local +- `tb local status`: Check Tinybird Local status +- `tb local remove`: Remove Tinybird Local completely +- `tb local version`: Show Tinybird Local version +- `tb local clear`: Clear local workspace state + +## Workspace + +- `tb workspace ls`: List all workspaces +- `tb workspace current`: Show current workspace +- `tb workspace clear --yes`: Clear workspace state + +## Authentication + +- `tb login`: Authenticate via browser +- `tb logout`: Remove authentication +- `tb update`: Update CLI to latest version diff --git a/.agents/skills/tinybird-cli-guidelines/rules/data-operations.md b/.agents/skills/tinybird-cli-guidelines/rules/data-operations.md new file mode 100644 index 00000000000..c5b5bab820a --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/data-operations.md @@ -0,0 +1,63 @@ +# Data Operations (Replace & Delete) + +Operations for updating and removing data from Data Sources. + +## Delete Data Selectively + +Delete rows matching a SQL condition: + +```bash +tb datasource delete events --sql-condition "toDate(date) >= '2019-11-01' AND toDate(date) <= '2019-11-30'" +``` + +- Runs asynchronously (returns job ID); use `--wait` to block until complete +- **Does not cascade** to downstream Materialized Views—delete from MVs separately +- Requires ADMIN token scope +- Safe to run while actively ingesting data + +## Truncate Data Source + +Delete all rows from a Data Source: + +```bash +tb datasource truncate events +``` + +Use `--cascade` to also truncate dependent Data Sources attached via Materialized Views. + +## Replace Data Selectively (Partial Replace) + +Replace only data matching a condition: + +```bash +tb datasource replace events data.csv --sql-condition "toDate(date) >= '2019-11-01' AND toDate(date) <= '2019-11-30'" +``` + +**⚠️ Critical**: Never replace data in partitions where you are actively ingesting. You may lose data inserted during the operation. + +**Rules**: + +- **Always include the partition key** in the SQL condition +- The condition determines: (1) which partitions to operate on, (2) which rows from new data to append +- **Cascades automatically** to downstream Materialized Views (all must have compatible partition keys) +- Schema of new data must match existing Data Source exactly + +### Why Partition Key Matters + +If your Data Source uses `ENGINE_PARTITION_KEY "country"` and you run: + +```bash +tb datasource replace events data.csv --sql-condition "status='active'" +``` + +This will **not work as expected**—the replace process uses payload rows to identify partitions. Always match the partition key. + +## Replace Data Completely (Full Replace) + +Replace entire Data Source contents (no `--sql-condition`): + +```bash +tb datasource replace events data.csv +``` + +**⚠️ Critical**: Do not run while actively ingesting—you may lose data. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/development-workflows.md b/.agents/skills/tinybird-cli-guidelines/rules/development-workflows.md new file mode 100644 index 00000000000..c8713cdc85a --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/development-workflows.md @@ -0,0 +1,80 @@ +# Development Workflows + +Tinybird supports three development workflows. Choose based on your team size, infrastructure, and iteration speed needs. + +## Workflow Comparison + +| Workflow | Best for | Requires | Data | +| -------------------------- | ------------------------------------------- | --------------- | ----------------------------- | +| Local (`dev_mode=local`) | Solo dev, fast iteration, offline work | Docker | Fixtures or manually appended | +| Branch (`dev_mode=branch`) | Team collaboration, production-like testing | Cloud workspace | Optional copy from production | +| Cloud direct | Simple projects, quick prototyping | Cloud workspace | Production data | + +## Recommended: Branch Workflow + +For most projects, use `dev_mode=branch`. It provides isolated environments backed by Tinybird Cloud, with optional access to production data. + +```json +{ + "dev_mode": "branch" +} +``` + +1. Create a git branch for your feature +2. Run `tb dev` — a Cloud branch is created automatically from the git branch name, file changes are watched and auto-rebuilt +3. Develop and test: `tb endpoint data ` +4. Push, create PR — CI runs `tb --cloud deploy --check` +5. Merge — CD runs `tb --cloud deploy` + +See `rules/branch-development.md` for details on branch tokens and `--last-partition`. + +## Local Workflow + +Use `dev_mode=local` for fast iteration without network dependencies. Good for developing SQL logic and testing with fixture data. + +```json +{ + "dev_mode": "local" +} +``` + +1. Start Tinybird Local: `tb local start` +2. Run `tb dev` in a new terminal — watches files and auto-rebuilds +3. Append test data: `tb datasource append --file fixtures/.ndjson` +4. Test endpoints: `tb endpoint data ` +5. Deploy when ready: `tb --cloud deploy` + +See `rules/local-development.md` for Tinybird Local commands and troubleshooting. + +## Cloud Direct Workflow + +For simple projects or quick prototyping, you can work directly against Cloud. Use `tb --cloud deploy` to deploy, or the two-step process for explicit confirmation: + +``` +tb --cloud deployment create --wait +tb --cloud deployment promote +``` + +Or the combined shorthand: + +``` +tb --cloud deploy +``` + +## Choosing a Workflow + +- **Starting a new project?** Start with Local for fast bootstrapping, switch to Branch when you need production data or team collaboration. +- **Team project with shared workspace?** Use Branch. Each developer gets an isolated environment. +- **Quick prototype or demo?** Cloud direct is fine. +- **CI/CD pipeline?** Use Tinybird Local for CI build/test, then `tb --cloud deploy` for production. See `rules/ci-cd.md`. + +## Testing Endpoints + +Use `tb endpoint data` to test endpoint output: + +``` +tb endpoint data my_endpoint +tb endpoint data my_endpoint --start_date 2024-01-01 --end_date 2024-01-31 +``` + +Use `tb endpoint data`, not `tb pipe data`. The `endpoint data` command calls the endpoint as an API consumer would, including parameter validation and output formatting. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/local-development.md b/.agents/skills/tinybird-cli-guidelines/rules/local-development.md new file mode 100644 index 00000000000..ad29f161153 --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/local-development.md @@ -0,0 +1,50 @@ +# Tinybird Local Development + +## Overview + +- Tinybird Local runs as a Docker container managed by the Tinybird CLI. +- In CLI 4.0, `tb build` uses `dev_mode` from `tinybird.config.json`. +- Use Tinybird Local for fast local iteration (`dev_mode=local`), then deploy with `tb deploy`. + +## Commands + +- `tb local start` + - Options: `--use-aws-creds`, `--volumes-path `, `--skip-new-version`, `--user-token`, `--workspace-token`, `--daemon`. +- `tb local stop` +- `tb local restart` + - Options: `--use-aws-creds`, `--volumes-path`, `--skip-new-version`, `--yes`. +- `tb local status` +- `tb local remove` +- `tb local version` +- `tb local generate-tokens` + +Notes: + +- If you remove the container without a persisted volume, local data is lost. +- Manual flags (`--local`, `--cloud`, `--branch`) still work as overrides. + +## Local-First Workflow + +1. `tb local start` +2. Set `dev_mode` to `local` in `tinybird.config.json` +3. Run `tb dev` in a new terminal — watches for file changes and auto-rebuilds +4. Test endpoints/queries locally with `tb endpoint data ` +5. Run `tb deploy` only when user explicitly requests production deployment + +Use `--volumes-path` to persist data between restarts. + +`tb dev` is the recommended development command. It watches your project files and automatically rebuilds Data Sources and Endpoints when changes are detected. + +## Connecting to the Tinybird UI + +- `tb dev --ui`: Builds in watch mode and connects the local project to the Tinybird UI for visual exploration and debugging. +- `tb open`: Opens the workspace in the browser. + +These are useful for visually inspecting query results, exploring Data Source schemas, or debugging pipe logic. + +## Troubleshooting + +- If status shows unhealthy, run `tb local restart` and re-check. +- If authentication is not ready, wait or restart the container. +- If memory warnings appear in status, increase Docker memory allocation. +- If Local is not running, start it with `tb local start`. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/mock-data.md b/.agents/skills/tinybird-cli-guidelines/rules/mock-data.md new file mode 100644 index 00000000000..259ad24ee5f --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/mock-data.md @@ -0,0 +1,36 @@ +# Mock Data Generation + +Tinybird mock data flow (as implemented by the agent) for a datasource: + +1. Build a SQL query that returns mock rows. +2. Execute locally with a limit and format using `tb --output=json|csv '' --rows-limit ` command. +3. Preview the generated output. +4. Confirm creation of a fixture file under `fixtures/`. +5. Write the fixture file: + - `fixtures/.ndjson` or `fixtures/.csv` +6. Confirm append. +7. Append the fixture to the datasource in Tinybird Local. + +## Example Mock Query + +``` +SELECT + rand() % 1000 AS experience_gained, + 1 + rand() % 100 AS level, + rand() % 500 AS monster_kills, + concat('player_', toString(rand() % 10000)) AS player_id, + rand() % 50 AS pvp_kills, + rand() % 200 AS quest_completions, + now() - rand() % 86400 AS timestamp +FROM numbers(ROWS) +``` + +Notes: + +- The query must return exactly `ROWS` rows via `FROM numbers(ROWS)`. +- Do not add FORMAT or a trailing semicolon in the mock query itself. + +## Error Handling Notes + +- If the datasource is in quarantine, query `_quarantine` and surface the first 5 rows. +- If append fails with "must be created first with 'mode=create'", rebuild the project and retry. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/secrets.md b/.agents/skills/tinybird-cli-guidelines/rules/secrets.md new file mode 100644 index 00000000000..2f34e9e1956 --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/secrets.md @@ -0,0 +1,27 @@ +# Secrets + +## Usage in Files + +- Secret syntax: `{{ tb_secret("SECRET_NAME", "DEFAULT_VALUE_OPTIONAL") }}`. +- Use secrets for credentials in connections and pipe SQL. +- Secrets in pipe files do not allow default values. +- Secrets in connection files may include default values. +- Do not replace secrets with dynamic parameters when secrets are required. + +## CLI: tb secret + +- List secrets: + - `tb secret ls` + - `tb secret ls --match _test` + +- Set or update a secret: + - `tb secret set SECRET_NAME SECRET_VALUE` + - `tb secret set SECRET_NAME` (prompts securely) + - `tb secret set SECRET_NAME --multiline` (opens editor) + +- Remove a secret: + - `tb secret rm SECRET_NAME` + +## Local Secrets + +- If a `.env.local` file is present, its secrets are loaded automatically in Tinybird Local. diff --git a/.agents/skills/tinybird-cli-guidelines/rules/tokens.md b/.agents/skills/tinybird-cli-guidelines/rules/tokens.md new file mode 100644 index 00000000000..f197dd49b18 --- /dev/null +++ b/.agents/skills/tinybird-cli-guidelines/rules/tokens.md @@ -0,0 +1,49 @@ +# Tokens + +- Resource-scoped tokens are defined in datafiles. +- Tinybird tracks and updates resource-scoped tokens from datafile contents. + +Scopes and usage: + +- DATASOURCES:READ:datasource_name => `TOKEN READ` in `.datasource` files +- DATASOURCES:APPEND:datasource_name => `TOKEN APPEND` in `.datasource` files +- PIPES:READ:pipe_name => `TOKEN READ` in `.pipe` files + +Examples: + +``` +TOKEN app_read READ +TOKEN landing_append APPEND +``` + +For operational tokens (not tied to resources): + +``` +tb token create static new_admin_token --scope +``` + +Scopes: `TOKENS`, `ADMIN`, `ORG_DATASOURCES:READ`, `WORKSPACE:READ_ALL`. + +## JWT Tokens + +JWT tokens have a TTL and can only use `PIPES:READ` or `DATASOURCES:READ` scopes. They are intended for end users calling endpoints or reading datasources without exposing a master API key. + +Create a JWT token: + +``` +tb token create jwt my_jwt_token --ttl 1h --scope PIPES:READ --resource my_pipe +``` + +Datasource read with filter: + +``` +tb token create jwt my_jwt_token --ttl 1h --scope DATASOURCES:READ --resource my_datasource --filter "column = 'value'" +``` + +Multiple scopes and resources (counts must match), with optional fixed params for PIPES:READ: + +``` +tb token create jwt my_jwt_token --ttl 1h \ + --scope PIPES:READ --resource my_pipe --fixed-params "k1=v1,k2=v2" \ + --scope DATASOURCES:READ --resource my_datasource --filter "column = 'value'" +``` diff --git a/.agents/skills/tinybird/SKILL.md b/.agents/skills/tinybird/SKILL.md new file mode 100644 index 00000000000..4c057cd3413 --- /dev/null +++ b/.agents/skills/tinybird/SKILL.md @@ -0,0 +1,45 @@ +--- +name: tinybird +description: Tinybird file formats, SQL rules, optimization patterns, and best practices for datasources, pipes, endpoints, and materialized views. +--- + +# Tinybird Best Practices + +Guidance for Tinybird file formats, SQL rules, optimization patterns, and data modeling. Use this skill when creating or editing Tinybird datafiles. + +## When to Apply + +- Creating or updating Tinybird resources (.datasource, .pipe, .connection) +- Writing or optimizing SQL queries +- Designing endpoint schemas and data models +- Organizing project structure and data layers +- Working with materialized views or copy pipes +- Implementing deduplication patterns +- Reviewing or refactoring Tinybird project files + +## Rule Files + +- `rules/project-files.md` +- `rules/build-deploy.md` +- `rules/datasource-files.md` +- `rules/pipe-files.md` +- `rules/endpoint-files.md` +- `rules/materialized-files.md` +- `rules/materialized-join-prefilter.md` +- `rules/sink-files.md` +- `rules/copy-files.md` +- `rules/connection-files.md` +- `rules/sql.md` +- `rules/endpoint-optimization.md` +- `rules/tests.md` +- `rules/deduplication-patterns.md` + +## Quick Reference + +- Project local files are the source of truth. +- Build target comes from `tinybird.config.json` `dev_mode` (`local` or `branch`). +- `tb deploy` targets Tinybird Cloud production. +- Commands like `tb sql` and `tb logs` default to local unless `--cloud` or `--branch=` is set. +- SQL is SELECT-only with Tinybird templating rules and strict parameter handling. +- Use MergeTree by default; AggregatingMergeTree for materialized targets. +- Filter early, select only needed columns, push complex work later in the pipeline. diff --git a/.agents/skills/tinybird/rules/build-deploy.md b/.agents/skills/tinybird/rules/build-deploy.md new file mode 100644 index 00000000000..9b03e9f81d7 --- /dev/null +++ b/.agents/skills/tinybird/rules/build-deploy.md @@ -0,0 +1,56 @@ +# Build & Deploy Targeting + +Start new projects with `tb init`. + +Use `tinybird.config.json` as the source of truth for `tb build` targeting. + +Example: + +```json +{ + "dev_mode": "branch", + "include": [ + "tinybird" + ] +} +``` + +## Build/Deploy Flow + +1. Read `dev_mode` from `tinybird.config.json`. +2. Run `tb build` against the configured development target. +3. Run `tb deploy` only when deployment to cloud production is explicitly requested. + +## `tb build` Targeting + +- `dev_mode: "local"` -> `tb build` runs against Tinybird Local. +- `dev_mode: "branch"` -> `tb build` runs against a Tinybird Cloud branch. + +## `tb deploy` Targeting + +- `tb --cloud deploy` deploys to Tinybird Cloud production. It creates a staging deployment, migrates data, and promotes to live. +- `tb deploy` is equivalent to `tb --cloud deploy`. +- Do not treat `tb build` as a production deployment. +- Use `tb --cloud deploy --check` to validate a deployment without applying it. Recommended for CI. +- For explicit confirmation, use `tb --cloud deployment create --wait` followed by `tb --cloud deployment promote`. + +## Non-Build Command Targeting + +Commands like `tb sql` and `tb logs` run against local by default. + +Use explicit overrides to target other environments: + +- `--cloud` for cloud +- `--branch=` for a specific branch + +Examples: + +```bash +tb sql "SELECT 1" +tb sql --cloud "SELECT 1" +tb sql --branch=feature_metrics "SELECT 1" + +tb logs +tb logs --cloud +tb logs --branch=feature_metrics +``` diff --git a/.agents/skills/tinybird/rules/connection-files.md b/.agents/skills/tinybird/rules/connection-files.md new file mode 100644 index 00000000000..d227edfeee4 --- /dev/null +++ b/.agents/skills/tinybird/rules/connection-files.md @@ -0,0 +1,41 @@ +# Connection Files + +- Content cannot be empty. +- Connection names must be unique. +- No indentation for property names. +- Supported types: kafka, gcs, s3. +- If user requests an unsupported type, report it and do not create it. + +Kafka example: + +``` +TYPE kafka +KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("PRODUCTION_KAFKA_SERVERS", "localhost:9092") }} +KAFKA_SECURITY_PROTOCOL SASL_SSL +KAFKA_SASL_MECHANISM PLAIN +KAFKA_KEY {{ tb_secret("PRODUCTION_KAFKA_USERNAME", "") }} +KAFKA_SECRET {{ tb_secret("PRODUCTION_KAFKA_PASSWORD", "") }} +``` + +S3 example: + +``` +TYPE s3 +S3_REGION {{ tb_secret("PRODUCTION_S3_REGION", "") }} +S3_ARN {{ tb_secret("PRODUCTION_S3_ARN", "") }} +``` + +GCS service account example: + +``` +TYPE gcs +GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON", "") }} +``` + +GCS HMAC example: + +``` +TYPE gcs +GCS_HMAC_ACCESS_ID {{ tb_secret("gcs_hmac_access_id") }} +GCS_HMAC_SECRET {{ tb_secret("gcs_hmac_secret") }} +``` diff --git a/.agents/skills/tinybird/rules/copy-files.md b/.agents/skills/tinybird/rules/copy-files.md new file mode 100644 index 00000000000..a76989f693f --- /dev/null +++ b/.agents/skills/tinybird/rules/copy-files.md @@ -0,0 +1,27 @@ +# Copy Pipe Files + +- Do not create by default unless requested. +- Create under `/copies`. +- Do not include COPY_SCHEDULE unless explicitly requested. +- Use TYPE COPY and TARGET_DATASOURCE. +- The default `copy_mode` is `append`; but it's better if you set it explicitly. The other option is `replace` + +Example: + +``` +DESCRIPTION Copy Pipe to export sales hour every hour to the sales_hour_copy Data Source + +NODE daily_sales +SQL > + % + SELECT toStartOfDay(starting_date) day, country, sum(sales) as total_sales + FROM teams + WHERE day BETWEEN toStartOfDay(now()) - interval 1 day AND toStartOfDay(now()) + and country = {{ String(country, 'US')}} + GROUP BY day, country + +TYPE COPY +TARGET_DATASOURCE sales_hour_copy +COPY_SCHEDULE 0 * * * * +COPY_MODE append +``` diff --git a/.agents/skills/tinybird/rules/datasource-files.md b/.agents/skills/tinybird/rules/datasource-files.md new file mode 100644 index 00000000000..efc2cf97821 --- /dev/null +++ b/.agents/skills/tinybird/rules/datasource-files.md @@ -0,0 +1,101 @@ +# Datasource Files + +- Content cannot be empty. +- Datasource names must be unique. +- No indentation for property names (DESCRIPTION, SCHEMA, ENGINE, etc.). +- Use MergeTree by default. +- Use AggregatingMergeTree for materialized targets. +- Always use JSON paths for schema (example: `user_id` String `json:$.user_id`). +- Array syntax: `items` Array(String) `json:$.items[:]`. +- DateTime64 requires precision (use DateTime64(3)). +- Only include ENGINE_PARTITION_KEY and ENGINE_PRIMARY_KEY when explicitly requested. +- Import configuration: + - S3/GCS: set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI, IMPORT_SCHEDULE (GCS supports @on-demand only, S3 supports @auto). + - Kafka: set KAFKA_CONNECTION_NAME, KAFKA_TOPIC, KAFKA_GROUP_ID. +- For landing datasources created from a .ndjson file with no schema specified, use: + - `SCHEMA >` + - `` `data` String `json:$` `` + +Example: + +``` +DESCRIPTION > + Some meaningful description of the datasource + +SCHEMA > + `column_name_1` Type `json:$.column_name_1`, + `column_name_2` Type `json:$.column_name_2` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "partition_key" +ENGINE_SORTING_KEY "sorting_key_1, sorting_key_2" +``` + +## Updating Data Source Schemas (Cloud) + +If a schema change is incompatible with the deployed Cloud Data Source, add a `FORWARD_QUERY` to transform existing data to the new schema. The query is a SELECT list only (no FROM/WHERE). It runs over existing data at read time until the next deploy compacts it. + +### When to use `FORWARD_QUERY` + +- Adding a new column that requires a default value for existing rows +- Changing a column type (e.g., String to UUID, Int32 to Int64) +- Renaming a column +- Removing a column (just omit it from the SELECT) + +### Examples + +Adding a new column with a default: + +``` +FORWARD_QUERY > + SELECT *, 'unknown' as source +``` + +Changing a column type: + +``` +FORWARD_QUERY > + SELECT timestamp, accurateCastOrDefault(session_id, 'UUID') as session_id, action, version, payload +``` + +Renaming a column: + +``` +FORWARD_QUERY > + SELECT old_name as new_name, other_column +``` + +### After migration + +Once the deploy applies the `FORWARD_QUERY` and the schema change is live, the `FORWARD_QUERY` has done its job. You can remove it from the datafile in a subsequent deploy if no further schema changes are pending. Keeping stale `FORWARD_QUERY` blocks around adds unnecessary complexity. + +## TTL and Partition Key Alignment + +Apply when a datasource sets both `ENGINE_TTL` and `ENGINE_PARTITION_KEY`: use a partition granularity equal to or finer than the TTL window (e.g. daily partitions for a TTL in days), so partitions expire and drop as whole units instead of ClickHouse rewriting them. A partition coarser than the TTL window (e.g. yearly partitions with a 65-day TTL) never fully expires, forcing constant rewrites instead of cheap drops. + +``` +# Bad: yearly partition, 65-day TTL — partition never fully expires +ENGINE_PARTITION_KEY "toYYYY(timestamp)" +ENGINE_TTL "toDateTime(timestamp) + toIntervalDay(65)" + +# Good: daily partition matches the TTL window — old partitions drop whole +ENGINE_PARTITION_KEY "toDate(timestamp)" +ENGINE_TTL "toDate(timestamp) + toIntervalDay(65)" +ENGINE_SETTINGS "ttl_only_drop_parts=1" +``` + +When possible, set `ttl_only_drop_parts=1` in `ENGINE_SETTINGS` — it makes ClickHouse only drop whole expired parts instead of rewriting partially-expired ones, which is much cheaper. + +## Sharing Datasources + +``` +SHARED_WITH > + destination_workspace, + other_destination_workspace +``` + +Limitations: + +- Shared datasources are read-only. +- You cannot share a shared datasource. +- You cannot create a materialized view from a shared datasource. diff --git a/.agents/skills/tinybird/rules/deduplication-patterns.md b/.agents/skills/tinybird/rules/deduplication-patterns.md new file mode 100644 index 00000000000..1daf467b00f --- /dev/null +++ b/.agents/skills/tinybird/rules/deduplication-patterns.md @@ -0,0 +1,117 @@ +# Deduplication and Lambda Architecture + +Strategies for handling duplicates and combining batch with real-time processing. + +## Deduplication Strategy Selection + +| Strategy | When to use | +| ------------------------------------------- | -------------------------------------------------------------- | +| Query-time (`argMax`, `LIMIT BY`, subquery) | Prototyping or small datasets | +| ReplacingMergeTree | Large datasets, need latest row per key | +| Periodic snapshots (Copy Pipes) | Freshness not critical, need rollups or different sorting keys | +| Lambda architecture | Need freshness + complex transformations that MVs can't handle | + +For dimensional/small tables, periodic full replace is usually best. + +## Query-time Deduplication + +```sql +-- argMax: get latest value per key +SELECT post_id, argMax(views, updated_at) as views +FROM posts GROUP BY post_id + +-- LIMIT BY +SELECT * FROM posts ORDER BY updated_at DESC LIMIT 1 BY post_id + +-- Subquery +SELECT * FROM posts WHERE (post_id, updated_at) IN ( + SELECT post_id, max(updated_at) FROM posts GROUP BY post_id +) +``` + +## ReplacingMergeTree + +``` +ENGINE "ReplacingMergeTree" +ENGINE_SORTING_KEY "unique_id" +ENGINE_VER "updated_at" +ENGINE_IS_DELETED "is_deleted" -- optional, UInt8: 1=deleted, 0=active +``` + +- Always query with `FINAL` or use alternative deduplication method +- Deduplication happens during merges (asynchronous, uncontrollable) +- **Do not** build AggregatingMergeTree MVs on top of ReplacingMergeTree—MVs only see incoming blocks, not merged state, so duplicates persist + +```sql +SELECT * FROM posts FINAL WHERE post_id = {{Int64(post_id)}} +``` + +## Snapshot-based Deduplication (Copy Pipes) + +Use Copy Pipes when: + +- ReplacingMergeTree + FINAL is too slow +- You need different sorting keys that change with updates +- You need downstream Materialized Views for rollups +- The default `copy_mode` is `append`. +- Use `COPY_MODE replace` for full refreshes when the table is not massive and you don't control when duplicates can occur. +- Keep `COPY_MODE append` (default) when you do control duplicate generation and can process incrementally. + +``` +NODE generate_snapshot +SQL > + SELECT post_id, argMax(views, updated_at) as views, max(updated_at) as updated_at + FROM posts_raw + GROUP BY post_id + +TYPE COPY +TARGET_DATASOURCE posts_snapshot +COPY_SCHEDULE 0 * * * * +COPY_MODE replace +``` + +## Lambda Architecture + +Combine batch snapshots with real-time queries when: + +- Aggregating over ReplacingMergeTree (MVs fail—they only see blocks, not merged state) +- Window functions requiring full table scans +- CDC workloads +- `uniqState` performance is problematic +- endpoints that require JOINs at query time + +### Pattern + +1. **Batch layer**: Copy Pipe creates periodic deduplicated snapshots or intermediate tables. +2. **Real-time layer**: Query fresh data since last snapshot +3. **Serving layer**: UNION ALL combines both + +```sql +SELECT * FROM posts_snapshot +UNION ALL +SELECT post_id, argMax(views, updated_at) as views, max(updated_at) as updated_at +FROM posts_raw +WHERE updated_at > (SELECT max(updated_at) FROM posts_snapshot) +GROUP BY post_id +``` + +### Freshness vs Cost Trade-off + +- More frequent Copy Pipe runs = fresher snapshots but higher cost +- Less frequent = stale batch layer but real-time layer covers the gap +- Balance based on query patterns and data volume + +## argMax with Null Values + +**Warning**: `argMaxMerge` prefers non-null values over null, even with lower timestamps. + +Workaround—convert nulls to epoch before aggregation: + +```sql +SELECT post_id, + argMaxState(CASE WHEN flagged_at IS NULL THEN toDateTime('1970-01-01 00:00:00') ELSE flagged_at END, updated_at) as flagged_at +FROM posts +GROUP BY post_id +``` + +Handle the sentinel value in downstream queries. diff --git a/.agents/skills/tinybird/rules/endpoint-files.md b/.agents/skills/tinybird/rules/endpoint-files.md new file mode 100644 index 00000000000..e1738a31d6e --- /dev/null +++ b/.agents/skills/tinybird/rules/endpoint-files.md @@ -0,0 +1,43 @@ +# Endpoint Files + +Endpoint files are `.pipe` files with `TYPE endpoint` and should live under `/endpoints`. + +- Follow all general pipe rules. +- Ensure SQL follows Tinybird SQL rules (templating, SELECT-only, parameters). +- Include the output node in TYPE or in the last node. + +Example: + +``` +DESCRIPTION > + Some meaningful description of the endpoint + +NODE endpoint_node +SQL > + SELECT ... +TYPE endpoint +``` + +## Testing Endpoints + +Use `tb endpoint data` to test endpoint output: + +``` +tb endpoint data my_endpoint +tb endpoint data my_endpoint --start_date 2024-01-01 --end_date 2024-01-31 +``` + +Use `tb endpoint data`, not `tb pipe data`. The `endpoint data` command calls the endpoint as a consumer would, including parameter validation and output formatting. + +## Endpoint URLs + +- Run `tb endpoint ls` to list all endpoints and their URLs. +- Include dynamic parameters when needed. +- Date formats: + - DateTime64: `YYYY-MM-DD HH:MM:SS.MMM` + - DateTime: `YYYY-MM-DD HH:MM:SS` + - Date: `YYYYMMDD` + +## OpenAPI Definitions + +- curl `/v0/pipes/openapi.json?token=` to get the OpenAPI definition for all endpoints. diff --git a/.agents/skills/tinybird/rules/endpoint-optimization.md b/.agents/skills/tinybird/rules/endpoint-optimization.md new file mode 100644 index 00000000000..df8a8aaa2f1 --- /dev/null +++ b/.agents/skills/tinybird/rules/endpoint-optimization.md @@ -0,0 +1,178 @@ +# Endpoint Optimization + +Use this checklist when optimizing endpoints. + +## Gathering Runtime Data + +Before optimizing, collect evidence from these sources: + +- **Endpoint source code**: SQL, datasources, materialized views, and pipes in the workspace. +- **`pipe_stats_rt`**: Query `SELECT * FROM tinybird.pipe_stats_rt WHERE pipe_name = 'endpoint_name'` to check execution duration percentiles (p50, p90, p95, p99), read_bytes, rows_read, and error counts. +- **Query plan**: Call the endpoint with `?explain=true` (e.g., `https://$TB_HOST/v0/pipes/endpoint_name?explain=true`) to inspect join strategies, aggregation stages, index usage, and partition pruning. + +Ignore datasources with fewer than 10,000 rows or less than 50 MB of data. + +1. Aggregations at query time? + +- Fix: Move to materialized views when possible, to snapshots (copy pipes) or lambda architecture if MVs do not fit. + +## Structural Rules + +Schema, query-shape, or data-layout issues. Apply whenever detected — no runtime evidence needed. + +### Selecting unnecessary columns + +- `SELECT *` or unused columns increase I/O, decompression cost, and cache pressure. +- Fix: Explicitly select only required columns. + +### Oversized data types + +- Larger types than necessary reduce compression and increase CPU/memory usage. +- Fix: Use smallest safe types. Use `LowCardinality` for low-unique strings, defaults instead of `Nullable`. + +### Unnecessary Nullable columns + +- `Nullable` adds overhead from null bitmaps and extra checks. +- Fix: Replace `Nullable(T)` with `T` when the column never contains nulls. + +### Inefficient ORDER BY key ordering + +- High-cardinality columns first in `ORDER BY` reduce sparse index effectiveness and data skipping. +- Fix: Start `ORDER BY` with low-cardinality and/or time columns. Avoid timestamp as first key in multi-tenant cases. + +### Unnecessary casting + +- Casting a column to its existing type wastes CPU and can block partition pruning. +- Fix: Remove redundant casts; fix types at ingestion time if needed. + +### Excessive string materialization + +- Materializing full `String` values when only metadata is needed wastes memory and CPU. +- Fix: Extract required string properties at ingestion time into typed columns. + +### Filter before join/aggregation + +- Applying filters after joins or aggregations increases input size and cost. +- Fix: Push filters as early as possible in the query pipeline. + +## Runtime-Dependent Rules + +Apply only when runtime thresholds are exceeded, based on `pipe_stats_rt` and `EXPLAIN` data. + +### Aggregations at query time + +- **When**: p95 > 5s, or aggregation dominates `EXPLAIN`, or memory > 60%, or OOM/timeout errors. +- Fix: Precompute via materialized view. + +### JOINs at query time + +- **When**: p95 > 5s, or join dominates `EXPLAIN`, or memory spikes, or OOM/timeout errors. +- Fix: Move join to ingestion time via materialized view, or denormalize. + +### Incorrect or missing sorting keys + +- **When**: reads > 10% of granules, and p95 > 3s or rows_read/rows_returned > 100x. +- Fix: Rebuild datasource with `ORDER BY` aligned to selective filters. + +### PREWHERE for early filtering + +- **When**: rows_read/rows_returned > 50x, or p95 > 3s, or `EXPLAIN` shows late filtering. +- Fix: Push selective filters into `PREWHERE`. + +### Data skipping indexes + +- **When**: filters on non-primary-key columns, and rows_read/rows_returned > 100x, or p95 > 3s. +- Fix: Add appropriate skip indexes and validate with `EXPLAIN`. + +### Large GROUP BY at query time + +- **When**: p95 > 5s, or aggregation memory > 50%, or OOM/timeout errors. +- Fix: Pre-aggregate at ingestion time using materialized view. + +### Regex at query time + +- **When**: p95 > 3s, or CPU > 70%. +- Fix: Move regex logic to ingestion time. + +### Unbounded history without TTL + +- **When**: p95 increases week-over-week, or rows_read grows for identical queries. +- Fix: Create a TTL-backed datasource via materialized view. + +### Misaligned or missing partition pruning + +- **When**: >20% of partitions scanned, or p95 > 3s, or `EXPLAIN` shows ineffective pruning. +- Fix: Recreate datasource with an aligned partitioning key. Include partition key column in query filters. + +### ORDER BY with LIMIT without pushdown + +- **When**: rows sorted >> LIMIT (>100x), or p95 > 3s. +- Fix: Restructure query or pre-materialize top-k at ingestion time. + +### DISTINCT instead of GROUP BY + +- **When**: p95 > 5s, or memory > 50%. +- Fix: Replace with ingestion-time aggregation or `GROUP BY`. + +### Overuse of FINAL + +- **When**: p95 > 3s, or rows_read >> rows_returned. +- Fix: Remove `FINAL` by enforcing correctness at ingestion time (lambda architecture). + +### Expensive JSON extraction at query time + +- **When**: p95 > 3s, or CPU > 70%. +- Fix: Extract JSON fields into typed columns at ingestion time. If many fields are extracted from the same JSON payload (per-field `visitParamExtract*`/`JSONExtract*` calls), see the single-parse JSON pattern in `materialized-files.md` — it applies here too, but matters most in materialized views since the parse cost compounds over every ingested row. + +### Large IN lists + +- **When**: p95 > 3s, or query planning time is high. +- Fix: Replace with lookup datasource or ingestion-time materialization. + +### Approximate uniques + +- **When**: exact `COUNT(DISTINCT)` with p95 > 5s, or memory > 50%, or OOM errors. +- Fix: Use `uniqHLL12` or similar approximate functions when acceptable. + +## Monitoring and Validation + +- Track `tinybird.pipe_stats_rt` and `tinybird.pipe_stats`. +- Success metrics: lower latency, lower read_bytes, improved read_bytes/write_bytes ratio. +- If for any reason these two datasources don't contain the needed information, check `system.query_log` + +## Query Explain + +- For more details, call the endpoint with explain=true parameter to understand the query plan. E.g: https://$TB_HOST/v0/pipes/endpoint_name?explain=true + +## Templates + +Materialized view: + +``` +NODE materialized_view_name +SQL > + SELECT toDate(timestamp) as date, customer_id, countState(*) as event_count + FROM source_table + GROUP BY date, customer_id + +TYPE materialized +DATASOURCE mv_datasource_name +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "customer_id, date" +``` + +Optimized query: + +``` +NODE endpoint_query +SQL > + % + SELECT date, sum(amount) as daily_total + FROM events + WHERE customer_id = {{ String(customer_id) }} + AND date >= {{ Date(start_date) }} + AND date <= {{ Date(end_date) }} + GROUP BY date + ORDER BY date DESC +``` diff --git a/.agents/skills/tinybird/rules/materialized-files.md b/.agents/skills/tinybird/rules/materialized-files.md new file mode 100644 index 00000000000..7a0156caea8 --- /dev/null +++ b/.agents/skills/tinybird/rules/materialized-files.md @@ -0,0 +1,93 @@ +# Materialized Pipe Files + +- Do not create by default unless requested. +- Create under `/materializations`. +- Use TYPE MATERIALIZED and set DATASOURCE to the target datasource. +- Use State modifiers in the pipe; use AggregateFunction in the target datasource. +- Use Merge modifiers when reading AggregateFunction columns. +- Put all dimensions in ENGINE_SORTING_KEY, ordered from least to most cardinality. + +Example: + +``` +NODE daily_sales +SQL > + SELECT toStartOfDay(starting_date) day, country, sumState(sales) as total_sales + FROM teams + GROUP BY day, country + +TYPE MATERIALIZED +DATASOURCE sales_by_hour +``` + +Target datasource example: + +``` +SCHEMA > + `total_sales` AggregateFunction(sum, Float64), + `sales_count` AggregateFunction(count, UInt64), + `dimension_1` String, + `dimension_2` String, + `date` DateTime + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date, dimension_1, dimension_2" +``` + +## JSON extraction: parse once, not once per field + +- **When to apply**: the query calls `JSONExtractString`/`JSONExtractInt`/`JSONExtractBool`/`JSONExtractFloat`/`simpleJSONExtractString`/`visitParam*` multiple times against the same JSON/string column — one call per field. Each call re-parses the raw JSON from scratch, so N fields means N full parses per row. This is most costly in materialized views, since it runs on every inserted block for the pipe's lifetime. +- **How to apply**: parse the JSON once into a typed `Tuple` with `JSONExtract(...)`, then read each field from it with `getSubcolumn`. + +Bad (one parse per field): + +``` +NODE typed_events +SQL > + SELECT + at AS timestamp, + visitParamExtractString(payload, 'field_a') AS field_a, + visitParamExtractInt(payload, 'field_b') AS field_b, + visitParamExtractBool(payload, 'field_c') AS field_c, + simpleJSONExtractString(payload, 'field_d') AS field_d + FROM raw_events + +TYPE MATERIALIZED +DATASOURCE typed_events_ds +``` + +Good (one parse total): + +``` +NODE typed_events +SQL > + WITH + JSONExtract(payload, 'Tuple( + field_a String, + field_b Int64, + field_c Bool, + field_d String + )') AS payload_json + SELECT + at AS timestamp, + getSubcolumn(payload_json, 'field_a') AS field_a, + getSubcolumn(payload_json, 'field_b') AS field_b, + getSubcolumn(payload_json, 'field_c') AS field_c, + getSubcolumn(payload_json, 'field_d') AS field_d + FROM raw_events + +TYPE MATERIALIZED +DATASOURCE typed_events_ds +``` + +- Missing fields default to their type's default value. +- Reuse a field via a `WITH` alias if multiple derived expressions depend on it. + +## Usual gotchas + +- Materialized Views work as insert triggers, which means a delete or truncate operation on your original Data Source doesn't affect the related Materialized Views. + +- As transformation and ingestion in the Materialized View is done on each block of inserted data in the original Data Source, some operations such as GROUP BY, ORDER BY, DISTINCT and LIMIT might need a specific engine, such as AggregatingMergeTree or SummingMergeTree, which can handle data aggregations. + +- The Data Source resulting from a Materialized View generated using JOIN is automatically updated only if and when a new operation is performed over the Data Source in the FROM. diff --git a/.agents/skills/tinybird/rules/materialized-join-prefilter.md b/.agents/skills/tinybird/rules/materialized-join-prefilter.md new file mode 100644 index 00000000000..12b329ec833 --- /dev/null +++ b/.agents/skills/tinybird/rules/materialized-join-prefilter.md @@ -0,0 +1,137 @@ +# Pre-filter Right-Side JOINs in Materialized Views + +Materialized views run as **insert triggers**: on every block inserted +into the source datasource (the left-most table in `FROM`), Tinybird +re-executes the pipe SQL with that block as the `FROM` source. Any +table on the right side of a `JOIN` / `ASOF JOIN`, however, is scanned +**in full** unless explicitly restricted. As the right-side table grows, +each insert becomes more expensive and ingestion can stall or fail. + +## When to Apply + +Apply this pattern to any `TYPE materialized` pipe where: + +- The right side of a JOIN is a datasource that grows unbounded over time. +- Symptoms: slow inserts, ingestion lag, memory spikes on the source + datasource, `MEMORY_LIMIT_EXCEEDED` errors on the MV. +- The JOIN already has selective conditions (equality on keys, time + bounds) — those conditions are what we promote into a pre-filter. + +## The Pattern + +Replace the right-side datasource with a subquery that restricts it +to rows that **could possibly match** the current insert batch. +Two filters compose: + +1. **Key pre-filter** — keep only right-side rows whose join keys + appear in the inserting batch from the left-side source. +2. **Time pre-filter** — for `ASOF` joins with `left.time >= right.time`, + bound `right.time` to `[min(left.time) - INTERVAL N , max(left.time)]` + of the inserting batch. The lower bound is the **maximum acceptable + gap** between the right-side row and the left-side row — make + it an obvious, configurable constant so it can be tuned later. + +The left-side reference inside the subquery (the same datasource that +appears in the outer `FROM`) resolves to the inserting block, not the +full table — that is exactly what makes the pre-filter cheap. + +## Example + +Before — `enrichment_table` is scanned in full on every insert: + +``` +NODE mv_node +SQL > + SELECT + e.tenant_id, + e.entity_id, + e.event_name, + e.event_time, + x.source_time AS resolved_time + FROM events_table e + ASOF LEFT JOIN enrichment_table x + ON e.tenant_id = x.tenant_id + AND e.entity_id = x.entity_id + AND e.ref_id = x.ref_id + AND e.event_time >= x.source_time + WHERE e.event_name IN ('event_x', 'event_y') + +TYPE materialized +DATASOURCE mv_target +``` + +After — `enrichment_table` is restricted by keys present in the batch +and by a 30-day time window relative to the batch's event times: + +``` +NODE mv_node +SQL > + SELECT + e.tenant_id, + e.entity_id, + e.event_name, + e.event_time, + x.source_time AS resolved_time + FROM events_table e + ASOF LEFT JOIN ( + SELECT tenant_id, entity_id, ref_id, source_time + FROM enrichment_table + WHERE source_time >= ( + SELECT min(event_time) + FROM events_table + WHERE event_name IN ('event_x', 'event_y') + ) - INTERVAL 30 DAY + AND source_time <= ( + SELECT max(event_time) + FROM events_table + WHERE event_name IN ('event_x', 'event_y') + ) + AND (tenant_id, entity_id, ref_id) IN ( + SELECT tenant_id, entity_id, ref_id + FROM events_table + WHERE event_name IN ('event_x', 'event_y') + ) + ) x + ON e.tenant_id = x.tenant_id + AND e.entity_id = x.entity_id + AND e.ref_id = x.ref_id + AND e.event_time >= x.source_time + WHERE e.event_name IN ('event_x', 'event_y') + +TYPE materialized +DATASOURCE mv_target +``` + +Repeat the same wrapping for every right-side JOIN — one independent +subquery per right-side table. + +## Checklist + +- [ ] Subquery on the right-side table selects only the columns used by + the JOIN and the SELECT. +- [ ] Key pre-filter uses an `IN` tuple of the join columns from the + left-side source, replicating the same `WHERE` that bounds the MV. +- [ ] For `ASOF`/time-bounded joins, include `right.time BETWEEN +(min(left.time) - INTERVAL N ) AND max(left.time)`. +- [ ] The `INTERVAL N ` constant is a single, obvious literal — + not buried in arithmetic — so the maximum gap is tunable. +- [ ] The `WHERE` filter inside the inner subqueries over the left-side + source matches the outer pipe's `WHERE` so the inserting block is + read consistently. + +## Gotchas + +- **The lower time bound trades cost for correctness.** Any right-side + row older than `min(left.time) - N ` will be excluded — even if + it would have been the correct `ASOF` match. Pick `N` large enough to + cover the realistic gap between right-side rows and the left-side rows + that reference them. Document it. +- **Multiple right-side JOINs need independent pre-filters.** Each + right-side table has its own keys and time semantics; do not share + one subquery across them. +- **Key extraction inside the inner subquery** must mirror the outer + extraction exactly (same casts, same `JSONExtract` / `toInt64OrZero` + wrappers, etc.), otherwise the `IN` tuple won't match. +- **The pre-filter does not change MV correctness for in-window data** + but it does change it for out-of-window data — make this explicit in + a pipe-level `DESCRIPTION`. diff --git a/.agents/skills/tinybird/rules/pipe-files.md b/.agents/skills/tinybird/rules/pipe-files.md new file mode 100644 index 00000000000..81b1f5076f0 --- /dev/null +++ b/.agents/skills/tinybird/rules/pipe-files.md @@ -0,0 +1,19 @@ +# Pipe Files (General) + +- Pipe names must be unique. +- Node names must differ from the pipe name and any resource name. +- No indentation for property names (DESCRIPTION, NODE, SQL, TYPE, etc.). +- Allowed TYPE values: endpoint, copy, materialized, sink. +- Add the output node in the TYPE section or in the last node. + +Example: + +``` +DESCRIPTION > + Some meaningful description of the pipe + +NODE node_1 +SQL > + SELECT ... +TYPE endpoint +``` diff --git a/.agents/skills/tinybird/rules/project-files.md b/.agents/skills/tinybird/rules/project-files.md new file mode 100644 index 00000000000..0b02862c145 --- /dev/null +++ b/.agents/skills/tinybird/rules/project-files.md @@ -0,0 +1,99 @@ +# Project Files + +## Project Root + +- By default, create a `tinybird/` folder at the project root and nest Tinybird folders under it. +- Ensure the `.tinyb` credentials file is at the same level where the CLI commands are run. +- The `tinybird.config.json` file in the project root controls build/deploy behavior. + +## tb info + +Use `tb info` to confirm CLI context, especially for credentials issues. + +It reports information about Local and Cloud environments: + +- Where the CLI is loading the `.tinyb` file from +- Current logged workspace +- API URL +- UI URL +- ClickHouse HTTP interface URL + +It can show values for both Cloud and Local environments. + +## File Locations + +Default locations (use these unless the project uses a different structure): + +- Endpoints: `/endpoints` +- Materialized pipes: `/materializations` +- Sink pipes: `/sinks` +- Copy pipes: `/copies` +- Connections: `/connections` +- Datasources: `/datasources` +- Fixtures: `/fixtures` +- Tests: `/tests` + +## Organizing Larger Projects + +As projects grow, consider organizing endpoints and datasources by domain or consumer. The `include` field in `tinybird.config.json` controls which directories are included in builds. + +For example, a project with multiple consumers might use: + +``` +tinybird/ +├── datasources/ +├── endpoints/ # General-purpose API endpoints +├── endpoints_dashboard/ # Dashboard-specific endpoints +├── endpoints_public/ # Public-facing endpoints +├── materializations/ +├── copies/ +├── connections/ +└── fixtures/ +``` + +This pattern helps when different teams or applications consume different sets of endpoints, and keeps the endpoint count manageable per directory. + +## Data Layer Architecture + +For complex projects, organizing datasources and pipes into logical data layers improves clarity: + +| Layer | Purpose | Example | +| ----------- | ----------------------------------------------- | ------------------------------------ | +| Landing | Raw ingested data from external sources | `raw_events`, `s3_import_logs` | +| Cleaned | Deduplicated or transformed data | `events_dedup`, `normalized_logs` | +| Dimensions | Lookup and reference tables | `dim_organizations`, `dim_users` | +| Aggregation | Materialized views for pre-computed metrics | `mv_events_daily`, `mv_usage_hourly` | +| API | Endpoint pipes that serve the final queries | `kpis`, `top_pages`, `user_activity` | +| Export | Sink pipes for sending data to external systems | `sink_to_s3`, `sink_to_kafka` | + +Not every project needs all layers. Start simple and add layers as complexity grows. + +## Tinybird Terminology + +When writing descriptions, comments, or documentation for Tinybird resources, use consistent capitalization: + +- **Data Source** (not datasource or data source in prose) +- **Pipe** +- **Endpoint** or **API Endpoint** +- **Materialized View** +- **Token** +- **Workspace** +- **Sink** +- **Copy Pipe** +- **Connection** + +Datafile instructions should be referenced in uppercase: `FORWARD_QUERY`, `ENGINE_SORTING_KEY`, `ENGINE_PARTITION_KEY`, `COPY_SCHEDULE`, `COPY_MODE`, `TYPE`, `SCHEMA`, `DESCRIPTION`. + +## File-Specific Rules + +See these rule files for detailed requirements: + +- `rules/datasource-files.md` +- `rules/pipe-files.md` +- `rules/endpoint-files.md` +- `rules/materialized-files.md` +- `rules/sink-files.md` +- `rules/copy-files.md` +- `rules/connection-files.md` + +After making changes in the project files, check `rules/build-deploy.md` for next steps. diff --git a/.agents/skills/tinybird/rules/sink-files.md b/.agents/skills/tinybird/rules/sink-files.md new file mode 100644 index 00000000000..2a38fe57cab --- /dev/null +++ b/.agents/skills/tinybird/rules/sink-files.md @@ -0,0 +1,32 @@ +# Sink Pipe Files + +- Do not create by default unless requested. +- Create under `/sinks`. +- Valid external systems: Kafka, S3, GCS. +- Sink pipes depend on a connection; reuse existing connections when possible. +- Do not include EXPORT_SCHEDULE unless explicitly requested. +- Use TYPE SINK and set EXPORT_CONNECTION_NAME. + +Example: + +``` +DESCRIPTION Sink Pipe to export sales hour every hour using my_connection + +NODE daily_sales +SQL > + % + SELECT toStartOfDay(starting_date) day, country, sum(sales) as total_sales + FROM teams + WHERE day BETWEEN toStartOfDay(now()) - interval 1 day AND toStartOfDay(now()) + and country = {{ String(country, 'US')}} + GROUP BY day, country + +TYPE sink +EXPORT_CONNECTION_NAME "my_connection" +EXPORT_BUCKET_URI "s3://tinybird-sinks" +EXPORT_FILE_TEMPLATE "daily_prices" +EXPORT_SCHEDULE "*/5 * * * *" +EXPORT_FORMAT "csv" +EXPORT_COMPRESSION "gz" +EXPORT_STRATEGY "truncate" +``` diff --git a/.agents/skills/tinybird/rules/sql.md b/.agents/skills/tinybird/rules/sql.md new file mode 100644 index 00000000000..5fc94c0beb2 --- /dev/null +++ b/.agents/skills/tinybird/rules/sql.md @@ -0,0 +1,70 @@ +# SQL Rules + +## Core Principles + +1. Filter early and read as little data as possible. +2. Select only needed columns. +3. Do complex work later in the pipeline. +4. Prefer ClickHouse functions; only supported functions are allowed. + +## Query Requirements + +- SQL must be valid ClickHouse SQL with Tinybird templating (Tornado). +- Only SELECT statements are allowed. +- Avoid CTEs; use nodes or subqueries instead. +- Do not use system tables (system.tables, system.datasources, information_schema.tables). +- Do not use CREATE/INSERT/DELETE/TRUNCATE or currentDatabase(). + +## Parameter and Templating Rules + +- If parameters are used, the query must start with `%` on its own line. +- Parameter functions: String, DateTime, Date, Float32, Float64, Int, Integer, UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256. +- Parameter names must be different from column names. +- Default values must be hardcoded. +- Parameters are never quoted. +- In `defined()` checks, do not quote the parameter name. + +Bad: + +``` +SELECT * FROM events WHERE session_id={{String(my_param, "default")}} +``` + +Good: + +``` +% +SELECT * FROM events WHERE session_id={{String(my_param, "default")}} +``` + +## Join and Aggregation Rules + +- Filter before JOINs and GROUP BY. +- Avoid joining tables with >1M rows without filtering. +- Avoid nested aggregates; use subqueries instead. +- Use AggregateFunction columns with -Merge combinators. + +## Operation Order + +1. WHERE filters +2. Select needed columns +3. JOIN +4. GROUP BY / aggregates +5. ORDER BY +6. LIMIT + +## External Tables + +Iceberg: + +``` +FROM iceberg('s3://bucket/path/to/table', {{tb_secret('aws_access_key_id')}}, {{tb_secret('aws_secret_access_key')}}) +``` + +Postgres: + +``` +FROM postgresql({{ tb_secret("db_host_port") }}, 'database', 'table', {{tb_secret('db_username')}}, {{tb_secret('db_password')}}, 'schema_optional') +``` + +Do not split host and port into multiple secrets. diff --git a/.agents/skills/tinybird/rules/tests.md b/.agents/skills/tinybird/rules/tests.md new file mode 100644 index 00000000000..823a0d5f198 --- /dev/null +++ b/.agents/skills/tinybird/rules/tests.md @@ -0,0 +1,38 @@ +# Tests + +- Test file name must match the pipe name. +- Scenario names must be unique inside a test file. +- Parameters format: `param1=value1¶m2=value2`. +- Preserve case and formatting when user provides parameters. +- If no parameters, create a single test with empty parameters. +- Use fixture data for expected results; do not query endpoints or SQL to infer data. +- Before creating tests, analyze fixture files used by the endpoint tables. +- `expected_result` should always be an empty string; the tool fills it. +- Only create tests when explicitly requested (e.g. "Create tests for this endpoint"). +- If asked to "test" or "call" an endpoint, use `tb endpoint data` instead of creating tests. + +Test format: + +``` +- name: kpis_single_day + description: Test hourly granularity for a single day + parameters: date_from=2024-01-01&date_to=2024-01-01 + expected_result: '' +``` + +## Fixture Data + +Fixtures live under `/fixtures` and provide sample data for testing. + +- Name fixture files to match the Data Source they populate: `fixtures/.ndjson` or `.csv`. +- Load fixtures into a local or branch environment with `tb datasource append --file fixtures/.ndjson`. +- Design fixture data to cover the scenarios your tests need (edge cases, date ranges, different parameter values). +- Keep fixtures small and deterministic. They should be committed to version control. + +## Running Tests + +``` +tb test run # Run all tests +tb test run tests/my_endpoint # Run specific test file +tb test update tests/my_endpoint # Update expected results from current output +``` diff --git a/.claude/skills/tinybird b/.claude/skills/tinybird new file mode 120000 index 00000000000..1042ef00243 --- /dev/null +++ b/.claude/skills/tinybird @@ -0,0 +1 @@ +../../.agents/skills/tinybird \ No newline at end of file diff --git a/.claude/skills/tinybird-cli-guidelines b/.claude/skills/tinybird-cli-guidelines new file mode 120000 index 00000000000..7cb9a1d6244 --- /dev/null +++ b/.claude/skills/tinybird-cli-guidelines @@ -0,0 +1 @@ +../../.agents/skills/tinybird-cli-guidelines \ No newline at end of file diff --git a/.github/actions/setup-node-pnpm/action.yml b/.github/actions/setup-node-pnpm/action.yml new file mode 100644 index 00000000000..95b1b534d0d --- /dev/null +++ b/.github/actions/setup-node-pnpm/action.yml @@ -0,0 +1,57 @@ +name: Set up Node and pnpm +description: Install pnpm and Node, then install workspace dependencies. + +inputs: + node-version: + description: Node version to install. + required: true + install: + description: Run `pnpm install`. Set to 'false' for jobs that only need the toolchain on PATH. + default: 'true' + install-args: + description: >- + Extra arguments appended to `pnpm install --frozen-lockfile`, e.g. + `--filter @tryghost/e2e...`, `--force`, `--prod`, `--ignore-scripts`. + Word-split, so quote nothing here that a shell would have to strip. + default: '' + trust-lockfile: + description: >- + Pass `--trust-lockfile`, which skips re-applying the supply-chain policies in + pnpm-workspace.yaml (minimumReleaseAge, trustPolicy, tarball-URL binding, tarball + integrity) to every lockfile entry. pnpm runs that pass on every install and + `--filter` does not narrow it, so it is pure duplicated work — but only once some + earlier job in the same run has verified this exact lockfile. Defaults to off so a + new workflow verifies unless it opts out. + default: 'false' + store-cache: + description: Let setup-node restore and save the pnpm store. + default: 'true' + +runs: + using: composite + steps: + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + env: + FORCE_COLOR: 0 + with: + node-version: ${{ inputs.node-version }} + cache: ${{ inputs.store-cache == 'true' && 'pnpm' || '' }} + + - name: Install dependencies + if: inputs.install == 'true' + shell: bash + # Args arrive as env rather than ${{ }} so nothing job-controlled + # (a matrix value, a job output) is interpolated into the script. + env: + INSTALL_ARGS: ${{ inputs.install-args }} + TRUST_LOCKFILE: ${{ inputs.trust-lockfile }} + run: | + args=(--frozen-lockfile) + if [ "$TRUST_LOCKFILE" = "true" ]; then + args+=(--trust-lockfile) + fi + # shellcheck disable=SC2206 # deliberate word split: install-args is an argument list + args+=($INSTALL_ARGS) + pnpm install "${args[@]}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3175df20e8c..c2941317350 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,17 +78,27 @@ jobs: echo "GITHUB_EVENT_NAME: ${{ github.event_name }}" echo "GITHUB_CONTEXT: ${{ toJson(github.event) }}" - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + # pnpm's cache dir, not the store dir that setup-node's `cache: pnpm` + # already handles. On a hit `pnpm install` reuses the cached verdict and + # skips verification entirely; on a miss it checks all ~4.4k lockfile + # entries (~7s, and ~670MB of registry metadata pulled into the cache + # dir) and the post-step saves the new verdict. + - name: Restore pnpm lockfile verification cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/pnpm/lockfile-verified.jsonl + key: pnpm-lockfile-verified-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml') }} + + # The only job that verifies the lockfile against the supply-chain + # policies in pnpm-workspace.yaml (minimumReleaseAge, trustPolicy, + # tarball-URL binding, tarball integrity). pnpm re-runs that whole pass + # on every install and --filter does not narrow it, so every other job — + # all of which need this one and install the same lockfile — sets + # trust-lockfile to skip the redundant re-verification. + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile --ignore-scripts + install-args: --ignore-scripts # Replaced nrwl/nx-set-shas, which verified each candidate commit over the # API and hid the errors — see scripts/nx-set-shas.js. @@ -328,18 +338,13 @@ jobs: - name: Fetch main branch run: git fetch --no-tags origin main - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - # The script's only runtime dep is semver, so install @internal/scripts - # alone rather than the whole workspace — ~0.5s and a single package. - - name: Install scripts dependencies - run: pnpm install --frozen-lockfile --filter @internal/scripts --prod --ignore-scripts + trust-lockfile: 'true' + # The script's only runtime dep is semver, so install @internal/scripts + # alone rather than the whole workspace — ~0.5s and a single package. + install-args: --filter @internal/scripts --prod --ignore-scripts - name: Check app version bump env: @@ -386,16 +391,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 1000 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: @@ -447,11 +446,12 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} + # Only needs pnpm and node on PATH for the checker scripts. + install: 'false' + store-cache: 'false' - name: Check internal package golden path run: node scripts/check-internal-packages.js @@ -468,14 +468,11 @@ jobs: || needs.job_setup.outputs.changed_i18n_apps == 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile --filter @tryghost/i18n... --ignore-scripts + trust-lockfile: 'true' + install-args: --filter @tryghost/i18n... --ignore-scripts - name: Run i18n tests run: pnpm --filter @tryghost/i18n test @@ -494,14 +491,10 @@ jobs: COVERAGE: ${{ needs.job_setup.outputs.coverage_enabled }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' - run: pnpm nx run ghost-admin:test env: @@ -542,18 +535,13 @@ jobs: # Boot activates the default theme, which is a submodule submodules: true - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - # --force for the same reason as job_unit-tests: better-sqlite3 is an - # optionalDependency and boot uses it for the development database. - run: pnpm install --frozen-lockfile --force + trust-lockfile: 'true' + # --force for the same reason as job_unit-tests: better-sqlite3 is an + # optionalDependency and boot uses it for the development database. + install-args: --force - name: Install hyperfine uses: ./.github/actions/install-hyperfine @@ -685,20 +673,15 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 1000 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ matrix.node }} - cache: pnpm - - - name: Install dependencies - # better-sqlite3 is an optionalDependency. Without --force, pnpm may skip - # installing/linking it when restoring from a cached store. --force - # ensures all optional deps are installed regardless. - # (ghost core's test:unit job requires better-sqlite3) - run: pnpm install --frozen-lockfile --force + trust-lockfile: 'true' + # better-sqlite3 is an optionalDependency. Without --force, pnpm may skip + # installing/linking it when restoring from a cached store. --force + # ensures all optional deps are installed regardless. + # (ghost core's test:unit job requires better-sqlite3) + install-args: --force - name: Set timezone (non-UTC) uses: szenius/set-timezone@1f9716b0f7120e344f0c62bb7b1ee98819aefd42 # v2.0 @@ -751,7 +734,9 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} job_acceptance-tests: - runs-on: ubuntu-latest + # Private copies of this repo get 2-core runners, where vitest's DB suite + # falls to a single worker; the public repo's runner is already 4-core. + runs-on: ${{ (github.repository_owner == 'TryGhost' && github.repository != 'TryGhost/Ghost') && 'ubuntu-latest-4-cores' || 'ubuntu-latest' }} needs: [job_setup] if: needs.job_setup.outputs.is_tag == 'true' || needs.job_setup.outputs.changed_core == 'true' services: @@ -785,16 +770,10 @@ jobs: name: Acceptance tests (Node ${{ needs.job_setup.outputs.node_version }}, mysql8) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ needs.job_setup.outputs.node_version }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' - name: Set timezone (non-UTC) uses: szenius/set-timezone@1f9716b0f7120e344f0c62bb7b1ee98819aefd42 # v2.0 @@ -891,16 +870,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: submodules: true - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ needs.job_setup.outputs.node_version }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' - name: Set env vars (MySQL) run: | @@ -932,7 +905,9 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} job_apps_acceptance-tests: - runs-on: ubuntu-latest + # Admin is by far the largest suite here and is CPU-bound; the other apps + # are not starved on the default runner. See job_acceptance-tests above. + runs-on: ${{ (github.repository_owner == 'TryGhost' && github.repository != 'TryGhost/Ghost' && matrix.app == '@tryghost/admin') && 'ubuntu-latest-4-cores' || 'ubuntu-latest' }} needs: [job_setup] if: needs.job_setup.outputs.affected_playwright_projects != '[]' name: App Playwright Acceptance Tests @@ -944,20 +919,15 @@ jobs: CI: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - # Each matrix leg only Playwright-tests one app, so scope the install to that - # app's dependency subgraph instead of the whole monorepo (which drags in - # ghost/core, ghost-admin and unrelated apps). nx and the app's workspace deps - # are still installed; the nx project name matches the package name. - run: pnpm install --frozen-lockfile --filter ${{ matrix.app }}... + trust-lockfile: 'true' + # Each matrix leg only Playwright-tests one app, so scope the install to that + # app's dependency subgraph instead of the whole monorepo (which drags in + # ghost/core, ghost-admin and unrelated apps). nx and the app's workspace deps + # are still installed; the nx project name matches the package name. + install-args: --filter ${{ matrix.app }}... - name: Setup Playwright uses: ./.github/actions/setup-playwright @@ -1144,16 +1114,10 @@ jobs: name: Stripe fixture checks steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' # Needs no Ghost, no Docker and no browser, so it does not belong in the e2e # matrix that waits on the image. Run through nx so the target pulls in the @@ -1180,19 +1144,13 @@ jobs: with: persist-credentials: false - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - # Admin's nx build fans out across the whole frontend graph (ghost-admin, - # admin-x-*, shade, koenig-lexical) via nx dependsOn rather than package - # deps, so a filtered install would miss pieces — install the full workspace. - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' + # Admin's nx build fans out across the whole frontend graph (ghost-admin, + # admin-x-*, shade, koenig-lexical) via nx dependsOn rather than package + # deps, so a filtered install would miss pieces — install the full workspace. - name: Build admin # IS_SHIPPING enables the Sentry vite plugin in koenig-lexical: it @@ -1269,18 +1227,13 @@ jobs: # archive — without them the tarball ships empty theme dirs. submodules: true - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - # pack only needs ghost and its dependency subgraph (dev deps included, to - # build the prod closure below) — not the whole monorepo. - run: pnpm install --frozen-lockfile --filter "ghost..." + trust-lockfile: 'true' + # pack only needs ghost and its dependency subgraph (dev deps included, to + # build the prod closure below) — not the whole monorepo. + install-args: --filter ghost... - name: Verify tag matches package.json if: startsWith(github.ref, 'refs/tags/v') @@ -1930,17 +1883,10 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' - name: Build public apps for E2E run: pnpm --filter @tryghost/e2e build:apps @@ -1970,6 +1916,20 @@ jobs: # Inherits the run_e2e gate transitively via job_docker (which builds the # ghost-e2e image). if: needs.job_docker.result == 'success' + permissions: + contents: read + packages: read # read the internal tinybird-local-slim package + env: + # Use the distilled Tinybird image instead of upstream: it needs several GB + # less runner disk, which is what keeps the analytics shards inside the disk + # budget. See docker/tinybird-local-slim/README.md. + # + # Its GHCR package is internal (the upstream licence only permits + # distribution within our own organization), so it is unreadable from a + # cross-repo PR's scoped token — those runs stay on the upstream image. + # infra-up.sh falls back on a failed pull regardless, so an unexpected + # permission gap degrades rather than breaks. + GHOST_E2E_TINYBIRD_SLIM: ${{ github.repository_owner == 'TryGhost' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} strategy: fail-fast: true matrix: @@ -2041,6 +2001,16 @@ jobs: - name: Setup Docker Registry Mirrors uses: ./.github/actions/setup-docker-registry-mirrors + - name: Log in to GitHub Container Registry + # Needed to read the internal tinybird-local-slim package. + if: matrix.analytics == 'true' && env.GHOST_E2E_TINYBIRD_SLIM == 'true' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + continue-on-error: true + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull Tinybird CLI image id: pull_tb_cli if: matrix.analytics == 'true' && needs.job_setup.outputs.changed_tb_cli != 'true' @@ -2077,17 +2047,14 @@ jobs: image-tags: ${{ needs.job_docker.outputs.image-e2e-tags }} artifact-name: docker-image-e2e - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - # The Playwright container runs against the host's bind-mounted node_modules, - # but only needs @tryghost/e2e's dependency subgraph — not the whole monorepo - # (admin, apps, ghost/core). Scope the install to cut shard setup time. - run: pnpm install --frozen-lockfile --filter @tryghost/e2e... + trust-lockfile: 'true' + # The Playwright container runs against the host's bind-mounted node_modules, + # but only needs @tryghost/e2e's dependency subgraph — not the whole monorepo + # (admin, apps, ghost/core). Scope the install to cut shard setup time. + install-args: --filter @tryghost/e2e... - name: Report Analytics runner disk usage if: matrix.analytics == 'true' @@ -2160,14 +2127,10 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' - name: Download blob reports from GitHub Actions Artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -2348,15 +2311,10 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile + trust-lockfile: 'true' - name: Determine release version id: release diff --git a/.github/workflows/koenig-demo.yml b/.github/workflows/koenig-demo.yml index 517581c4569..71ca1d442f1 100644 --- a/.github/workflows/koenig-demo.yml +++ b/.github/workflows/koenig-demo.yml @@ -34,15 +34,10 @@ jobs: - name: Checkout repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile --filter @tryghost/koenig-lexical... + install-args: --filter @tryghost/koenig-lexical... - name: Build demo run: pnpm nx run @tryghost/koenig-lexical:build:demo diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index f48ee9b885b..b228371ccd8 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -67,15 +67,11 @@ jobs: # the token, so don't leave it in .git/config for later steps. persist-credentials: false - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - package-manager-cache: false - - - name: Install dependencies - run: pnpm install --frozen-lockfile + # Publishing runs against the public registry with a cold store on purpose. + store-cache: 'false' - name: Configure .npmrc run: | diff --git a/.github/workflows/publish-tinybird-local-slim.yml b/.github/workflows/publish-tinybird-local-slim.yml new file mode 100644 index 00000000000..d777fdfb6d5 --- /dev/null +++ b/.github/workflows/publish-tinybird-local-slim.yml @@ -0,0 +1,100 @@ +name: Publish tinybird-local-slim Image + +# Builds a distilled tinybird-local image for use as the Tinybird service in +# CI/E2E: ~0.7GB pulled and ~2.4GB on disk, against ~2.1GB and ~6.9GB for +# upstream, which is what keeps the analytics E2E jobs inside the runner disk +# budget. See docker/tinybird-local-slim/README.md. +# +# The upstream image is proprietary (Tinybird License, Self-Managed). It permits +# derivative works but only allows distributing them within our own organization, +# so the GHCR package must stay internal — never public. + +on: + workflow_dispatch: # Manual trigger from GitHub UI or CLI (e.g. after a digest bump) + push: + branches: [main] + paths: + - 'docker/tinybird-local-slim/**' + - '.github/workflows/publish-tinybird-local-slim.yml' + # The upstream digest this image is distilled from lives here. + - 'compose.dev.analytics.yaml' + +permissions: + contents: read + packages: write + +jobs: + publish: + name: Build and push tinybird-local-slim to GHCR + runs-on: ubuntu-latest + # Publish from main, or from any branch via a manual dispatch (for PR testing). + if: github.repository == 'TryGhost/Ghost' && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') + concurrency: + group: publish-tinybird-local-slim-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Free up runner disk space + # The upstream base image on its own is larger than the free space a stock + # runner has left after its preinstalled toolchains. + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup + df -h / + + - name: Resolve upstream image + id: upstream + # compose.dev.analytics.yaml is the single source of truth for the digest, + # so the published image can never drift from what local dev runs. + run: | + ref=$(grep -oE 'tinybirdco/tinybird-local:[^ ]+' compose.dev.analytics.yaml) + echo "Upstream image: $ref" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4 + + - name: Login to GHCR + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + file: docker/tinybird-local-slim/Dockerfile + build-args: TINYBIRD_LOCAL_REF=${{ steps.upstream.outputs.ref }} + platforms: linux/amd64 # E2E CI runners are amd64; upstream also publishes arm64 + provenance: false # keep a clean single-platform manifest (no attestation entries) + load: true + tags: tinybird-local-slim:verify + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Verify runtime config matches upstream + # The flatten discards upstream's image config, so the Dockerfile restates + # it by hand. Catch an upstream release that changes it before publishing. + run: | + docker pull --platform linux/amd64 "${{ steps.upstream.outputs.ref }}" + bash docker/tinybird-local-slim/verify-config.sh \ + "${{ steps.upstream.outputs.ref }}" tinybird-local-slim:verify + + - name: Push + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + file: docker/tinybird-local-slim/Dockerfile + build-args: TINYBIRD_LOCAL_REF=${{ steps.upstream.outputs.ref }} + platforms: linux/amd64 + provenance: false + push: true + # Always tag with the immutable commit SHA (PR-test runs reference this); + # only move :latest on main. Empty lines are ignored by the action. + tags: | + ghcr.io/tryghost/tinybird-local-slim:${{ github.sha }} + ${{ github.ref == 'refs/heads/main' && 'ghcr.io/tryghost/tinybird-local-slim:latest' || '' }} + cache-from: type=gha diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 14a24979e84..0bbf7914053 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,16 +62,9 @@ jobs: # Ghost only and can't authenticate against Casper/Source over SSH - run: git submodule update --init - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - env: - FORCE_COLOR: 0 + - uses: ./.github/actions/setup-node-pnpm with: node-version: ${{ env.NODE_VERSION }} - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile - name: Set up Git run: | diff --git a/apps/activitypub/src/components/layout/header/header.tsx b/apps/activitypub/src/components/layout/header/header.tsx index 9d53b378184..e3a4135c0cd 100644 --- a/apps/activitypub/src/components/layout/header/header.tsx +++ b/apps/activitypub/src/components/layout/header/header.tsx @@ -69,7 +69,10 @@ const Header: React.FC = ({ onToggleMobileSidebar, showBorder = tru {!backActive && } ) : ( -
+
diff --git a/apps/activitypub/src/components/layout/host-context.tsx b/apps/activitypub/src/components/layout/host-context.tsx new file mode 100644 index 00000000000..b4cbe0c8b1d --- /dev/null +++ b/apps/activitypub/src/components/layout/host-context.tsx @@ -0,0 +1,13 @@ +import { createContext, useContext } from 'react'; + +interface ActivityPubHostLayout { + contentClassName?: string; + contentGutter?: string; +} + +// The embedding shell supplies layout dimensions without coupling Network to Admin. +// Standalone Network has no host and keeps its existing layout. +const ActivityPubHostLayoutContext = createContext(undefined); + +export const ActivityPubHostLayoutProvider = ActivityPubHostLayoutContext.Provider; +export const useActivityPubHostLayout = () => useContext(ActivityPubHostLayoutContext); diff --git a/apps/activitypub/src/components/layout/layout.tsx b/apps/activitypub/src/components/layout/layout.tsx index ffe777ade0e..284f07afea6 100644 --- a/apps/activitypub/src/components/layout/layout.tsx +++ b/apps/activitypub/src/components/layout/layout.tsx @@ -4,13 +4,21 @@ import Onboarding, { useOnboardingStatus } from './onboarding'; import React, { useRef, useState } from 'react'; import Sidebar from './sidebar'; import { Navigate, ScrollRestoration } from '@tryghost/admin-x-framework'; +import { cn } from '@tryghost/shade/utils'; +import { useActivityPubHostLayout } from './host-context'; import { useAppBasePath } from '@src/hooks/use-app-base-path'; import { useCurrentPage } from '@src/hooks/use-current-page'; import { useCurrentUser } from '@tryghost/admin-x-framework/api/current-user'; import { useKeyboardShortcuts } from '@hooks/use-keyboard-shortcuts'; import { useTopicsForUser } from '@src/hooks/use-activity-pub-queries'; -const Layout: React.FC> = ({ children, ...props }) => { +const Layout: React.FC> = ({ + children, + className, + style, + ...props +}) => { + const hostLayout = useActivityPubHostLayout(); const { isOnboarded } = useOnboardingStatus(); const basePath = useAppBasePath(); const { data: currentUser, isLoading } = useCurrentUser(); @@ -46,7 +54,15 @@ const Layout: React.FC> = ({ children, ...p data-scrollable-container > -
+
{isOnboarded ? ( <>
@@ -57,7 +73,7 @@ const Layout: React.FC> = ({ children, ...p } onToggleMobileSidebar={toggleMobileSidebar} /> -
{children}
+
{children}
= ({ isMobileSidebarOpen }) => { return (
diff --git a/apps/activitypub/src/index.tsx b/apps/activitypub/src/index.tsx index 7d22d9b44c3..a2c166eef7d 100644 --- a/apps/activitypub/src/index.tsx +++ b/apps/activitypub/src/index.tsx @@ -5,3 +5,5 @@ export { default as AdminXApp } from './app'; export { routes } from './routes'; export { FeatureFlagsProvider } from './lib/feature-flags'; export { useNotificationsCountForUser } from './hooks/use-activity-pub-queries'; + +export { ActivityPubHostLayoutProvider } from './components/layout/host-context'; diff --git a/apps/admin-x-framework/src/api/member-custom-fields.ts b/apps/admin-x-framework/src/api/member-custom-fields.ts index 22741bb9107..ab2c2e7710d 100644 --- a/apps/admin-x-framework/src/api/member-custom-fields.ts +++ b/apps/admin-x-framework/src/api/member-custom-fields.ts @@ -1,13 +1,15 @@ import { + FIELD_TYPES, FIELD_TYPE_IDS, subFieldsOf, + type FieldKind, type FieldType, type PartsOf, } from '@tryghost/custom-field-types'; import { csvColumnsForField } from '@tryghost/custom-field-types/csv'; import { Meta, createMutation, createQuery } from '../utils/api/hooks'; -// Re-exported so the import mapping can recognise a custom_fields.* column (same reason +// Re-exported so the import mapping can recognize a custom_fields.* column (same reason // as the re-exports below). export { isCustomFieldColumn } from '@tryghost/custom-field-types/csv'; @@ -16,6 +18,8 @@ export { isCustomFieldColumn } from '@tryghost/custom-field-types/csv'; // catalog package — the framework is their surface for everything custom-fields. export type { Address as MemberCustomFieldAddress } from '@tryghost/custom-field-types'; export { FIELD_TYPES as MEMBER_CUSTOM_FIELD_TYPES } from '@tryghost/custom-field-types'; +export { FIELD_KINDS as MEMBER_CUSTOM_FIELD_KINDS } from '@tryghost/custom-field-types'; +export type { FieldKind as MemberCustomFieldKind } from '@tryghost/custom-field-types'; export type MemberCustomField = { // Fields are addressed by their immutable key; the DB id is never exposed. @@ -135,7 +139,7 @@ export type MemberCustomFieldCsvColumn = { /** * The CSV import mapping targets for a set of custom fields: one per column the export - * writes, labelled for the field (and sub-field, for a composite). Column names come from + * writes, labeled for the field (and sub-field, for a composite). Column names come from * the shared codec the exporter writes and the importer reads, so a target is exactly a * round-tripping column rather than one hand-kept in sync. */ @@ -281,6 +285,8 @@ export const formatMemberCustomFieldValue = (type: FieldType, value: unknown): s .join(', '); }; +export const memberCustomFieldKind = (type: FieldType): FieldKind => FIELD_TYPES[type].kind; + export interface MemberCustomFieldsResponseType { meta?: Meta; members_custom_fields: MemberCustomField[]; diff --git a/apps/admin/src/analytics/analytics.acceptance.test.tsx b/apps/admin/src/analytics/analytics.acceptance.test.tsx index 6be77376850..23ef7eb7853 100644 --- a/apps/admin/src/analytics/analytics.acceptance.test.tsx +++ b/apps/admin/src/analytics/analytics.acceptance.test.tsx @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { page } from 'vitest/browser'; import { TINYBIRD_SITE_UUID, @@ -157,6 +158,24 @@ describe('Analytics overview', () => { await expect.element(analyticsScreen.activeVisitors()).toHaveTextContent('12 online'); }); + it('uses Admin 7 typography in the portalled trend tooltip', async () => { + seedAnalyticsWorld(); + seedTopPostsViews(); + await renderAdminApp('/analytics', { + labs: { admin7PageChrome: true }, + boot: webAnalyticsBootOverrides(), + }); + await expect.element(analyticsScreen.membersValue()).toHaveTextContent('175'); + await expect.poll(() => document.querySelector('#root .admin7')).not.toBeNull(); + await analyticsScreen.membersCard().getByTestId('kpi-card-header-diff').hover(); + const tooltip = page.getByRole('tooltip'); + await expect.element(tooltip).toHaveTextContent(/trending/); + expect(tooltip.element().closest('#root')).toBeNull(); + await expect + .poll(() => getComputedStyle(tooltip.element()).fontFamily) + .toContain('Inter Admin 7'); + }); + it('re-queries Tinybird when the date range changes', async () => { const { kpisApi } = seedAnalyticsWorld(); seedTopPostsViews(); diff --git a/apps/admin/src/analytics/components/layout/main-layout.tsx b/apps/admin/src/analytics/components/layout/main-layout.tsx index 3fb22fd0961..e08538e515a 100644 --- a/apps/admin/src/analytics/components/layout/main-layout.tsx +++ b/apps/admin/src/analytics/components/layout/main-layout.tsx @@ -4,7 +4,9 @@ const MainLayout: React.FC> = ({ children, return (
-
{children}
+
+ {children} +
); diff --git a/apps/admin/src/analytics/views/stats/layout/stats-header.tsx b/apps/admin/src/analytics/views/stats/layout/stats-header.tsx index 90f1355d30f..8c27d62da80 100644 --- a/apps/admin/src/analytics/views/stats/layout/stats-header.tsx +++ b/apps/admin/src/analytics/views/stats/layout/stats-header.tsx @@ -29,9 +29,9 @@ const StatsHeader: React.FC = ({ children }) => { return ( <> -
+

{ return ( -
-
{children}
+
+
{children}
); diff --git a/apps/admin/src/app-root.tsx b/apps/admin/src/app-root.tsx index d87ff4984fe..54d8ffa13c8 100644 --- a/apps/admin/src/app-root.tsx +++ b/apps/admin/src/app-root.tsx @@ -15,7 +15,7 @@ function ThemedAdminApp() { const { resolvedTheme } = useThemeContext(); return ( - + ); diff --git a/apps/admin/src/assets/fonts/inter-admin-7/InterVariable-Italic.woff2 b/apps/admin/src/assets/fonts/inter-admin-7/InterVariable-Italic.woff2 new file mode 100644 index 00000000000..b3530f3f526 Binary files /dev/null and b/apps/admin/src/assets/fonts/inter-admin-7/InterVariable-Italic.woff2 differ diff --git a/apps/admin/src/assets/fonts/inter-admin-7/InterVariable.woff2 b/apps/admin/src/assets/fonts/inter-admin-7/InterVariable.woff2 new file mode 100644 index 00000000000..5a8d3e72ad7 Binary files /dev/null and b/apps/admin/src/assets/fonts/inter-admin-7/InterVariable.woff2 differ diff --git a/apps/admin/src/assets/fonts/inter-admin-7/fonts.css b/apps/admin/src/assets/fonts/inter-admin-7/fonts.css new file mode 100644 index 00000000000..d3dffb54ee8 --- /dev/null +++ b/apps/admin/src/assets/fonts/inter-admin-7/fonts.css @@ -0,0 +1,17 @@ +/* Unmodified Inter v4.1 variable fonts. */ + +@font-face { + font-family: 'Inter Admin 7'; + font-style: normal; + font-display: swap; + font-weight: 100 900; + src: url('./InterVariable.woff2') format('woff2'); +} + +@font-face { + font-family: 'Inter Admin 7'; + font-style: italic; + font-display: swap; + font-weight: 100 900; + src: url('./InterVariable-Italic.woff2') format('woff2'); +} diff --git a/apps/admin/src/comments/comment-fields.test.ts b/apps/admin/src/comments/comment-fields.test.ts index 6d628e5c086..216cfc93fe9 100644 --- a/apps/admin/src/comments/comment-fields.test.ts +++ b/apps/admin/src/comments/comment-fields.test.ts @@ -1,5 +1,7 @@ import nql from '@tryghost/nql-lang'; -import { commentFields } from '@/comments/comment-fields'; +import { COMMENT_FIELD_CATALOG } from '@/comments/comment-filter-catalog'; + +const commentFields = COMMENT_FIELD_CATALOG; import { describe, expect, it } from 'vitest'; import type { CodecContext, FilterPredicate } from '@/shared/filters'; diff --git a/apps/admin/src/comments/comment-fields.ts b/apps/admin/src/comments/comment-fields.ts index d0a722765ad..7b4c1a87d38 100644 --- a/apps/admin/src/comments/comment-fields.ts +++ b/apps/admin/src/comments/comment-fields.ts @@ -1,134 +1,80 @@ -import { - DATE_FILTER_OPERATORS, - DEFAULT_DATE_OPERATOR, - type FilterCodec, - dateCodec, - defineFields, - extractComparator, - scalarCodec, - textCodec, - withPastRelativeOperator, -} from '@/shared/filters'; +import { PAST_TIMESTAMP_OPERATORS, columnAddressing } from '@/shared/filters'; +import type { FieldDescriptor } from '@/shared/filters'; -const reportedCodec: FilterCodec = { - parse(node, ctx) { - const comparator = extractComparator(node as Record); - - if (!comparator || comparator.field !== 'count.reports') { - return null; - } - - if (comparator.operator === '$eq' && comparator.value === 0) { - return { - field: ctx.key, - operator: 'is', - values: ['false'], - }; - } - - if (comparator.operator === '$gt' && comparator.value === 0) { - return { - field: ctx.key, - operator: 'is', - values: ['true'], - }; - } - - return null; - }, - serialize(predicate) { - const value = predicate.values[0]; - - if (predicate.operator !== 'is') { - return null; - } - - if (value === 'true') { - return ['count.reports:>0']; - } - - if (value === 'false') { - return ['count.reports:0']; - } - - return null; - }, -}; - -export const commentFields = defineFields({ - status: { +const COMMENT_FIELDS: FieldDescriptor[] = [ + { + key: 'status', + icon: 'circle', + type: 'scalar', operators: ['is'], - ui: { - label: 'Status', - type: 'select', - searchable: false, - hideOperatorSelect: true, - }, options: [ { value: 'published', label: 'Published' }, { value: 'hidden', label: 'Hidden' }, ], - codec: scalarCodec(), + ui: { label: 'Status', searchable: false, hideOperatorSelect: true }, }, - created_at: withPastRelativeOperator({ - operators: DATE_FILTER_OPERATORS, - ui: { - label: 'Date', - defaultOperator: DEFAULT_DATE_OPERATOR, - type: 'date', - }, - codec: dateCodec(), - }), - body: { + { + key: 'created_at', + icon: 'calendar', + type: 'timestamp', + operators: PAST_TIMESTAMP_OPERATORS, + ui: { label: 'Date' }, + }, + { + key: 'body', + icon: 'message-text', + type: 'text', operators: ['contains', 'does-not-contain'], + addressing: columnAddressing({ field: 'html' }), parseKeys: ['html'], ui: { label: 'Text', - type: 'text', placeholder: 'Search comment text...', - defaultOperator: 'contains', className: 'w-full max-w-48', popoverContentClassName: 'w-full max-w-48', }, - codec: textCodec({ field: 'html' }), }, - post: { + { + key: 'post', + icon: 'file-text', + type: 'scalar', operators: ['is', 'is-not'], + addressing: columnAddressing({ field: 'post_id' }), parseKeys: ['post_id'], ui: { label: 'Post', - type: 'select', searchable: true, className: 'w-full max-w-80', popoverContentClassName: 'w-full max-w-[calc(100vw-32px)] max-w-80', }, - codec: scalarCodec({ field: 'post_id' }), }, - author: { + { + key: 'author', + icon: 'person', + type: 'scalar', operators: ['is', 'is-not'], + addressing: columnAddressing({ field: 'member_id' }), parseKeys: ['member_id'], ui: { label: 'Author', - type: 'select', searchable: true, className: 'w-80', popoverContentClassName: 'w-80', }, - codec: scalarCodec({ field: 'member_id' }), }, - reported: { - operators: ['is'], + { + key: 'reported', + icon: 'flag', + type: 'count', + valueConfig: { threshold: 0, absentForm: 'equals' }, + addressing: columnAddressing({ field: 'count.reports' }), parseKeys: ['count.reports'], - ui: { - label: 'Reported', - type: 'select', - searchable: false, - hideOperatorSelect: true, - }, options: [ { value: 'true', label: 'Yes' }, { value: 'false', label: 'No' }, ], - codec: reportedCodec, + ui: { label: 'Reported', type: 'select', searchable: false, hideOperatorSelect: true }, }, -}); +]; + +export { COMMENT_FIELDS }; diff --git a/apps/admin/src/comments/comment-filter-catalog.ts b/apps/admin/src/comments/comment-filter-catalog.ts new file mode 100644 index 00000000000..f22706e1def --- /dev/null +++ b/apps/admin/src/comments/comment-filter-catalog.ts @@ -0,0 +1,7 @@ +import { buildCatalog } from '@/shared/filters'; +import { COMMENT_FIELDS } from './comment-fields'; +import type { FilterField } from '@/shared/filters'; + +export type CommentFields = Record; + +export const COMMENT_FIELD_CATALOG: CommentFields = buildCatalog(COMMENT_FIELDS); diff --git a/apps/admin/src/comments/comment-filter-query.ts b/apps/admin/src/comments/comment-filter-query.ts index 5c84d7524ab..17839819abd 100644 --- a/apps/admin/src/comments/comment-filter-query.ts +++ b/apps/admin/src/comments/comment-filter-query.ts @@ -1,30 +1,20 @@ import { - type AstNode, type FilterPredicate, type ParsedPredicate, - dispatchSimpleNodes, getFieldKeysByType, hasFieldKey, + isPredicateEnabled as isEnabled, parseFilterToAst, - resolveField, + parseNodeToPredicates, serializePredicates, stampPredicates, } from '@/shared/filters'; -import { commentFields } from './comment-fields'; +import { COMMENT_FIELD_CATALOG } from './comment-filter-catalog'; -const TIMEZONE_SENSITIVE_COMMENT_FIELDS = getFieldKeysByType(commentFields, 'date'); +const TIMEZONE_SENSITIVE_COMMENT_FIELDS = getFieldKeysByType(COMMENT_FIELD_CATALOG, 'date'); function isPredicateEnabled(predicate: ParsedPredicate): boolean { - const resolved = resolveField(commentFields, predicate.field, 'UTC'); - return resolved?.definition.operators.includes(predicate.operator) ?? false; -} - -function parseCommentNode(node: AstNode, timezone: string): ParsedPredicate[] { - if (Array.isArray(node.$and)) { - return (node.$and as AstNode[]).flatMap((child) => parseCommentNode(child, timezone)); - } - - return dispatchSimpleNodes([node], commentFields, timezone); + return isEnabled(predicate, COMMENT_FIELD_CATALOG); } export function parseCommentFilter( @@ -37,7 +27,9 @@ export function parseCommentFilter( return []; } - return stampPredicates(parseCommentNode(ast, timezone).filter(isPredicateEnabled)); + return stampPredicates( + parseNodeToPredicates(ast, COMMENT_FIELD_CATALOG, timezone).filter(isPredicateEnabled), + ); } export function hasTimezoneSensitiveCommentFilter(filter: string | undefined): boolean { @@ -54,5 +46,9 @@ export function serializeCommentFilters( predicates: FilterPredicate[], timezone: string, ): string | undefined { - return serializePredicates(predicates.filter(isPredicateEnabled), commentFields, timezone); + return serializePredicates( + predicates.filter(isPredicateEnabled), + COMMENT_FIELD_CATALOG, + timezone, + ); } diff --git a/apps/admin/src/comments/use-comment-filter-fields.ts b/apps/admin/src/comments/use-comment-filter-fields.ts index 4646ae0a804..fd3c171d29b 100644 --- a/apps/admin/src/comments/use-comment-filter-fields.ts +++ b/apps/admin/src/comments/use-comment-filter-fields.ts @@ -1,15 +1,16 @@ -import React, { useMemo } from 'react'; +import { useMemo } from 'react'; import { DATE_OPERATOR_LABELS, + FIELD_ICONS, RELATIVE_DATE_OPERATOR_LABELS, createOperatorOptions, createRelativeDateRenderer, fieldHasRelativeOperator, getTodayInTimezone, } from '@/shared/filters'; +import type { FieldIcon } from '@/shared/filters'; import type { FilterFieldConfig, ValueSource } from '@tryghost/shade/patterns'; -import { LucideIcon } from '@tryghost/shade/utils'; -import { commentFields } from './comment-fields'; +import { COMMENT_FIELD_CATALOG } from './comment-filter-catalog'; interface UseCommentFilterFieldsOptions { postValueSource: ValueSource; @@ -24,25 +25,6 @@ const COMMENT_OPERATOR_LABELS = { ...RELATIVE_DATE_OPERATOR_LABELS, }; -function getFieldIcon(key: string) { - switch (key) { - case 'author': - return React.createElement(LucideIcon.User, { className: 'size-4' }); - case 'post': - return React.createElement(LucideIcon.FileText, { className: 'size-4' }); - case 'body': - return React.createElement(LucideIcon.MessageSquareText, { className: 'size-4' }); - case 'status': - return React.createElement(LucideIcon.Circle, { className: 'size-4' }); - case 'reported': - return React.createElement(LucideIcon.Flag, { className: 'size-4' }); - case 'created_at': - return React.createElement(LucideIcon.Calendar, { className: 'size-4' }); - default: - return undefined; - } -} - export function useCommentFilterFields({ postValueSource, memberValueSource, @@ -52,7 +34,7 @@ export function useCommentFilterFields({ const today = getTodayInTimezone(siteTimezone); return COMMENT_FIELD_ORDER.map((key) => { - const field = commentFields[key]; + const field = COMMENT_FIELD_CATALOG[key]; const dateConfig = key === 'created_at' ? { @@ -66,7 +48,7 @@ export function useCommentFilterFields({ return { key, ...field.ui, - icon: getFieldIcon(key), + icon: FIELD_ICONS[field.ui.icon as FieldIcon], operators: createOperatorOptions(field.operators, { labels: COMMENT_OPERATOR_LABELS }), ...('options' in field && field.options ? { options: field.options } : {}), ...dateConfig, diff --git a/apps/admin/src/hooks/use-theme.test.tsx b/apps/admin/src/hooks/use-theme.test.tsx index d082041d38b..ece4bfeb7ce 100644 --- a/apps/admin/src/hooks/use-theme.test.tsx +++ b/apps/admin/src/hooks/use-theme.test.tsx @@ -12,6 +12,7 @@ import type { UsersResponseType, } from '@tryghost/admin-x-framework/api/users'; import type { SetupServer } from 'msw/node'; +import * as emberBridge from '@/ember-bridge'; // Constants const USERS_API_URL = '/ghost/api/admin/users/me/'; @@ -96,6 +97,19 @@ afterEach(() => { }); describe('useTheme (standalone)', () => { + themeTest( + 'does not report a ready theme until preferences have loaded', + async ({ server, wrapper, animationFrames }) => { + mockPreferences(server, 'dark'); + const { result } = renderHook(() => useTheme(), { wrapper }); + + expect(result.current.isThemeReady).toBe(false); + await waitFor(() => expect(result.current.isThemeReady).toBe(true)); + expect(result.current.resolvedTheme).toBe('dark'); + flushAnimationFrames(animationFrames); + }, + ); + themeTest( 'applies the persisted theme with transition suppression', async ({ server, wrapper, animationFrames }) => { @@ -158,3 +172,41 @@ describe('useTheme (standalone)', () => { }, ); }); + +describe('useTheme (Ember-managed)', () => { + themeTest( + 'tracks system changes without applying the DOM theme', + async ({ server, wrapper, animationFrames }) => { + mockPreferences(server, 'system'); + const mediaQuery = Object.assign(new EventTarget(), { matches: false }) as MediaQueryList; + const mediaSpy = vi.spyOn(window, 'matchMedia').mockReturnValue(mediaQuery); + const managedSpy = vi.spyOn(emberBridge, 'isEmberThemeManaged').mockReturnValue(true); + const removeListenerSpy = vi.spyOn(mediaQuery, 'removeEventListener'); + + try { + const { result, unmount } = renderHook(() => useTheme(), { wrapper }); + await waitFor(() => expect(result.current.theme).toBe('system')); + expect(result.current.resolvedTheme).toBe('light'); + + act(() => { + mediaQuery.dispatchEvent(Object.assign(new Event('change'), { matches: true })); + }); + expect(result.current.resolvedTheme).toBe('dark'); + expect(document.documentElement.classList.contains('dark')).toBe(false); + expect(animationFrames.size).toBe(0); + + act(() => { + mediaQuery.dispatchEvent(Object.assign(new Event('change'), { matches: false })); + }); + expect(result.current.resolvedTheme).toBe('light'); + + unmount(); + expect(removeListenerSpy).toHaveBeenCalledWith('change', expect.any(Function)); + } finally { + mediaSpy.mockRestore(); + managedSpy.mockRestore(); + removeListenerSpy.mockRestore(); + } + }, + ); +}); diff --git a/apps/admin/src/hooks/use-theme.ts b/apps/admin/src/hooks/use-theme.ts index 907e7fcee42..3d692846bf5 100644 --- a/apps/admin/src/hooks/use-theme.ts +++ b/apps/admin/src/hooks/use-theme.ts @@ -38,9 +38,9 @@ function applyThemeClass(resolvedTheme: ResolvedThemeMode) { }); } -// The class-toggling and media-query effects below are only a fallback for -// running this hook standalone (no EmberBridge), so they must not fight Ember -// when it manages the DOM theme — see isEmberThemeManaged. +// Applying the DOM theme is only a fallback for running without EmberBridge. +// React still tracks system preference changes so resolvedTheme stays current +// for consumers even when Ember owns the DOM — see isEmberThemeManaged. function applyAdminTheme(mode: ThemeMode, resolvedTheme: ResolvedThemeMode) { if (!applyEmberAdminThemePreference(mode)) { applyThemeClass(resolvedTheme); @@ -63,11 +63,7 @@ export function useTheme() { const resolvedTheme: ResolvedThemeMode = theme === 'system' ? systemTheme : theme; useEffect(() => { - if ( - isEmberThemeManaged() || - typeof window === 'undefined' || - typeof window.matchMedia !== 'function' - ) { + if (typeof window === 'undefined' || typeof window.matchMedia !== 'function') { return; } @@ -144,6 +140,7 @@ export function useTheme() { return { theme, resolvedTheme, + isThemeReady: preferences !== undefined, setTheme, isSettingTheme: isEditingPreferences || isPendingTheme, } as const; diff --git a/apps/admin/src/hooks/user-preferences.test.tsx b/apps/admin/src/hooks/user-preferences.test.tsx index 1217f204d57..df654f3b12f 100644 --- a/apps/admin/src/hooks/user-preferences.test.tsx +++ b/apps/admin/src/hooks/user-preferences.test.tsx @@ -214,11 +214,11 @@ describe('useUserPreferences', () => { }); }); - queryTest('errors when invalid JSON', async ({ setup }) => { + queryTest('uses defaults when accessibility contains invalid JSON', async ({ setup }) => { const result = await setup({ accessibility: '{invalid json' }); - expect(result.current.isError).toBe(true); - expect(result.current.error).toBeInstanceOf(Error); + expect(result.current.isError).toBe(false); + expect(result.current.data).toEqual(fixtures.defaults); }); queryTest('gracefully handles invalid schema values', async ({ setup }) => { diff --git a/apps/admin/src/hooks/user-preferences.ts b/apps/admin/src/hooks/user-preferences.ts index 7989d83ec80..bef1b243b9f 100644 --- a/apps/admin/src/hooks/user-preferences.ts +++ b/apps/admin/src/hooks/user-preferences.ts @@ -79,7 +79,12 @@ const userPreferencesQueryKey = (user: User | undefined) => function parsePreferences(user: User): Preferences { const raw = user.accessibility || '{}'; - const parsedRaw: unknown = JSON.parse(raw); + let parsedRaw: unknown; + try { + parsedRaw = JSON.parse(raw); + } catch { + parsedRaw = {}; + } const parsed: Record = parsedRaw && typeof parsedRaw === 'object' && !Array.isArray(parsedRaw) ? (parsedRaw as Record) diff --git a/apps/admin/src/index.css b/apps/admin/src/index.css index 7b7bb3f7432..e67dd43e0d9 100644 --- a/apps/admin/src/index.css +++ b/apps/admin/src/index.css @@ -4,6 +4,53 @@ @source "../node_modules/@tryghost/kg-unsplash-selector/dist/**/*.js"; @import '@tryghost/shade/styles.css'; +@import './assets/fonts/inter-admin-7/fonts.css'; + +@custom-variant admin7 (&:where(.admin7, .admin7 *)); + +/* The shell owns rollout eligibility. Admin overlays mount beside #root, so + mirror typography into Shade portals and the three legacy overlay hosts. + Do not set body fonts or change component weights, sizes, or line heights. */ +.admin7, +:where(body.react-admin:has(> #root .admin7)) + > :is( + .shade.shade-admin, + #ember-basic-dropdown-wormhole, + #ember-modal-wormhole, + #ember-liquid-wormhole, + #ember-alerts-wormhole, + #ember-notifications-wormhole + ) { + --font-sans: + 'Inter Admin 7', Inter, -apple-system, BlinkMacSystemFont, avenir next, avenir, helvetica neue, + helvetica, ubuntu, roboto, noto, segoe ui, arial, sans-serif; + --font-family: var(--font-sans); + --font-feature-settings: 'dlig' 1, 'zero' 1, 'ss01' 1, 'cv05' 1; + font-family: var(--font-sans); + font-optical-sizing: none; + font-variation-settings: 'opsz' 14; + font-feature-settings: var(--font-feature-settings); +} + +/* Browser form-control defaults can reset these inherited font properties. + Keep the rule weaker than any explicit component typography choice. */ +:where( + .admin7, + body.react-admin:has(> #root .admin7) + > :is( + .shade.shade-admin, + #ember-basic-dropdown-wormhole, + #ember-modal-wormhole, + #ember-liquid-wormhole, + #ember-alerts-wormhole, + #ember-notifications-wormhole + ) + ) + :where(button, input, optgroup, select, textarea) { + font-family: inherit; + font-feature-settings: inherit; + font-variation-settings: inherit; +} /* Site custom-font families for the settings font pickers. The font-* utilities must be generated in this Tailwind lane; the font files themselves load with @@ -168,6 +215,13 @@ body.react-admin #root .shade.shade-admin { overflow: hidden; } +/* Keep the server-rendered loader visible while the React bundle boots. Once + React commits its mount marker, hide a still-detached Ember root until the + bridge relocates it into the shell. */ +body.react-admin:has(> #root [data-react-admin-mounted]) > #ember-app { + visibility: hidden; +} + /* iOS safe area handling for mobile navbar */ body.react-admin .safe-area-inset-bottom { padding-bottom: env(safe-area-inset-bottom); diff --git a/apps/admin/src/layout/admin-layout.tsx b/apps/admin/src/layout/admin-layout.tsx index 2653afe49ff..1079ce84ad9 100644 --- a/apps/admin/src/layout/admin-layout.tsx +++ b/apps/admin/src/layout/admin-layout.tsx @@ -1,12 +1,51 @@ +import { ActivityPubHostLayoutProvider } from '@tryghost/activitypub/api'; import React from 'react'; import { SidebarInset, SidebarProvider } from '@tryghost/shade/components'; import { useCurrentUser } from '@tryghost/admin-x-framework/api/current-user'; import { isContributorUser } from '@tryghost/admin-x-framework/api/users'; import { useAdminSidebarVisibility } from '@/layout/sidebar-visibility'; +import { useAdmin7 } from '@/layout/use-admin7'; +import { cn } from '@tryghost/shade/utils'; import AppSidebar from './app-sidebar'; import { MobileNavBar } from './app-sidebar/mobile-nav-bar'; import { ContributorUserMenu } from './app-sidebar/user-menu'; +const networkPageChrome = { + contentClassName: 'admin7:max-w-(--content-width)', + contentGutter: 'var(--page-gutter)', +}; + +const admin7PageChromeClassName = [ + 'admin7:[&_.max-w-page]:max-w-(--content-width)', + 'admin7:[&_[data-list-page=list-page]]:px-(--page-gutter)', + 'admin7:[&_[data-detail-page=detail-page]]:px-(--page-gutter)', + 'admin7:[&_[data-list-page=header]]:-mx-(--page-gutter)', + 'admin7:[&_[data-list-page=header]]:px-(--page-gutter)', + 'admin7:[&_[data-list-page=header]]:pt-[28px]', + 'admin7:[&_[data-detail-page=header]]:pt-[28px]', + 'admin7:[&_[data-network-header=header]]:pt-[8px]', + 'admin7:[&_[data-page-header=main]]:flex-wrap', + 'admin7:[&_[data-page-header=left]]:h-auto', + 'admin7:[&_[data-page-header=left]]:max-w-full', + 'admin7:[&_.admin-x-container-error]:bg-background', + 'admin7:[&_.gh-canvas]:max-w-(--content-width)', + 'admin7:[&_.gh-canvas]:px-(--page-gutter)', + 'admin7:[&_.gh-main-width]:max-w-(--content-width)', + 'admin7:[&_.gh-main-width]:px-(--page-gutter)', + 'admin7:[&_.gh-canvas-header]:-mx-(--page-gutter)', + 'admin7:[&_.gh-canvas-header]:px-(--page-gutter)', + 'admin7:[&_.gh-canvas-header]:pt-[28px]!', + 'admin7:[&_.gh-canvas-header]:pb-[28px]!', + 'admin7:[&_[data-view-site-preview]]:inset-y-2!', + 'admin7:[&_[data-view-site-preview]]:right-2!', + 'admin7:[&_[data-view-site-preview]]:left-0!', + 'admin7:[&_[data-view-site-preview]]:h-[calc(100%-16px)]!', + 'admin7:[&_[data-view-site-preview]]:w-[calc(100%-8px)]!', + 'admin7:[&_[data-view-site-preview]]:rounded-xl!', + 'admin7:[&_[data-view-site-preview]]:border!', + 'admin7:[&_[data-view-site-preview]]:border-[var(--border-subtle)]!', +].join(' '); + interface AdminLayoutProps { children: React.ReactNode; } @@ -15,6 +54,14 @@ export function AdminLayout({ children }: AdminLayoutProps) { const { data: currentUser } = useCurrentUser(); const sidebarVisible = useAdminSidebarVisibility(); const isContributor = currentUser && isContributorUser(currentUser); + const { + isReady: admin7Ready, + enabled: admin7Enabled, + pageChromeEnabled, + } = useAdmin7({ + hasNavigation: sidebarVisible, + isEligibleUser: !!currentUser && !isContributor, + }); // Contributors get a floating profile menu instead of the full sidebar if (isContributor) { @@ -31,12 +78,27 @@ export function AdminLayout({ children }: AdminLayoutProps) { } return ( - - {sidebarVisible && } + main]:min-w-0', + )} + open={!!currentUser && sidebarVisible} + style={ + pageChromeEnabled ? ({ '--sidebar-width': '316px' } as React.CSSProperties) : undefined + } + > + {sidebarVisible && } -
{children}
+
+ + {children} + +
diff --git a/apps/admin/src/layout/sidebar.acceptance.test.tsx b/apps/admin/src/layout/sidebar.acceptance.test.tsx index bdcf8a38be9..ae0282974bd 100644 --- a/apps/admin/src/layout/sidebar.acceptance.test.tsx +++ b/apps/admin/src/layout/sidebar.acceptance.test.tsx @@ -4,6 +4,7 @@ import type { StateBridge } from '@/ember-bridge'; import { activeThemeResponse, allowUnhandledRequests, + configResponse, currentRoute, fakeAdminEndpoint, fakeEndpoint, @@ -55,6 +56,155 @@ afterEach(() => { }); describe('Sidebar navigation', () => { + it('keeps the shell hidden until Admin 7 eligibility is known', async () => { + fakeTags([]); + let resolveConfig!: (value: ReturnType) => void; + const pendingConfig = new Promise>((resolve) => { + resolveConfig = resolve; + }); + + await renderAdminApp('/tags', { + boot: { browseConfig: { response: () => pendingConfig } }, + }); + + const getShell = () => + document.querySelector('[data-sidebar="sidebar"]')?.closest('.group\\/sidebar-wrapper'); + await expect.poll(getShell).toBeTruthy(); + const shell = getShell()!; + expect(shell).toHaveClass('invisible'); + + resolveConfig(configResponse({ labs: { admin7PageChrome: true } })); + await expect.poll(() => shell?.classList.contains('invisible')).toBe(false); + expect(shell).toHaveClass('admin7'); + }); + + it('shows the existing shell when Admin 7 config cannot be loaded', async () => { + fakeTags([]); + await renderAdminApp('/tags', { + boot: { + browseConfig: { + response: { errors: [{ message: 'Config unavailable' }] }, + responseStatus: 400, + }, + }, + }); + + const getShell = () => + document + .querySelector('[data-sidebar="sidebar"]') + ?.closest('[class~="group/sidebar-wrapper"]'); + await expect.poll(getShell).toBeTruthy(); + await expect.poll(() => getShell()?.classList.contains('invisible')).toBe(false); + expect(getShell()).not.toHaveClass('admin7'); + }); + + it('uses default preferences when accessibility JSON is malformed', async () => { + fakeTags([]); + const me = currentUserResponse(); + me.users[0].accessibility = '{invalid json'; + + await renderAdminApp('/tags', { + labs: { admin7PageChrome: true }, + boot: { browseMe: { response: me } }, + }); + + const getShell = () => + document + .querySelector('[data-sidebar="sidebar"]') + ?.closest('[class~="group/sidebar-wrapper"]'); + await expect.poll(getShell).toBeTruthy(); + await expect.poll(() => getShell()?.classList.contains('invisible')).toBe(false); + expect(getShell()).toHaveClass('admin7'); + }); + + it('uses the static Admin 7 shell without reading the saved menu visibility', async () => { + fakeTags([]); + const me = currentUserResponse(); + me.users[0].accessibility = JSON.stringify({ + navigation: { expanded: { posts: true, members: true }, menu: { visible: false } }, + nightShift: 'light', + }); + + await renderAdminApp('/tags', { + labs: { admin7PageChrome: true }, + boot: { browseMe: { response: me } }, + }); + + await expect.element(sidebarScreen.shellNav()).toBeVisible(); + await expect.poll(() => document.querySelector('.admin7')).not.toBeNull(); + expect(document.querySelector('[data-state="collapsed"]')).toBeNull(); + expect(document.querySelector('[aria-label="Hide sidebar"]')).toBeNull(); + }); + + it('applies the Admin 7 page chrome in dark mode', async () => { + fakeTags([]); + const me = currentUserResponse(); + me.users[0].accessibility = JSON.stringify({ nightShift: 'dark' }); + + await renderAdminApp('/tags', { + labs: { admin7PageChrome: true }, + boot: { browseMe: { response: me } }, + }); + + await expect.poll(() => document.documentElement.classList.contains('dark')).toBe(true); + await expect.poll(() => document.querySelector('.admin7')).not.toBeNull(); + expect(document.querySelector('[data-sidebar="sidebar"]')).not.toBeNull(); + }); + + it('applies Admin 7 typography to legacy alert and notification portals', async () => { + fakeTags([]); + await renderAdminApp('/tags', { labs: { admin7PageChrome: true } }); + await expect.poll(() => document.querySelector('.admin7')).not.toBeNull(); + + const shell = document.querySelector('.admin7')!; + const createdHosts: HTMLElement[] = []; + const hosts = ['ember-alerts-wormhole', 'ember-notifications-wormhole'].map((id) => { + const existing = document.getElementById(id); + if (existing) { + return existing; + } + const host = document.createElement('div'); + host.id = id; + document.body.appendChild(host); + createdHosts.push(host); + return host; + }); + + try { + for (const host of hosts) { + expect(getComputedStyle(host).fontFamily).toBe(getComputedStyle(shell).fontFamily); + } + } finally { + createdHosts.forEach((host) => host.remove()); + } + }); + + it('keeps the boot loader visible until React commits its mount marker', async () => { + await renderAdminApp('/site'); + + const marker = document.querySelector('[data-react-admin-mounted]')!; + const emberApp = document.getElementById('ember-app')!; + const bridgeHost = emberApp.parentElement!; + + try { + document.body.appendChild(emberApp); + expect(getComputedStyle(emberApp).visibility).toBe('hidden'); + marker.removeAttribute('data-react-admin-mounted'); + expect(getComputedStyle(emberApp).visibility).toBe('visible'); + } finally { + marker.setAttribute('data-react-admin-mounted', ''); + bridgeHost.appendChild(emberApp); + } + }); + + it('keeps the existing sidebar treatment when Admin 7 page chrome is disabled', async () => { + fakeTags([]); + await renderAdminApp('/tags', { labs: { admin7PageChrome: false } }); + + await expect.element(sidebarScreen.shellNav()).toBeVisible(); + expect(document.querySelector('.admin7')).toBeNull(); + }); + it('renders the navigation for the current user', async () => { await renderAdminApp('/site'); diff --git a/apps/admin/src/layout/use-admin7.ts b/apps/admin/src/layout/use-admin7.ts new file mode 100644 index 00000000000..d6a23a1c739 --- /dev/null +++ b/apps/admin/src/layout/use-admin7.ts @@ -0,0 +1,29 @@ +import { useLocation } from '@tryghost/admin-x-framework'; +import { useBrowseConfig } from '@tryghost/admin-x-framework/api/config'; +import { useIsMobile } from '@tryghost/shade/utils'; +import { useThemeContext } from '@/providers/theme-context'; + +interface Admin7Eligibility { + hasNavigation: boolean; + isEligibleUser: boolean; +} + +export function useAdmin7({ hasNavigation, isEligibleUser }: Admin7Eligibility) { + const { data: config, isPending: isConfigPending } = useBrowseConfig({ + refetchOnMount: false, + }); + const flagEnabled = config?.config.labs?.admin7PageChrome === true; + const isMobile = useIsMobile(); + const { isThemeReady } = useThemeContext(); + const { pathname } = useLocation(); + const isReady = + !isEligibleUser || isMobile || (!isConfigPending && (!flagEnabled || isThemeReady)); + const enabled = flagEnabled && isEligibleUser && !isMobile && isThemeReady; + const isSettings = /^\/settings(?:\/|$)/.test(pathname); + + return { + isReady, + enabled, + pageChromeEnabled: enabled && hasNavigation && !isSettings, + }; +} diff --git a/apps/admin/src/members/components/members-filters.tsx b/apps/admin/src/members/components/members-filters.tsx index e09274cdfd5..06eb2a365bf 100644 --- a/apps/admin/src/members/components/members-filters.tsx +++ b/apps/admin/src/members/components/members-filters.tsx @@ -1,3 +1,5 @@ +import { CUSTOM_FIELDS_PREFIX } from '@/members/member-fields'; +import { keyBelow } from '@/shared/filters'; import ManageViewPopover from './manage-view-popover'; import React, { useCallback, useMemo } from 'react'; import { Button } from '@tryghost/shade/components'; @@ -9,7 +11,6 @@ import { toOfferFilterDisplayValues, useMemberFilterFields, } from '@/members/use-member-filter-fields'; -import { CUSTOM_FIELDS_PREFIX } from '@/members/member-fields'; import { useBrowseSettings, useEmailTrackClicks, @@ -22,10 +23,7 @@ import { getSiteTimezone } from '@tryghost/admin-x-framework/utils/get-site-time import { useBrowseNewsletters } from '@tryghost/admin-x-framework/api/newsletters'; import { useBrowseOffers } from '@tryghost/admin-x-framework/api/offers'; import { useFeatureFlag } from '@tryghost/admin-x-framework/hooks'; -import { - useBrowseMemberCustomFields, - useBrowseMemberCustomFieldsIncludingArchived, -} from '@tryghost/admin-x-framework/api/member-custom-fields'; +import { useBrowseMemberCustomFieldsIncludingArchived } from '@tryghost/admin-x-framework/api/member-custom-fields'; import type { MemberCustomField } from '@tryghost/admin-x-framework/api/member-custom-fields'; import { useEmailPostValueSource, @@ -47,6 +45,31 @@ interface MembersFiltersProps { const EMPTY_OFFERS: typeof buildOfferOptions extends (offers: infer T) => unknown ? T : never = []; const EMPTY_CUSTOM_FIELDS: MemberCustomField[] = []; +const EMPTY_NEWSLETTERS: NonNullable< + ReturnType['data'] +>['newsletters'] = []; +const NO_KEYS: string[] = []; + +// The keys a set of filters names under a given prefix. +// +// Keyed on the keys themselves rather than on the filters holding them, because these feed the +// field catalog, and rebuilding that means rebuilding every field's codec. Editing a filter +// almost never changes which newsletters or custom fields are named, and when it doesn't, this +// hands back the identical array and the catalog is left alone. +function useReferencedKeys(filters: Filter[], prefix: string): string[] { + const signature = [ + ...new Set( + filters + .map((filter) => filter.field) + .map((field) => keyBelow(field, prefix)) + .filter((name) => name !== null), + ), + ] + .sort() + .join('\n'); + + return useMemo(() => (signature ? signature.split('\n') : NO_KEYS), [signature]); +} function mapOfferRedemptionFilters(filters: Filter[], mapValues: (values: string[]) => string[]) { return filters.map((filter) => { @@ -83,23 +106,13 @@ const MembersFilters: React.FC = ({ const emailTrackClicks = useEmailTrackClicks() === true; const siteTimezone = getSiteTimezone(settings); - const newsletters = newslettersData?.newsletters || []; + const newsletters = newslettersData?.newsletters ?? EMPTY_NEWSLETTERS; const offers = useMemo(() => offersData?.offers ?? EMPTY_OFFERS, [offersData?.offers]); const offersOptions = useMemo(() => { return buildOfferOptions(offers); }, [offers]); - const hydratedNewsletterSlugs = useMemo(() => { - return [ - ...new Set( - filters - .map((filter) => filter.field) - .filter((field) => field.startsWith('newsletters.')) - .map((field) => field.slice('newsletters.'.length)) - .filter(Boolean), - ), - ]; - }, [filters]); + const hydratedNewsletterSlugs = useReferencedKeys(filters, 'newsletters.'); const displayFilters = useMemo(() => { return mapOfferRedemptionFilters(filters, (values) => @@ -123,32 +136,33 @@ const MembersFilters: React.FC = ({ const labelValueSource = useLabelValueSource(); const { valueSource: tierValueSource, hasMultipleTiers } = useTierValueSource(); const customFieldsEnabled = useFeatureFlag('membersCustomFields'); - // The picker lists active fields — the endpoint the members page has always used. - const { data: customFieldsData } = useBrowseMemberCustomFields({ enabled: customFieldsEnabled }); - const customFields = customFieldsData?.members_custom_fields ?? EMPTY_CUSTOM_FIELDS; + // The archived-inclusive browse, fetched eagerly: this is the query the hydration gate + // in Members waits on once a filter names a custom field, and a pill reaches the URL on + // the first keystroke — if the gate finds this cache cold it unmounts the whole page to + // a spinner mid-interaction. A field can only be picked after this has answered, so + // fetching it here is what keeps that wait confined to fresh page loads. Archived + // fields ride along so a saved segment on a since-archived field still renders its + // read-only pill. + const { data: customFieldsData } = useBrowseMemberCustomFieldsIncludingArchived({ + enabled: customFieldsEnabled, + }); + const catalogCustomFields = customFieldsData?.members_custom_fields ?? EMPTY_CUSTOM_FIELDS; + // The picker offers active fields only. + const customFields = useMemo( + () => catalogCustomFields.filter((field) => field.status === 'active'), + [catalogCustomFields], + ); + const referencedCustomFieldNames = useReferencedKeys(filters, CUSTOM_FIELDS_PREFIX); const referencedCustomFieldKeys = useMemo( - () => - new Set( - filters - .map((filter) => filter.field) - .filter((field) => field.startsWith(CUSTOM_FIELDS_PREFIX)) - .map((field) => field.slice(CUSTOM_FIELDS_PREFIX.length)) - .filter(Boolean), - ), - [filters], + () => new Set(referencedCustomFieldNames), + [referencedCustomFieldNames], ); - // Only when the current filter references a custom field do we also pull the archived - // ones, so a saved segment on a since-archived field still renders its read-only pill. - // Skipped otherwise, so the common members view makes no extra request. - const { data: archivedCustomFieldsData } = useBrowseMemberCustomFieldsIncludingArchived({ - enabled: customFieldsEnabled && referencedCustomFieldKeys.size > 0, - }); const archivedCustomFields = useMemo( () => - (archivedCustomFieldsData?.members_custom_fields ?? EMPTY_CUSTOM_FIELDS) + catalogCustomFields .filter((field) => field.status === 'archived' && referencedCustomFieldKeys.has(field.key)) .map((field) => ({ key: field.key, name: field.name })), - [archivedCustomFieldsData, referencedCustomFieldKeys], + [catalogCustomFields, referencedCustomFieldKeys], ); const filterFields = useMemberFilterFields({ diff --git a/apps/admin/src/members/custom-fields/addressing.ts b/apps/admin/src/members/custom-fields/addressing.ts new file mode 100644 index 00000000000..adcf2145d86 --- /dev/null +++ b/apps/admin/src/members/custom-fields/addressing.ts @@ -0,0 +1,183 @@ +import { escapeNqlString } from '@tryghost/nql-string'; +import { + keyBelow, + PRESENCE_OPERATORS, + getCompoundChildren, + readNegatedString, + toComparator, +} from '@/shared/filters'; +import type { CompoundMatch, PresenceAddressing } from '@/shared/filters'; + +const RELATION = 'custom_fields'; +const KEY_ATTRIBUTE = `${RELATION}.key`; +const VALUE_ATTRIBUTE = `${RELATION}.value`; +const PATH_ATTRIBUTE = `${RELATION}.path`; + +export const CUSTOM_FIELD_KEY_PREFIX = 'custom_fields.'; + +export const CUSTOM_FIELD_SET_OPERATORS = PRESENCE_OPERATORS; + +function keyClause(fieldKey: string): string { + return `${KEY_ATTRIBUTE}:${escapeNqlString(fieldKey)}`; +} + +function readValues(values: unknown[]): { subfield: string; value: unknown } { + const [subfield, value] = values; + + return { + subfield: typeof subfield === 'string' ? subfield : '', + value, + }; +} + +export function customFieldAddressing(boundKey?: string): PresenceAddressing { + return { + presenceOperators: CUSTOM_FIELD_SET_OPERATORS, + + address(predicate, ctx) { + const fieldKey = boundKey ?? ctx.params.key; + const { subfield, value } = readValues(predicate.values); + + if (!fieldKey) { + return null; + } + + return { + valueKey: subfield ? `${VALUE_ATTRIBUTE}.${subfield}` : VALUE_ATTRIBUTE, + companions: [keyClause(fieldKey)], + values: [value], + }; + }, + + // The shape of these clauses is not ours to choose. The members API rewrites them into a + // single lookup over the rows holding custom field values, and it only accepts two forms: + // a lone key clause, or a key clause grouped with one path or value clause. See + // ghost/core/core/server/services/members-custom-fields/filter.ts. + // + // A minus sign inside the group also negates the whole lookup, so `key:'x'+path:-'country'` + // asks for members with no x/country value at all, not for one whose part is something + // else. Anything else is either a 400 or a quietly wrong set of members. + addressPresence(predicate, ctx) { + const fieldKey = boundKey ?? ctx.params.key; + const { subfield } = readValues(predicate.values); + + if (!fieldKey) { + return null; + } + + if (predicate.operator === 'is-set') { + return subfield + ? [`(${keyClause(fieldKey)}+${PATH_ATTRIBUTE}:${escapeNqlString(subfield)})`] + : [keyClause(fieldKey)]; + } + + return subfield + ? [`(${keyClause(fieldKey)}+${PATH_ATTRIBUTE}:-${escapeNqlString(subfield)})`] + : [`${KEY_ATTRIBUTE}:-${escapeNqlString(fieldKey)}`]; + }, + + match() { + return null; + }, + + matchCompound(node): CompoundMatch | null { + const children = getCompoundChildren(node, '$and'); + + if (!children) { + const keyValue = node[KEY_ATTRIBUTE]; + + if (typeof keyValue === 'string') { + return { + kind: 'predicate', + predicate: { + field: `${CUSTOM_FIELD_KEY_PREFIX}${keyValue}`, + operator: 'is-set', + values: ['', ''], + }, + }; + } + + const negatedKey = readNegatedString(keyValue); + + if (negatedKey !== null) { + return { + kind: 'predicate', + predicate: { + field: `${CUSTOM_FIELD_KEY_PREFIX}${negatedKey}`, + operator: 'is-not-set', + values: ['', ''], + }, + }; + } + + return null; + } + + if (children.length !== 2) { + return null; + } + + let fieldKey: string | undefined; + let valueEntry: { subfield: string; raw: unknown } | undefined; + let pathEntry: { subfield: string; negated: boolean } | undefined; + + for (const child of children) { + if (typeof child[KEY_ATTRIBUTE] === 'string') { + fieldKey = child[KEY_ATTRIBUTE]; + } + + for (const childKey of Object.keys(child)) { + if (childKey === VALUE_ATTRIBUTE) { + valueEntry = { subfield: '', raw: child[childKey] }; + } else if (keyBelow(childKey, VALUE_ATTRIBUTE)) { + valueEntry = { + subfield: keyBelow(childKey, VALUE_ATTRIBUTE) ?? '', + raw: child[childKey], + }; + } else if (childKey === PATH_ATTRIBUTE) { + const raw = child[childKey]; + const negatedPath = readNegatedString(raw); + + if (typeof raw === 'string') { + pathEntry = { subfield: raw, negated: false }; + } else if (negatedPath !== null) { + pathEntry = { subfield: negatedPath, negated: true }; + } + } + } + } + + if (!fieldKey) { + return null; + } + + if (pathEntry) { + return { + kind: 'predicate', + predicate: { + field: `${CUSTOM_FIELD_KEY_PREFIX}${fieldKey}`, + operator: pathEntry.negated ? 'is-not-set' : 'is-set', + values: [pathEntry.subfield, ''], + }, + }; + } + + if (!valueEntry) { + return null; + } + + const comparator = toComparator(valueEntry.raw); + + if (!comparator) { + return null; + } + + return { + kind: 'value', + field: `${CUSTOM_FIELD_KEY_PREFIX}${fieldKey}`, + leadingValues: [valueEntry.subfield], + comparator, + }; + }, + }; +} diff --git a/apps/admin/src/members/custom-fields/filter-fields.ts b/apps/admin/src/members/custom-fields/filter-fields.ts new file mode 100644 index 00000000000..5e554afca14 --- /dev/null +++ b/apps/admin/src/members/custom-fields/filter-fields.ts @@ -0,0 +1,56 @@ +import { CUSTOM_FIELD_SET_OPERATORS, customFieldAddressing } from './addressing'; +import { filterType } from '@/shared/filters'; +import { memberCustomFieldKind } from '@tryghost/admin-x-framework/api/member-custom-fields'; +import type { FieldDescriptor, FieldProvider, FilterTypeId } from '@/shared/filters'; +import type { + MemberCustomField, + MemberCustomFieldKind, +} from '@tryghost/admin-x-framework/api/member-custom-fields'; + +const FILTER_TYPE_FOR_KIND: Record = { + text: 'text', + date: 'plain_date', + number: 'number', + record: 'text', +}; + +export const CUSTOM_FIELD_CLAUSE = 'custom_fields.'; + +export interface CustomFieldDefinition { + key: string; + name: string; + type: MemberCustomField['type']; +} + +function filterTypeFor(type: MemberCustomField['type']): FilterTypeId { + return FILTER_TYPE_FOR_KIND[memberCustomFieldKind(type)]; +} + +export function customFieldDescriptor(definition: CustomFieldDefinition): FieldDescriptor { + const type = filterTypeFor(definition.type); + const isRecord = memberCustomFieldKind(definition.type) === 'record'; + + return { + key: `custom_fields.${definition.key}`, + icon: 'text', + type, + addressing: customFieldAddressing(definition.key), + ui: { + label: definition.name, + type: 'custom', + defaultOperator: isRecord + ? CUSTOM_FIELD_SET_OPERATORS[0] + : (filterType(type).defaultOperator ?? CUSTOM_FIELD_SET_OPERATORS[0]), + }, + } as FieldDescriptor; +} + +export function customFieldProvider( + definitions: readonly CustomFieldDefinition[] | undefined, +): FieldProvider { + return { + resolved: definitions !== undefined, + claims: [CUSTOM_FIELD_CLAUSE], + fields: (definitions ?? []).map(customFieldDescriptor), + }; +} diff --git a/apps/admin/src/members/custom-field-filter-renderer.tsx b/apps/admin/src/members/custom-fields/filter-renderer.tsx similarity index 57% rename from apps/admin/src/members/custom-field-filter-renderer.tsx rename to apps/admin/src/members/custom-fields/filter-renderer.tsx index 089f4c24b6f..69f65b5d497 100644 --- a/apps/admin/src/members/custom-field-filter-renderer.tsx +++ b/apps/admin/src/members/custom-fields/filter-renderer.tsx @@ -1,24 +1,14 @@ import React, { useEffect } from 'react'; -import { - CUSTOM_FIELDS_PREFIX, - CUSTOM_FIELD_OPERATORS, - CUSTOM_FIELD_SET_OPERATORS, -} from './member-fields'; +import { CUSTOM_FIELDS_PREFIX, CUSTOM_FIELD_OPERATORS } from '@/members/member-fields'; +import { CUSTOM_FIELD_SET_OPERATORS } from './addressing'; import { FilterSegmentInput, FilterSegmentSelect } from '@tryghost/shade/patterns'; -import { createOperatorOptions } from '@/shared/filters'; +import { createOperatorOptions, listsOperator } from '@/shared/filters'; import { memberCustomFieldParts, useBrowseMemberCustomFieldsIncludingArchived, } from '@tryghost/admin-x-framework/api/member-custom-fields'; import type { CustomRendererProps } from '@tryghost/shade/patterns'; -// The dropdown entry has already chosen the field (its key is in `field.key` as -// `custom_fields.`), so this renders only what's left in the pill: for a -// composite field a part selector (with "Any" for the whole field), then the -// operator, then the value. The predicate carries [subfield, value]; subfield is '' -// for a scalar field or the "Any" whole-field set/unset case. The operator lives here -// because its valid set depends on the part chosen here. - const CustomFieldFilterRenderer: React.FC> = ({ field, values, @@ -27,47 +17,34 @@ const CustomFieldFilterRenderer: React.FC> = ({ onOperatorChange, readOnly, }) => { - // Include-archived so an archived composite field's pill can still resolve its parts - // and show which one the saved segment filters on. const { data } = useBrowseMemberCustomFieldsIncludingArchived(); const definitions = data?.members_custom_fields ?? []; const fieldKey = (field.key ?? '').slice(CUSTOM_FIELDS_PREFIX.length); const definition = definitions.find((candidate) => candidate.key === fieldKey); - // The shared catalog decides which parts a type has and what they are called; a scalar - // field has none. Its keys are the ones the predicate carries. const parts = definition ? (memberCustomFieldParts(definition.type) ?? []).map(({ key, label }) => ({ value: key, label, })) : []; - // Name the field in each segment's aria-label so two custom-field pills on one row - // are distinguishable to a screen reader rather than all reading "Operator"/"Value". const fieldLabel = field.label ?? definition?.name ?? 'Custom field'; const isComposite = parts.length > 0; const [subfield = '', value = ''] = values; const isWholeField = subfield === ''; - // A composite's "Any" (whole field) only supports set / not-set — "Any contains X" - // is meaningless. A specific part, and a scalar field, support the value operators - // and set / not-set. Only "Any" restricts the set, so only it needs the operator - // coerced when the part selection changes — done in an effect rather than the change - // handler, because the framework's filter update reads a stale list within a tick, so - // a value change and an operator change can't both land in the same one. const operators = isComposite && isWholeField ? CUSTOM_FIELD_SET_OPERATORS : CUSTOM_FIELD_OPERATORS; useEffect(() => { - // A read-only pill never rewrites its own operator; it just displays what's set. - if (readOnly || !onOperatorChange || operators.includes(operator)) { + if (readOnly || !onOperatorChange || listsOperator(operators, operator)) { return; } onOperatorChange('is-set'); }, [readOnly, operator, operators, onOperatorChange]); - const needsValue = !CUSTOM_FIELD_SET_OPERATORS.includes(operator); + const needsValue = !listsOperator(CUSTOM_FIELD_SET_OPERATORS, operator); const partOptions = [{ value: '', label: 'Any' }, ...parts]; return ( diff --git a/apps/admin/src/members/detail/member-subscriptions-section.tsx b/apps/admin/src/members/detail/member-subscriptions-section.tsx index 83e5cf57ab9..0d1647b0782 100644 --- a/apps/admin/src/members/detail/member-subscriptions-section.tsx +++ b/apps/admin/src/members/detail/member-subscriptions-section.tsx @@ -2,6 +2,7 @@ import MemberAddCompModal from './member-add-comp-modal'; import MemberSubscriptionActions from './member-subscription-actions'; import MemberSubscriptionCompActions from './member-subscription-comp-actions'; import React from 'react'; +import { isSafeHref } from './is-safe-href'; import moment from 'moment-timezone'; import { Badge, Button, Card, CardContent, EmptyIndicator } from '@tryghost/shade/components'; import { LucideIcon, cn } from '@tryghost/shade/utils'; @@ -78,7 +79,7 @@ const SubscriptionDetails: React.FC<{ sub: MemberSubscription }> = ({ sub }) => {page && (

Page —{' '} - {page.url ? ( + {isSafeHref(page.url) ? ( { - it('waits for timezone resolution when date filters are present', () => { +describe('shouldDelayMembersFilterHydration', () => { + const DATE_FILTER = "created_at:<='2024-02-01T22:59:59.999Z'"; + const resolved = { + hasResolvedSettings: true, + isLoadingSettings: false, + newsletters: [], + customFields: [], + }; + + it('waits for the timezone when the filter says a date', () => { expect( - shouldDelayMembersDateFilterHydration("created_at:<='2024-02-01T22:59:59.999Z'", false, true), + shouldDelayMembersFilterHydration(DATE_FILTER, { + ...resolved, + hasResolvedSettings: false, + isLoadingSettings: true, + }), ).toBe(true); }); - it('does not wait for unsupported non-date filters', () => { - expect(shouldDelayMembersDateFilterHydration('status:paid,label:vip', false, true)).toBe(false); + it('does not wait for the timezone when no date is named', () => { + expect( + shouldDelayMembersFilterHydration('status:paid,label:vip', { + ...resolved, + hasResolvedSettings: false, + isLoadingSettings: true, + }), + ).toBe(false); }); it('does not wait once the site timezone is resolved', () => { + expect(shouldDelayMembersFilterHydration(DATE_FILTER, resolved)).toBe(false); + }); + + it('waits for the definitions a filter names, and only those', () => { + // A custom field can only be read for what it holds once the site says what it holds, so + // the page waits rather than reading it as text and answering a wider question. + const customFieldFilter = "(custom_fields.key:'joined'+custom_fields.value:<'2024-01-01')"; + + expect( + shouldDelayMembersFilterHydration(customFieldFilter, { + ...resolved, + customFields: undefined, + }), + ).toBe(true); + expect(shouldDelayMembersFilterHydration(customFieldFilter, resolved)).toBe(false); + + expect( + shouldDelayMembersFilterHydration('(newsletters.slug:weekly+email_disabled:0)', { + ...resolved, + newsletters: undefined, + }), + ).toBe(true); + expect( + shouldDelayMembersFilterHydration('status:paid', { + ...resolved, + newsletters: undefined, + customFields: undefined, + }), + ).toBe(false); + }); + + it('does not wait for a source only a quoted value mentions', () => { + expect( + shouldDelayMembersFilterHydration("name:~'custom_fields.'", { + ...resolved, + customFields: undefined, + }), + ).toBe(false); + }); + + it('does not wait for definitions that are never coming', () => { + // An empty list means nothing is on its way, so waiting would never end. + const customFieldFilter = "(custom_fields.key:'company'+custom_fields.value:'Ghost')"; + + expect( + shouldDelayMembersFilterHydration(customFieldFilter, { ...resolved, customFields: [] }), + ).toBe(false); expect( - shouldDelayMembersDateFilterHydration("created_at:<='2024-02-01T22:59:59.999Z'", true, false), + shouldDelayMembersFilterHydration('(newsletters.slug:weekly+email_disabled:0)', { + ...resolved, + newsletters: [], + }), ).toBe(false); }); - it('does not wait if settings loading has already stopped', () => { + it('does not wait when there is no filter at all', () => { expect( - shouldDelayMembersDateFilterHydration( - "created_at:<='2024-02-01T22:59:59.999Z'", - false, - false, - ), + shouldDelayMembersFilterHydration(undefined, { + ...resolved, + newsletters: undefined, + customFields: undefined, + }), ).toBe(false); }); }); @@ -372,3 +440,38 @@ describe('useMembersFilterState', () => { expect(result.current.hasFilterOrSearch).toBe(false); }); }); + +describe('useMembersFilterState — once its sources have arrived', () => { + const CUSTOM_FIELD_FILTER = "(custom_fields.key:'company'+custom_fields.value:'Ghost')"; + + function renderWithSources(sources: { + newsletters?: { slug: string; name: string }[]; + customFields?: { key: string; name: string; type: 'short_text' }[]; + }) { + return renderHook( + () => { + const state = useMembersFilterState('UTC', sources.newsletters, sources.customFields); + const [searchParams] = useSearchParams(); + + return { ...state, query: searchParams.toString() }; + }, + { + wrapper: createWrapper(`/?filter=${encodeURIComponent(CUSTOM_FIELD_FILTER)}`), + }, + ); + } + + it('reads and writes normally once they have', async () => { + const { result } = renderWithSources({ + customFields: [{ key: 'company', name: 'Company', type: 'short_text' }], + }); + + await waitFor(() => { + expect(result.current.filters).toHaveLength(1); + }); + + expect(result.current.filters[0].field).toBe('custom_fields.company'); + expect(result.current.nql).toBe(CUSTOM_FIELD_FILTER); + expect(decodeURIComponent(result.current.query)).toContain("custom_fields.key:'company'"); + }); +}); diff --git a/apps/admin/src/members/hooks/use-members-filter-state.ts b/apps/admin/src/members/hooks/use-members-filter-state.ts index 52cf73e266c..51c6193768e 100644 --- a/apps/admin/src/members/hooks/use-members-filter-state.ts +++ b/apps/admin/src/members/hooks/use-members-filter-state.ts @@ -1,5 +1,5 @@ import { type Filter } from '@tryghost/shade/patterns'; -import { getMemberFields } from '@/members/member-fields'; +import { buildMemberFields, canReadMemberFilter } from '@/members/member-filter-catalog'; import { hasTimezoneSensitiveMemberFilter, isPredicateEnabled, @@ -8,7 +8,9 @@ import { } from '@/members/member-filter-query'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useSearchParams } from '@tryghost/admin-x-framework'; -import type { MemberFields } from '@/members/member-fields'; +import type { CustomFieldDefinition } from '@/members/custom-fields/filter-fields'; +import type { MemberFields } from '@/members/member-filter-catalog'; +import type { NewsletterDefinition } from '@/members/newsletter-filter-fields'; interface SetFiltersOptions { replace?: boolean; @@ -33,24 +35,46 @@ interface ToSearchParamsOptions { fields: MemberFields; } +/** What a filter may depend on before it can be read for what it says. */ +export interface MemberFilterPrerequisites { + /** Settings carry the site timezone, which is what a date in a filter is relative to. */ + hasResolvedSettings: boolean; + isLoadingSettings?: boolean; + /** The site's own newsletters and custom fields. Undefined until they arrive. */ + newsletters?: readonly NewsletterDefinition[]; + customFields?: readonly CustomFieldDefinition[]; +} + /** - * Should the page hold off parsing the URL filter until more data is in? + * Whether the page should wait before touching the filter at all. * - * Parsing a date-sensitive filter needs the timezone from settings. If we parse - * before it resolves, the writeback effect can round-trip the date in UTC - * instead of site time. + * A filter can only be read for what it says once everything it leans on has arrived. The site's + * timezone decides which day a date means; the site's own definitions decide that a custom field + * holds dates rather than text. Read one early and the parts that cannot be understood yet are + * simply not there, so the page answers a wider question than was asked — and, on the next write, + * replaces what the publisher wrote with that wider version. + * + * The page therefore waits, rather than each place that reads or writes a filter remembering to + * check. Waiting is only ever for what this particular filter names: one that mentions no + * newsletter does not wait for newsletters, and one with no date does not wait for the timezone. */ -export function shouldDelayMembersDateFilterHydration( +export function shouldDelayMembersFilterHydration( filterParam: string | undefined, - hasResolvedDependencies: boolean, - isLoadingDependencies: boolean = !hasResolvedDependencies, + { + hasResolvedSettings, + isLoadingSettings = !hasResolvedSettings, + newsletters, + customFields, + }: MemberFilterPrerequisites, ): boolean { - return ( - Boolean(filterParam) && - isLoadingDependencies && - !hasResolvedDependencies && - hasTimezoneSensitiveMemberFilter(filterParam) - ); + if (!filterParam) { + return false; + } + + const waitingForTimezone = + isLoadingSettings && !hasResolvedSettings && hasTimezoneSensitiveMemberFilter(filterParam); + + return waitingForTimezone || !canReadMemberFilter(filterParam, { newsletters, customFields }); } function getEnabledFilters(filters: Filter[], fields: MemberFields): Filter[] { @@ -65,7 +89,7 @@ function toSearchParams({ fields, }: ToSearchParamsOptions): URLSearchParams { const params = new URLSearchParams(baseSearchParams); - const filter = serializeMemberFilters(getEnabledFilters(filters, fields), timezone); + const filter = serializeMemberFilters(getEnabledFilters(filters, fields), timezone, fields); params.delete('filter'); params.delete('search'); @@ -81,15 +105,22 @@ function toSearchParams({ return params; } -export function useMembersFilterState(timezone: string): UseMembersFilterStateReturn { - const fields = useMemo(() => getMemberFields(), []); +export function useMembersFilterState( + timezone: string, + newsletters?: readonly NewsletterDefinition[], + customFields?: readonly CustomFieldDefinition[], +): UseMembersFilterStateReturn { + const fields = useMemo( + () => buildMemberFields({ newsletters, customFields }), + [newsletters, customFields], + ); const [searchParams, setSearchParams] = useSearchParams(); const lastWrittenQueryRef = useRef(null); const filterParam = useMemo(() => searchParams.get('filter') ?? undefined, [searchParams]); const currentQuery = useMemo(() => searchParams.toString(), [searchParams]); const parsedFilters = useMemo(() => { - return getEnabledFilters(parseMemberFilter(filterParam, timezone), fields); + return getEnabledFilters(parseMemberFilter(filterParam, timezone, fields), fields); }, [filterParam, timezone, fields]); const [filters, setDraftFilters] = useState(parsedFilters); @@ -98,7 +129,7 @@ export function useMembersFilterState(timezone: string): UseMembersFilterStateRe }, [searchParams]); const nql = useMemo(() => { - return serializeMemberFilters(getEnabledFilters(filters, fields), timezone); + return serializeMemberFilters(getEnabledFilters(filters, fields), timezone, fields); }, [filters, timezone, fields]); useEffect(() => { diff --git a/apps/admin/src/members/import-members-gate.acceptance.test.tsx b/apps/admin/src/members/import-members-gate.acceptance.test.tsx index a85d9a10809..22bcc27af4c 100644 --- a/apps/admin/src/members/import-members-gate.acceptance.test.tsx +++ b/apps/admin/src/members/import-members-gate.acceptance.test.tsx @@ -36,6 +36,32 @@ async function openMappingStep(labs: Record) { } describe('Import members gate', () => { + it.each([false, true])( + 'inherits portal typography in the import flow (redesigned: %s)', + async (membersImportRedesign) => { + await openMappingStep({ admin7PageChrome: true, membersImportRedesign }); + const modal = membersScreen.dialog(); + await expect.element(modal).toBeVisible(); + expect(modal.element().closest('#root')).toBeNull(); + const select = page.getByRole('combobox').first(); + await expect.element(select).toBeVisible(); + const hasFont = () => getComputedStyle(modal.element()).fontFamily.includes('Inter Admin 7'); + await expect.poll(hasFont).toBe(true); + + const originalModal = modal.element(); + try { + await page.viewport(800, 800); + await expect.element(modal).toBeVisible(); + expect(modal.element()).toBe(originalModal); + await expect.poll(hasFont).toBe(false); + await page.viewport(801, 800); + await expect.poll(hasFont).toBe(true); + } finally { + await page.viewport(1280, 800); + } + }, + ); + it('serves the redesigned import when the flag is on', async () => { await openMappingStep({ membersImportRedesign: true }); diff --git a/apps/admin/src/members/member-fields.test.ts b/apps/admin/src/members/member-fields.test.ts index 0e7b033b3c2..0f8eca91217 100644 --- a/apps/admin/src/members/member-fields.test.ts +++ b/apps/admin/src/members/member-fields.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { getMemberFields, memberFields } from './member-fields'; +import { describeField } from '@/shared/filters'; import type { CodecContext, FilterPredicate } from '@/shared/filters'; const dateContext: CodecContext = { @@ -26,7 +27,6 @@ describe('memberFields', () => { 'last_seen_at', 'created_at', 'signup', - 'newsletters.:slug', 'tier_id', 'status', 'subscriptions.plan_interval', @@ -43,6 +43,7 @@ describe('memberFields', () => { 'newsletter_feedback', 'offer_redemptions', 'count.active_stripe_customers', + 'newsletters.:slug', 'custom_fields.:key', ]); }); @@ -74,7 +75,7 @@ describe('memberFields', () => { expect(memberFields['subscriptions.current_period_end'].operators).toEqual(futureDateOperators); }); - it('always appends the past/future relative operator to member date fields', () => { + it('gives each member date field the direction it can actually point', () => { const fields = getMemberFields(); expect(fields.created_at.operators).toContain('in-the-last'); @@ -83,6 +84,18 @@ describe('memberFields', () => { expect(fields['subscriptions.current_period_end'].operators).toContain('in-the-next'); }); + it('offers both relative directions to a date field that does not narrow them', () => { + const both = describeField({ + key: 'x', + icon: 'calendar', + type: 'timestamp', + ui: { label: 'X' }, + }); + + expect(both.operators).toContain('in-the-last'); + expect(both.operators).toContain('in-the-next'); + }); + it('keeps the expected subscription status options', () => { expect(memberFields['subscriptions.status'].options).toEqual([ { value: 'active', label: 'Active' }, @@ -203,7 +216,7 @@ describe('multipleActiveSubscriptionsCodec', () => { }); describe('newsletterCodec', () => { - it('serializes newsletter subscription state from a pattern field', () => { + it('serializes newsletter subscription state from the field it belongs to', () => { const predicate: FilterPredicate = { id: '1', field: 'newsletters.weekly', diff --git a/apps/admin/src/members/member-fields.ts b/apps/admin/src/members/member-fields.ts index ed50916ba1a..8b61d74a21e 100644 --- a/apps/admin/src/members/member-fields.ts +++ b/apps/admin/src/members/member-fields.ts @@ -1,28 +1,17 @@ +import { CUSTOM_FIELD_SET_OPERATORS, customFieldAddressing } from './custom-fields/addressing'; import { - DATE_FILTER_OPERATORS, - DEFAULT_DATE_OPERATOR, - type FilterCodec, - dateCodec, - defineFields, - extractComparator, - numberCodec, - scalarCodec, - setCodec, - textCodec, - withFutureRelativeOperator, - withPastRelativeOperator, + FUTURE_TIMESTAMP_OPERATORS, + PAST_TIMESTAMP_OPERATORS, + FILTER_TYPES, + type FieldDescriptor, + buildCatalog, + domainField, + columnAddressing, } from '@/shared/filters'; -import { escapeNqlString } from '@tryghost/nql-string'; -import { - MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD, - MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FILTER, - NO_MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FILTER, -} from './multiple-active-subscriptions'; +import { NEWSLETTER_FIELD } from './newsletter-filter-fields'; +import { feedbackSemantics, subscriptionSemantics } from './member-value-semantics'; +import { MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD } from './multiple-active-subscriptions'; -const TEXT_OPERATORS = ['is', 'contains', 'does-not-contain', 'starts-with', 'ends-with'] as const; -const NUMBER_OPERATORS = ['is', 'is-greater', 'is-less'] as const; -const SCALAR_OPERATORS = ['is', 'is-not'] as const; -const SET_OPERATORS = ['is-any', 'is-not-any'] as const; const SUBSCRIPTION_STATUS_OPTIONS: Array<{ value: string; label: string }> = [ { value: 'active', label: 'Active' }, { value: 'trialing', label: 'Trialing' }, @@ -33,488 +22,217 @@ const SUBSCRIPTION_STATUS_OPTIONS: Array<{ value: string; label: string }> = [ { value: 'incomplete_expired', label: 'Incomplete - Expired' }, ]; -const subscribedCodec: FilterCodec = { - parse() { - return null; - }, - serialize(predicate) { - const value = predicate.values[0]; - - if (predicate.operator !== 'is' && predicate.operator !== 'is-not') { - return null; - } - - if (value === 'email-disabled') { - return predicate.operator === 'is' ? ['(email_disabled:1)'] : ['(email_disabled:0)']; - } - - if (value === 'subscribed') { - return predicate.operator === 'is' - ? ['(subscribed:true+email_disabled:0)'] - : ['(subscribed:false,email_disabled:1)']; - } - - if (value === 'unsubscribed') { - return predicate.operator === 'is' - ? ['(subscribed:false+email_disabled:0)'] - : ['(subscribed:true,email_disabled:1)']; - } - - return null; - }, -}; - -const newsletterCodec: FilterCodec = { - parse() { - return null; - }, - serialize(predicate, ctx) { - const slug = ctx.params.slug; - const value = predicate.values[0]; - - if (!slug || predicate.operator !== 'is') { - return null; - } - - if (value === 'subscribed') { - return [`(newsletters.slug:${slug}+email_disabled:0)`]; - } - - if (value === 'unsubscribed') { - return [`(newsletters.slug:-${slug},email_disabled:1)`]; - } - - return null; - }, -}; - -const feedbackCodec: FilterCodec = { - parse() { - return null; - }, - serialize(predicate) { - const postId = predicate.values[0]; - - if ( - typeof postId !== 'string' || - !postId || - (predicate.operator !== '1' && predicate.operator !== '0') - ) { - return null; - } - - return [`(feedback.post_id:${escapeNqlString(postId)}+feedback.score:${predicate.operator})`]; - }, -}; - -const multipleActiveSubscriptionsCodec: FilterCodec = { - parse(node, ctx) { - const comparator = extractComparator(node as Record); - - if (!comparator || comparator.field !== ctx.key) { - return null; - } - - if (comparator.operator === '$gt' && comparator.value === 1) { - return { - field: ctx.key, - operator: 'is', - values: ['true'], - }; - } - - if (comparator.operator === '$lt' && comparator.value === 2) { - return { - field: ctx.key, - operator: 'is', - values: ['false'], - }; - } - - return null; - }, - serialize(predicate) { - const value = predicate.values[0]; - - if (predicate.operator !== 'is') { - return null; - } - - if (value === 'true') { - return [MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FILTER]; - } - - if (value === 'false') { - return [NO_MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FILTER]; - } - - return null; - }, -}; - -// Presence operators: the extra an optional, per-member field has that a table column does -// not — a column is always set, so no built-in field offers these. -export const CUSTOM_FIELD_SET_OPERATORS: readonly string[] = ['is-set', 'is-not-set']; - -// A custom text field's operators, composed from the shared groups so the members filter -// keeps one vocabulary: the equality pair (is / is-not) the scalar fields use, then the -// text matching operators (contains, starts-with, …) with their duplicate `is` dropped, -// then presence. Labels come from the shared createOperatorOptions default (dash to space), -// which reads every one of these correctly, so no label map is needed. -export const CUSTOM_FIELD_OPERATORS: readonly string[] = [ - ...SCALAR_OPERATORS, - ...TEXT_OPERATORS.filter((op) => !(SCALAR_OPERATORS as readonly string[]).includes(op)), - ...CUSTOM_FIELD_SET_OPERATORS, -]; - -/** - * The namespace custom fields are addressed under, in filter field keys and list column - * keys alike, so one field reads the same wherever it is named. - * - * Named rather than spelled out at each use because the namespace describes a bag of - * fields, and there is one bag today. An extension bringing its own bag would bring its - * own namespace, which is a change to what this resolves to rather than to its callers. - */ -export const CUSTOM_FIELDS_PREFIX = 'custom_fields.'; - -// NQL operator symbol for each value operator. The field is named in the value -// position (`custom_fields.key:'…'`) so its key can carry hyphens; the value is -// matched on `custom_fields.value` (scalar) or `custom_fields.value.` -// (address), which the members filter relation maps onto the real columns. -const CUSTOM_FIELD_VALUE_SYMBOLS: Record = { - is: '', - 'is-not': '-', - contains: '~', - 'does-not-contain': '-~', - 'starts-with': '~^', - 'ends-with': '~$', -}; - -const customFieldsCodec: FilterCodec = { - // Parsing a grouped custom-field expression back to a predicate is bespoke — - // its field and part are spread across a `(key + value)` pair — so it's handled - // by a compound matcher in member-filter-query.ts, not here. - parse() { - return null; - }, - // The field's stable key comes from the dropdown entry (`custom_fields.`, - // resolved into `ctx.params.key`); the predicate carries only [subfield, value], - // with subfield '' for a scalar field or the "Any" (whole-field set/unset) case. - serialize(predicate, ctx) { - const fieldKey = ctx.params.key; - const [subfield, value] = predicate.values as [string, string]; - - if (!fieldKey) { - return null; - } - - const keyClause = `custom_fields.key:${escapeNqlString(fieldKey)}`; - - // set / not-set target a part's presence when a part is chosen (`path`), or the - // whole field otherwise (the bare key / its negation). - if (predicate.operator === 'is-set') { - return subfield - ? [`(${keyClause}+custom_fields.path:${escapeNqlString(subfield)})`] - : [keyClause]; - } - - if (predicate.operator === 'is-not-set') { - return subfield - ? [`(${keyClause}+custom_fields.path:-${escapeNqlString(subfield)})`] - : [`custom_fields.key:-${escapeNqlString(fieldKey)}`]; - } - - const symbol = CUSTOM_FIELD_VALUE_SYMBOLS[predicate.operator]; - - if (symbol === undefined || value === undefined || value === null || value === '') { - return null; - } - - const valueKey = subfield ? `custom_fields.value.${subfield}` : 'custom_fields.value'; - - return [`(${keyClause}+${valueKey}:${symbol}${escapeNqlString(String(value))})`]; - }, -}; - -const baseMemberFields = defineFields({ - name: { - operators: TEXT_OPERATORS, - ui: { - label: 'Name', - type: 'text', - placeholder: 'Enter name...', - defaultOperator: 'contains', - className: 'w-48', - }, - codec: textCodec(), - }, - email: { - operators: TEXT_OPERATORS, - ui: { - label: 'Email', - type: 'text', - placeholder: 'Enter email...', - defaultOperator: 'contains', - className: 'w-48', - }, - codec: textCodec(), - }, - label: { - operators: SET_OPERATORS, - ui: { - label: 'Label', - type: 'multiselect', - searchable: true, - className: 'w-64', - defaultOperator: 'is-any', - }, - metadata: { - activeColumn: { - key: 'labels', - label: 'Labels', - }, - columnInclude: 'labels', - }, - codec: setCodec(), - }, - subscribed: { - operators: SCALAR_OPERATORS, - ui: { - label: 'Newsletter subscription', - type: 'select', - searchable: false, - }, +const MEMBER_FIELDS = [ + { + key: 'name', + icon: 'person', + type: 'text', + ui: { label: 'Name', placeholder: 'Enter name...', className: 'w-48' }, + }, + { + key: 'email', + icon: 'mail', + type: 'text', + ui: { label: 'Email', placeholder: 'Enter email...', className: 'w-48' }, + }, + { + key: 'label', + icon: 'tag', + type: 'set', + ui: { label: 'Label', searchable: true, className: 'w-64' }, + metadata: { activeColumn: { key: 'labels', label: 'Labels' }, columnInclude: 'labels' }, + }, + domainField({ + key: 'subscribed', + icon: 'mail', + semantics: subscriptionSemantics(), + operators: FILTER_TYPES.scalar.operators, + ui: { label: 'Newsletter subscription', type: 'select', searchable: false }, options: [ { value: 'subscribed', label: 'Subscribed' }, { value: 'unsubscribed', label: 'Unsubscribed' }, { value: 'email-disabled', label: 'Email disabled' }, ], - codec: subscribedCodec, - }, - last_seen_at: { - operators: DATE_FILTER_OPERATORS, - ui: { - label: 'Last seen', - type: 'date', - defaultOperator: DEFAULT_DATE_OPERATOR, - }, - codec: dateCodec(), - }, - created_at: { - operators: DATE_FILTER_OPERATORS, - ui: { - label: 'Created', - type: 'date', - defaultOperator: DEFAULT_DATE_OPERATOR, - }, - codec: dateCodec(), - }, - signup: { - operators: SCALAR_OPERATORS, + }), + { + key: 'last_seen_at', + icon: 'eye', + type: 'timestamp', + operators: PAST_TIMESTAMP_OPERATORS, + ui: { label: 'Last seen' }, + }, + { + key: 'created_at', + icon: 'calendar', + type: 'timestamp', + operators: PAST_TIMESTAMP_OPERATORS, + ui: { label: 'Created' }, + }, + { + key: 'signup', + icon: 'person-plus', + type: 'scalar', + valueConfig: { quoteStrings: true }, ui: { label: 'Signed up on post/page', - type: 'select', searchable: true, placeholder: 'Select a post or page...', className: 'w-64', }, - codec: scalarCodec({ quoteStrings: true }), }, - 'newsletters.:slug': { - operators: ['is'], - ui: { - label: 'Newsletter', - type: 'select', - searchable: false, - hideOperatorSelect: true, - }, - options: [ - { value: 'subscribed', label: 'Subscribed' }, - { value: 'unsubscribed', label: 'Unsubscribed' }, - ], - codec: newsletterCodec, + { + key: 'tier_id', + icon: 'card', + type: 'set', + ui: { label: 'Membership tier', searchable: true, className: 'w-64' }, + metadata: { activeColumn: { key: 'tiers', label: 'Tiers' }, columnInclude: 'tiers' }, }, - tier_id: { - operators: SET_OPERATORS, - ui: { - label: 'Membership tier', - type: 'multiselect', - searchable: true, - className: 'w-64', - defaultOperator: 'is-any', - }, - metadata: { - activeColumn: { - key: 'tiers', - label: 'Tiers', - }, - columnInclude: 'tiers', - }, - codec: setCodec(), - }, - status: { - operators: SCALAR_OPERATORS, - ui: { - label: 'Member status', - type: 'select', - searchable: false, - }, + { + key: 'status', + icon: 'person-circle', + type: 'scalar', + ui: { label: 'Member status', searchable: false }, options: [ { value: 'paid', label: 'Paid' }, { value: 'free', label: 'Free' }, { value: 'comped', label: 'Complimentary' }, ], - codec: scalarCodec(), }, - 'subscriptions.plan_interval': { - operators: SCALAR_OPERATORS, - ui: { - label: 'Billing period', - type: 'select', - searchable: false, - }, + { + key: 'subscriptions.plan_interval', + icon: 'calendar-clock', + type: 'scalar', + ui: { label: 'Billing period', searchable: false }, options: [ { value: 'month', label: 'Monthly' }, { value: 'year', label: 'Yearly' }, ], metadata: { - activeColumn: { - key: 'subscriptions.plan_interval', - label: 'Billing period', - }, + activeColumn: { key: 'subscriptions.plan_interval', label: 'Billing period' }, columnInclude: 'subscriptions', }, - codec: scalarCodec(), }, - 'subscriptions.status': { - operators: SCALAR_OPERATORS, - ui: { - label: 'Stripe subscription status', - type: 'select', - searchable: false, - }, + { + key: 'subscriptions.status', + icon: 'card', + type: 'scalar', + ui: { label: 'Stripe subscription status', searchable: false }, options: SUBSCRIPTION_STATUS_OPTIONS, metadata: { - activeColumn: { - key: 'subscriptions.status', - label: 'Subscription status', - }, + activeColumn: { key: 'subscriptions.status', label: 'Subscription status' }, columnInclude: 'subscriptions', }, - codec: scalarCodec(), }, - 'subscriptions.start_date': { - operators: DATE_FILTER_OPERATORS, - ui: { - label: 'Paid start date', - type: 'date', - defaultOperator: DEFAULT_DATE_OPERATOR, - }, + { + key: 'subscriptions.start_date', + icon: 'calendar-start', + type: 'timestamp', + operators: PAST_TIMESTAMP_OPERATORS, + ui: { label: 'Paid start date' }, metadata: { - activeColumn: { - key: 'subscriptions.start_date', - label: 'Paid start date', - }, + activeColumn: { key: 'subscriptions.start_date', label: 'Paid start date' }, columnInclude: 'subscriptions', }, - codec: dateCodec(), }, - 'subscriptions.current_period_end': { - operators: DATE_FILTER_OPERATORS, - ui: { - label: 'Next billing date', - type: 'date', - defaultOperator: DEFAULT_DATE_OPERATOR, - }, + { + key: 'subscriptions.current_period_end', + icon: 'calendar-end', + type: 'timestamp', + operators: FUTURE_TIMESTAMP_OPERATORS, + ui: { label: 'Next billing date' }, metadata: { - activeColumn: { - key: 'subscriptions.current_period_end', - label: 'Next billing date', - }, + activeColumn: { key: 'subscriptions.current_period_end', label: 'Next billing date' }, columnInclude: 'subscriptions', }, - codec: dateCodec(), }, - conversion: { - operators: SCALAR_OPERATORS, + { + key: 'conversion', + icon: 'arrows', + type: 'scalar', + valueConfig: { quoteStrings: true }, ui: { label: 'Subscription started on post/page', - type: 'select', searchable: true, placeholder: 'Select a post or page...', className: 'w-64', }, - codec: scalarCodec({ quoteStrings: true }), }, - email_count: { - operators: NUMBER_OPERATORS, + { + key: 'email_count', + icon: 'send', + type: 'number', ui: { label: 'Emails sent (all time)', - type: 'number', defaultOperator: 'is-greater', min: 0, className: 'w-24', }, - codec: numberCodec(), }, - email_opened_count: { - operators: NUMBER_OPERATORS, + { + key: 'email_opened_count', + icon: 'mail-open', + type: 'number', ui: { label: 'Emails opened (all time)', - type: 'number', defaultOperator: 'is-greater', min: 0, className: 'w-24', }, - codec: numberCodec(), }, - email_open_rate: { - operators: NUMBER_OPERATORS, + { + key: 'email_open_rate', + icon: 'percent', + type: 'number', ui: { label: 'Open rate (all time)', - type: 'number', defaultOperator: 'is-greater', min: 0, max: 100, suffix: '%', className: 'w-24', }, - codec: numberCodec(), }, - 'emails.post_id': { - operators: SCALAR_OPERATORS, + { + key: 'emails.post_id', + icon: 'send', + type: 'scalar', + valueConfig: { quoteStrings: true }, ui: { label: 'Sent email', - type: 'select', searchable: true, placeholder: 'Select an email...', className: 'w-64', }, - codec: scalarCodec({ quoteStrings: true }), }, - 'opened_emails.post_id': { - operators: SCALAR_OPERATORS, + { + key: 'opened_emails.post_id', + icon: 'mail-open', + type: 'scalar', + valueConfig: { quoteStrings: true }, ui: { label: 'Opened email', - type: 'select', searchable: true, placeholder: 'Select an email...', className: 'w-64', }, - codec: scalarCodec({ quoteStrings: true }), }, - 'clicked_links.post_id': { - operators: SCALAR_OPERATORS, + { + key: 'clicked_links.post_id', + icon: 'click', + type: 'scalar', + valueConfig: { quoteStrings: true }, ui: { label: 'Clicked email', - type: 'select', searchable: true, placeholder: 'Select an email...', className: 'w-64', }, - codec: scalarCodec({ quoteStrings: true }), }, - newsletter_feedback: { + domainField({ + key: 'newsletter_feedback', + icon: 'message', + semantics: feedbackSemantics(), + addressing: columnAddressing({ field: 'feedback.post_id' }), operators: ['1', '0'], ui: { label: 'Responded with feedback', @@ -524,27 +242,20 @@ const baseMemberFields = defineFields({ className: 'w-64', defaultOperator: '1', }, - codec: feedbackCodec, - }, - offer_redemptions: { - operators: SET_OPERATORS, - ui: { - label: 'Offer', - type: 'multiselect', - searchable: true, - className: 'w-64', - defaultOperator: 'is-any', - }, - metadata: { - activeColumn: { - key: 'offer_redemptions', - label: 'Offer', - }, - }, - codec: setCodec({ quoteStrings: true, serializeSingletonAsScalar: true }), - }, - [MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD]: { - operators: ['is'], + }), + { + key: 'offer_redemptions', + icon: 'ticket', + type: 'set', + valueConfig: { quoteStrings: true, serializeSingletonAsScalar: true }, + ui: { label: 'Offer', searchable: true, className: 'w-64' }, + metadata: { activeColumn: { key: 'offer_redemptions', label: 'Offer' } }, + }, + { + key: MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD, + icon: 'layers', + type: 'count', + valueConfig: { threshold: 1, absentForm: 'below' }, ui: { label: 'Multiple active subscriptions', type: 'select', @@ -555,46 +266,58 @@ const baseMemberFields = defineFields({ { value: 'true', label: 'Yes' }, { value: 'false', label: 'No' }, ], - codec: multipleActiveSubscriptionsCodec, }, - // Each defined custom field is its own filter, named directly in the dropdown - // (`custom_fields.`), so this template supplies the shared operators and codec; - // use-member-filter-fields builds one entry per field from the definitions. - 'custom_fields.:key': { - operators: CUSTOM_FIELD_OPERATORS, - ui: { - label: 'Custom field', - type: 'custom', - component: 'custom-field', - }, - metadata: { - // One column per field filtered on, named after the field itself. A value a - // publisher collected varies member to member, which is what earns a column; - // it is shown whatever the operator, the way Label is. No name resolved means - // no such field for this site (or the flag is off), so no column either. - activeColumn: ({ params, label }) => - label ? { key: `${CUSTOM_FIELDS_PREFIX}${params.key}`, label } : null, - // Asked for as soon as a custom field is filtered on, without waiting for the - // names: the values are what the column will hold, and they travel on the same - // request the filter already sends. The API takes this include whether or not - // the flag is on, and returns values only when it is. - columnInclude: 'custom_fields', - }, - codec: customFieldsCodec, +] as const satisfies readonly FieldDescriptor[]; + +export const CUSTOM_FIELD_OPERATORS = [ + ...FILTER_TYPES.text.operators, + ...CUSTOM_FIELD_SET_OPERATORS, +]; + +/** + * The namespace custom fields are addressed under, in filter field keys and list column + * keys alike, so one field reads the same wherever it is named. + * + * Named rather than spelled out at each use because the namespace describes a bag of + * fields, and there is one bag today. An extension bringing its own bag would bring its + * own namespace, which is a change to what this resolves to rather than to its callers. + */ +export const CUSTOM_FIELDS_PREFIX = 'custom_fields.'; + +const CUSTOM_FIELD: FieldDescriptor = { + key: 'custom_fields.:key', + icon: 'text', + type: 'text', + addressing: customFieldAddressing(), + operators: CUSTOM_FIELD_OPERATORS, + ui: { + label: 'Custom field', + type: 'custom', + }, + metadata: { + // One column per field filtered on, named after the field itself. A value a + // publisher collected varies member to member, which is what earns a column; + // it is shown whatever the operator, the way Label is. No name resolved means + // no such field for this site (or the flag is off), so no column either. + activeColumn: ({ params, label }) => + label ? { key: `${CUSTOM_FIELDS_PREFIX}${params.key}`, label } : null, + // Asked for as soon as a custom field is filtered on, without waiting for the + // names: the values are what the column will hold, and they travel on the same + // request the filter already sends. The API takes this include whether or not + // the flag is on, and returns values only when it is. + columnInclude: 'custom_fields', }, -}); +}; + +export type StaticMemberFieldKey = (typeof MEMBER_FIELDS)[number]['key']; + +export const MEMBER_FIELD_DESCRIPTORS: FieldDescriptor[] = [ + ...MEMBER_FIELDS, + NEWSLETTER_FIELD, + CUSTOM_FIELD, +]; -export const memberFields = defineFields({ - ...baseMemberFields, - last_seen_at: withPastRelativeOperator(baseMemberFields.last_seen_at), - created_at: withPastRelativeOperator(baseMemberFields.created_at), - 'subscriptions.start_date': withPastRelativeOperator( - baseMemberFields['subscriptions.start_date'], - ), - 'subscriptions.current_period_end': withFutureRelativeOperator( - baseMemberFields['subscriptions.current_period_end'], - ), -}); +export const memberFields = buildCatalog(MEMBER_FIELD_DESCRIPTORS); export type MemberFields = typeof memberFields; diff --git a/apps/admin/src/members/member-filter-catalog.test.ts b/apps/admin/src/members/member-filter-catalog.test.ts new file mode 100644 index 00000000000..53ce476778c --- /dev/null +++ b/apps/admin/src/members/member-filter-catalog.test.ts @@ -0,0 +1,114 @@ +import { buildMemberFields, canReadMemberFilter } from './member-filter-catalog'; +import { describe, expect, it } from 'vitest'; +import { parseMemberFilter, serializeMemberFilters } from './member-filter-query'; +import type { MemberFields } from './member-filter-catalog'; + +const SOURCES = { + newsletters: [ + { slug: 'weekly', name: 'Weekly' }, + { slug: 'daily', name: 'Daily' }, + ], + customFields: [ + { key: 'company', name: 'Company', type: 'short_text' as const }, + { key: 'shipping_address', name: 'Shipping address', type: 'address' as const }, + ], +}; + +const resolved = buildMemberFields(SOURCES); +const pending = buildMemberFields(); + +function roundTrip(fields: MemberFields, nql: string) { + return serializeMemberFilters(parseMemberFilter(nql, 'UTC', fields), 'UTC', fields); +} + +describe("a site's own definitions add precision, not correctness", () => { + it('reads every newsletter as itself, whichever entry is tried first', () => { + for (const slug of ['weekly', 'daily']) { + const nql = `(newsletters.slug:${slug}+email_disabled:0)`; + + expect(roundTrip(resolved, nql)).toBe(nql); + } + }); + + it('reads a newsletter the site no longer has', () => { + const nql = '(newsletters.slug:retired+email_disabled:0)'; + + expect(roundTrip(resolved, nql)).toBe(nql); + }); + + it('keeps a clause naming a custom field that no longer exists', () => { + const nql = "(custom_fields.key:'deleted'+custom_fields.value:~'x')"; + + expect(roundTrip(resolved, nql)).toBe(nql); + }); + + it('reads every shape before the definitions arrive, losing nothing', () => { + for (const nql of [ + '(newsletters.slug:daily+email_disabled:0)', + "(custom_fields.key:'company'+custom_fields.value:'Ghost')", + "custom_fields.key:'company'", + ]) { + expect(roundTrip(pending, nql)).toBe(nql); + } + }); + + it('gives a known custom field its own entry, and an unknown one the shared entry', () => { + expect(resolved['custom_fields.company']).toBeDefined(); + expect(pending['custom_fields.company']).toBeUndefined(); + expect(pending['custom_fields.:key']).toBeDefined(); + }); +}); + +describe('waiting protects precision, not the clauses', () => { + const customFieldFilter = "(custom_fields.key:'company'+custom_fields.value:'Ghost')"; + + it('waits while a source the filter names is in flight', () => { + expect(canReadMemberFilter(customFieldFilter, { customFields: undefined })).toBe(false); + }); + + it('does not wait once that source has resolved, even to nothing', () => { + expect(canReadMemberFilter(customFieldFilter, { customFields: [] })).toBe(true); + }); + + it('never waits for a filter naming no dynamic clause', () => { + expect(canReadMemberFilter('status:paid', {})).toBe(true); + expect(canReadMemberFilter(undefined, {})).toBe(true); + }); + + it('does not wait for sources a quoted value only happens to mention', () => { + // Someone searching their members for the literal text "custom_fields." or + // "newsletters.slug" is not asking about custom fields or newsletters, and waiting for + // those to load would leave the filter unreadable until something else resolved them. + expect(canReadMemberFilter("name:~'custom_fields.'", { customFields: undefined })).toBe(true); + expect(canReadMemberFilter("name:~'newsletters.slug'", { newsletters: undefined })).toBe(true); + }); + + it('still waits when the source is named as a clause key, wherever it sits', () => { + expect(canReadMemberFilter("custom_fields.key:'company'", { customFields: undefined })).toBe( + false, + ); + expect( + canReadMemberFilter("(status:paid+custom_fields.key:'company')", { customFields: undefined }), + ).toBe(false); + expect( + canReadMemberFilter("status:paid,custom_fields.key:'company'", { customFields: undefined }), + ).toBe(false); + expect(canReadMemberFilter("custom_fields.key:-'company'", { customFields: undefined })).toBe( + false, + ); + }); + + it('waits for newsletters the same way', () => { + const nql = '(newsletters.slug:weekly+email_disabled:0)'; + + expect(canReadMemberFilter(nql, { newsletters: undefined })).toBe(false); + expect(canReadMemberFilter(nql, { newsletters: [] })).toBe(true); + }); + + it('parses the filter it is waiting on anyway', () => { + const parsed = parseMemberFilter(customFieldFilter, 'UTC', pending); + + expect(parsed).toHaveLength(1); + expect(parsed[0].field).toBe('custom_fields.company'); + }); +}); diff --git a/apps/admin/src/members/member-filter-catalog.ts b/apps/admin/src/members/member-filter-catalog.ts new file mode 100644 index 00000000000..b28fdeb1b16 --- /dev/null +++ b/apps/admin/src/members/member-filter-catalog.ts @@ -0,0 +1,38 @@ +import { buildProvidedCatalog, catalogCanRead } from '@/shared/filters'; +import { customFieldProvider, type CustomFieldDefinition } from './custom-fields/filter-fields'; +import { MEMBER_FIELD_DESCRIPTORS, memberFields } from './member-fields'; +import { newsletterProvider, type NewsletterDefinition } from './newsletter-filter-fields'; +import type { FieldProvider, FilterField } from '@/shared/filters'; + +export type MemberFields = Record; + +export interface MemberCatalogSources { + newsletters?: readonly NewsletterDefinition[]; + customFields?: readonly CustomFieldDefinition[]; +} + +export function memberFieldProviders({ + newsletters, + customFields, +}: MemberCatalogSources = {}): FieldProvider[] { + return [ + { resolved: true, fields: MEMBER_FIELD_DESCRIPTORS }, + newsletterProvider(newsletters), + customFieldProvider(customFields), + ]; +} + +export function buildMemberFields(sources: MemberCatalogSources = {}): MemberFields { + if (!sources.newsletters && !sources.customFields) { + return memberFields; + } + + return buildProvidedCatalog(memberFieldProviders(sources)); +} + +export function canReadMemberFilter( + filter: string | undefined, + sources: MemberCatalogSources = {}, +): boolean { + return catalogCanRead(filter, memberFieldProviders(sources)); +} diff --git a/apps/admin/src/members/member-filter-query.test.ts b/apps/admin/src/members/member-filter-query.test.ts index 97f7a4b1e2b..27ca5e096ee 100644 --- a/apps/admin/src/members/member-filter-query.test.ts +++ b/apps/admin/src/members/member-filter-query.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { getMemberFields } from './member-fields'; +import { getMemberFields, memberFields } from './member-fields'; import { hasTimezoneSensitiveMemberFilter, isPredicateEnabled, @@ -18,90 +18,126 @@ function stripIds(predicates: FilterPredicate[]) { describe('member-filter-query', () => { it('parses subscribed lifecycle compounds and legacy email-disabled filters', () => { - expect(stripIds(parseMemberFilter('(subscribed:true+email_disabled:0)', 'UTC'))).toEqual([ - { field: 'subscribed', operator: 'is', values: ['subscribed'] }, - ]); + expect( + stripIds(parseMemberFilter('(subscribed:true+email_disabled:0)', 'UTC', memberFields)), + ).toEqual([{ field: 'subscribed', operator: 'is', values: ['subscribed'] }]); - expect(stripIds(parseMemberFilter('(subscribed:false,email_disabled:1)', 'UTC'))).toEqual([ - { field: 'subscribed', operator: 'is-not', values: ['subscribed'] }, - ]); + expect( + stripIds(parseMemberFilter('(subscribed:false,email_disabled:1)', 'UTC', memberFields)), + ).toEqual([{ field: 'subscribed', operator: 'is-not', values: ['subscribed'] }]); - expect(stripIds(parseMemberFilter('(email_disabled:0)', 'UTC'))).toEqual([ + expect(stripIds(parseMemberFilter('(email_disabled:0)', 'UTC', memberFields))).toEqual([ { field: 'subscribed', operator: 'is-not', values: ['email-disabled'] }, ]); }); - it('parses newsletter and feedback compounds', () => { - expect( - stripIds(parseMemberFilter('(newsletters.slug:weekly+email_disabled:0)', 'UTC')), - ).toEqual([{ field: 'newsletters.weekly', operator: 'is', values: ['subscribed'] }]); + it('leaves comparative clauses unread rather than reading them as their opposite', () => { + const comparative = [ + 'subscribed:-true', + 'email_disabled:>0', + 'email_disabled:-1', + "(feedback.post_id:'post_123'+feedback.score:-1)", + "(feedback.post_id:-'post_123'+feedback.score:1)", + 'newsletters.slug:>weekly', + ]; - expect( - stripIds(parseMemberFilter("(feedback.post_id:'post_123'+feedback.score:1)", 'UTC')), - ).toEqual([{ field: 'newsletter_feedback', operator: '1', values: ['post_123'] }]); + for (const filter of comparative) { + expect(stripIds(parseMemberFilter(filter, 'UTC', memberFields))).toEqual([]); + } }); - it('reads newsletter subscription state from the slug clause polarity', () => { - expect( - stripIds(parseMemberFilter('(newsletters.slug:-weekly,email_disabled:1)', 'UTC')), - ).toEqual([{ field: 'newsletters.weekly', operator: 'is', values: ['unsubscribed'] }]); - - // Hand-written shapes pairing the slug with the "wrong" join and - // email_disabled value still follow the slug's polarity. - expect( - stripIds(parseMemberFilter('(newsletters.slug:-weekly+email_disabled:0)', 'UTC')), - ).toEqual([{ field: 'newsletters.weekly', operator: 'is', values: ['unsubscribed'] }]); + it('does not claim a pair whose partner clause is comparative', () => { + for (const filter of [ + '(subscribed:-true+email_disabled:0)', + '(newsletters.slug:>weekly+email_disabled:0)', + ]) { + expect(stripIds(parseMemberFilter(filter, 'UTC', memberFields))).toEqual([ + { field: 'subscribed', operator: 'is-not', values: ['email-disabled'] }, + ]); + } + }); + it('parses newsletter and feedback compounds', () => { expect( - stripIds(parseMemberFilter('(newsletters.slug:weekly,email_disabled:1)', 'UTC')), + stripIds( + parseMemberFilter('(newsletters.slug:weekly+email_disabled:0)', 'UTC', memberFields), + ), ).toEqual([{ field: 'newsletters.weekly', operator: 'is', values: ['subscribed'] }]); - }); - - it('round-trips a mismatched unsubscribed compound to the canonical shape', () => { - const parsed = parseMemberFilter('(newsletters.slug:-weekly+email_disabled:0)', 'UTC'); - expect(serializeMemberFilters(parsed, 'UTC')).toBe( - '(newsletters.slug:-weekly,email_disabled:1)', - ); + expect( + stripIds( + parseMemberFilter("(feedback.post_id:'post_123'+feedback.score:1)", 'UTC', memberFields), + ), + ).toEqual([{ field: 'newsletter_feedback', operator: '1', values: ['post_123'] }]); }); it('parses legacy scalar set filters and preserves singleton offer ids', () => { - const parsed = parseMemberFilter("offer_redemptions:'offer_123'", 'UTC'); + const parsed = parseMemberFilter("offer_redemptions:'offer_123'", 'UTC', memberFields); expect(stripIds(parsed)).toEqual([ { field: 'offer_redemptions', operator: 'is-any', values: ['offer_123'] }, ]); - expect(serializeMemberFilters(parsed, 'UTC')).toBe("offer_redemptions:'offer_123'"); + expect(serializeMemberFilters(parsed, 'UTC', memberFields)).toBe( + "offer_redemptions:'offer_123'", + ); }); it('parses legacy scalar label filters into set predicates', () => { - expect(stripIds(parseMemberFilter('label:vip', 'UTC'))).toEqual([ + expect(stripIds(parseMemberFilter('label:vip', 'UTC', memberFields))).toEqual([ { field: 'label', operator: 'is-any', values: ['vip'] }, ]); }); it('best-effort parses compat subscribed booleans into subscribed filters', () => { - expect(stripIds(parseMemberFilter('subscribed:true', 'UTC'))).toEqual([ + expect(stripIds(parseMemberFilter('subscribed:true', 'UTC', memberFields))).toEqual([ { field: 'subscribed', operator: 'is', values: ['subscribed'] }, ]); - expect(stripIds(parseMemberFilter('subscribed:false', 'UTC'))).toEqual([ + expect(stripIds(parseMemberFilter('subscribed:false', 'UTC', memberFields))).toEqual([ { field: 'subscribed', operator: 'is', values: ['unsubscribed'] }, ]); }); + it('keeps the other clauses when an unwrapped compound carries more than the pair', () => { + expect( + stripIds( + parseMemberFilter("subscribed:true+email_disabled:0+name:~'x'", 'UTC', memberFields), + ).map((p) => p.field), + ).toEqual(['subscribed', 'subscribed', 'name']); + + expect( + stripIds( + parseMemberFilter( + "subscribed:true+email_disabled:0+name:~'x'+status:paid", + 'UTC', + memberFields, + ), + ).map((p) => p.field), + ).toEqual(['subscribed', 'subscribed', 'name', 'status']); + }); + + it('drops an ungrouped feedback pair, because neither of its clauses reads alone', () => { + expect( + stripIds( + parseMemberFilter("feedback.post_id:'p1'+feedback.score:1+name:~'x'", 'UTC', memberFields), + ).map((p) => p.field), + ).toEqual(['name']); + }); + it('parses unwrapped Ember compounds at the root', () => { - expect(stripIds(parseMemberFilter('subscribed:true+email_disabled:0', 'UTC'))).toEqual([ - { field: 'subscribed', operator: 'is', values: ['subscribed'] }, - ]); + expect( + stripIds(parseMemberFilter('subscribed:true+email_disabled:0', 'UTC', memberFields)), + ).toEqual([{ field: 'subscribed', operator: 'is', values: ['subscribed'] }]); - expect(stripIds(parseMemberFilter('newsletters.slug:weekly+email_disabled:0', 'UTC'))).toEqual([ - { field: 'newsletters.weekly', operator: 'is', values: ['subscribed'] }, - ]); + expect( + stripIds(parseMemberFilter('newsletters.slug:weekly+email_disabled:0', 'UTC', memberFields)), + ).toEqual([{ field: 'newsletters.weekly', operator: 'is', values: ['subscribed'] }]); expect( - stripIds(parseMemberFilter("feedback.post_id:'post_123'+feedback.score:1", 'UTC')), + stripIds( + parseMemberFilter("feedback.post_id:'post_123'+feedback.score:1", 'UTC', memberFields), + ), ).toEqual([{ field: 'newsletter_feedback', operator: '1', values: ['post_123'] }]); }); @@ -112,29 +148,41 @@ describe('member-filter-query', () => { { id: '3', field: 'newsletters.weekly', operator: 'is', values: ['subscribed'] }, ]; - expect(serializeMemberFilters(predicates, 'UTC')).toBe( + expect(serializeMemberFilters(predicates, 'UTC', memberFields)).toBe( "(newsletters.slug:weekly+email_disabled:0)+emails.post_id:'post_123'+status:paid", ); }); it('canonicalizes compat subscribed booleans to member filter compounds', () => { - const parsed = parseMemberFilter('subscribed:true', 'UTC'); + const parsed = parseMemberFilter('subscribed:true', 'UTC', memberFields); - expect(serializeMemberFilters(parsed, 'UTC')).toBe('(subscribed:true+email_disabled:0)'); + expect(serializeMemberFilters(parsed, 'UTC', memberFields)).toBe( + '(subscribed:true+email_disabled:0)', + ); }); it('parses and serializes member date boundaries', () => { - const parsed = parseMemberFilter("created_at:<='2024-01-01T23:59:59.999Z'", 'UTC'); + const parsed = parseMemberFilter( + "created_at:<='2024-01-01T23:59:59.999Z'", + 'UTC', + memberFields, + ); expect(stripIds(parsed)).toEqual([ { field: 'created_at', operator: 'is-or-less', values: ['2024-01-01'] }, ]); - expect(serializeMemberFilters(parsed, 'UTC')).toBe("created_at:<='2024-01-01T23:59:59.999Z'"); + expect(serializeMemberFilters(parsed, 'UTC', memberFields)).toBe( + "created_at:<='2024-01-01T23:59:59.999Z'", + ); }); it('parses legacy Ember member date URLs without ISO timezone markers', () => { - const parsed = parseMemberFilter("subscriptions.start_date:<='2022-02-01 23:59:59'", 'UTC'); + const parsed = parseMemberFilter( + "subscriptions.start_date:<='2022-02-01 23:59:59'", + 'UTC', + memberFields, + ); expect(stripIds(parsed)).toEqual([ { field: 'subscriptions.start_date', operator: 'is-or-less', values: ['2022-02-01'] }, @@ -145,6 +193,7 @@ describe('member-filter-query', () => { const parsed = parseMemberFilter( "subscriptions.start_date:<='2022-02-01 23:59:59'", 'Europe/Stockholm', + memberFields, ); expect(stripIds(parsed)).toEqual([ @@ -153,13 +202,17 @@ describe('member-filter-query', () => { }); it('round-trips member date boundaries in site timezones', () => { - const parsed = parseMemberFilter("created_at:<='2024-02-01T22:59:59.999Z'", 'Europe/Stockholm'); + const parsed = parseMemberFilter( + "created_at:<='2024-02-01T22:59:59.999Z'", + 'Europe/Stockholm', + memberFields, + ); expect(stripIds(parsed)).toEqual([ { field: 'created_at', operator: 'is-or-less', values: ['2024-02-01'] }, ]); - expect(serializeMemberFilters(parsed, 'Europe/Stockholm')).toBe( + expect(serializeMemberFilters(parsed, 'Europe/Stockholm', memberFields)).toBe( "created_at:<='2024-02-01T22:59:59.999Z'", ); }); @@ -170,21 +223,23 @@ describe('member-filter-query', () => { { id: '1', field: 'label', operator: 'is-any', values: ['vip', 'alpha'] }, ]; - expect(serializeMemberFilters(predicates, 'UTC')).toBe('label:[alpha,vip]+status:paid'); + expect(serializeMemberFilters(predicates, 'UTC', memberFields)).toBe( + 'label:[alpha,vip]+status:paid', + ); }); it('round-trips canonical member examples', () => { const filter = "(feedback.post_id:'post_123'+feedback.score:1)+status:paid+subscriptions.current_period_end:<='2024-01-01T23:59:59.999Z'"; - const parsed = parseMemberFilter(filter, 'UTC'); + const parsed = parseMemberFilter(filter, 'UTC', memberFields); - expect(serializeMemberFilters(parsed, 'UTC')).toBe( + expect(serializeMemberFilters(parsed, 'UTC', memberFields)).toBe( "(feedback.post_id:'post_123'+feedback.score:1)+status:paid+subscriptions.current_period_end:<='2024-01-01T23:59:59.999Z'", ); }); it('prefers grouped compound parsing over simple node fallback', () => { - const parsed = parseMemberFilter('(subscribed:false,email_disabled:1)', 'UTC'); + const parsed = parseMemberFilter('(subscribed:false,email_disabled:1)', 'UTC', memberFields); expect(stripIds(parsed)).toEqual([ { field: 'subscribed', operator: 'is-not', values: ['subscribed'] }, @@ -195,6 +250,7 @@ describe('member-filter-query', () => { const parsed = parseMemberFilter( "(subscribed:true+email_disabled:0)+(newsletters.slug:weekly+email_disabled:0)+(feedback.post_id:'post_123'+feedback.score:1)+status:paid", 'UTC', + memberFields, ); expect(stripIds(parsed)).toEqual([ @@ -206,42 +262,48 @@ describe('member-filter-query', () => { }); it('drops unsupported OR compounds during parse', () => { - expect(parseMemberFilter('status:paid,label:vip', 'UTC')).toEqual([]); + expect(parseMemberFilter('status:paid,label:vip', 'UTC', memberFields)).toEqual([]); }); it('keeps supported siblings when unsupported OR compounds are present', () => { expect( stripIds( - parseMemberFilter("(status:paid,label:vip)+created_at:<='2024-02-01T23:59:59.999Z'", 'UTC'), + parseMemberFilter( + "(status:paid,label:vip)+created_at:<='2024-02-01T23:59:59.999Z'", + 'UTC', + memberFields, + ), ), ).toEqual([{ field: 'created_at', operator: 'is-or-less', values: ['2024-02-01'] }]); }); it('ignores malformed NQL input', () => { - expect(parseMemberFilter('status:(', 'UTC')).toEqual([]); + expect(parseMemberFilter('status:(', 'UTC', memberFields)).toEqual([]); }); it('drops invalid member date values during parse', () => { - expect(parseMemberFilter("created_at:<='not-a-date'", 'UTC')).toEqual([]); + expect(parseMemberFilter("created_at:<='not-a-date'", 'UTC', memberFields)).toEqual([]); }); it('parses relative past-date filters into in-the-last predicates for every supported field', () => { - expect(stripIds(parseMemberFilter('created_at:>=now-7d', 'UTC'))).toEqual([ + expect(stripIds(parseMemberFilter('created_at:>=now-7d', 'UTC', memberFields))).toEqual([ { field: 'created_at', operator: 'in-the-last', values: [7] }, ]); - expect(stripIds(parseMemberFilter('last_seen_at:>=now-30d', 'UTC'))).toEqual([ + expect(stripIds(parseMemberFilter('last_seen_at:>=now-30d', 'UTC', memberFields))).toEqual([ { field: 'last_seen_at', operator: 'in-the-last', values: [30] }, ]); - expect(stripIds(parseMemberFilter('subscriptions.start_date:>=now-90d', 'UTC'))).toEqual([ - { field: 'subscriptions.start_date', operator: 'in-the-last', values: [90] }, - ]); + expect( + stripIds(parseMemberFilter('subscriptions.start_date:>=now-90d', 'UTC', memberFields)), + ).toEqual([{ field: 'subscriptions.start_date', operator: 'in-the-last', values: [90] }]); }); it('parses relative future-date filters into in-the-next predicates', () => { expect( - stripIds(parseMemberFilter('subscriptions.current_period_end:<=now+14d', 'UTC')), + stripIds( + parseMemberFilter('subscriptions.current_period_end:<=now+14d', 'UTC', memberFields), + ), ).toEqual([ { field: 'subscriptions.current_period_end', operator: 'in-the-next', values: [14] }, ]); @@ -252,6 +314,7 @@ describe('member-filter-query', () => { serializeMemberFilters( [{ id: '1', field: 'created_at', operator: 'in-the-last', values: [7] }], 'UTC', + memberFields, ), ).toBe('created_at:>=now-7d'); @@ -266,20 +329,23 @@ describe('member-filter-query', () => { }, ], 'UTC', + memberFields, ), ).toBe('subscriptions.current_period_end:<=now+14d'); }); it('round-trips relative created_at predicates alongside other clauses', () => { const filter = 'created_at:>=now-30d+status:paid'; - const parsed = parseMemberFilter(filter, 'UTC'); + const parsed = parseMemberFilter(filter, 'UTC', memberFields); expect(stripIds(parsed)).toEqual([ { field: 'created_at', operator: 'in-the-last', values: [30] }, { field: 'status', operator: 'is', values: ['paid'] }, ]); - expect(serializeMemberFilters(parsed, 'UTC')).toBe('created_at:>=now-30d+status:paid'); + expect(serializeMemberFilters(parsed, 'UTC', memberFields)).toBe( + 'created_at:>=now-30d+status:paid', + ); }); it('drops invalid relative date predicates on serialize', () => { @@ -287,6 +353,7 @@ describe('member-filter-query', () => { serializeMemberFilters( [{ id: '1', field: 'created_at', operator: 'in-the-last', values: [0] }], 'UTC', + memberFields, ), ).toBeUndefined(); @@ -294,6 +361,7 @@ describe('member-filter-query', () => { serializeMemberFilters( [{ id: '1', field: 'created_at', operator: 'in-the-last', values: ['7'] }], 'UTC', + memberFields, ), ).toBeUndefined(); }); @@ -302,7 +370,7 @@ describe('member-filter-query', () => { // Top-level OR isn't flattened by parseMemberNode, so every clause — // including the non-relative `status:paid` — is dropped. Pinned to // catch silent regressions if OR support is added later. - const parsed = parseMemberFilter('created_at:>=now-7d,status:paid', 'UTC'); + const parsed = parseMemberFilter('created_at:>=now-7d,status:paid', 'UTC', memberFields); expect(stripIds(parsed)).toEqual([]); }); @@ -310,11 +378,13 @@ describe('member-filter-query', () => { it('drops the degenerate now-0d form', () => { // The codec only accepts a relative-day count > 0 — both directions // (parse and serialize) defend the same predicate-shape invariant. - expect(parseMemberFilter('created_at:>=now-0d', 'UTC')).toEqual([]); + expect(parseMemberFilter('created_at:>=now-0d', 'UTC', memberFields)).toEqual([]); }); it('rejects relative day counts outside the safe-integer range on parse', () => { - expect(parseMemberFilter('created_at:>=now-9999999999999999d', 'UTC')).toEqual([]); + expect(parseMemberFilter('created_at:>=now-9999999999999999d', 'UTC', memberFields)).toEqual( + [], + ); }); it('reports relative-date filters as timezone-sensitive', () => { @@ -371,7 +441,7 @@ describe('member-filter-query - custom fields', () => { // Serialize each operator to NQL, then parse it back, and confirm the predicate // survives the round trip a saved segment relies on. Each field is its own // predicate keyed `custom_fields.`; `values` is [subfield, value]. - const cases: Array<{ field: string; operator: string; values: [string, string]; nql: string }> = [ + const cases: Array<{ field: string; operator: string; values: string[]; nql: string }> = [ { field: 'custom_fields.company', operator: 'is', @@ -458,7 +528,11 @@ describe('member-filter-query - custom fields', () => { it.each(cases)( 'serializes $field $operator to the expected NQL', ({ field, operator, values, nql }) => { - const serialized = serializeMemberFilters([{ id: 'x', field, operator, values }], 'UTC'); + const serialized = serializeMemberFilters( + [{ id: 'x', field, operator, values }], + 'UTC', + memberFields, + ); expect(serialized).toBe(nql); }, ); @@ -466,7 +540,9 @@ describe('member-filter-query - custom fields', () => { it.each(cases)( 'parses $field $operator back into the same predicate', ({ field, operator, values, nql }) => { - expect(stripIds(parseMemberFilter(nql, 'UTC'))).toEqual([{ field, operator, values }]); + expect(stripIds(parseMemberFilter(nql, 'UTC', memberFields))).toEqual([ + { field, operator, values }, + ]); }, ); @@ -474,8 +550,10 @@ describe('member-filter-query - custom fields', () => { 'round-trips $field $operator (predicate -> nql -> predicate)', ({ field, operator, values }) => { const predicate: FilterPredicate = { id: 'x', field, operator, values }; - const nql = serializeMemberFilters([predicate], 'UTC'); - expect(stripIds(parseMemberFilter(nql, 'UTC'))).toEqual([{ field, operator, values }]); + const nql = serializeMemberFilters([predicate], 'UTC', memberFields); + expect(stripIds(parseMemberFilter(nql, 'UTC', memberFields))).toEqual([ + { field, operator, values }, + ]); }, ); diff --git a/apps/admin/src/members/member-filter-query.ts b/apps/admin/src/members/member-filter-query.ts index f3febb4d0e2..c6712d0a334 100644 --- a/apps/admin/src/members/member-filter-query.ts +++ b/apps/admin/src/members/member-filter-query.ts @@ -1,400 +1,42 @@ +import { memberFields } from './member-fields'; import { - type AstNode, type FilterPredicate, type ParsedPredicate, - dispatchSimpleNodes, getFieldKeysByType, hasFieldKey, + isPredicateEnabled as isEnabled, parseFilterToAst, - resolveField, + parseNodeToPredicates, serializePredicates, stampPredicates, } from '@/shared/filters'; -import { memberFields } from './member-fields'; import type { MemberFields } from './member-fields'; -type CompoundMatcher = (node: AstNode) => ParsedPredicate | null; const TIMEZONE_SENSITIVE_MEMBER_FIELDS = getFieldKeysByType(memberFields, 'date'); -/** - * Is this predicate's operator one the field currently advertises? - * - * This returns `false` for predicates the user can't reach in the UI because - * the field never declares the operator. Hooks call this to drop unreachable - * predicates before serializing or after parsing. The parser/serializer - * themselves stay pure. - */ -export function isPredicateEnabled(predicate: ParsedPredicate, fields: MemberFields): boolean { - const resolved = resolveField(fields, predicate.field, 'UTC'); - return resolved?.definition.operators.includes(predicate.operator) ?? false; -} - -function getCompoundChildren( - node: AstNode, -): { operator: '$and' | '$or'; children: AstNode[] } | null { - if (Array.isArray(node.$and)) { - return { operator: '$and', children: node.$and as AstNode[] }; - } - - if (Array.isArray(node.$or)) { - return { operator: '$or', children: node.$or as AstNode[] }; - } - - return null; -} - -function matchSubscribedNode(node: AstNode): ParsedPredicate | null { - if (typeof node.subscribed === 'boolean') { - return { - field: 'subscribed', - operator: 'is', - values: [node.subscribed ? 'subscribed' : 'unsubscribed'], - }; - } - - if (typeof node.email_disabled === 'number') { - if (node.email_disabled === 1) { - return { - field: 'subscribed', - operator: 'is', - values: ['email-disabled'], - }; - } - - if (node.email_disabled === 0) { - return { - field: 'subscribed', - operator: 'is-not', - values: ['email-disabled'], - }; - } - } - - const compound = getCompoundChildren(node); - - if (!compound || compound.children.length !== 2) { - return null; - } - - let subscribedValue: boolean | undefined; - let emailDisabledValue: number | undefined; - - for (const child of compound.children) { - if (typeof child.subscribed === 'boolean') { - subscribedValue = child.subscribed; - } - - if (typeof child.email_disabled === 'number') { - emailDisabledValue = child.email_disabled; - } - } - - if (compound.operator === '$and' && emailDisabledValue === 0 && subscribedValue !== undefined) { - return { - field: 'subscribed', - operator: 'is', - values: [subscribedValue ? 'subscribed' : 'unsubscribed'], - }; - } - - if (compound.operator === '$or' && emailDisabledValue === 1 && subscribedValue !== undefined) { - return { - field: 'subscribed', - operator: 'is-not', - values: [subscribedValue ? 'unsubscribed' : 'subscribed'], - }; - } - - return null; -} - -function matchNewsletterGroupedNode(node: AstNode): ParsedPredicate | null { - const compound = getCompoundChildren(node); - - if (!compound || compound.children.length !== 2) { - return null; - } - - let slug: string | undefined; - let slugNegated = false; - let hasEmailDisabled = false; - - for (const child of compound.children) { - const newsletterSlug = child['newsletters.slug']; - - if (typeof newsletterSlug === 'string') { - slug = newsletterSlug; - slugNegated = false; - } - - if ( - newsletterSlug && - typeof newsletterSlug === 'object' && - !Array.isArray(newsletterSlug) && - typeof (newsletterSlug as Record).$ne === 'string' - ) { - slug = (newsletterSlug as Record).$ne; - slugNegated = true; - } - - if (typeof child.email_disabled === 'number') { - hasEmailDisabled = true; - } - } - - if (!slug || !hasEmailDisabled) { - return null; - } - - // The slug clause's polarity is the subscription state. Serialize pairs it - // with a fixed join + email_disabled shape, but hand-written filters may - // pair them differently; the email_disabled clause only marks the compound - // as a newsletter subscription filter and never flips its meaning. - return { - field: `newsletters.${slug}`, - operator: 'is', - values: [slugNegated ? 'unsubscribed' : 'subscribed'], - }; -} - -function matchFeedbackGroupedNode(node: AstNode): ParsedPredicate | null { - const compound = getCompoundChildren(node); - - if (!compound || compound.operator !== '$and' || compound.children.length !== 2) { - return null; - } - - let postId: string | undefined; - let score: number | undefined; - - for (const child of compound.children) { - if (typeof child['feedback.post_id'] === 'string') { - postId = child['feedback.post_id']; - } - - if (typeof child['feedback.score'] === 'number') { - score = child['feedback.score']; - } - } - - if (!postId || (score !== 0 && score !== 1)) { - return null; - } - - return { - field: 'newsletter_feedback', - operator: String(score), - values: [postId], - }; -} - -// A trailing `$` is an end anchor only when it isn't escaped: a value containing -// a literal `$` (contains `5$`) reaches here as the source `5\$`, which must not -// be read as ends-with. An odd run of backslashes before the `$` escapes it. -function endsWithAnchor(source: string): boolean { - if (!source.endsWith('$')) { - return false; - } - let backslashes = 0; - for (let i = source.length - 2; i >= 0 && source[i] === '\\'; i -= 1) { - backslashes += 1; - } - return backslashes % 2 === 0; -} - -// A regex value read back into a text operator by its anchors, mirroring the -// serialize symbols in member-fields.ts (`~` contains, `~^` starts, `~$` ends). -// A literal `^` in a value is escaped to `\^` so it never leads, but a literal -// `$` escapes to `\$` and still ends the source, hence the anchor check above. -// `$not` is only ever emitted by `does-not-contain` (an unanchored regex), so a -// negated pattern always maps back to that operator. -function regexToOperator(pattern: RegExp, negated: boolean): { operator: string; value: string } { - const source = pattern.source; - const startsWith = source.startsWith('^'); - const endsWith = endsWithAnchor(source); - - let base: string; - let body: string; - if (startsWith && !endsWith) { - base = 'starts-with'; - body = source.slice(1); - } else if (endsWith && !startsWith) { - base = 'ends-with'; - body = source.slice(0, -1); - } else { - base = 'contains'; - body = source; - } - - const value = body.replace(/\\([\\.^$|?*+()[\]{}/-])/g, '$1'); - return { operator: negated ? 'does-not-contain' : base, value }; -} - -// The value NQL a custom-field predicate carries, read back into a (operator, -// value) pair. nql represents `:~x` as {$regex: /x/} and its negation as -// {$not: /x/}; a bare string is `is` and {$ne} is `is-not`. Returns null for a -// shape we don't emit. -function interpretCustomFieldValue(raw: unknown): { operator: string; value: string } | null { - if (typeof raw === 'string') { - return { operator: 'is', value: raw }; - } - - if (raw && typeof raw === 'object' && !Array.isArray(raw)) { - const object = raw as Record; - - if (typeof object.$ne === 'string') { - return { operator: 'is-not', value: object.$ne }; - } - - if (object.$regex instanceof RegExp) { - return regexToOperator(object.$regex, false); - } - - if (object.$not instanceof RegExp) { - return regexToOperator(object.$not, true); - } - } - - return null; -} - -// A custom-field filter is `(custom_fields.key:''+custom_fields.value[.sub]:)`, -// or the flat `custom_fields.key:''` / `:-''` for set / not set. Each field -// is its own predicate keyed `custom_fields.`, so the field's stable key becomes -// part of the predicate field and the remaining `values` are [subfield, value] -// (subfield '' for a scalar field or the whole-field set/unset case). This can't ride -// the generic parser because the key lives in the value of the key clause, not the key. -function matchCustomFieldNode(node: AstNode): ParsedPredicate | null { - const compound = getCompoundChildren(node); - - if (!compound) { - const keyValue = node['custom_fields.key']; - - if (typeof keyValue === 'string') { - return { field: `custom_fields.${keyValue}`, operator: 'is-set', values: ['', ''] }; - } - - if ( - keyValue && - typeof keyValue === 'object' && - !Array.isArray(keyValue) && - typeof (keyValue as Record).$ne === 'string' - ) { - return { - field: `custom_fields.${(keyValue as Record).$ne}`, - operator: 'is-not-set', - values: ['', ''], - }; - } - - return null; - } - - if (compound.operator !== '$and' || compound.children.length !== 2) { - return null; - } - - let fieldKey: string | undefined; - let valueEntry: { subfield: string; raw: unknown } | undefined; - let pathEntry: { subfield: string; negated: boolean } | undefined; - - for (const child of compound.children) { - if (typeof child['custom_fields.key'] === 'string') { - fieldKey = child['custom_fields.key']; - } - - for (const childKey of Object.keys(child)) { - if (childKey === 'custom_fields.value') { - valueEntry = { subfield: '', raw: child[childKey] }; - } else if (childKey.startsWith('custom_fields.value.')) { - valueEntry = { - subfield: childKey.slice('custom_fields.value.'.length), - raw: child[childKey], - }; - } else if (childKey === 'custom_fields.path') { - const raw = child[childKey]; - - if (typeof raw === 'string') { - pathEntry = { subfield: raw, negated: false }; - } else if ( - raw && - typeof raw === 'object' && - !Array.isArray(raw) && - typeof (raw as Record).$ne === 'string' - ) { - pathEntry = { subfield: (raw as Record).$ne, negated: true }; - } - } - } - } - - if (!fieldKey) { - return null; - } - - // A `path` clause is a part's set / not-set: its presence, carrying no value. - if (pathEntry) { - return { - field: `custom_fields.${fieldKey}`, - operator: pathEntry.negated ? 'is-not-set' : 'is-set', - values: [pathEntry.subfield, ''], - }; - } - - if (!valueEntry) { - return null; - } - - const interpreted = interpretCustomFieldValue(valueEntry.raw); - - if (!interpreted) { - return null; - } - - return { - field: `custom_fields.${fieldKey}`, - operator: interpreted.operator, - values: [valueEntry.subfield, interpreted.value], - }; -} - -const MEMBER_COMPOUND_MATCHERS: CompoundMatcher[] = [ - matchSubscribedNode, - matchNewsletterGroupedNode, - matchFeedbackGroupedNode, - matchCustomFieldNode, -]; - -function parseMemberNode(node: AstNode, timezone: string): ParsedPredicate[] { - for (const matcher of MEMBER_COMPOUND_MATCHERS) { - const parsed = matcher(node); - - if (parsed) { - return [parsed]; - } - } - - const compound = getCompoundChildren(node); - - if (compound?.operator === '$and') { - return compound.children.flatMap((child) => parseMemberNode(child, timezone)); - } - - return dispatchSimpleNodes([node], memberFields, timezone); +export function isPredicateEnabled( + predicate: ParsedPredicate, + fields: MemberFields = memberFields, +): boolean { + return isEnabled(predicate, fields); } /** * Parses NQL into predicates. Pure: callers are responsible for filtering the * output via `isPredicateEnabled` against the field map they want to enforce. */ -export function parseMemberFilter(filter: string | undefined, timezone: string): FilterPredicate[] { +export function parseMemberFilter( + filter: string | undefined, + timezone: string, + fields: MemberFields = memberFields, +): FilterPredicate[] { const ast = parseFilterToAst(filter ?? ''); if (!ast) { return []; } - return stampPredicates(parseMemberNode(ast, timezone)); + return stampPredicates(parseNodeToPredicates(ast, fields, timezone)); } export function hasTimezoneSensitiveMemberFilter(filter: string | undefined): boolean { @@ -415,6 +57,7 @@ export function hasTimezoneSensitiveMemberFilter(filter: string | undefined): bo export function serializeMemberFilters( predicates: FilterPredicate[], timezone: string, + fields: MemberFields = memberFields, ): string | undefined { - return serializePredicates(predicates, memberFields, timezone); + return serializePredicates(predicates, fields, timezone); } diff --git a/apps/admin/src/members/member-query-params.ts b/apps/admin/src/members/member-query-params.ts index 8af9b66e95f..6775167143f 100644 --- a/apps/admin/src/members/member-query-params.ts +++ b/apps/admin/src/members/member-query-params.ts @@ -18,7 +18,7 @@ export type MemberColumnCustomField = Pick clause.key === key); +} + +export type SubscriptionOperator = 'is' | 'is-not'; + +export function subscriptionSemantics(): ValueSemantics { + return { + operators: ['is', 'is-not'], + serialize({ operator, values }): SerializedValue | null { + const value = values[0]; + const affirmative = operator === 'is'; + + if (operator !== 'is' && operator !== 'is-not') { + return null; + } + + if (value === 'email-disabled') { + return { fragments: [{ key: EMAIL_DISABLED, expression: affirmative ? '1' : '0' }] }; + } + + if (value !== 'subscribed' && value !== 'unsubscribed') { + return null; + } + + const optedIn = value === 'subscribed'; + + return affirmative + ? { + join: 'and', + fragments: [{ expression: String(optedIn) }, { key: EMAIL_DISABLED, expression: '0' }], + } + : { + join: 'or', + fragments: [{ expression: String(!optedIn) }, { key: EMAIL_DISABLED, expression: '1' }], + }; + }, + parse() { + return null; + }, + parseClauses(group): SemanticValue | null { + const disabled = clauseFor(group, EMAIL_DISABLED); + const subscribed = clauseFor(group, 'subscribed'); + + if (disabled && !subscribed && group.clauses.length === 1) { + if (disabled.value === 1) { + return { operator: 'is', values: ['email-disabled'] }; + } + + if (disabled.value === 0) { + return { operator: 'is-not', values: ['email-disabled'] }; + } + + return null; + } + + if (!subscribed || typeof subscribed.value !== 'boolean') { + return null; + } + + if (!disabled && group.clauses.length === 1) { + return { operator: 'is', values: [subscribed.value ? 'subscribed' : 'unsubscribed'] }; + } + + if (!disabled) { + return null; + } + + if (group.clauses.length !== 2) { + return null; + } + + if (group.join === 'and' && disabled.value === 0) { + return { operator: 'is', values: [subscribed.value ? 'subscribed' : 'unsubscribed'] }; + } + + if (group.join === 'or' && disabled.value === 1) { + return { operator: 'is-not', values: [subscribed.value ? 'unsubscribed' : 'subscribed'] }; + } + + return null; + }, + }; +} + +export type FeedbackOperator = '1' | '0'; + +export function feedbackSemantics(): ValueSemantics { + return { + operators: ['1', '0'], + serialize({ operator, values }): SerializedValue | null { + const postId = values[0]; + + if (typeof postId !== 'string' || !postId) { + return null; + } + + return { + join: 'and', + fragments: [ + { expression: escapeNqlString(postId) }, + { key: 'feedback.score', expression: operator }, + ], + }; + }, + parse() { + return null; + }, + parseClauses(group): SemanticValue | null { + const post = clauseFor(group, 'feedback.post_id'); + const score = clauseFor(group, 'feedback.score'); + + if ( + group.clauses.length !== 2 || + group.join !== 'and' || + !post || + !score || + typeof post.value !== 'string' + ) { + return null; + } + + if (score.value !== 0 && score.value !== 1) { + return null; + } + + return { operator: score.value === 1 ? '1' : '0', values: [post.value] }; + }, + }; +} diff --git a/apps/admin/src/members/members-filtering.acceptance.test.tsx b/apps/admin/src/members/members-filtering.acceptance.test.tsx index 25fa3e86eca..d66c4e91633 100644 --- a/apps/admin/src/members/members-filtering.acceptance.test.tsx +++ b/apps/admin/src/members/members-filtering.acceptance.test.tsx @@ -1,6 +1,14 @@ import { describe, expect, it } from 'vitest'; - -import { fakeMembers, label, member, renderAdminApp, tier } from '@test-utils/acceptance'; +import { page } from 'vitest/browser'; + +import { + fakeMemberCustomFields, + fakeMembers, + label, + member, + renderAdminApp, + tier, +} from '@test-utils/acceptance'; import { membersScreen } from './members.screen'; describe('Members list', () => { @@ -75,6 +83,47 @@ describe('Members list', () => { await expect.element(membersScreen.link('Silver Member')).toBeVisible(); }); + it('builds a custom field filter without losing the page to the hydration gate', async () => { + const fieldsApi = fakeMemberCustomFields([ + { + key: 'employer', + name: 'Employer', + type: 'short_text', + status: 'active', + created_at: '2026-08-05T00:00:00.000Z', + updated_at: null, + }, + ]); + const membersApi = fakeMembers(({ filter }) => + filter ? [member({ name: 'Acme Member' })] : [member({ name: 'Acme Member' }), member()], + ); + await renderAdminApp('/members', { labs: { membersCustomFields: true } }); + + await expect(membersScreen.memberRows()).toHaveCount(2); + + // The filter bar fetches the archived-inclusive catalog up front: it is the query the + // hydration gate waits on once a filter names a custom field, so it must be answered + // before a field can be picked — a cold cache here unmounts the page to a spinner on + // the first keystroke of the value. + await expect(fieldsApi).toHaveSentFilter('status:[active,archived]'); + + await membersScreen.openFilterField('Employer'); + const valueInput = page.getByRole('textbox', { name: 'Employer value' }); + await valueInput.fill('Acme'); + + // Still here: typing the value must not unmount the filter UI. + await expect.element(valueInput).toBeVisible(); + await expect(membersApi).toHaveSentFilter( + "(custom_fields.key:'employer'+custom_fields.value:~'Acme')", + ); + await expect(membersScreen.memberRows()).toHaveCount(1); + + // Every catalog browse this flow made was the archived-inclusive one the gate shares. + expect(fieldsApi.requests.map((request) => request.filter)).toEqual( + fieldsApi.requests.map(() => 'status:[active,archived]'), + ); + }); + it('builds a name filter through the filters UI', async () => { const membersApi = fakeMembers(({ filter }) => filter diff --git a/apps/admin/src/members/members.tsx b/apps/admin/src/members/members.tsx index b25da4121a1..86626c11193 100644 --- a/apps/admin/src/members/members.tsx +++ b/apps/admin/src/members/members.tsx @@ -12,7 +12,6 @@ import { Box, Container } from '@tryghost/shade/primitives'; import { ListPage } from '@tryghost/shade/page-templates'; import { keepPreviousData } from '@tanstack/react-query'; import { LucideIcon, cn, formatNumber } from '@tryghost/shade/utils'; -import { CUSTOM_FIELDS_PREFIX } from './member-fields'; import { buildMemberListSearchParams, getMemberActiveColumns } from './member-query-params'; import { canBulkDeleteMembers, shouldShowMembersLoading } from './members-view-state'; import { @@ -22,15 +21,14 @@ import { } from '@tryghost/admin-x-framework/api/settings'; import { getSiteTimezone } from '@tryghost/admin-x-framework/utils/get-site-timezone'; import { - shouldDelayMembersDateFilterHydration, + shouldDelayMembersFilterHydration, useMembersFilterState, } from './hooks/use-members-filter-state'; +import { useMemberFilterSources } from './hooks/use-member-filter-sources'; import { useActiveMemberView, useMemberViews } from './hooks/use-member-views'; import { useBrowseConfig } from '@tryghost/admin-x-framework/api/config'; -import { useBrowseMemberCustomFieldsIncludingArchived } from '@tryghost/admin-x-framework/api/member-custom-fields'; import { useBrowseMembersInfinite } from '@tryghost/admin-x-framework/api/members'; import { useDebouncedCallback } from 'use-debounce'; -import { useFeatureFlag } from '@tryghost/admin-x-framework/hooks'; import { useLocation, useSearchParams } from '@tryghost/admin-x-framework'; import { useMultipleActiveSubscriptionsCount } from './hooks/use-multiple-active-subscriptions-count'; @@ -54,8 +52,16 @@ const MembersPage: React.FC = ({ const setHeaderContentRef = useCallback((node: HTMLDivElement | null) => { headerRef.current = node?.closest('[data-list-page="header"]') as HTMLDivElement | null; }, []); + // Names the custom field columns, and gives the filter catalog the definitions that let a + // saved filter on a custom field be read precisely. Archived fields are included so a filter + // on one still shows its values, matching the read-only pill the filter bar renders for it. + // + const [filterSearchParams] = useSearchParams(); + const { newsletters, customFields } = useMemberFilterSources( + filterSearchParams.get('filter') ?? undefined, + ); const { filters, nql, search, setFilters, setSearch, hasFilterOrSearch, clearAll } = - useMembersFilterState(timezone); + useMembersFilterState(timezone, newsletters, customFields); const location = useLocation(); const savedViews = useMemberViews(); const activeView = useActiveMemberView(savedViews, nql); @@ -73,23 +79,6 @@ const MembersPage: React.FC = ({ hasResolvedCount: hasResolvedMultipleActiveSubscriptionsCount, } = useMultipleActiveSubscriptionsCount({ enabled: hasStripeEnabled }); - // Names the custom field columns. Archived fields are included so a filter on one - // still shows its values, matching the read-only pill the filter bar renders for it. - // - // Only a filter on a custom field earns a column, and naming one is all these are for, - // so the fetch waits for a filter rather than riding every visit to the members list. - const customFieldsEnabled = useFeatureFlag('membersCustomFields'); - const hasCustomFieldFilter = useMemo( - () => filters.some((filter) => filter.field.startsWith(CUSTOM_FIELDS_PREFIX)), - [filters], - ); - const { data: customFieldsData } = useBrowseMemberCustomFieldsIncludingArchived({ - enabled: customFieldsEnabled && hasCustomFieldFilter, - }); - // Left undefined until the fetch lands (and while the flag is off) rather than defaulted - // to an empty array, so the identity the memos below depend on stays stable. - const customFields = customFieldsData?.members_custom_fields; - const activeColumns = useMemo(() => { return getMemberActiveColumns(filters, { customFields }); }, [filters, customFields]); @@ -326,11 +315,16 @@ const Members: React.FC = () => { const { data: configData, isLoading: isConfigLoading } = useBrowseConfig(); const filterParam = searchParams.get('filter') ?? undefined; const hasResolvedSettings = Boolean(settingsData?.settings); - const shouldDelayHydration = shouldDelayMembersDateFilterHydration( - filterParam, + + const { newsletters: gateNewsletters, customFields: gateCustomFields } = + useMemberFilterSources(filterParam); + + const shouldDelayHydration = shouldDelayMembersFilterHydration(filterParam, { hasResolvedSettings, - isSettingsLoading, - ); + isLoadingSettings: isSettingsLoading, + newsletters: gateNewsletters, + customFields: gateCustomFields, + }); if ( isSettingsLoading || diff --git a/apps/admin/src/members/newsletter-filter-fields.test.ts b/apps/admin/src/members/newsletter-filter-fields.test.ts new file mode 100644 index 00000000000..dda6cfb5b8f --- /dev/null +++ b/apps/admin/src/members/newsletter-filter-fields.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { parseMemberFilter } from './member-filter-query'; +import type { FilterPredicate } from '@/shared/filters'; + +function stripIds(predicates: FilterPredicate[]) { + return predicates.map((predicate) => ({ + field: predicate.field, + operator: predicate.operator, + values: predicate.values, + })); +} + +describe('newsletter subscription filters', () => { + it('reads newsletter subscription state from the slug clause polarity', () => { + expect( + stripIds(parseMemberFilter('(newsletters.slug:-weekly,email_disabled:1)', 'UTC')), + ).toEqual([{ field: 'newsletters.weekly', operator: 'is', values: ['unsubscribed'] }]); + + expect( + stripIds(parseMemberFilter('(newsletters.slug:weekly+email_disabled:0)', 'UTC')), + ).toEqual([{ field: 'newsletters.weekly', operator: 'is', values: ['subscribed'] }]); + }); + + it('leaves a newsletter pair unread when its parts are arranged to mean something else', () => { + // Being on the list *or* having bounced is a wider set of members than being on the + // list and not having bounced, so these are not the filter they resemble. Reading them + // as one would answer with a different set of members and then save that back. + for (const filter of [ + '(newsletters.slug:weekly,email_disabled:1)', + '(newsletters.slug:-weekly+email_disabled:0)', + '(newsletters.slug:weekly+email_disabled:1)', + ]) { + expect(stripIds(parseMemberFilter(filter, 'UTC'))).not.toContainEqual( + expect.objectContaining({ field: 'newsletters.weekly' }), + ); + } + }); +}); diff --git a/apps/admin/src/members/newsletter-filter-fields.ts b/apps/admin/src/members/newsletter-filter-fields.ts new file mode 100644 index 00000000000..5407059ba98 --- /dev/null +++ b/apps/admin/src/members/newsletter-filter-fields.ts @@ -0,0 +1,170 @@ +import { domainField, getCompoundChildren, readNegatedString } from '@/shared/filters'; +import type { + AstNode, + CompoundMatch, + PlainAddressing, + FieldProvider, + SemanticValue, + SerializedValue, + ValueSemantics, +} from '@/shared/filters'; + +const KEY_PREFIX = 'newsletters.'; +const SLUG_ATTRIBUTE = 'newsletters.slug'; +const EMAIL_DISABLED = 'email_disabled'; + +const OPTIONS = [ + { value: 'subscribed', label: 'Subscribed' }, + { value: 'unsubscribed', label: 'Unsubscribed' }, +]; + +export function newsletterSubscriptionSemantics(slug?: string): ValueSemantics<'is'> { + return { + operators: ['is'], + serialize({ operator, values }, ctx): SerializedValue | null { + const writing = slug ?? ctx.params.slug; + const value = values[0]; + + if (!writing || operator !== 'is') { + return null; + } + + if (value === 'subscribed') { + return { + join: 'and', + fragments: [{ expression: writing }, { key: EMAIL_DISABLED, expression: '0' }], + }; + } + + if (value === 'unsubscribed') { + return { + join: 'or', + fragments: [{ expression: `-${writing}` }, { key: EMAIL_DISABLED, expression: '1' }], + }; + } + + return null; + }, + parse(): SemanticValue<'is'> | null { + return null; + }, + }; +} + +export function newsletterAddressing(slug?: string): PlainAddressing { + return { + address(predicate, ctx) { + return (slug ?? ctx.params.slug) + ? { valueKey: SLUG_ATTRIBUTE, values: predicate.values } + : null; + }, + + match() { + return null; + }, + + matchCompound(node: AstNode): CompoundMatch | null { + for (const join of ['and', 'or'] as const) { + const children = getCompoundChildren(node, join === 'and' ? '$and' : '$or'); + + if (!children || children.length !== 2) { + continue; + } + + let named: string | undefined; + let negated = false; + let disabled: number | undefined; + + for (const child of children) { + const raw = child[SLUG_ATTRIBUTE]; + + if (typeof raw === 'string') { + named = raw; + negated = false; + } + + const denied = readNegatedString(raw); + + if (denied !== null) { + named = denied; + negated = true; + } + + if (typeof child[EMAIL_DISABLED] === 'number') { + disabled = child[EMAIL_DISABLED]; + } + } + + // The pair means the pair. "Subscribed" is written as being on the list and not + // bounced, joined by and; "unsubscribed" is the denial of exactly that, joined by + // or. Any other arrangement of the same three parts says something else — being + // on the list *or* having bounced is a wider set of members than being on it — + // so it is left unread rather than answered for. + const subscribed = !negated && join === 'and' && disabled === 0; + const unsubscribed = negated && join === 'or' && disabled === 1; + + if (!named || (!subscribed && !unsubscribed)) { + continue; + } + + return { + kind: 'predicate', + predicate: { + field: `${KEY_PREFIX}${named}`, + operator: 'is', + values: [subscribed ? 'subscribed' : 'unsubscribed'], + }, + }; + } + + return null; + }, + }; +} + +export interface NewsletterDefinition { + slug: string; + name: string; +} + +export function newsletterDescriptor(newsletter: NewsletterDefinition) { + return domainField({ + key: `${KEY_PREFIX}${newsletter.slug}`, + icon: 'newspaper', + semantics: newsletterSubscriptionSemantics(newsletter.slug), + addressing: newsletterAddressing(newsletter.slug), + operators: ['is'], + options: OPTIONS, + ui: { + label: newsletter.name, + type: 'select', + searchable: false, + hideOperatorSelect: true, + }, + }); +} + +export function newsletterProvider( + newsletters: readonly NewsletterDefinition[] | undefined, +): FieldProvider { + return { + resolved: newsletters !== undefined, + claims: [SLUG_ATTRIBUTE], + fields: (newsletters ?? []).map(newsletterDescriptor), + }; +} + +export const NEWSLETTER_FIELD = domainField({ + key: `${KEY_PREFIX}:slug`, + icon: 'newspaper', + semantics: newsletterSubscriptionSemantics(), + addressing: newsletterAddressing(), + operators: ['is'], + options: OPTIONS, + ui: { + label: 'Newsletter', + type: 'select', + searchable: false, + hideOperatorSelect: true, + }, +}); diff --git a/apps/admin/src/members/use-member-filter-fields.ts b/apps/admin/src/members/use-member-filter-fields.ts index 017ec1b5711..5d7f002839e 100644 --- a/apps/admin/src/members/use-member-filter-fields.ts +++ b/apps/admin/src/members/use-member-filter-fields.ts @@ -1,24 +1,28 @@ import React, { useMemo } from 'react'; import { - DATE_OPERATOR_LABELS, RELATIVE_DATE_OPERATOR_LABELS, createOperatorOptions, createRelativeDateRenderer, fieldHasRelativeOperator, getTodayInTimezone, + FIELD_ICONS, } from '@/shared/filters'; +import type { FieldIcon } from '@/shared/filters'; +import { CUSTOM_FIELDS_PREFIX } from '@/members/member-fields'; +import { keyIsUnder } from '@/shared/filters'; +import type { StaticMemberFieldKey } from '@/members/member-fields'; import { type FilterFieldConfig, type FilterFieldGroup, type FilterOption, type ValueSource, } from '@tryghost/shade/patterns'; -import CustomFieldFilterRenderer from './custom-field-filter-renderer'; +import CustomFieldFilterRenderer from './custom-fields/filter-renderer'; import CustomFieldIcon from '@/shared/member-custom-fields/custom-field-icon'; import { LabelFilterRenderer } from '@/members/label-picker'; import { LucideIcon } from '@tryghost/shade/utils'; import { MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD } from './multiple-active-subscriptions'; -import { CUSTOM_FIELDS_PREFIX, getMemberFields } from './member-fields'; +import { buildMemberFields } from './member-filter-catalog'; import type { MemberCustomField } from '@tryghost/admin-x-framework/api/member-custom-fields'; import type { Offer } from '@tryghost/admin-x-framework/api/offers'; @@ -49,78 +53,71 @@ interface UseMemberFilterFieldsOptions { type OfferOption = FilterOption; type SearchableFieldOverrides = Pick; +type PickerKey = StaticMemberFieldKey | `newsletters.${string}` | `custom_fields.${string}`; + +const BASIC_ORDER = [ + 'name', + 'email', + 'label', + 'subscribed', + 'last_seen_at', + 'created_at', + 'signup', +] as const; + +const SUBSCRIPTION_ORDER = [ + 'tier_id', + 'status', + MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD, + 'subscriptions.plan_interval', + 'subscriptions.status', + 'subscriptions.start_date', + 'subscriptions.current_period_end', + 'conversion', + 'offer_redemptions', +] as const; + +const EMAIL_ORDER = [ + 'email_count', + 'email_opened_count', + 'email_open_rate', + 'emails.post_id', + 'opened_emails.post_id', + 'clicked_links.post_id', + 'newsletter_feedback', +] as const; + +type PlacedFieldKey = + | (typeof BASIC_ORDER)[number] + | (typeof SUBSCRIPTION_ORDER)[number] + | (typeof EMAIL_ORDER)[number]; + +const EVERY_DECLARED_FIELD_IS_PLACED: Exclude extends never + ? true + : { fieldsWithNoGroup: Exclude } = true; + +void EVERY_DECLARED_FIELD_IS_PLACED; + // How many custom fields the picker shows before "Show more" — the same preview // size the settings list uses. The rest stay searchable and resolvable. +const NO_NEWSLETTERS: NonNullable = []; +const NO_SLUGS: string[] = []; +const NO_OFFERS: NonNullable = []; +const NO_CUSTOM_FIELDS: NonNullable = []; +const NO_ARCHIVED_CUSTOM_FIELDS: NonNullable = + []; + const CUSTOM_FIELDS_PREVIEW_LIMIT = 5; -const MEMBER_OPERATOR_LABELS: Record = { +const MEMBER_OPERATOR_LABELS: Partial> = { 'is-any': 'is any of', 'is-not-any': 'is none of', 'does-not-contain': 'does not contain', - ...DATE_OPERATOR_LABELS, ...RELATIVE_DATE_OPERATOR_LABELS, 1: 'More like this', 0: 'Less like this', }; -const NUMBER_OPERATOR_LABELS: Record = { - 'is-greater': 'is greater than', - 'is-less': 'is less than', -}; - -function getFieldIcon(key: string) { - switch (key) { - case 'name': - return React.createElement(LucideIcon.User, { className: 'size-4' }); - case 'email': - case 'subscribed': - return React.createElement(LucideIcon.Mail, { className: 'size-4' }); - case 'label': - return React.createElement(LucideIcon.Tag, { className: 'size-4' }); - case 'last_seen_at': - return React.createElement(LucideIcon.Eye, { className: 'size-4' }); - case 'created_at': - return React.createElement(LucideIcon.Calendar, { className: 'size-4' }); - case 'signup': - return React.createElement(LucideIcon.UserPlus, { className: 'size-4' }); - case 'tier_id': - case 'subscriptions.status': - return React.createElement(LucideIcon.CreditCard, { className: 'size-4' }); - case 'status': - return React.createElement(LucideIcon.UserCircle, { className: 'size-4' }); - case 'subscriptions.plan_interval': - return React.createElement(LucideIcon.CalendarClock, { className: 'size-4' }); - case 'subscriptions.start_date': - return React.createElement(LucideIcon.CalendarPlus, { className: 'size-4' }); - case 'subscriptions.current_period_end': - return React.createElement(LucideIcon.CalendarArrowDown, { className: 'size-4' }); - case 'conversion': - return React.createElement(LucideIcon.ArrowRightLeft, { className: 'size-4' }); - case 'email_count': - case 'emails.post_id': - return React.createElement(LucideIcon.Send, { className: 'size-4' }); - case 'email_opened_count': - case 'opened_emails.post_id': - return React.createElement(LucideIcon.MailOpen, { className: 'size-4' }); - case 'email_open_rate': - return React.createElement(LucideIcon.Percent, { className: 'size-4' }); - case 'clicked_links.post_id': - return React.createElement(LucideIcon.MousePointerClick, { className: 'size-4' }); - case 'newsletter_feedback': - return React.createElement(LucideIcon.MessageSquare, { className: 'size-4' }); - case 'offer_redemptions': - return React.createElement(LucideIcon.Ticket, { className: 'size-4' }); - case MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD: - return React.createElement(LucideIcon.Layers, { className: 'size-4' }); - default: - if (key.startsWith('newsletters.')) { - return React.createElement(LucideIcon.Newspaper, { className: 'size-4' }); - } - - return undefined; - } -} - function createSearchableFieldOverrides( options: FilterOption[], valueSource?: ValueSource, @@ -272,57 +269,74 @@ function renderOfferFilterValues( export function useMemberFilterFields({ labelValueSource, tierValueSource, - newsletters = [], - hydratedNewsletterSlugs = [], + newsletters = NO_NEWSLETTERS, + hydratedNewsletterSlugs = NO_SLUGS, hasMultipleTiers = false, paidMembersEnabled = false, emailFiltersEnabled = false, postValueSource, emailValueSource, - offers = [], + offers = NO_OFFERS, multipleActiveSubscriptionsCount = 0, membersTrackSources = false, emailTrackOpens = false, emailTrackClicks = false, customFieldsEnabled = false, - customFields = [], - archivedCustomFields = [], + customFields = NO_CUSTOM_FIELDS, + archivedCustomFields = NO_ARCHIVED_CUSTOM_FIELDS, siteTimezone = 'UTC', }: UseMemberFilterFieldsOptions): FilterFieldGroup[] { - return useMemo(() => { - const fields = getMemberFields(); - type MemberFieldKey = keyof typeof fields; + // Which fields exist is decided by the site's own data, and nothing else. Kept apart from the + // picker below because building it builds a codec for every field, while the picker changes + // whenever the view does — on every filter edit, among other things. + const fields = useMemo(() => { + const catalogNewsletters = [ + ...newsletters, + ...hydratedNewsletterSlugs + .filter((slug) => !newsletters.some((newsletter) => newsletter.slug === slug)) + .map((slug) => ({ slug, name: slug })), + ]; + + return buildMemberFields({ + newsletters: catalogNewsletters, + customFields: [ + ...customFields, + ...archivedCustomFields.map((field) => ({ ...field, type: 'short_text' as const })), + ], + }); + }, [newsletters, hydratedNewsletterSlugs, customFields, archivedCustomFields]); + return useMemo(() => { function createFieldConfig( - key: string, + key: PickerKey, overrides: Partial = {}, - operatorLabels: Record = MEMBER_OPERATOR_LABELS, + operatorLabels: Partial> = MEMBER_OPERATOR_LABELS, ): FilterFieldConfig { - let field; - if (key.startsWith('newsletters.')) { - field = fields['newsletters.:slug']; - } else if (key.startsWith(CUSTOM_FIELDS_PREFIX)) { - field = fields['custom_fields.:key']; - } else { - field = fields[key as MemberFieldKey]; - } + const parameterised = keyIsUnder(key, 'newsletters') + ? fields['newsletters.:slug'] + : keyIsUnder(key, CUSTOM_FIELDS_PREFIX) + ? fields['custom_fields.:key'] + : undefined; + const field = fields[key] ?? parameterised; return { key, ...field.ui, - icon: getFieldIcon(key), - operators: createOperatorOptions(field.operators, { labels: operatorLabels }), + icon: FIELD_ICONS[field.ui.icon as FieldIcon], + operators: createOperatorOptions(field.operators, { + labels: { ...operatorLabels, ...field.operatorLabels }, + }), ...('options' in field && field.options ? { options: field.options } : {}), ...overrides, }; } function createDateFieldConfig( - key: string, + key: PickerKey, today: string, overrides: Partial = {}, ): FilterFieldConfig { - const field = fields[key as MemberFieldKey]; + const field = fields[key]; const config = createFieldConfig(key, { defaultValue: today, ...overrides }); return fieldHasRelativeOperator(field) @@ -348,47 +362,45 @@ export function useMemberFilterFields({ const offerLabels = createOfferLabelMap(offers); const today = getTodayInTimezone(siteTimezone); - const basicFields: FilterFieldConfig[] = [ - createFieldConfig('name'), - createFieldConfig('email'), - ]; - - if (labelValueSource) { - basicFields.push( - createFieldConfig('label', { - ...createSearchableFieldOverrides([], labelValueSource), - customRenderer: (props) => - React.createElement( - LabelFilterRenderer, - props as React.ComponentProps, - ), - }), - ); - } - - if (activeNewsletters.length <= 1) { - basicFields.push(createFieldConfig('subscribed')); - - for (const newsletter of visibleHydratedNewsletters) { - basicFields.push( - createFieldConfig(`newsletters.${newsletter.slug}`, { - label: newsletter.name, - }), - ); - } - } - - basicFields.push( - createDateFieldConfig('last_seen_at', today), - createDateFieldConfig('created_at', today), + const oneNewsletterOrFewer = activeNewsletters.length <= 1; + const basicShown: Partial> = { + label: Boolean(labelValueSource), + subscribed: oneNewsletterOrFewer, + signup: membersTrackSources, + }; + + const basicFields = BASIC_ORDER.filter((key) => basicShown[key] ?? true).flatMap( + (key): FilterFieldConfig[] => { + switch (key) { + case 'label': + return [ + createFieldConfig(key, { + ...createSearchableFieldOverrides([], labelValueSource), + customRenderer: (props) => + React.createElement( + LabelFilterRenderer, + props as React.ComponentProps, + ), + }), + ]; + case 'subscribed': + return [ + createFieldConfig(key), + ...visibleHydratedNewsletters.map((newsletter) => + createFieldConfig(`newsletters.${newsletter.slug}`, { label: newsletter.name }), + ), + ]; + case 'last_seen_at': + case 'created_at': + return [createDateFieldConfig(key, today)]; + case 'signup': + return [createFieldConfig(key, createSearchableFieldOverrides([], postValueSource))]; + default: + return [createFieldConfig(key)]; + } + }, ); - if (membersTrackSources) { - basicFields.push( - createFieldConfig('signup', createSearchableFieldOverrides([], postValueSource)), - ); - } - groups.push({ group: 'Basic', fields: basicFields }); // Each defined custom field is its own named entry, so a publisher can search @@ -402,9 +414,6 @@ export function useMemberFilterFields({ // The dropdown entry and the added filter show the field type's own icon // rather than a generic custom-field mark. icon: React.createElement(CustomFieldIcon, { type: field.type, className: 'size-4' }), - // Text fields default to "contains" to match native Name/Email; a - // composite defaults to whole-field "is set" (the renderer coerces it). - defaultOperator: 'contains', // The field's type decides its parts and operators, so the operator // control lives in the renderer, after any part is chosen. renderOperatorInValue: true, @@ -482,87 +491,57 @@ export function useMemberFilterFields({ } if (paidMembersEnabled) { - const subscriptionFields: FilterFieldConfig[] = []; - - if (hasMultipleTiers) { - subscriptionFields.push( - createFieldConfig('tier_id', createSearchableFieldOverrides([], tierValueSource)), - ); - } - - subscriptionFields.push( - createFieldConfig('status', { - options: [...fields.status.options, { value: 'gift', label: 'Gift subscription' }], - }), - ); - - if (multipleActiveSubscriptionsCount > 0) { - subscriptionFields.push(createFieldConfig(MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD)); - } - - subscriptionFields.push( - createFieldConfig('subscriptions.plan_interval'), - createFieldConfig('subscriptions.status'), - createDateFieldConfig('subscriptions.start_date', today), - createDateFieldConfig('subscriptions.current_period_end', today), - ); - - if (membersTrackSources) { - subscriptionFields.push( - createFieldConfig('conversion', createSearchableFieldOverrides([], postValueSource)), - ); - } + const subscriptionShown: Partial> = { + tier_id: hasMultipleTiers, + [MULTIPLE_ACTIVE_STRIPE_CUSTOMERS_FIELD]: multipleActiveSubscriptionsCount > 0, + conversion: membersTrackSources, + offer_redemptions: offers.length > 0, + }; - if (offers.length > 0) { - subscriptionFields.push( - createFieldConfig('offer_redemptions', { - options: offerOptions, - customValueRenderer: (values) => - renderOfferFilterValues(values as string[], offerOptions, offerLabels), - }), - ); - } + const subscriptionFields = SUBSCRIPTION_ORDER.filter( + (key) => subscriptionShown[key] ?? true, + ).map((key) => { + switch (key) { + case 'tier_id': + return createFieldConfig(key, createSearchableFieldOverrides([], tierValueSource)); + case 'status': + return createFieldConfig(key, { + options: [ + ...(fields.status?.options ?? []), + { value: 'gift', label: 'Gift subscription' }, + ], + }); + case 'subscriptions.start_date': + case 'subscriptions.current_period_end': + return createDateFieldConfig(key, today); + case 'conversion': + return createFieldConfig(key, createSearchableFieldOverrides([], postValueSource)); + case 'offer_redemptions': + return createFieldConfig(key, { + options: offerOptions, + customValueRenderer: (values) => + renderOfferFilterValues(values as string[], offerOptions, offerLabels), + }); + default: + return createFieldConfig(key); + } + }); groups.push({ group: 'Subscription', fields: subscriptionFields }); } if (emailFiltersEnabled) { - const emailFields: FilterFieldConfig[] = [ - createFieldConfig('email_count', {}, NUMBER_OPERATOR_LABELS), - createFieldConfig('email_opened_count', {}, NUMBER_OPERATOR_LABELS), - ]; - - if (emailTrackOpens) { - emailFields.push(createFieldConfig('email_open_rate', {}, NUMBER_OPERATOR_LABELS)); - } - - emailFields.push( - createFieldConfig('emails.post_id', createSearchableFieldOverrides([], emailValueSource)), - ); - - if (emailTrackOpens) { - emailFields.push( - createFieldConfig( - 'opened_emails.post_id', - createSearchableFieldOverrides([], emailValueSource), - ), - ); - } - - if (emailTrackClicks) { - emailFields.push( - createFieldConfig( - 'clicked_links.post_id', - createSearchableFieldOverrides([], emailValueSource), - ), - ); - } + const emailShown: Partial> = { + email_open_rate: emailTrackOpens, + 'opened_emails.post_id': emailTrackOpens, + 'clicked_links.post_id': emailTrackClicks, + }; + const emailCounts = new Set(['email_count', 'email_opened_count', 'email_open_rate']); - emailFields.push( - createFieldConfig( - 'newsletter_feedback', - createSearchableFieldOverrides([], emailValueSource), - ), + const emailFields = EMAIL_ORDER.filter((key) => emailShown[key] ?? true).map((key) => + emailCounts.has(key) + ? createFieldConfig(key) + : createFieldConfig(key, createSearchableFieldOverrides([], emailValueSource)), ); groups.push({ group: 'Email', fields: emailFields }); @@ -570,11 +549,10 @@ export function useMemberFilterFields({ return groups; }, [ + fields, emailFiltersEnabled, emailValueSource, customFieldsEnabled, - customFields, - archivedCustomFields, emailTrackClicks, emailTrackOpens, hasMultipleTiers, diff --git a/apps/admin/src/posts/analytics/components/layout/post-analytics-layout.tsx b/apps/admin/src/posts/analytics/components/layout/post-analytics-layout.tsx index af6c239d014..28c6084fc10 100644 --- a/apps/admin/src/posts/analytics/components/layout/post-analytics-layout.tsx +++ b/apps/admin/src/posts/analytics/components/layout/post-analytics-layout.tsx @@ -5,8 +5,8 @@ const PostAnalyticsLayout: React.FC> = ({ c return ( -

-
{children}
+
+
{children}
diff --git a/apps/admin/src/posts/analytics/components/post-analytics-header.tsx b/apps/admin/src/posts/analytics/components/post-analytics-header.tsx index edde200b78c..fa3866d427f 100644 --- a/apps/admin/src/posts/analytics/components/post-analytics-header.tsx +++ b/apps/admin/src/posts/analytics/components/post-analytics-header.tsx @@ -151,13 +151,13 @@ const PostAnalyticsHeader: React.FC = ({ currentTab, c return ( <> -
+
-
+
diff --git a/apps/admin/src/posts/analytics/post-analytics.acceptance.test.tsx b/apps/admin/src/posts/analytics/post-analytics.acceptance.test.tsx index 974d5fd7243..6f8228b1d94 100644 --- a/apps/admin/src/posts/analytics/post-analytics.acceptance.test.tsx +++ b/apps/admin/src/posts/analytics/post-analytics.acceptance.test.tsx @@ -128,6 +128,16 @@ function seedEmptyPostAnalyticsWorld() { } describe('Post analytics overview', () => { + it('applies the Admin 7 chrome on post analytics', async () => { + seedPostAnalyticsWorld(); + await renderAdminApp(`/posts/analytics/${POST_ID}`, { + labs: { admin7PageChrome: true }, + boot: webAnalyticsBootOverrides(), + }); + await expect.element(postAnalyticsScreen.postTitle('Attack of the Clones')).toBeVisible(); + await expect.poll(() => document.querySelector('#root .admin7')).not.toBeNull(); + }); + it('renders the seeded post with web and growth sections', async () => { const { postsApi } = seedPostAnalyticsWorld(); await renderAdminApp(`/posts/analytics/${POST_ID}`, { boot: webAnalyticsBootOverrides() }); diff --git a/apps/admin/src/settings/advanced/advanced.acceptance.test.tsx b/apps/admin/src/settings/advanced/advanced.acceptance.test.tsx index 5f666efa3a0..4747a170680 100644 --- a/apps/admin/src/settings/advanced/advanced.acceptance.test.tsx +++ b/apps/admin/src/settings/advanced/advanced.acceptance.test.tsx @@ -11,6 +11,7 @@ import { settingsResponse, currentUserResponse, currentRoute, + configResponse, type StaffUser, } from '@test-utils/acceptance'; import { settingsScreen } from '@/settings/settings.screen'; @@ -37,6 +38,58 @@ function advancedSettings(overrides: Record) { } describe('Advanced settings', () => { + it('offers the page chrome toggle only in developer experiments', async () => { + fakeSettingsScreens(); + const response = configResponse(); + response.config.enableDeveloperExperiments = false; + await renderAdminApp('/settings/labs', { boot: { browseConfig: { response } } }); + + const section = settingsScreen.section('labs'); + await section.getByRole('button', { name: 'Open' }).click(); + await expect + .element(section.getByRole('tab', { name: 'Private features' })) + .not.toBeInTheDocument(); + await expect + .element(section.getByRole('switch', { name: 'Admin 7 page chrome' })) + .not.toBeInTheDocument(); + }); + + it('saves the private page chrome flag without changing other Labs settings', async () => { + fakeSettingsScreens(); + const settingsApi = fakeEditSettings(); + const labs = { admin7PageChrome: false, tagsX: true }; + const response = configResponse(); + response.config.enableDeveloperExperiments = true; + await renderAdminApp('/settings/labs', { labs, boot: { browseConfig: { response } } }); + + const section = settingsScreen.section('labs'); + await section.getByRole('button', { name: 'Open' }).click(); + await section.getByRole('tab', { name: 'Private features' }).click(); + const toggle = section.getByRole('switch', { name: 'Admin 7 page chrome' }); + await expect.element(toggle).not.toBeChecked(); + await toggle.click(); + await expect(settingsApi).toHaveEditedSettings([ + { + key: 'labs', + value: String( + settingsResponse({ labs: { ...labs, admin7PageChrome: true } }).settings.find( + (setting) => setting.key === 'labs', + )!.value, + ), + }, + ]); + await expect.element(toggle).toBeChecked(); + await toggle.click(); + await expect(settingsApi).toHaveEditedSettings([ + { + key: 'labs', + value: String( + settingsResponse({ labs }).settings.find((setting) => setting.key === 'labs')!.value, + ), + }, + ]); + }); + it('saves header and footer code injection', async () => { fakeSettingsScreens(); const settingsApi = fakeEditSettings(); diff --git a/apps/admin/src/settings/advanced/labs/private-features.tsx b/apps/admin/src/settings/advanced/labs/private-features.tsx index 979ec31736a..66d19bf00ab 100644 --- a/apps/admin/src/settings/advanced/labs/private-features.tsx +++ b/apps/admin/src/settings/advanced/labs/private-features.tsx @@ -44,6 +44,11 @@ const features: Feature[] = [ description: 'Enable Admin UI refresh (exploration)', flag: 'adminUIRefresh', }, + { + title: 'Admin 7 page chrome', + description: 'Enable the new Admin page chrome on desktop in light mode.', + flag: 'admin7PageChrome', + }, { title: 'Tags X', description: 'Enables the new Tags UI', diff --git a/apps/admin/src/settings/layout.acceptance.test.tsx b/apps/admin/src/settings/layout.acceptance.test.tsx index 511218f3db1..8d8c3ee62a2 100644 --- a/apps/admin/src/settings/layout.acceptance.test.tsx +++ b/apps/admin/src/settings/layout.acceptance.test.tsx @@ -1,8 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { userEvent } from 'vitest/browser'; +import { page, userEvent } from 'vitest/browser'; import { currentRoute, + currentUserResponse, fakeAnalyticsOverview, fakeSettingsScreens, fakeTiers, @@ -13,6 +14,80 @@ import { import { settingsScreen } from './settings.screen'; describe('Settings layout', () => { + it.each([true, false])( + 'uses Admin 7 typography in Settings only when enabled (%s)', + async (enabled) => { + fakeSettingsScreens(); + await renderAdminApp('/settings', { labs: { admin7PageChrome: enabled } }); + + await expect.element(settingsScreen.search()).toBeVisible(); + const titleAndDescriptionEdit = settingsScreen + .titleAndDescription() + .getByRole('button', { name: 'Edit' }); + await expect.element(titleAndDescriptionEdit).toBeVisible(); + const elements = [ + ...page.getByRole('heading', { name: 'General settings', exact: true }).elements(), + settingsScreen.search().element(), + titleAndDescriptionEdit.element(), + ]; + for (const element of elements) { + await expect + .poll(() => getComputedStyle(element).fontFamily.includes('Inter Admin 7')) + .toBe(enabled); + } + expect(document.querySelector('.admin7') !== null).toBe(enabled); + expect( + getComputedStyle(document.querySelector('#root > div')!).getPropertyValue( + '--content-width', + ), + ).toBe(''); + expect(getComputedStyle(document.body).fontFamily).not.toContain('Inter Admin 7'); + await expect.element(settingsScreen.sidebar()).toBeVisible(); + await expect.element(settingsScreen.exitButton()).toBeVisible(); + }, + ); + + it('uses Admin 7 typography without page chrome in dark mode', async () => { + fakeSettingsScreens(); + const me = currentUserResponse(); + me.users[0].accessibility = JSON.stringify({ nightShift: 'dark' }); + await renderAdminApp('/settings', { + labs: { admin7PageChrome: true }, + boot: { browseMe: { response: me } }, + }); + await expect.element(settingsScreen.search()).toBeVisible(); + await expect.poll(() => document.documentElement.classList.contains('dark')).toBe(true); + expect(getComputedStyle(settingsScreen.search().element()).fontFamily).toContain( + 'Inter Admin 7', + ); + expect(document.querySelector('.admin7')).not.toBeNull(); + expect( + getComputedStyle(document.querySelector('.admin7')!).getPropertyValue('--content-width'), + ).toBe(''); + }); + + it('limits Settings typography to desktop without opting into page chrome', async () => { + fakeSettingsScreens(); + await renderAdminApp('/settings', { labs: { admin7PageChrome: true } }); + await expect.element(settingsScreen.search()).toBeVisible(); + const hasNewFont = () => + getComputedStyle(settingsScreen.search().element()).fontFamily.includes('Inter Admin 7'); + await expect.poll(hasNewFont).toBe(true); + try { + await page.viewport(800, 800); + await expect.poll(hasNewFont).toBe(false); + expect(document.querySelector('.admin7')).toBeNull(); + await page.viewport(801, 800); + await expect.poll(hasNewFont).toBe(true); + expect(document.querySelector('.admin7')).not.toBeNull(); + expect( + getComputedStyle(document.querySelector('.admin7')!).getPropertyValue('--content-width'), + ).toBe(''); + } finally { + await page.viewport(1280, 800); + } + }); + it('leaves immediately when the page is clean', async () => { fakeSettingsScreens(); fakeAnalyticsOverview(); @@ -62,6 +137,7 @@ describe('Settings layout', () => { fakeSettingsScreens(); fakeTiers([tier({ name: 'Supporter' })]); await renderAdminApp('/settings/portal/edit', { + labs: { admin7PageChrome: true }, boot: { browseSettings: { response: settingsResponse({ @@ -76,6 +152,9 @@ describe('Settings layout', () => { const modal = settingsScreen.portalModal(); await expect.element(modal).toBeVisible(); + await expect + .poll(() => getComputedStyle(modal.element()).fontFamily) + .toContain('Inter Admin 7'); await modal.getByLabelText('Default price at signup').click(); await expect.element(settingsScreen.selectOptionExact('Yearly')).toBeVisible(); await userEvent.keyboard('{Escape}'); diff --git a/apps/admin/src/settings/site/theme.acceptance.test.tsx b/apps/admin/src/settings/site/theme.acceptance.test.tsx index 2c6c7347158..efe55eb8683 100644 --- a/apps/admin/src/settings/site/theme.acceptance.test.tsx +++ b/apps/admin/src/settings/site/theme.acceptance.test.tsx @@ -1,5 +1,6 @@ import { describe, expect, it, onTestFinished, vi } from 'vitest'; import { page, userEvent } from 'vitest/browser'; +import { EditorView } from '@uiw/react-codemirror'; import { configResponse, @@ -133,10 +134,24 @@ function problemLists(): HTMLElement[] { return [...dialog.querySelectorAll(`[data-testid="${THEME_PROBLEM_LIST_TESTID}"]`)]; } +/** + * Replaces the editor's contents through CodeMirror itself. Playwright's + * `fill()` costs ~2s per freshly mounted editor here; a transaction drives the + * same onChange path in ~50ms. + */ async function editorTextbox() { const editor = settingsScreen.themeCodeEditorModal().getByRole('textbox').first(); await expect.element(editor).toBeVisible(); - return editor; + return Object.assign(editor, { + setContent: (text: string) => { + const view = EditorView.findFromDOM(editor.element() as HTMLElement); + if (!view) { + throw new Error('CodeMirror EditorView not found for the theme editor'); + } + view.focus(); + view.dispatch({ changes: { from: 0, to: view.state.doc.length, insert: text } }); + }, + }); } describe('Theme settings', () => { @@ -636,7 +651,7 @@ describe('Theme settings', () => { await expect.element(modal).toHaveTextContent(/Edit theme/); await expect.element(modal).toHaveTextContent(/json/i); const editor = await editorTextbox(); - await editor.fill('{"name":"edition","version":"1.0.0"}\n'); + editor.setContent('{"name":"edition","version":"1.0.0"}\n'); await expect.element(modal).toHaveTextContent(/1 file modified/); await modal.getByRole('button', { name: 'Save' }).click(); await settingsScreen @@ -658,7 +673,10 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/edition'); const editor = await editorTextbox(); - await editor.fill('{"name":"edition","version":"1.0.0"}\n'); + editor.setContent('{"name":"edition","version":"1.0.0"}\n'); + await expect + .element(settingsScreen.themeCodeEditorModal()) + .toHaveTextContent(/1 file modified/); window.dispatchEvent( new KeyboardEvent('keydown', { key: 's', ctrlKey: true, bubbles: true, cancelable: true }), ); @@ -701,7 +719,7 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/edition'); const editor = await editorTextbox(); - await editor.fill('{"name":"edition","version":"1.0.0"}\n'); + editor.setContent('{"name":"edition","version":"1.0.0"}\n'); await settingsScreen.themeCodeEditorModal().getByRole('button', { name: 'Save' }).click(); await settingsScreen .themeEditorConfirmModal() @@ -733,7 +751,7 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/edition'); const editor = await editorTextbox(); - await editor.fill('{"name":"edition","version":"1.0.0"}\n'); + editor.setContent('{"name":"edition","version":"1.0.0"}\n'); await settingsScreen.themeCodeEditorModal().getByRole('button', { name: 'Save' }).click(); await settingsScreen .themeEditorConfirmModal() @@ -765,7 +783,7 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/edition'); const editor = await editorTextbox(); - await editor.fill('{"name":"edition","version":"1.0.0"}\n'); + editor.setContent('{"name":"edition","version":"1.0.0"}\n'); await settingsScreen.themeCodeEditorModal().getByRole('button', { name: 'Save' }).click(); await settingsScreen .themeEditorConfirmModal() @@ -788,7 +806,7 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/edition'); const editor = await editorTextbox(); - await editor.fill('{"name":"edition","version":"1.0.0"}\n'); + editor.setContent('{"name":"edition","version":"1.0.0"}\n'); await settingsScreen.themeCodeEditorModal().getByRole('button', { name: 'Save' }).click(); await settingsScreen .themeEditorConfirmModal() @@ -806,7 +824,7 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/casper'); const editor = await editorTextbox(); - await editor.fill('{"name":"casper","version":"1.0.0"}\n'); + editor.setContent('{"name":"casper","version":"1.0.0"}\n'); await settingsScreen.themeCodeEditorModal().getByRole('button', { name: 'Save' }).click(); const inputModal = settingsScreen.themeEditorInputModal(); await inputModal.getByLabelText('Theme name').fill('Foo Bar!'); @@ -830,7 +848,7 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/casper'); const editor = await editorTextbox(); - await editor.fill('{"name":"casper","version":"1.0.0"}\n'); + editor.setContent('{"name":"casper","version":"1.0.0"}\n'); await settingsScreen.themeCodeEditorModal().getByRole('button', { name: 'Save' }).click(); await settingsScreen.themeEditorInputModal().getByLabelText('Theme name').fill('casper-edited'); await settingsScreen.themeEditorInputModal().getByRole('button', { name: 'Continue' }).click(); @@ -984,7 +1002,7 @@ describe('Theme settings', () => { await renderAdminApp('/settings/theme/edit/edition'); const editor = await editorTextbox(); - await editor.fill('{"name":"edition","version":"1.0.0"}\n'); + editor.setContent('{"name":"edition","version":"1.0.0"}\n'); await settingsScreen.themeCodeEditorModal().getByRole('button', { name: 'Close' }).click(); await expect .element(settingsScreen.themeEditorConfirmModal()) diff --git a/apps/admin/src/shared/filters/filter-codecs.test.ts b/apps/admin/src/shared/filters/codec-composition.test.ts similarity index 75% rename from apps/admin/src/shared/filters/filter-codecs.test.ts rename to apps/admin/src/shared/filters/codec-composition.test.ts index 9826348537e..c453c42530e 100644 --- a/apps/admin/src/shared/filters/filter-codecs.test.ts +++ b/apps/admin/src/shared/filters/codec-composition.test.ts @@ -1,8 +1,36 @@ -import nql from '@tryghost/nql-lang'; -import { dateCodec, numberCodec, scalarCodec, setCodec, textCodec } from './filter-codecs'; +import { columnAddressing, composeCodec } from './filter-addressing'; +import { + dateSemantics, + numberSemantics, + scalarSemantics, + setSemantics, + textSemantics, +} from './semantics'; +import type { ValueConfig } from './semantics'; +import { parseFilterToAst } from './filter-query-core'; +type CodecConfig = ValueConfig & { field?: string }; +const textCodec = (config?: CodecConfig) => composeCodec(columnAddressing(config), textSemantics()); +const scalarCodec = (config?: CodecConfig) => + composeCodec(columnAddressing(config), scalarSemantics(config)); +const setCodec = (config?: CodecConfig) => + composeCodec(columnAddressing(config), setSemantics(config)); +const numberCodec = (config?: CodecConfig) => + composeCodec(columnAddressing(config), numberSemantics()); +const dateCodec = (config?: CodecConfig) => composeCodec(columnAddressing(config), dateSemantics()); + import { describe, expect, it } from 'vitest'; import type { CodecContext, FilterPredicate } from './filter-types'; +function ast(filter: string) { + const node = parseFilterToAst(filter); + + if (!node) { + throw new Error(`could not parse: ${filter}`); + } + + return node; +} + const statusContext: CodecContext = { key: 'status', pattern: 'status', @@ -61,12 +89,12 @@ const dateContext: CodecContext = { describe('scalarCodec', () => { it('parses simple scalar comparisons', () => { - expect(scalarCodec().parse(nql.parse('status:paid') as never, statusContext)).toEqual({ + expect(scalarCodec().parse(ast('status:paid'), statusContext)).toEqual({ field: 'status', operator: 'is', values: ['paid'], }); - expect(scalarCodec().parse(nql.parse('status:-paid') as never, statusContext)).toEqual({ + expect(scalarCodec().parse(ast('status:-paid'), statusContext)).toEqual({ field: 'status', operator: 'is-not', values: ['paid'], @@ -98,7 +126,7 @@ describe('scalarCodec', () => { it('supports mapped NQL field names', () => { const authorCodec = scalarCodec({ field: 'member_id' }); - expect(authorCodec.parse(nql.parse('member_id:abc123') as never, authorContext)).toEqual({ + expect(authorCodec.parse(ast('member_id:abc123'), authorContext)).toEqual({ field: 'author', operator: 'is', values: ['abc123'], @@ -155,12 +183,12 @@ describe('scalarCodec', () => { describe('textCodec', () => { it('parses regex-based text operators', () => { - expect(textCodec().parse(nql.parse("email:~'ghost'") as never, emailContext)).toEqual({ + expect(textCodec().parse(ast("email:~'ghost'"), emailContext)).toEqual({ field: 'email', operator: 'contains', values: ['ghost'], }); - expect(textCodec().parse(nql.parse("email:-~$'ghost'") as never, emailContext)).toEqual({ + expect(textCodec().parse(ast("email:-~$'ghost'"), emailContext)).toEqual({ field: 'email', operator: 'does-not-end-with', values: ['ghost'], @@ -168,13 +196,13 @@ describe('textCodec', () => { }); it('preserves regex escape sequences while unescaping literal punctuation', () => { - expect(textCodec().parse(nql.parse("email:~'g.ost'") as never, emailContext)).toEqual({ + expect(textCodec().parse(ast("email:~'g.ost'"), emailContext)).toEqual({ field: 'email', operator: 'contains', values: ['g.ost'], }); - expect(textCodec().parse(nql.parse("email:~'\\d'") as never, emailContext)).toEqual({ + expect(textCodec().parse(ast("email:~'\\d'"), emailContext)).toEqual({ field: 'email', operator: 'contains', values: ['\\d'], @@ -182,9 +210,7 @@ describe('textCodec', () => { }); it('parses and serializes exact text operators', () => { - expect( - textCodec().parse(nql.parse("email:'ghost@example.com'") as never, emailContext), - ).toEqual({ + expect(textCodec().parse(ast("email:'ghost@example.com'"), emailContext)).toEqual({ field: 'email', operator: 'is', values: ['ghost@example.com'], @@ -211,17 +237,28 @@ describe('textCodec', () => { expect(textCodec().serialize(predicate, emailContext)).toEqual(["email:~^'can\\'t'"]); }); - it('returns null for invalid text operators', () => { + it('returns null for operators outside the text vocabulary', () => { const predicate: FilterPredicate = { id: '1', field: 'email', - operator: 'is-not', + operator: 'is-greater', values: ['ghost'], }; expect(textCodec().serialize(predicate, emailContext)).toBeNull(); }); + it('serializes the equality pair the vocabulary supports', () => { + const predicate: FilterPredicate = { + id: '1', + field: 'email', + operator: 'is-not', + values: ['ghost'], + }; + + expect(textCodec().serialize(predicate, emailContext)).toEqual(["email:-'ghost'"]); + }); + it('does not serialize empty text values', () => { const predicate: FilterPredicate = { id: '1', @@ -236,7 +273,7 @@ describe('textCodec', () => { it('supports mapped NQL field names', () => { const bodyCodec = textCodec({ field: 'html' }); - expect(bodyCodec.parse(nql.parse("html:~'ghost'") as never, bodyContext)).toEqual({ + expect(bodyCodec.parse(ast("html:~'ghost'"), bodyContext)).toEqual({ field: 'body', operator: 'contains', values: ['ghost'], @@ -258,12 +295,12 @@ describe('textCodec', () => { describe('setCodec', () => { it('parses set membership operators', () => { - expect(setCodec().parse(nql.parse('label:[vip,alpha]') as never, labelContext)).toEqual({ + expect(setCodec().parse(ast('label:[vip,alpha]'), labelContext)).toEqual({ field: 'label', operator: 'is-any', values: ['vip', 'alpha'], }); - expect(setCodec().parse(nql.parse('label:-[vip,alpha]') as never, labelContext)).toEqual({ + expect(setCodec().parse(ast('label:-[vip,alpha]'), labelContext)).toEqual({ field: 'label', operator: 'is-not-any', values: ['vip', 'alpha'], @@ -271,12 +308,12 @@ describe('setCodec', () => { }); it('parses singleton set values through scalar NQL operators', () => { - expect(setCodec().parse(nql.parse('label:vip') as never, labelContext)).toEqual({ + expect(setCodec().parse(ast('label:vip'), labelContext)).toEqual({ field: 'label', operator: 'is-any', values: ['vip'], }); - expect(setCodec().parse(nql.parse('label:-vip') as never, labelContext)).toEqual({ + expect(setCodec().parse(ast('label:-vip'), labelContext)).toEqual({ field: 'label', operator: 'is-not-any', values: ['vip'], @@ -324,12 +361,12 @@ describe('setCodec', () => { describe('numberCodec', () => { it('parses numeric comparison operators', () => { - expect(numberCodec().parse(nql.parse('email_count:>5') as never, countContext)).toEqual({ + expect(numberCodec().parse(ast('email_count:>5'), countContext)).toEqual({ field: 'email_count', operator: 'is-greater', values: [5], }); - expect(numberCodec().parse(nql.parse('email_count:10') as never, countContext)).toEqual({ + expect(numberCodec().parse(ast('email_count:10'), countContext)).toEqual({ field: 'email_count', operator: 'is', values: [10], @@ -372,17 +409,13 @@ describe('numberCodec', () => { describe('dateCodec', () => { it('parses date comparison operators', () => { - expect( - dateCodec().parse(nql.parse("created_at:<='2024-01-01T23:59:59.999Z'") as never, dateContext), - ).toEqual({ + expect(dateCodec().parse(ast("created_at:<='2024-01-01T23:59:59.999Z'"), dateContext)).toEqual({ field: 'created_at', operator: 'is-or-less', values: ['2024-01-01'], }); - expect( - dateCodec().parse(nql.parse("created_at:>'2024-01-01T23:59:59.999Z'") as never, dateContext), - ).toEqual({ + expect(dateCodec().parse(ast("created_at:>'2024-01-01T23:59:59.999Z'"), dateContext)).toEqual({ field: 'created_at', operator: 'is-greater', values: ['2024-01-01'], @@ -407,9 +440,7 @@ describe('dateCodec', () => { }); it('returns null for invalid date values', () => { - expect( - dateCodec().parse(nql.parse("created_at:<='not-a-date'") as never, dateContext), - ).toBeNull(); + expect(dateCodec().parse(ast("created_at:<='not-a-date'"), dateContext)).toBeNull(); expect( dateCodec().serialize( { diff --git a/apps/admin/src/shared/filters/field-icons.ts b/apps/admin/src/shared/filters/field-icons.ts new file mode 100644 index 00000000000..1efc035aa05 --- /dev/null +++ b/apps/admin/src/shared/filters/field-icons.ts @@ -0,0 +1,33 @@ +import React from 'react'; +import { LucideIcon } from '@tryghost/shade/utils'; +import type { FieldIcon } from './filter-providers'; + +// Total over `FieldIcon`, so naming an icon on a field is enough to make it render, and adding a +// name to the union fails to compile until there is something to draw for it. +export const FIELD_ICONS: Record = { + arrows: React.createElement(LucideIcon.ArrowRightLeft, { className: 'size-4' }), + calendar: React.createElement(LucideIcon.Calendar, { className: 'size-4' }), + 'calendar-clock': React.createElement(LucideIcon.CalendarClock, { className: 'size-4' }), + 'calendar-end': React.createElement(LucideIcon.CalendarArrowDown, { className: 'size-4' }), + 'calendar-start': React.createElement(LucideIcon.CalendarPlus, { className: 'size-4' }), + card: React.createElement(LucideIcon.CreditCard, { className: 'size-4' }), + click: React.createElement(LucideIcon.MousePointerClick, { className: 'size-4' }), + eye: React.createElement(LucideIcon.Eye, { className: 'size-4' }), + layers: React.createElement(LucideIcon.Layers, { className: 'size-4' }), + mail: React.createElement(LucideIcon.Mail, { className: 'size-4' }), + 'mail-open': React.createElement(LucideIcon.MailOpen, { className: 'size-4' }), + message: React.createElement(LucideIcon.MessageSquare, { className: 'size-4' }), + newspaper: React.createElement(LucideIcon.Newspaper, { className: 'size-4' }), + percent: React.createElement(LucideIcon.Percent, { className: 'size-4' }), + person: React.createElement(LucideIcon.User, { className: 'size-4' }), + 'person-circle': React.createElement(LucideIcon.UserCircle, { className: 'size-4' }), + 'person-plus': React.createElement(LucideIcon.UserPlus, { className: 'size-4' }), + send: React.createElement(LucideIcon.Send, { className: 'size-4' }), + tag: React.createElement(LucideIcon.Tag, { className: 'size-4' }), + text: React.createElement(LucideIcon.Type, { className: 'size-4' }), + ticket: React.createElement(LucideIcon.Ticket, { className: 'size-4' }), + circle: React.createElement(LucideIcon.Circle, { className: 'size-4' }), + 'file-text': React.createElement(LucideIcon.FileText, { className: 'size-4' }), + flag: React.createElement(LucideIcon.Flag, { className: 'size-4' }), + 'message-text': React.createElement(LucideIcon.MessageSquareText, { className: 'size-4' }), +}; diff --git a/apps/admin/src/shared/filters/filter-addressing.ts b/apps/admin/src/shared/filters/filter-addressing.ts new file mode 100644 index 00000000000..f2330840751 --- /dev/null +++ b/apps/admin/src/shared/filters/filter-addressing.ts @@ -0,0 +1,225 @@ +import { extractComparator, getCompoundChildren } from './filter-ast'; +import type { AstNode } from './filter-ast'; +import type { CodecContext, FilterCodec, FilterPredicate, ParsedPredicate } from './filter-types'; +import { listsOperator } from './filter-operators'; +import type { PresenceOperator } from './filter-operators'; +import type { + ClauseGroup, + EqualityClause, + SemanticValue, + SerializedValue, + ValueComparator, + ValueSemantics, +} from './semantics'; + +export interface FieldAddress { + valueKey: string; + companions?: string[]; + values: unknown[]; +} + +export interface MatchedValue { + comparator: ValueComparator; + field?: string; + leadingValues?: unknown[]; +} + +export type CompoundMatch = + | { kind: 'predicate'; predicate: ParsedPredicate } + | ({ kind: 'value' } & MatchedValue); + +interface FieldAddressingBase { + address: (predicate: FilterPredicate, ctx: CodecContext) => FieldAddress | null; + match: (node: AstNode, ctx: CodecContext) => MatchedValue | null; + matchCompound?: (node: AstNode) => CompoundMatch | null; +} + +export type PlainAddressing = FieldAddressingBase & { + presenceOperators?: undefined; + addressPresence?: undefined; +}; + +export type PresenceAddressing = FieldAddressingBase & { + presenceOperators: readonly PresenceOperator[]; + addressPresence: (predicate: FilterPredicate, ctx: CodecContext) => string[] | null; +}; + +export type FieldAddressing = PlainAddressing | PresenceAddressing; + +// The brackets are not decoration. A clause naming which custom field we mean has to reach the +// backend inside the same bracket as the value it qualifies, and an OR has to keep its comma +// inside brackets or the surrounding pluses steal it: `a+b,c` reads as `(a+b),c`, which is a +// wider set of members than anyone asked for. +function combine(clauses: string[], join: 'and' | 'or' = 'and', group = false): string { + if (clauses.length === 1 && !group) { + return clauses[0]; + } + + return `(${clauses.join(join === 'and' ? '+' : ',')})`; +} + +function writtenClauses( + written: SerializedValue, + valueKey: string, +): { clauses: string[]; join: 'and' | 'or' } { + if (typeof written === 'string') { + return { clauses: [`${valueKey}:${written}`], join: 'and' }; + } + + return { + clauses: written.fragments.map( + (fragment) => `${fragment.key ?? valueKey}:${fragment.expression}`, + ), + join: written.join ?? 'and', + }; +} + +export function columnAddressing(config?: { field?: string }): PlainAddressing { + const keyFor = (ctx: CodecContext) => config?.field ?? ctx.key; + + return { + address(predicate, ctx) { + return { valueKey: keyFor(ctx), values: predicate.values }; + }, + match(node, ctx) { + const comparator = extractComparator(node); + + if (!comparator || comparator.field !== keyFor(ctx)) { + return null; + } + + return { comparator: { operator: comparator.operator, value: comparator.value } }; + }, + }; +} + +function toEqualityClause(node: AstNode): EqualityClause | null { + const comparator = extractComparator(node); + + if (!comparator || comparator.operator !== '$eq') { + return null; + } + + return { key: comparator.field, value: comparator.value }; +} + +function toClauseGroup(node: AstNode): ClauseGroup | null { + for (const join of ['and', 'or'] as const) { + const children = getCompoundChildren(node, join === 'and' ? '$and' : '$or'); + + if (children) { + const clauses = children + .map((child) => toEqualityClause(child)) + .filter((clause) => clause !== null); + + return clauses.length === children.length ? { join, clauses } : null; + } + } + + const clause = toEqualityClause(node); + + return clause ? { join: 'and', clauses: [clause] } : null; +} + +function toPredicate( + matched: MatchedValue, + parsed: SemanticValue, + ctx: CodecContext, +): ParsedPredicate { + return { + field: matched.field ?? ctx.key, + operator: parsed.operator, + values: [...(matched.leadingValues ?? []), ...parsed.values], + }; +} + +export function composeCodec( + addressing: FieldAddressing, + semantics: ValueSemantics, +): FilterCodec { + return { + parse(node, ctx) { + const matched = addressing.match(node, ctx); + + if (!matched) { + return null; + } + + const parsed = semantics.parse(matched.comparator, ctx); + + if (!parsed) { + return null; + } + + return toPredicate(matched, parsed, ctx); + }, + serialize(predicate, ctx) { + if ( + addressing.presenceOperators && + listsOperator(addressing.presenceOperators, predicate.operator) + ) { + return addressing.addressPresence(predicate, ctx); + } + + const address = addressing.address(predicate, ctx); + + if (!address) { + return null; + } + + const operator = semantics.operators.find((candidate) => candidate === predicate.operator); + + if (operator === undefined) { + return null; + } + + const written = semantics.serialize({ operator, values: address.values }, ctx); + + if (written === null) { + return null; + } + + const { clauses, join } = writtenClauses(written, address.valueKey); + const grouped = typeof written !== 'string'; + + return [combine([...(address.companions ?? []), ...clauses], join, grouped)]; + }, + parseCompound: semantics.parseClauses + ? (node, ctx) => { + const group = toClauseGroup(node); + + if (!group) { + return null; + } + + const parsed = semantics.parseClauses?.(group, ctx); + + if (!parsed) { + return null; + } + + return { field: ctx.key, operator: parsed.operator, values: parsed.values }; + } + : addressing.matchCompound + ? (node, ctx) => { + const matched = addressing.matchCompound?.(node); + + if (!matched) { + return null; + } + + if (matched.kind === 'predicate') { + return matched.predicate; + } + + const parsed = semantics.parse(matched.comparator, ctx); + + if (!parsed) { + return null; + } + + return toPredicate(matched, parsed, ctx); + } + : undefined, + }; +} diff --git a/apps/admin/src/shared/filters/filter-ast.test.ts b/apps/admin/src/shared/filters/filter-ast.test.ts index 1f4ae07f7d3..8229e397f4e 100644 --- a/apps/admin/src/shared/filters/filter-ast.test.ts +++ b/apps/admin/src/shared/filters/filter-ast.test.ts @@ -1,17 +1,28 @@ -import nql from '@tryghost/nql-lang'; import { describe, expect, it } from 'vitest'; import { extractComparator, extractFieldName } from './filter-ast'; +import { parseFilterToAst } from './filter-query-core'; +import type { AstNode } from './filter-ast'; + +function ast(filter: string): AstNode { + const node = parseFilterToAst(filter); + + if (!node) { + throw new Error(`could not parse: ${filter}`); + } + + return node; +} describe('filter-ast helpers', () => { it('extracts simple field names', () => { - const node = nql.parse('status:paid') as Record; + const node = ast('status:paid'); expect(extractFieldName(node)).toBe('status'); }); it('extracts comparators from simple nodes', () => { - const lessThanNode = nql.parse("created_at:<'2024-01-01'") as Record; - const equalNode = nql.parse('status:paid') as Record; + const lessThanNode = ast("created_at:<'2024-01-01'"); + const equalNode = ast('status:paid'); expect(extractComparator(lessThanNode)).toEqual({ field: 'created_at', @@ -26,13 +37,13 @@ describe('filter-ast helpers', () => { }); it('preserves grouped nodes in the parsed AST', () => { - const compoundNode = nql.parse("(status:paid+email:~'ghost')") as Record; + const compoundNode = ast("(status:paid+email:~'ghost')"); expect(compoundNode.$and).toEqual([{ status: 'paid' }, { email: { $regex: /ghost/i } }]); }); it('returns undefined for non-simple nodes', () => { - const compoundNode = nql.parse("(status:paid+email:~'ghost')") as Record; + const compoundNode = ast("(status:paid+email:~'ghost')"); expect(extractFieldName(compoundNode)).toBeUndefined(); expect(extractComparator(compoundNode)).toBeUndefined(); diff --git a/apps/admin/src/shared/filters/filter-ast.ts b/apps/admin/src/shared/filters/filter-ast.ts index 1a082ccd09a..79f4d622cef 100644 --- a/apps/admin/src/shared/filters/filter-ast.ts +++ b/apps/admin/src/shared/filters/filter-ast.ts @@ -1,6 +1,6 @@ export type AstNode = Record; -function isPlainObject(value: unknown): value is Record { +export function isAstNode(value: unknown): value is AstNode { return ( typeof value === 'object' && value !== null && @@ -9,6 +9,24 @@ function isPlainObject(value: unknown): value is Record { ); } +export function getCompoundChildren(node: AstNode, operator: '$and' | '$or'): AstNode[] | null { + const children = node[operator]; + + if (!Array.isArray(children) || !children.every(isAstNode)) { + return null; + } + + return children; +} + +export function readNegatedString(value: unknown): string | null { + if (!isAstNode(value)) { + return null; + } + + return typeof value.$ne === 'string' ? value.$ne : null; +} + export function extractFieldName(node: AstNode): string | undefined { const keys = Object.keys(node); @@ -25,18 +43,8 @@ export function extractFieldName(node: AstNode): string | undefined { return field; } -export function extractComparator( - node: AstNode, -): { field: string; operator: string; value: unknown } | undefined { - const field = extractFieldName(node); - - if (!field) { - return undefined; - } - - const value = node[field]; - - if (isPlainObject(value)) { +export function toComparator(value: unknown): { operator: string; value: unknown } | undefined { + if (isAstNode(value)) { const entries = Object.entries(value); if (entries.length !== 1) { @@ -44,12 +52,29 @@ export function extractComparator( } const [operator, comparatorValue] = entries[0]; - return { field, operator, value: comparatorValue }; + return { operator, value: comparatorValue }; } return { - field, operator: '$eq', value, }; } + +export function extractComparator( + node: AstNode, +): { field: string; operator: string; value: unknown } | undefined { + const field = extractFieldName(node); + + if (!field) { + return undefined; + } + + const comparator = toComparator(node[field]); + + if (!comparator) { + return undefined; + } + + return { field, ...comparator }; +} diff --git a/apps/admin/src/shared/filters/filter-codec-roundtrip.test.ts b/apps/admin/src/shared/filters/filter-codec-roundtrip.test.ts index 68c2f3a702a..a5783e25a77 100644 --- a/apps/admin/src/shared/filters/filter-codec-roundtrip.test.ts +++ b/apps/admin/src/shared/filters/filter-codec-roundtrip.test.ts @@ -1,14 +1,31 @@ -import nql from '@tryghost/nql-lang'; -import { dateCodec, numberCodec, scalarCodec, setCodec, textCodec } from './filter-codecs'; +import { columnAddressing, composeCodec } from './filter-addressing'; +import { + dateSemantics, + numberSemantics, + scalarSemantics, + setSemantics, + textSemantics, +} from './semantics'; +import type { ValueConfig } from './semantics'; +import { parseFilterToAst } from './filter-query-core'; +type CodecConfig = ValueConfig & { field?: string }; +const textCodec = (config?: CodecConfig) => composeCodec(columnAddressing(config), textSemantics()); +const scalarCodec = (config?: CodecConfig) => + composeCodec(columnAddressing(config), scalarSemantics(config)); +const setCodec = (config?: CodecConfig) => + composeCodec(columnAddressing(config), setSemantics(config)); +const numberCodec = (config?: CodecConfig) => + composeCodec(columnAddressing(config), numberSemantics()); +const dateCodec = (config?: CodecConfig) => composeCodec(columnAddressing(config), dateSemantics()); + import { describe, expect, it } from 'vitest'; import type { CodecContext, FilterCodec, FilterPredicate } from './filter-types'; -// What a saved segment actually relies on: a predicate the publisher built in the UI -// is serialized to NQL, stored, and read back the next time the page loads. Every -// codec must survive that trip for every operator it advertises, whatever the value -// holds — the per-codec tests above assert one direction at a time against hand-written -// NQL, which is how the anchor readers in this engine and in member-filter-query.ts -// drifted apart without a test noticing. +// A filter the publisher saves has to come back meaning the same thing when the page reloads. +// Writing it and reading it back are separate pieces of code, so they can drift apart and each +// still look right on its own — that is how values ending in a dollar sign were once lost. The +// only way to catch that is to send a value out and back and check it survived, which is what +// these do, using the values most likely to break the trip. function context(key: string, timezone = 'UTC'): CodecContext { return { key, pattern: key, params: {}, timezone }; @@ -21,14 +38,18 @@ function roundTrip(codec: FilterCodec, predicate: Omit, c throw new Error(`serialize returned null for ${predicate.operator}`); } - const node = nql.parse(clauses.join('+'), { preserveRelativeDates: true }); + const node = parseFilterToAst(clauses.join('+')); + + if (!node) { + throw new Error(`could not parse: ${clauses.join('+')}`); + } return codec.parse(node, ctx); } -// Values chosen for what they do to the regex the text codec builds: `$` and `^` are -// the anchors the parse side reads operators from, so a value containing one is the -// case where escaping and anchoring have to be told apart. +// Chosen because they collide with the characters the text codec uses to mean something: +// `$` and `^` are how "ends with" and "starts with" are marked, so a value containing one +// is where escaping and marking have to be told apart. const TEXT_VALUES = [ 'Ghost', 'two words', @@ -44,6 +65,7 @@ const TEXT_VALUES = [ const TEXT_OPERATORS = [ 'is', + 'is-not', 'contains', 'does-not-contain', 'starts-with', diff --git a/apps/admin/src/shared/filters/filter-codecs.ts b/apps/admin/src/shared/filters/filter-codecs.ts deleted file mode 100644 index 55a0b3fb768..00000000000 --- a/apps/admin/src/shared/filters/filter-codecs.ts +++ /dev/null @@ -1,475 +0,0 @@ -import { DATE_FILTER_OPERATORS } from './filter-date'; -import { escapeNqlString } from '@tryghost/nql-string'; -import { formatDateInTimezone, getDayBoundsInUtc } from './filter-normalization'; -import { extractComparator } from './filter-ast'; -import type { FilterCodec } from './filter-types'; - -type DateOperator = (typeof DATE_FILTER_OPERATORS)[number]; - -const SCALAR_OPERATORS: Record = { - $eq: 'is', - $ne: 'is-not', -}; - -const NUMBER_OPERATORS: Record = { - $eq: 'is', - $gt: 'is-greater', - $gte: 'is-or-greater', - $lt: 'is-less', - $lte: 'is-or-less', -}; - -const DATE_OPERATORS: Record = { - $lt: 'is-less', - $lte: 'is-or-less', - $gt: 'is-greater', - $gte: 'is-or-greater', -}; - -const TEXT_OPERATOR_SYMBOLS: Record = { - contains: '~', - 'does-not-contain': '-~', - 'starts-with': '~^', - 'does-not-start-with': '-~^', - 'ends-with': '~$', - 'does-not-end-with': '-~$', -}; - -const NUMBER_OPERATOR_SYMBOLS: Record = { - is: '', - 'is-greater': '>', - 'is-or-greater': '>=', - 'is-less': '<', - 'is-or-less': '<=', -}; - -const DATE_OPERATOR_SYMBOLS: Record = { - 'is-less': '<', - 'is-or-less': '<=', - 'is-greater': '>', - 'is-or-greater': '>=', -}; - -const SET_OPERATOR_SYMBOLS: Record = { - 'is-any': '', - 'is-not-any': '-', -}; - -const UNQUOTED_TOKEN_PATTERN = /^[A-Za-z0-9_.-]+$/; - -interface CodecConfig { - field?: string; - quoteStrings?: boolean; - serializeSingletonAsScalar?: boolean; -} - -function getCodecField(config: CodecConfig | undefined, key: string): string { - return config?.field ?? key; -} - -function normalizeMultiValue(values: unknown[]): string[] { - return values.map((value) => String(value)).sort((left, right) => left.localeCompare(right)); -} - -function serializeScalarValue(value: unknown, config?: CodecConfig): string { - if (typeof value === 'string') { - if (config?.quoteStrings || value.startsWith('-') || !UNQUOTED_TOKEN_PATTERN.test(value)) { - return escapeNqlString(value); - } - - return value; - } - - return String(value); -} - -// A trailing `$` anchors the regex only when it isn't itself escaped: a value holding a -// literal `$` (contains `5$`) reaches here as the source `5\$`, which still ends in `$`. -// An odd run of backslashes before it means it is escaped, so it is part of the value. -// A literal `^` is always escaped to `\^`, so a leading `^` needs no such check. -function hasEndAnchor(source: string): boolean { - if (!source.endsWith('$')) { - return false; - } - - let backslashes = 0; - - for (let index = source.length - 2; index >= 0 && source[index] === '\\'; index -= 1) { - backslashes += 1; - } - - return backslashes % 2 === 0; -} - -// Which anchors a regex carries, and the value left once they are removed. Read together -// rather than one at a time: the operator and the value are two answers to the same -// question, and deciding the anchors twice is how a value could keep a `$` the operator -// had already consumed. -function decomposeRegex(pattern: RegExp): { - anchorStart: boolean; - anchorEnd: boolean; - value: string; -} { - const source = pattern.source; - const anchorStart = source.startsWith('^'); - const anchorEnd = hasEndAnchor(source); - const body = source.slice(anchorStart ? 1 : 0, anchorEnd ? -1 : undefined); - - return { - anchorStart, - anchorEnd, - value: body.replace(/\\([\\.^$|?*+()[\]{}/-])/g, '$1'), - }; -} - -// Anchors read back into the operator that would have produced them. Both anchors is not -// an operator this codec emits, so it falls back to the unanchored reading. -function anchorsToOperator(anchorStart: boolean, anchorEnd: boolean, negated: boolean): string { - if (anchorStart && !anchorEnd) { - return negated ? 'does-not-start-with' : 'starts-with'; - } - - if (anchorEnd && !anchorStart) { - return negated ? 'does-not-end-with' : 'ends-with'; - } - - return negated ? 'does-not-contain' : 'contains'; -} - -export function scalarCodec(config?: CodecConfig): FilterCodec { - return { - parse(node, ctx) { - const comparator = extractComparator(node as Record); - const field = getCodecField(config, ctx.key); - - if (!comparator || comparator.field !== field) { - return null; - } - - const operator = SCALAR_OPERATORS[comparator.operator]; - - if (!operator) { - return null; - } - - return { - field: ctx.key, - operator, - values: [comparator.value], - }; - }, - serialize(predicate, ctx) { - const value = predicate.values[0]; - const field = getCodecField(config, ctx.key); - - if (value === undefined || value === null || value === '') { - return null; - } - - if (predicate.operator === 'is') { - return [`${field}:${serializeScalarValue(value, config)}`]; - } - - if (predicate.operator === 'is-not') { - return [`${field}:-${serializeScalarValue(value, config)}`]; - } - - return null; - }, - }; -} - -export function textCodec(config?: CodecConfig): FilterCodec { - return { - parse(node, ctx) { - const comparator = extractComparator(node as Record); - const field = getCodecField(config, ctx.key); - - if (!comparator || comparator.field !== field) { - return null; - } - - if (comparator.operator === '$eq' && typeof comparator.value === 'string') { - return { - field: ctx.key, - operator: 'is', - values: [comparator.value], - }; - } - - if (comparator.operator === '$regex' && comparator.value instanceof RegExp) { - const { anchorStart, anchorEnd, value } = decomposeRegex(comparator.value); - - return { - field: ctx.key, - operator: anchorsToOperator(anchorStart, anchorEnd, false), - values: [value], - }; - } - - if (comparator.operator === '$not' && comparator.value instanceof RegExp) { - const { anchorStart, anchorEnd, value } = decomposeRegex(comparator.value); - - return { - field: ctx.key, - operator: anchorsToOperator(anchorStart, anchorEnd, true), - values: [value], - }; - } - - return null; - }, - serialize(predicate, ctx) { - const rawValue = predicate.values[0]; - const field = getCodecField(config, ctx.key); - - if (typeof rawValue !== 'string' || rawValue === '') { - return null; - } - - if (predicate.operator === 'is') { - return [`${field}:${escapeNqlString(rawValue)}`]; - } - - const operator = TEXT_OPERATOR_SYMBOLS[predicate.operator]; - - if (!operator) { - return null; - } - - return [`${field}:${operator}${escapeNqlString(rawValue)}`]; - }, - }; -} - -export function setCodec(config?: CodecConfig): FilterCodec { - return { - parse(node, ctx) { - const comparator = extractComparator(node as Record); - const field = getCodecField(config, ctx.key); - - if (!comparator || comparator.field !== field) { - return null; - } - - if (comparator.operator === '$in' && Array.isArray(comparator.value)) { - return { - field: ctx.key, - operator: 'is-any', - values: comparator.value, - }; - } - - if (comparator.operator === '$nin' && Array.isArray(comparator.value)) { - return { - field: ctx.key, - operator: 'is-not-any', - values: comparator.value, - }; - } - - if (comparator.operator === '$eq') { - return { - field: ctx.key, - operator: 'is-any', - values: [comparator.value], - }; - } - - if (comparator.operator === '$ne') { - return { - field: ctx.key, - operator: 'is-not-any', - values: [comparator.value], - }; - } - - return null; - }, - serialize(predicate, ctx) { - const field = getCodecField(config, ctx.key); - - if (!predicate.values.length) { - return null; - } - - const operator = SET_OPERATOR_SYMBOLS[predicate.operator]; - - if (operator === undefined) { - return null; - } - - const values = normalizeMultiValue(predicate.values); - - if (config?.serializeSingletonAsScalar && values.length === 1) { - return [`${field}:${operator}${serializeScalarValue(values[0], config)}`]; - } - - return [ - `${field}:${operator}[${values.map((value) => serializeScalarValue(value, config)).join(',')}]`, - ]; - }, - }; -} - -export function numberCodec(config?: CodecConfig): FilterCodec { - return { - parse(node, ctx) { - const comparator = extractComparator(node as Record); - const field = getCodecField(config, ctx.key); - - if (!comparator || comparator.field !== field || typeof comparator.value !== 'number') { - return null; - } - - const operator = NUMBER_OPERATORS[comparator.operator]; - - if (!operator) { - return null; - } - - return { - field: ctx.key, - operator, - values: [comparator.value], - }; - }, - serialize(predicate, ctx) { - const rawValue = predicate.values[0]; - const field = getCodecField(config, ctx.key); - const value = - typeof rawValue === 'string' ? (rawValue.trim() === '' ? NaN : Number(rawValue)) : rawValue; - - if (typeof value !== 'number' || Number.isNaN(value)) { - return null; - } - - const operator = NUMBER_OPERATOR_SYMBOLS[predicate.operator]; - - if (operator === undefined) { - return null; - } - - return [`${field}:${operator}${value}`]; - }, - }; -} - -interface RelativeDateTag { - $relativeDate: { - op: 'sub' | 'add'; - amount: number; - unit: string; - }; -} - -function isRelativeDateTag(value: unknown): value is RelativeDateTag { - if (!value || typeof value !== 'object') { - return false; - } - - const tag = (value as Record).$relativeDate; - - if (!tag || typeof tag !== 'object') { - return false; - } - - const { op, amount, unit } = tag as Record; - - return ( - (op === 'sub' || op === 'add') && - typeof amount === 'number' && - Number.isSafeInteger(amount) && - amount > 0 && - typeof unit === 'string' - ); -} - -export function dateCodec(config?: CodecConfig): FilterCodec { - return { - parse(node, ctx) { - const comparator = extractComparator(node as Record); - const field = getCodecField(config, ctx.key); - - if (!comparator || comparator.field !== field) { - return null; - } - - // Relative dates flow through as `{$gte: {$relativeDate: ...}}` - // when the parse caller opted in via `preserveRelativeDates: true` - // — we currently only render relative day counts in the UI, so any - // other unit (weeks, months, ...) falls through to absolute-date - // handling below. - if (isRelativeDateTag(comparator.value) && comparator.value.$relativeDate.unit === 'days') { - const { op, amount } = comparator.value.$relativeDate; - const isPast = op === 'sub' && comparator.operator === '$gte'; - const isFuture = op === 'add' && comparator.operator === '$lte'; - - if (isPast || isFuture) { - return { - field: ctx.key, - operator: isPast ? 'in-the-last' : 'in-the-next', - values: [amount], - }; - } - } - - if (typeof comparator.value !== 'string') { - return null; - } - - const operator = DATE_OPERATORS[comparator.operator]; - const value = formatDateInTimezone(comparator.value, ctx.timezone); - - if (!operator || !value) { - return null; - } - - return { - field: ctx.key, - operator, - values: [value], - }; - }, - serialize(predicate, ctx) { - const field = getCodecField(config, ctx.key); - - if (predicate.operator === 'in-the-last' || predicate.operator === 'in-the-next') { - const days = predicate.values[0]; - - if (typeof days !== 'number' || !Number.isSafeInteger(days) || days <= 0) { - return null; - } - - const sign = predicate.operator === 'in-the-last' ? '-' : '+'; - const op = predicate.operator === 'in-the-last' ? '>=' : '<='; - - return [`${field}:${op}now${sign}${days}d`]; - } - - const rawValue = predicate.values[0]; - - if (typeof rawValue !== 'string' || rawValue === '') { - return null; - } - - const value = formatDateInTimezone(rawValue, ctx.timezone); - - if (!value) { - return null; - } - - const { start, end } = getDayBoundsInUtc(value, ctx.timezone); - const operator = DATE_OPERATOR_SYMBOLS[predicate.operator]; - - if (operator === undefined) { - return null; - } - - const boundary = - predicate.operator === 'is-less' || predicate.operator === 'is-or-greater' ? start : end; - - return [`${field}:${operator}'${boundary}'`]; - }, - }; -} diff --git a/apps/admin/src/shared/filters/filter-keys.test.ts b/apps/admin/src/shared/filters/filter-keys.test.ts new file mode 100644 index 00000000000..d52a1855d0d --- /dev/null +++ b/apps/admin/src/shared/filters/filter-keys.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import { keyBelow, keyIsUnder } from './filter-keys'; + +describe('filter keys are paths', () => { + it('a longer step is not the step it starts with', () => { + expect(keyIsUnder('newsletters.slugfoo', 'newsletters.slug')).toBe(false); + expect(keyIsUnder('custom_fieldsfoo.value', 'custom_fields')).toBe(false); + }); + + it('a key is under itself, and under anything it hangs off', () => { + expect(keyIsUnder('newsletters.slug', 'newsletters.slug')).toBe(true); + expect(keyIsUnder('custom_fields.value.country', 'custom_fields')).toBe(true); + expect(keyIsUnder('custom_fields.value.country', 'custom_fields.value')).toBe(true); + }); + + it('reads a namespace written with or without its dot the same way', () => { + expect(keyIsUnder('custom_fields.key', 'custom_fields.')).toBe(true); + expect(keyIsUnder('custom_fields.key', 'custom_fields')).toBe(true); + }); + + it('names what sits below', () => { + expect(keyBelow('custom_fields.company', 'custom_fields.')).toBe('company'); + expect(keyBelow('custom_fields.value.country', 'custom_fields.value')).toBe('country'); + expect(keyBelow('newsletters.slug', 'newsletters.slug')).toBeNull(); + expect(keyBelow('newsletters.slugfoo', 'newsletters.slug')).toBeNull(); + }); +}); diff --git a/apps/admin/src/shared/filters/filter-keys.ts b/apps/admin/src/shared/filters/filter-keys.ts new file mode 100644 index 00000000000..b063a40b327 --- /dev/null +++ b/apps/admin/src/shared/filters/filter-keys.ts @@ -0,0 +1,31 @@ +/** + * A filter key is a path, not a piece of text: `custom_fields.value.country` is three steps. + * + * Everything that asks a question about a key asks it here, in steps. Asking in characters is + * what goes wrong — `newsletters.slug` is a prefix of the *text* `newsletters.slugfoo`, but it + * is not a prefix of its path, and only the second answer is the one anyone means. + * + * A namespace may be written with or without its trailing dot; both name the same steps. + */ +function steps(key: string): string[] { + return key.split('.').filter(Boolean); +} + +/** Whether `key` is the namespace itself, or sits somewhere beneath it. */ +export function keyIsUnder(key: string, namespace: string): boolean { + const under = steps(key); + const above = steps(namespace); + + return above.length <= under.length && above.every((step, index) => step === under[index]); +} + +/** What `key` is called beneath `namespace`, or null when it does not sit beneath it. */ +export function keyBelow(key: string, namespace: string): string | null { + if (!keyIsUnder(key, namespace)) { + return null; + } + + const rest = steps(key).slice(steps(namespace).length); + + return rest.length ? rest.join('.') : null; +} diff --git a/apps/admin/src/shared/filters/filter-operator-options.ts b/apps/admin/src/shared/filters/filter-operator-options.ts index cbdbd4316bc..80901fd49d4 100644 --- a/apps/admin/src/shared/filters/filter-operator-options.ts +++ b/apps/admin/src/shared/filters/filter-operator-options.ts @@ -4,7 +4,7 @@ interface OperatorOption { } interface CreateOperatorOptionsOptions { - labels?: Record; + labels?: Partial>; } export function createOperatorOptions( diff --git a/apps/admin/src/shared/filters/filter-operators.ts b/apps/admin/src/shared/filters/filter-operators.ts new file mode 100644 index 00000000000..c3f88b81f23 --- /dev/null +++ b/apps/admin/src/shared/filters/filter-operators.ts @@ -0,0 +1,35 @@ +export const VALUE_OPERATORS = [ + 'is', + 'is-not', + 'contains', + 'does-not-contain', + 'starts-with', + 'does-not-start-with', + 'ends-with', + 'does-not-end-with', + 'is-any', + 'is-not-any', + 'is-greater', + 'is-or-greater', + 'is-less', + 'is-or-less', + 'in-the-last', + 'in-the-next', +] as const; + +export type ValueOperator = (typeof VALUE_OPERATORS)[number]; + +export const PRESENCE_OPERATORS = ['is-set', 'is-not-set'] as const; + +export type PresenceOperator = (typeof PRESENCE_OPERATORS)[number]; + +// The operators the engine itself knows how to offer. A field whose vocabulary is its own — a +// newsletter feedback score, say — names its operators in that vocabulary instead of widening +// this, which is why anything holding a vocabulary is generic over `string` rather than this. +export type OperatorId = ValueOperator | PresenceOperator; + +// Asking whether a list of operators contains one. Spelled out rather than using `includes`, +// which refuses an arbitrary string when the list it is asked about is a fixed set. +export function listsOperator(operators: readonly string[], operator: string): boolean { + return operators.some((candidate) => candidate === operator); +} diff --git a/apps/admin/src/shared/filters/filter-providers.ts b/apps/admin/src/shared/filters/filter-providers.ts new file mode 100644 index 00000000000..44bad693cee --- /dev/null +++ b/apps/admin/src/shared/filters/filter-providers.ts @@ -0,0 +1,199 @@ +import { FILTER_TYPES } from './filter-registry'; +import { filterNamesKey } from './filter-query-core'; +import { columnAddressing, composeCodec } from './filter-addressing'; +import type { FieldAddressing, PlainAddressing, PresenceAddressing } from './filter-addressing'; +import type { ConfigOf, FilterTypeFacts, FilterTypeId } from './filter-registry'; +import type { FilterField } from './filter-types'; +import type { ValueSemantics } from './semantics'; +import type { PresenceOperator } from './filter-operators'; + +export type FieldIcon = + | 'arrows' + | 'calendar' + | 'calendar-clock' + | 'calendar-end' + | 'calendar-start' + | 'card' + | 'circle' + | 'click' + | 'eye' + | 'file-text' + | 'flag' + | 'layers' + | 'mail' + | 'mail-open' + | 'message' + | 'message-text' + | 'newspaper' + | 'percent' + | 'person' + | 'person-circle' + | 'person-plus' + | 'send' + | 'tag' + | 'ticket' + | 'text'; + +interface FieldDescriptorBase { + key: string; + icon: FieldIcon; + addressing?: FieldAddressing; + ui: Omit & { type?: FilterField['ui']['type'] }; + options?: FilterField['options']; + metadata?: FilterField['metadata']; + parseKeys?: readonly string[]; +} + +type WritableBy = ReturnType< + (typeof FILTER_TYPES)[TType]['semantics'] +>['operators'][number]; + +type TypeSpecific = { + type: TType; + valueConfig?: ConfigOf; +}; + +type TypedFieldDescriptor = { + [TType in FilterTypeId]: + | (FieldDescriptorBase & + TypeSpecific & { + addressing?: PlainAddressing; + operators?: readonly WritableBy[]; + }) + | (FieldDescriptorBase & + TypeSpecific & { + addressing: PresenceAddressing; + operators?: readonly (WritableBy | PresenceOperator)[]; + }); +}[FilterTypeId]; + +// This looks like an unused symbol and a pointless cast in `domainField()` below. Deleting +// either one silently removes a check. +// +// Nothing ever sets this property at runtime. It exists so that a plain object can never be a +// DomainFieldDescriptor, which forces every one of them through `domainField()` — the only +// place that compares a field's operators against the vocabulary that has to write them. Take +// it away and a field can list an operator its vocabulary cannot express: it compiles, no test +// fails, and the operator just quietly goes missing from the filter menu. +declare const CHECKED_AGAINST_ITS_VOCABULARY: unique symbol; + +export interface DomainFieldDescriptor< + TOperator extends string = string, +> extends FieldDescriptorBase { + type?: undefined; + valueConfig?: undefined; + semantics: ValueSemantics; + operators?: readonly (TOperator | PresenceOperator)[]; + readonly [CHECKED_AGAINST_ITS_VOCABULARY]: true; +} + +type DomainFieldCommon = { + key: TKey; + icon: FieldIcon; + semantics: ValueSemantics; + ui: FieldDescriptorBase['ui']; + options?: FieldDescriptorBase['options']; + metadata?: FieldDescriptorBase['metadata']; + parseKeys?: readonly string[]; +}; + +export function domainField( + descriptor: + | (DomainFieldCommon & { + addressing?: PlainAddressing; + operators?: readonly NoInfer[]; + }) + | (DomainFieldCommon & { + addressing: PresenceAddressing; + operators?: readonly (NoInfer | PresenceOperator)[]; + }), +): DomainFieldDescriptor & { key: TKey } { + return descriptor as DomainFieldDescriptor & { key: TKey }; +} + +export type FieldDescriptor = TypedFieldDescriptor | DomainFieldDescriptor; + +// Every type is handed its config, whether or not it currently takes one, so a type that grows +// a config parameter later starts receiving it rather than quietly ignoring it. The cast is +// needed only because TypeScript cannot see that `type` and `valueConfig` were checked against +// each other where the field was declared; it can't tell them apart once they arrive here as a +// union. +type SemanticsFactory = (config?: unknown) => ValueSemantics; + +function semanticsFor(descriptor: FieldDescriptor) { + if (descriptor.type === undefined) { + return descriptor.semantics; + } + + return (FILTER_TYPES[descriptor.type].semantics as SemanticsFactory)(descriptor.valueConfig); +} + +export function describeField(descriptor: FieldDescriptor): FilterField { + const addressing = descriptor.addressing ?? columnAddressing(); + const registered: FilterTypeFacts | undefined = descriptor.type + ? FILTER_TYPES[descriptor.type] + : undefined; + const semantics = semanticsFor(descriptor); + const presenceOperators = addressing.presenceOperators ?? []; + const encodable: readonly string[] = [...semantics.operators, ...presenceOperators]; + const offered = [...(registered?.operators ?? semantics.operators), ...presenceOperators]; + const chosen = + descriptor.operators?.filter((operator) => encodable.includes(operator)) ?? offered; + + const ui: FilterField['ui'] = { + ...descriptor.ui, + icon: descriptor.icon, + label: String(descriptor.ui.label), + type: descriptor.ui.type ?? registered?.control ?? 'text', + }; + + if (ui.defaultOperator === undefined && registered?.defaultOperator) { + ui.defaultOperator = registered.defaultOperator; + } + + return { + operators: chosen satisfies readonly string[], + codec: composeCodec(addressing, semantics), + ...(registered?.labels ? { operatorLabels: registered.labels } : {}), + ...(descriptor.options ? { options: descriptor.options } : {}), + ...(descriptor.metadata ? { metadata: descriptor.metadata } : {}), + ...(descriptor.parseKeys ? { parseKeys: descriptor.parseKeys } : {}), + ui, + }; +} + +export function buildCatalog(descriptors: readonly FieldDescriptor[]): Record { + const catalog: Record = {}; + + for (const descriptor of descriptors) { + catalog[descriptor.key] = describeField(descriptor); + } + + return catalog; +} + +export interface FieldProvider { + resolved: boolean; + claims?: readonly string[]; + fields: readonly FieldDescriptor[]; +} + +export function buildProvidedCatalog( + providers: readonly FieldProvider[], +): Record { + return buildCatalog(providers.flatMap((provider) => provider.fields)); +} + +export function catalogCanRead( + filter: string | undefined, + providers: readonly FieldProvider[], +): boolean { + if (!filter) { + return true; + } + + return providers.every( + (provider) => + provider.resolved || !(provider.claims ?? []).some((claim) => filterNamesKey(filter, claim)), + ); +} diff --git a/apps/admin/src/shared/filters/filter-query-core-names-key.test.ts b/apps/admin/src/shared/filters/filter-query-core-names-key.test.ts new file mode 100644 index 00000000000..4c54c1368ba --- /dev/null +++ b/apps/admin/src/shared/filters/filter-query-core-names-key.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { filterNamesKey } from './filter-query-core'; + +describe('filterNamesKey', () => { + it('finds a key wherever a clause sits', () => { + for (const filter of [ + 'newsletters.slug:weekly', + '(status:paid+newsletters.slug:weekly)', + 'status:paid,newsletters.slug:weekly', + 'newsletters.slug:-weekly', + ]) { + expect(filterNamesKey(filter, 'newsletters.slug')).toBe(true); + } + }); + + it('ignores anything inside a quoted value', () => { + // Someone searching their members for this text is not filtering on a newsletter, and + // a joiner inside their search term is their data rather than our structure. + for (const filter of [ + "email:~'newsletters.slug'", + "email:~'+newsletters.slug'", + "email:~'(newsletters.slug'", + 'email:~"+newsletters.slug"', + "name:'it\\'s +newsletters.slug'", + ]) { + expect(filterNamesKey(filter, 'newsletters.slug')).toBe(false); + } + }); + + it('still finds a real clause alongside a quoted decoy', () => { + expect( + filterNamesKey("(email:~'+newsletters.slug'+newsletters.slug:weekly)", 'newsletters.slug'), + ).toBe(true); + }); + + it('does not mistake a key that merely ends with the text', () => { + expect(filterNamesKey('other_newsletters.slug:weekly', 'newsletters.slug')).toBe(false); + }); + + it('finds a key alongside a value carrying an escaped control character', () => { + // `a\"b` is one value with a quote in it, not the start of a quoted section — which a + // reader looking at the text rather than parsing it gets wrong. + expect(filterNamesKey('name:a\\"b+newsletters.slug:weekly', 'newsletters.slug')).toBe(true); + }); + + it('matches any key beneath a namespace', () => { + expect(filterNamesKey("custom_fields.key:'company'", 'custom_fields.')).toBe(true); + expect(filterNamesKey("name:~'custom_fields.'", 'custom_fields.')).toBe(false); + }); + + it('names nothing when the filter cannot be parsed', () => { + expect(filterNamesKey('this is not a filter((', 'newsletters.slug')).toBe(false); + }); +}); diff --git a/apps/admin/src/shared/filters/filter-query-core.test.ts b/apps/admin/src/shared/filters/filter-query-core.test.ts index 8311c62b874..41aa72ae02b 100644 --- a/apps/admin/src/shared/filters/filter-query-core.test.ts +++ b/apps/admin/src/shared/filters/filter-query-core.test.ts @@ -1,4 +1,3 @@ -import { defineFields } from './filter-types'; import { describe, expect, it } from 'vitest'; import { dispatchSimpleNodes, @@ -7,9 +6,37 @@ import { parseFilterToAst, serializePredicates, } from './filter-query-core'; -import { numberCodec, scalarCodec } from './filter-codecs'; +import { getCompoundChildren } from './filter-ast'; +import { columnAddressing, composeCodec } from './filter-addressing'; +import { numberSemantics, scalarSemantics } from './semantics'; + +const scalarCodec = (config?: { field?: string }) => + composeCodec(columnAddressing(config), scalarSemantics()); +const numberCodec = (config?: { field?: string }) => + composeCodec(columnAddressing(config), numberSemantics()); +const defineFields = >(fields: T): T => fields; import type { AstNode } from './filter-ast'; -import type { FilterPredicate } from './filter-types'; +import type { FilterField, FilterPredicate } from './filter-types'; + +function ast(filter: string): AstNode { + const node = parseFilterToAst(filter); + + if (!node) { + throw new Error(`could not parse: ${filter}`); + } + + return node; +} + +function compound(filter: string): AstNode[] { + const children = getCompoundChildren(ast(filter), '$and'); + + if (!children) { + throw new Error(`not a compound: ${filter}`); + } + + return children; +} const fields = defineFields({ status: { @@ -58,9 +85,7 @@ const fields = defineFields({ describe('filter-query-core', () => { it('parses NQL into a traversable AST for surface-level composition', () => { - const ast = parseFilterToAst('status:paid+email_count:>5'); - - expect((ast as Record).$and).toEqual([ + expect(getCompoundChildren(ast('status:paid+email_count:>5'), '$and')).toEqual([ { status: 'paid' }, { email_count: { $gt: 5 } }, ]); @@ -71,12 +96,8 @@ describe('filter-query-core', () => { }); it('dispatches simple nodes into parsed predicates', () => { - const ast = parseFilterToAst('status:paid+email_count:>5'); - const predicates = dispatchSimpleNodes( - (ast as Record).$and as AstNode[], - fields, - 'UTC', - ); + const children = compound('status:paid+email_count:>5'); + const predicates = dispatchSimpleNodes(children, fields, 'UTC'); expect(predicates).toEqual([ { field: 'status', operator: 'is', values: ['paid'] }, @@ -85,19 +106,15 @@ describe('filter-query-core', () => { }); it('skips unknown simple nodes', () => { - const ast = parseFilterToAst('status:paid+unknown:test'); - const predicates = dispatchSimpleNodes( - (ast as Record).$and as AstNode[], - fields, - 'UTC', - ); + const children = compound('status:paid+unknown:test'); + const predicates = dispatchSimpleNodes(children, fields, 'UTC'); expect(predicates).toEqual([{ field: 'status', operator: 'is', values: ['paid'] }]); }); it('dispatches through declared parse aliases when the AST field name differs', () => { - const ast = parseFilterToAst('member_id:abc123'); - const predicates = dispatchSimpleNodes([ast as AstNode], fields, 'UTC'); + const node = ast('member_id:abc123'); + const predicates = dispatchSimpleNodes([node], fields, 'UTC'); expect(predicates).toEqual([{ field: 'author', operator: 'is', values: ['abc123'] }]); }); @@ -115,12 +132,8 @@ describe('filter-query-core', () => { }); it('round-trips simple predicates canonically', () => { - const ast = parseFilterToAst('status:paid+email_count:>5'); - const parsed = dispatchSimpleNodes( - (ast as Record).$and as AstNode[], - fields, - 'UTC', - ).map((predicate, index) => ({ + const children = compound('status:paid+email_count:>5'); + const parsed = dispatchSimpleNodes(children, fields, 'UTC').map((predicate, index) => ({ ...predicate, id: String(index + 1), })); @@ -129,12 +142,10 @@ describe('filter-query-core', () => { }); it('finds fields by UI type and declared parse aliases in nested AST nodes', () => { - const ast = parseFilterToAst( - "(status:paid,created_at_utc:<'2024-01-01T00:00:00.000Z')", - ) as AstNode; + const node = ast("(status:paid,created_at_utc:<'2024-01-01T00:00:00.000Z')"); const fieldKeys = getFieldKeysByType(fields, 'date'); expect([...fieldKeys]).toEqual(['created_at', 'created_at_utc']); - expect(hasFieldKey(ast, fieldKeys)).toBe(true); + expect(hasFieldKey(node, fieldKeys)).toBe(true); }); }); diff --git a/apps/admin/src/shared/filters/filter-query-core.ts b/apps/admin/src/shared/filters/filter-query-core.ts index 4b471332dcf..4e3da71592f 100644 --- a/apps/admin/src/shared/filters/filter-query-core.ts +++ b/apps/admin/src/shared/filters/filter-query-core.ts @@ -1,4 +1,7 @@ import nql from '@tryghost/nql-lang'; +import { getCompoundChildren, isAstNode } from './filter-ast'; +import { keyIsUnder } from './filter-keys'; +import { listsOperator } from './filter-operators'; import { resolveField } from './resolve-field'; import type { AstNode } from './filter-ast'; import type { FilterField, FilterPredicate, ParsedPredicate } from './filter-types'; @@ -9,6 +12,9 @@ export function parseFilterToAst(filter: string): AstNode | undefined { } try { + // Without this option nql turns `now-7d` into the exact timestamp it happens to be right + // now, and we would have no way of telling it apart from a date the user typed. The pill + // would stop saying "in the last 7 days" and freeze to a fixed day. return nql.parse(filter, { preserveRelativeDates: true }) as AstNode; } catch { return undefined; @@ -47,21 +53,66 @@ export function hasFieldKey(node: AstNode, fieldKeys: ReadonlySet): bool return Object.values(node).some((value) => { if (Array.isArray(value)) { - return value.some( - (child) => - child !== null && typeof child === 'object' && hasFieldKey(child as AstNode, fieldKeys), - ); + return value.some((child) => isAstNode(child) && hasFieldKey(child, fieldKeys)); } - return ( - value !== null && - typeof value === 'object' && - !(value instanceof RegExp) && - hasFieldKey(value as AstNode, fieldKeys) - ); + return isAstNode(value) && hasFieldKey(value, fieldKeys); }); } +/** + * Whether a filter names a key, or any key beneath a namespace. + * + * Answered by parsing rather than by looking through the text, because only the parser knows + * which characters are structure and which are somebody's data: `name:'+newsletters.slug'` is a + * search for that text, not a filter on a newsletter, and the quoting is what says so. Reading + * the filter as text means reimplementing quoting and escaping, which has been got wrong here + * more than once. + * + * A filter that cannot be parsed names nothing, which is the same answer the rest of the page + * gives it. + */ +export function filterNamesKey(filter: string | undefined, keyOrNamespace: string): boolean { + const ast = parseFilterToAst(filter ?? ''); + + return ast ? namesKey(ast, keyOrNamespace) : false; +} + +function namesKey(node: AstNode, keyOrNamespace: string): boolean { + if (Object.keys(node).some((key) => !key.startsWith('$') && keyIsUnder(key, keyOrNamespace))) { + return true; + } + + return Object.values(node).some((value) => { + if (Array.isArray(value)) { + return value.some((child) => isAstNode(child) && namesKey(child, keyOrNamespace)); + } + + return isAstNode(value) && namesKey(value, keyOrNamespace); + }); +} + +export function dispatchCompoundNode>( + node: AstNode, + fields: TFields, + timezone: string, +): ParsedPredicate | null { + for (const [key, definition] of Object.entries(fields)) { + const parsed = definition.codec.parseCompound?.(node, { + key, + pattern: key, + params: {}, + timezone, + }); + + if (parsed) { + return parsed; + } + } + + return null; +} + export function dispatchSimpleNodes>( nodes: AstNode[], fields: TFields, @@ -88,6 +139,54 @@ export function dispatchSimpleNodes> }); } +function getCompound(node: AstNode): { operator: '$and' | '$or'; children: AstNode[] } | null { + for (const operator of ['$and', '$or'] as const) { + const children = getCompoundChildren(node, operator); + + if (children) { + return { operator, children }; + } + } + + return null; +} + +// A whole filter read into predicates. A field addressed across several clauses gets first +// refusal on the node, because only it can recognize the combination; what is left is either an +// AND to walk into, or a single clause to look up by its key. An OR is not walked, because its +// branches are alternatives and a row of filter pills can only say AND. +export function parseNodeToPredicates>( + node: AstNode, + fields: TFields, + timezone: string, +): ParsedPredicate[] { + const addressed = dispatchCompoundNode(node, fields, timezone); + + if (addressed) { + return [addressed]; + } + + const compound = getCompound(node); + + if (compound?.operator === '$and') { + return compound.children.flatMap((child) => parseNodeToPredicates(child, fields, timezone)); + } + + return dispatchSimpleNodes([node], fields, timezone); +} + +// Whether the field still advertises the operator this predicate uses. A saved filter can name a +// pairing the UI no longer offers, and those are dropped rather than shown as a pill the user +// cannot operate. +export function isPredicateEnabled>( + predicate: ParsedPredicate, + fields: TFields, +): boolean { + const resolved = resolveField(fields, predicate.field, 'UTC'); + + return resolved ? listsOperator(resolved.definition.operators, predicate.operator) : false; +} + function canonicalizeClauses(clauses: string[]): string[] { return [...clauses].sort((left, right) => left.localeCompare(right)); } diff --git a/apps/admin/src/shared/filters/filter-registry.ts b/apps/admin/src/shared/filters/filter-registry.ts new file mode 100644 index 00000000000..3f0e669a951 --- /dev/null +++ b/apps/admin/src/shared/filters/filter-registry.ts @@ -0,0 +1,97 @@ +import { DATE_OPERATOR_LABELS, DEFAULT_DATE_OPERATOR } from './filter-date'; +import { + PLAIN_DATE_OPERATORS, + TIMESTAMP_OPERATORS, + SCALAR_VALUE_OPERATORS, + SET_VALUE_OPERATORS, + countSemantics, + dateSemantics, + numberSemantics, + plainDateSemantics, + scalarSemantics, + setSemantics, + textSemantics, +} from './semantics'; +import type { FilterControl } from './filter-types'; +import type { OperatorId } from './filter-operators'; +import type { ValueSemantics } from './semantics'; + +export interface FilterTypeDefinition { + semantics: (config?: TConfig) => ValueSemantics; + operators: readonly TOperator[]; + control: FilterControl; + labels?: Partial>; + defaultOperator?: OperatorId; +} + +function defineFilterType(definition: { + semantics: (config?: TConfig) => ValueSemantics; + operators: readonly NoInfer[]; + control: FilterControl; + labels?: Partial, string>>; + defaultOperator?: NoInfer; +}): FilterTypeDefinition { + return definition; +} + +export const FILTER_TYPES = { + text: defineFilterType({ + semantics: textSemantics, + operators: ['is', 'is-not', 'contains', 'does-not-contain', 'starts-with', 'ends-with'], + control: 'text', + defaultOperator: 'contains', + }), + scalar: defineFilterType({ + semantics: scalarSemantics, + operators: SCALAR_VALUE_OPERATORS, + control: 'select', + }), + set: defineFilterType({ + semantics: setSemantics, + operators: SET_VALUE_OPERATORS, + control: 'multiselect', + defaultOperator: 'is-any', + }), + number: defineFilterType({ + semantics: numberSemantics, + operators: ['is', 'is-greater', 'is-less'], + control: 'number', + labels: { 'is-greater': 'is greater than', 'is-less': 'is less than' }, + }), + timestamp: defineFilterType({ + semantics: dateSemantics, + operators: TIMESTAMP_OPERATORS, + control: 'date', + labels: DATE_OPERATOR_LABELS, + defaultOperator: DEFAULT_DATE_OPERATOR, + }), + plain_date: defineFilterType({ + semantics: plainDateSemantics, + operators: PLAIN_DATE_OPERATORS, + control: 'date', + labels: DATE_OPERATOR_LABELS, + defaultOperator: DEFAULT_DATE_OPERATOR, + }), + count: defineFilterType({ + semantics: countSemantics, + operators: ['is'], + control: 'select', + }), +} as const; + +export type FilterTypeId = keyof typeof FILTER_TYPES; + +export interface FilterTypeFacts { + operators: readonly OperatorId[]; + control: FilterControl; + labels?: Partial>; + defaultOperator?: OperatorId; +} + +export function filterType(id: FilterTypeId): FilterTypeFacts { + return FILTER_TYPES[id]; +} + +export type ConfigOf = Parameters< + (typeof FILTER_TYPES)[TType]['semantics'] +>[0]; diff --git a/apps/admin/src/shared/filters/filter-relative-date.ts b/apps/admin/src/shared/filters/filter-relative-date.ts index 2516a213a07..6e33d7e3761 100644 --- a/apps/admin/src/shared/filters/filter-relative-date.ts +++ b/apps/admin/src/shared/filters/filter-relative-date.ts @@ -25,16 +25,6 @@ export function fieldHasRelativeOperator(field: FilterField): boolean { return field.operators.some(isRelativeDateOperator); } -/** Returns the field with the past-leaning relative operator appended. */ -export function withPastRelativeOperator(field: T): T { - return { ...field, operators: [...field.operators, RELATIVE_PAST_OPERATOR] }; -} - -/** Returns the field with the future-leaning relative operator appended. */ -export function withFutureRelativeOperator(field: T): T { - return { ...field, operators: [...field.operators, RELATIVE_FUTURE_OPERATOR] }; -} - // `yyyymmdd` is a calendar date in the site's timezone. We construct a Date in the // browser's local zone purely to do calendar arithmetic — only the y/m/d fields are // read back via Intl, so the local-timezone Date is fine for display. diff --git a/apps/admin/src/shared/filters/filter-types.test.ts b/apps/admin/src/shared/filters/filter-types.test.ts deleted file mode 100644 index 0999a5809f9..00000000000 --- a/apps/admin/src/shared/filters/filter-types.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { defineFields } from './filter-types'; -import { describe, expect, expectTypeOf, it } from 'vitest'; -import type { CodecContext, FilterCodec, FilterField, ParsedPredicate } from './filter-types'; - -describe('defineFields', () => { - it('returns the same object shape at runtime', () => { - const fields = defineFields({ - status: { - operators: ['is'], - ui: { - label: 'Status', - type: 'select', - }, - codec: { - parse: () => null, - serialize: () => null, - }, - }, - }); - - expect(fields).toEqual({ - status: { - operators: ['is'], - ui: { - label: 'Status', - type: 'select', - }, - codec: { - parse: expect.any(Function) as unknown, - serialize: expect.any(Function) as unknown, - }, - }, - }); - }); - - it('supports object spread composition and preserves field keys', () => { - const baseFields = defineFields({ - status: { - operators: ['is'], - ui: { - label: 'Status', - type: 'select', - }, - codec: { - parse: () => null, - serialize: () => null, - }, - }, - }); - - const fields = defineFields({ - ...baseFields, - email: { - operators: ['contains'], - ui: { - label: 'Email', - type: 'text', - }, - codec: { - parse: () => null, - serialize: () => null, - }, - }, - }); - - expect(Object.keys(fields)).toEqual(['status', 'email']); - expectTypeOf(fields).toHaveProperty('status'); - expectTypeOf(fields).toHaveProperty('email'); - }); -}); - -describe('filter core types', () => { - it('exposes the minimal codec context contract', () => { - expectTypeOf().toMatchTypeOf<{ - key: string; - pattern: string; - params: Record; - timezone: string; - }>(); - }); - - it('keeps predicate ids out of parse-time codec results', () => { - const codec: FilterCodec = { - parse: () => { - const parsed: ParsedPredicate = { - field: 'status', - operator: 'is', - values: ['paid'], - }; - - return parsed; - }, - serialize: () => null, - }; - - const parsed = codec.parse( - {}, - { - key: 'status', - pattern: 'status', - params: {}, - timezone: 'UTC', - }, - ); - - expect(parsed).toEqual({ - field: 'status', - operator: 'is', - values: ['paid'], - }); - expect(parsed).not.toHaveProperty('id'); - expectTypeOf().not.toHaveProperty('id'); - }); - - it('accepts plain filter field definitions', () => { - expectTypeOf().toMatchTypeOf<{ - operators: readonly string[]; - ui: { - label: string; - type: string; - }; - codec: FilterCodec; - }>(); - }); -}); diff --git a/apps/admin/src/shared/filters/filter-types.ts b/apps/admin/src/shared/filters/filter-types.ts index 00f6ee79ea3..6a6f107b015 100644 --- a/apps/admin/src/shared/filters/filter-types.ts +++ b/apps/admin/src/shared/filters/filter-types.ts @@ -1,3 +1,8 @@ +import type { AstNode } from './filter-ast'; +import type { FilterFieldConfig } from '@tryghost/shade/patterns'; + +export type FilterControl = NonNullable; + export interface FilterPredicate { id: string; field: string; @@ -9,14 +14,16 @@ export type ParsedPredicate = Omit; export interface CodecContext { key: string; + /** The catalog entry that matched, which for a parameterised field is its pattern. */ pattern: string; params: Record; timezone: string; } export interface FilterCodec { - parse: (node: unknown, ctx: CodecContext) => ParsedPredicate | null; + parse: (node: AstNode, ctx: CodecContext) => ParsedPredicate | null; serialize: (predicate: FilterPredicate, ctx: CodecContext) => string[] | null; + parseCompound?: (node: AstNode, ctx: CodecContext) => ParsedPredicate | null; } /** A column the list appends while a filter on this field is active. */ @@ -36,11 +43,12 @@ export interface ActiveColumnContext extends CodecContext { export interface FilterField { operators: readonly string[]; + operatorLabels?: Partial>; parseKeys?: readonly string[]; - ui: { + ui: Partial> & { label: string; - type: 'text' | 'select' | 'multiselect' | 'date' | 'number' | 'custom'; - [key: string]: unknown; + type: FilterControl; + icon?: string; }; options?: Array<{ value: string; label: string }>; metadata?: { @@ -62,9 +70,3 @@ export interface FilterField { }; codec: FilterCodec; } - -export function defineFields>( - fields: TFields, -): TFields { - return fields; -} diff --git a/apps/admin/src/shared/filters/index.ts b/apps/admin/src/shared/filters/index.ts index e00224a67bf..bc05709e04e 100644 --- a/apps/admin/src/shared/filters/index.ts +++ b/apps/admin/src/shared/filters/index.ts @@ -1,12 +1,24 @@ // Filter AST / codec engine — shared across posts, comments and members domains. export * from './create-relative-date-renderer'; +export * from './filter-addressing'; export * from './filter-ast'; -export * from './filter-codecs'; +export * from './semantics'; export * from './filter-date'; export * from './filter-normalization'; export * from './filter-operator-options'; +export * from './filter-operators'; +export * from './nql-tokens'; +export * from './filter-providers'; +export * from './filter-keys'; export * from './filter-query-core'; +export * from './filter-registry'; export * from './filter-relative-date'; export * from './filter-types'; export * from './resolve-field'; + +export type { FieldIcon } from './filter-providers'; +export { FIELD_ICONS } from './field-icons'; + +export { domainField } from './filter-providers'; +export type { PlainAddressing, PresenceAddressing } from './filter-addressing'; diff --git a/apps/admin/src/shared/filters/nql-tokens.ts b/apps/admin/src/shared/filters/nql-tokens.ts new file mode 100644 index 00000000000..eb57c3bf84f --- /dev/null +++ b/apps/admin/src/shared/filters/nql-tokens.ts @@ -0,0 +1,31 @@ +export const NQL_SYMBOLS = [ + '', + '-', + '>', + '>=', + '<', + '<=', + '~', + '-~', + '~^', + '-~^', + '~$', + '-~$', +] as const; + +export type NqlSymbol = (typeof NQL_SYMBOLS)[number]; + +export const NQL_COMPARATORS = [ + '$eq', + '$ne', + '$gt', + '$gte', + '$lt', + '$lte', + '$in', + '$nin', + '$regex', + '$not', +] as const; + +export type NqlComparator = (typeof NQL_COMPARATORS)[number]; diff --git a/apps/admin/src/shared/filters/resolve-field.test.ts b/apps/admin/src/shared/filters/resolve-field.test.ts index 06b687895bb..346842551ed 100644 --- a/apps/admin/src/shared/filters/resolve-field.test.ts +++ b/apps/admin/src/shared/filters/resolve-field.test.ts @@ -1,6 +1,8 @@ -import { defineFields } from './filter-types'; import { describe, expect, it } from 'vitest'; import { resolveField } from './resolve-field'; +import type { FilterField } from './filter-types'; + +const defineFields = >(fields: T): T => fields; const fields = defineFields({ status: { @@ -73,8 +75,8 @@ describe('resolveField', () => { it('supports serialize-direction lookups for concrete keys', () => { const resolved = resolveField(fields, 'newsletters.weekly', 'UTC'); - expect(resolved?.context.pattern).toBe('newsletters.:slug'); expect(resolved?.context.key).toBe('newsletters.weekly'); + expect(resolved?.context.params).toEqual({ slug: 'weekly' }); }); it('resolves parse aliases back to their field definitions', () => { diff --git a/apps/admin/src/shared/filters/semantics/count.ts b/apps/admin/src/shared/filters/semantics/count.ts new file mode 100644 index 00000000000..0d0ceaf4609 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/count.ts @@ -0,0 +1,46 @@ +import type { ValueSemantics } from './types'; + +export interface CountConfig { + threshold: number; + absentForm: 'equals' | 'below'; +} + +export type CountOperator = 'is'; + +const DEFAULT_COUNT_CONFIG: CountConfig = { threshold: 0, absentForm: 'equals' }; + +export function countSemantics( + config: CountConfig = DEFAULT_COUNT_CONFIG, +): ValueSemantics { + const { threshold, absentForm } = config; + const absent = absentForm === 'equals' ? `${threshold}` : `<${threshold + 1}`; + + return { + operators: ['is'], + serialize({ operator, values }) { + const value = values[0]; + + if (operator !== 'is') { + return null; + } + + if (value === 'true') { + return `>${threshold}`; + } + + return value === 'false' ? absent : null; + }, + parse({ operator, value }) { + if (operator === '$gt' && value === threshold) { + return { operator: 'is', values: ['true'] }; + } + + const matchesAbsent = + absentForm === 'equals' + ? operator === '$eq' && value === threshold + : operator === '$lt' && value === threshold + 1; + + return matchesAbsent ? { operator: 'is', values: ['false'] } : null; + }, + }; +} diff --git a/apps/admin/src/shared/filters/semantics/date.ts b/apps/admin/src/shared/filters/semantics/date.ts new file mode 100644 index 00000000000..27260fb6ee2 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/date.ts @@ -0,0 +1,180 @@ +import { formatDateInTimezone, getDayBoundsInUtc } from '@/shared/filters/filter-normalization'; +import { isAstNode } from '@/shared/filters/filter-ast'; +import { bidirectional } from './operator-table'; +import type { OperatorTable } from './operator-table'; +import type { ValueSemantics } from './types'; + +export type PlainDateOperator = 'is-less' | 'is-or-less' | 'is-greater' | 'is-or-greater'; +export type TimestampOperator = PlainDateOperator | 'in-the-last' | 'in-the-next'; + +const DATE_TABLE: OperatorTable = { + 'is-less': { symbol: '<', comparator: '$lt' }, + 'is-or-less': { symbol: '<=', comparator: '$lte' }, + 'is-greater': { symbol: '>', comparator: '$gt' }, + 'is-or-greater': { symbol: '>=', comparator: '$gte' }, +}; + +const date = bidirectional(DATE_TABLE); + +export const PLAIN_DATE_OPERATORS = date.operators; +export const TIMESTAMP_OPERATORS = [ + ...PLAIN_DATE_OPERATORS, + 'in-the-last', + 'in-the-next', +] as const satisfies readonly TimestampOperator[]; + +/** + * The two useful trims of the above. A date that can only have happened offers "in the last"; + * one that can only be coming offers "in the next". A date that can be either offers both, which + * is what a field gets by saying nothing. + */ +export const PAST_TIMESTAMP_OPERATORS = [ + ...PLAIN_DATE_OPERATORS, + 'in-the-last', +] as const satisfies readonly TimestampOperator[]; + +export const FUTURE_TIMESTAMP_OPERATORS = [ + ...PLAIN_DATE_OPERATORS, + 'in-the-next', +] as const satisfies readonly TimestampOperator[]; + +const PLAIN_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/; + +export function plainDateSemantics(): ValueSemantics { + return { + operators: PLAIN_DATE_OPERATORS, + serialize({ operator, values }) { + const value = values[0]; + + if (typeof value !== 'string' || !PLAIN_DATE_PATTERN.test(value)) { + return null; + } + + const symbol = date.symbolFor(operator); + + if (symbol === undefined) { + return null; + } + + return `${symbol}'${value}'`; + }, + parse({ operator, value }) { + if (typeof value !== 'string' || !PLAIN_DATE_PATTERN.test(value)) { + return null; + } + + const parsed = date.operatorFor(operator); + + if (!parsed) { + return null; + } + + return { operator: parsed, values: [value] }; + }, + }; +} + +interface RelativeDateTag { + $relativeDate: { + op: 'sub' | 'add'; + amount: number; + unit: string; + }; +} + +function isRelativeDateTag(value: unknown): value is RelativeDateTag { + if (!isAstNode(value)) { + return false; + } + + const tag = value.$relativeDate; + + if (!isAstNode(tag)) { + return false; + } + + const { op, amount, unit } = tag; + + return ( + (op === 'sub' || op === 'add') && + typeof amount === 'number' && + Number.isSafeInteger(amount) && + amount > 0 && + typeof unit === 'string' + ); +} + +export function dateSemantics(): ValueSemantics { + return { + operators: TIMESTAMP_OPERATORS, + serialize({ operator, values }, ctx) { + if (operator === 'in-the-last' || operator === 'in-the-next') { + const days = values[0]; + + if (typeof days !== 'number' || !Number.isSafeInteger(days) || days <= 0) { + return null; + } + + const sign = operator === 'in-the-last' ? '-' : '+'; + const symbol = operator === 'in-the-last' ? '>=' : '<='; + + return `${symbol}now${sign}${days}d`; + } + + const rawValue = values[0]; + + if (typeof rawValue !== 'string' || rawValue === '') { + return null; + } + + const value = formatDateInTimezone(rawValue, ctx.timezone); + + if (!value) { + return null; + } + + const { start, end } = getDayBoundsInUtc(value, ctx.timezone); + const symbol = date.symbolFor(operator); + + if (symbol === undefined) { + return null; + } + + // The pairing looks wrong and is right. The user picks a whole day, but the column + // holds an exact moment, so each comparison has to land on whichever end of that day + // keeps the day itself in or out: before the day starts, from the moment it starts, + // after it ends, until it ends. Pair them up the tidy-looking way instead and every + // filter is off by a day at one end. + const boundary = operator === 'is-less' || operator === 'is-or-greater' ? start : end; + + return `${symbol}'${boundary}'`; + }, + parse({ operator, value }, ctx) { + if (isRelativeDateTag(value) && value.$relativeDate.unit === 'days') { + const { op, amount } = value.$relativeDate; + const isPast = op === 'sub' && operator === '$gte'; + const isFuture = op === 'add' && operator === '$lte'; + + if (isPast || isFuture) { + return { + operator: isPast ? 'in-the-last' : 'in-the-next', + values: [amount], + }; + } + } + + if (typeof value !== 'string') { + return null; + } + + const parsed = date.operatorFor(operator); + const formatted = formatDateInTimezone(value, ctx.timezone); + + if (!parsed || !formatted) { + return null; + } + + return { operator: parsed, values: [formatted] }; + }, + }; +} diff --git a/apps/admin/src/shared/filters/semantics/index.ts b/apps/admin/src/shared/filters/semantics/index.ts new file mode 100644 index 00000000000..1e92dab9446 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/index.ts @@ -0,0 +1,8 @@ +export * from './types'; +export * from './operator-table'; +export * from './text'; +export * from './scalar'; +export * from './set'; +export * from './number'; +export * from './date'; +export * from './count'; diff --git a/apps/admin/src/shared/filters/semantics/number.ts b/apps/admin/src/shared/filters/semantics/number.ts new file mode 100644 index 00000000000..d5b9fe7ce92 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/number.ts @@ -0,0 +1,44 @@ +import { bidirectional } from './operator-table'; +import type { OperatorTable } from './operator-table'; +import type { ValueSemantics } from './types'; + +export type NumberOperator = 'is' | 'is-greater' | 'is-or-greater' | 'is-less' | 'is-or-less'; + +const NUMBER_TABLE: OperatorTable = { + is: { symbol: '', comparator: '$eq' }, + 'is-greater': { symbol: '>', comparator: '$gt' }, + 'is-or-greater': { symbol: '>=', comparator: '$gte' }, + 'is-less': { symbol: '<', comparator: '$lt' }, + 'is-or-less': { symbol: '<=', comparator: '$lte' }, +}; + +const number = bidirectional(NUMBER_TABLE); + +export function numberSemantics(): ValueSemantics { + return { + operators: number.operators, + serialize({ operator, values }) { + const rawValue = values[0]; + const value = + typeof rawValue === 'string' ? (rawValue.trim() === '' ? NaN : Number(rawValue)) : rawValue; + const symbol = number.symbolFor(operator); + + // Finite, not merely not-NaN: `1e309` parses to Infinity, which would be written + // into the query as the word "Infinity" and could not be read back as a number. + if (typeof value !== 'number' || !Number.isFinite(value) || symbol === undefined) { + return null; + } + + return `${symbol}${value}`; + }, + parse({ operator, value }) { + if (typeof value !== 'number') { + return null; + } + + const parsed = number.operatorFor(operator); + + return parsed ? { operator: parsed, values: [value] } : null; + }, + }; +} diff --git a/apps/admin/src/shared/filters/semantics/operator-table.ts b/apps/admin/src/shared/filters/semantics/operator-table.ts new file mode 100644 index 00000000000..c95b8852795 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/operator-table.ts @@ -0,0 +1,37 @@ +import type { NqlComparator, NqlSymbol } from '@/shared/filters/nql-tokens'; + +export interface OperatorEncoding { + symbol: NqlSymbol; + comparator: NqlComparator; +} + +export type OperatorTable = Readonly>; + +export interface BidirectionalOperators { + operators: readonly TOperator[]; + symbolFor: (operator: string) => NqlSymbol | undefined; + operatorFor: (comparator: string) => TOperator | undefined; +} + +export function bidirectional( + table: OperatorTable, +): BidirectionalOperators { + const entries = Object.entries(table) as [TOperator, OperatorEncoding][]; + const byComparator = new Map(); + + for (const [operator, encoding] of entries) { + if (!byComparator.has(encoding.comparator)) { + byComparator.set(encoding.comparator, operator); + } + } + + return { + operators: entries.map(([operator]) => operator), + symbolFor(operator) { + return entries.find(([candidate]) => candidate === operator)?.[1].symbol; + }, + operatorFor(comparator) { + return byComparator.get(comparator); + }, + }; +} diff --git a/apps/admin/src/shared/filters/semantics/scalar.ts b/apps/admin/src/shared/filters/semantics/scalar.ts new file mode 100644 index 00000000000..a2581d06d58 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/scalar.ts @@ -0,0 +1,36 @@ +import { bidirectional } from './operator-table'; +import { serializeScalarValue } from './value'; +import type { OperatorTable } from './operator-table'; +import type { ValueConfig, ValueSemantics } from './types'; + +export type ScalarOperator = 'is' | 'is-not'; + +const SCALAR_TABLE: OperatorTable = { + is: { symbol: '', comparator: '$eq' }, + 'is-not': { symbol: '-', comparator: '$ne' }, +}; + +const scalar = bidirectional(SCALAR_TABLE); + +export const SCALAR_VALUE_OPERATORS = scalar.operators; + +export function scalarSemantics(config?: ValueConfig): ValueSemantics { + return { + operators: scalar.operators, + serialize({ operator, values }) { + const value = values[0]; + const symbol = scalar.symbolFor(operator); + + if (value === undefined || value === null || value === '' || symbol === undefined) { + return null; + } + + return `${symbol}${serializeScalarValue(value, config)}`; + }, + parse({ operator, value }) { + const parsed = scalar.operatorFor(operator); + + return parsed ? { operator: parsed, values: [value] } : null; + }, + }; +} diff --git a/apps/admin/src/shared/filters/semantics/set.ts b/apps/admin/src/shared/filters/semantics/set.ts new file mode 100644 index 00000000000..53faa56ffd9 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/set.ts @@ -0,0 +1,61 @@ +import { normalizeMultiValue, serializeScalarValue } from './value'; +import type { NqlSymbol } from '@/shared/filters/nql-tokens'; +import type { OperatorId } from '@/shared/filters/filter-operators'; +import type { ValueConfig, ValueSemantics } from './types'; + +const SET_OPERATOR_SYMBOLS = { + 'is-any': '', + 'is-not-any': '-', +} as const satisfies Partial>; + +const SET_SYMBOLS: Partial> = SET_OPERATOR_SYMBOLS; + +export type SetOperator = keyof typeof SET_OPERATOR_SYMBOLS; +export const SET_VALUE_OPERATORS = [ + 'is-any', + 'is-not-any', +] as const satisfies readonly SetOperator[]; + +export function setSemantics(config?: ValueConfig): ValueSemantics { + return { + operators: SET_VALUE_OPERATORS, + serialize({ operator, values }) { + if (!values.length) { + return null; + } + + const symbol = SET_SYMBOLS[operator]; + + if (symbol === undefined) { + return null; + } + + const sorted = normalizeMultiValue(values); + + if (config?.serializeSingletonAsScalar && sorted.length === 1) { + return `${symbol}${serializeScalarValue(sorted[0], config)}`; + } + + return `${symbol}[${sorted.map((value: string) => serializeScalarValue(value, config)).join(',')}]`; + }, + parse({ operator, value }) { + if (operator === '$in' && Array.isArray(value)) { + return { operator: 'is-any', values: value }; + } + + if (operator === '$nin' && Array.isArray(value)) { + return { operator: 'is-not-any', values: value }; + } + + if (operator === '$eq') { + return { operator: 'is-any', values: [value] }; + } + + if (operator === '$ne') { + return { operator: 'is-not-any', values: [value] }; + } + + return null; + }, + }; +} diff --git a/apps/admin/src/shared/filters/semantics/text.ts b/apps/admin/src/shared/filters/semantics/text.ts new file mode 100644 index 00000000000..7e770381d72 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/text.ts @@ -0,0 +1,127 @@ +import { escapeNqlString } from '@tryghost/nql-string'; +import type { NqlSymbol } from '@/shared/filters/nql-tokens'; +import type { OperatorId } from '@/shared/filters/filter-operators'; +import type { ValueSemantics } from './types'; + +const TEXT_OPERATOR_SYMBOLS = { + contains: '~', + 'does-not-contain': '-~', + 'starts-with': '~^', + 'does-not-start-with': '-~^', + 'ends-with': '~$', + 'does-not-end-with': '-~$', +} as const satisfies Partial>; + +const TEXT_SYMBOLS: Partial> = TEXT_OPERATOR_SYMBOLS; + +export type TextOperator = 'is' | 'is-not' | keyof typeof TEXT_OPERATOR_SYMBOLS; +export const TEXT_OPERATORS = [ + 'is', + 'is-not', + 'contains', + 'does-not-contain', + 'starts-with', + 'does-not-start-with', + 'ends-with', + 'does-not-end-with', +] as const satisfies readonly TextOperator[]; + +// Undoing what nql did on the way out. It escapes the user's text first and then adds the ^ or $ +// marking "starts with" / "ends with", so we have to take them off in the opposite order — and a +// trailing $ only means "ends with" if it wasn't itself escaped, which is what the backslash +// counting is for. Someone searching for a literal "5$" would otherwise get "ends with 5". +function hasEndAnchor(source: string): boolean { + if (!source.endsWith('$')) { + return false; + } + + let backslashes = 0; + + for (let index = source.length - 2; index >= 0 && source[index] === '\\'; index -= 1) { + backslashes += 1; + } + + return backslashes % 2 === 0; +} + +function decomposeRegex(pattern: RegExp): { + anchorStart: boolean; + anchorEnd: boolean; + value: string; +} { + const source = pattern.source; + const anchorStart = source.startsWith('^'); + const anchorEnd = hasEndAnchor(source); + const body = source.slice(anchorStart ? 1 : 0, anchorEnd ? -1 : undefined); + + return { + anchorStart, + anchorEnd, + value: body.replace(/\\([\\.^$|?*+()[\]{}/-])/g, '$1'), + }; +} + +function anchorsToOperator( + anchorStart: boolean, + anchorEnd: boolean, + negated: boolean, +): TextOperator { + if (anchorStart && !anchorEnd) { + return negated ? 'does-not-start-with' : 'starts-with'; + } + + if (anchorEnd && !anchorStart) { + return negated ? 'does-not-end-with' : 'ends-with'; + } + + return negated ? 'does-not-contain' : 'contains'; +} + +export function textSemantics(): ValueSemantics { + return { + operators: TEXT_OPERATORS, + serialize({ operator, values }) { + const rawValue = values[0]; + + if (typeof rawValue !== 'string' || rawValue === '') { + return null; + } + + if (operator === 'is') { + return escapeNqlString(rawValue); + } + + if (operator === 'is-not') { + return `-${escapeNqlString(rawValue)}`; + } + + const symbol = TEXT_SYMBOLS[operator]; + + if (!symbol) { + return null; + } + + return `${symbol}${escapeNqlString(rawValue)}`; + }, + parse({ operator, value }) { + if (operator === '$eq' && typeof value === 'string') { + return { operator: 'is', values: [value] }; + } + + if (operator === '$ne' && typeof value === 'string') { + return { operator: 'is-not', values: [value] }; + } + + if ((operator === '$regex' || operator === '$not') && value instanceof RegExp) { + const { anchorStart, anchorEnd, value: text } = decomposeRegex(value); + + return { + operator: anchorsToOperator(anchorStart, anchorEnd, operator === '$not'), + values: [text], + }; + } + + return null; + }, + }; +} diff --git a/apps/admin/src/shared/filters/semantics/types.ts b/apps/admin/src/shared/filters/semantics/types.ts new file mode 100644 index 00000000000..d75af682bbf --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/types.ts @@ -0,0 +1,45 @@ +import type { CodecContext } from '@/shared/filters/filter-types'; + +export interface ValueComparator { + operator: string; + value: unknown; +} + +export interface ClauseFragment { + key?: string; + expression: string; +} + +export interface WrittenValue { + join?: 'and' | 'or'; + fragments: ClauseFragment[]; +} + +export type SerializedValue = string | WrittenValue; + +export interface EqualityClause { + key: string; + value: unknown; +} + +export interface ClauseGroup { + join: 'and' | 'or'; + clauses: readonly EqualityClause[]; +} + +export interface SemanticValue { + operator: TOperator; + values: unknown[]; +} + +export interface ValueSemantics { + readonly operators: readonly TOperator[]; + serialize: (input: SemanticValue, ctx: CodecContext) => SerializedValue | null; + parse: (comparator: ValueComparator, ctx: CodecContext) => SemanticValue | null; + parseClauses?: (group: ClauseGroup, ctx: CodecContext) => SemanticValue | null; +} + +export interface ValueConfig { + quoteStrings?: boolean; + serializeSingletonAsScalar?: boolean; +} diff --git a/apps/admin/src/shared/filters/semantics/value.ts b/apps/admin/src/shared/filters/semantics/value.ts new file mode 100644 index 00000000000..2d118f6c8e7 --- /dev/null +++ b/apps/admin/src/shared/filters/semantics/value.ts @@ -0,0 +1,24 @@ +import { escapeNqlString } from '@tryghost/nql-string'; +import type { ValueConfig } from './types'; + +const UNQUOTED_TOKEN_PATTERN = /^[A-Za-z0-9_.-]+$/; + +export function normalizeMultiValue(values: unknown[]): string[] { + return values.map((value) => String(value)).sort((left, right) => left.localeCompare(right)); +} + +export function serializeScalarValue(value: unknown, config?: ValueConfig): string { + if (typeof value === 'string') { + // The leading-minus check looks redundant because the pattern below allows a minus, and + // it isn't: in a query a minus at the front of a value means "not this". A value that + // genuinely starts with one has to be quoted, or `status:'-paid'` turns into `status:-paid` + // and the filter starts meaning the opposite of what it says. + if (config?.quoteStrings || value.startsWith('-') || !UNQUOTED_TOKEN_PATTERN.test(value)) { + return escapeNqlString(value); + } + + return value; + } + + return String(value); +} diff --git a/apps/admin/test-utils/acceptance/resources.ts b/apps/admin/test-utils/acceptance/resources.ts index d286779e286..bdef6c49e32 100644 --- a/apps/admin/test-utils/acceptance/resources.ts +++ b/apps/admin/test-utils/acceptance/resources.ts @@ -492,7 +492,13 @@ export function fakeEditSettings(): EditSettingsCapture { requests.push(body); const overrides = Object.fromEntries(body.settings.map(({ key, value }) => [key, value])); - const response: SettingsResponse = settingsResponse({ settings: overrides }); + // The fixture accepts Labs separately; otherwise it overwrites the saved + // JSON with defaults and a feature toggle immediately appears unchecked. + const labs = + typeof overrides.labs === 'string' + ? (JSON.parse(overrides.labs) as Record) + : undefined; + const response: SettingsResponse = settingsResponse({ settings: overrides, labs }); return HttpResponse.json(response); }); diff --git a/apps/admin/test-utils/acceptance/setup.ts b/apps/admin/test-utils/acceptance/setup.ts index 7e74ce33e9a..5f8da72e39e 100644 --- a/apps/admin/test-utils/acceptance/setup.ts +++ b/apps/admin/test-utils/acceptance/setup.ts @@ -6,6 +6,19 @@ import { defaultBootResolver, defaultBootRoutes } from './boot'; import { resetFakeApi, settleRequests, startFakeApi, verifyNoUnhandledRequests } from './worker'; beforeAll(async () => { + // Playwright waits for an element to stop moving before it acts on it, so every + // click that opens an animated surface pays that animation — Shade's toaster-in + // alone is 0.8s, and a modal-opening click costs ~209ms against ~37ms without. + // Scoped to elements carrying an `animate-*` utility (Shade's --animate-* tokens + // plus Radix's data-[state]:animate-in), so transitions — which one analytics + // test asserts on — are untouched. + const style = document.createElement('style'); + style.textContent = `[class*="animate-"] { + animation-duration: 0s !important; + animation-delay: 0s !important; + }`; + document.head.appendChild(style); + await startFakeApi({ resolver: defaultBootResolver, routes: defaultBootRoutes() }); }); diff --git a/apps/admin/vitest.acceptance.config.ts b/apps/admin/vitest.acceptance.config.ts index 6a8ebbaef56..9176717869e 100644 --- a/apps/admin/vitest.acceptance.config.ts +++ b/apps/admin/vitest.acceptance.config.ts @@ -1,3 +1,5 @@ +import { availableParallelism } from 'node:os'; + import { defineConfig } from 'vitest/config'; import { playwright } from '@vitest/browser-playwright'; import type { PluginOption } from 'vite'; @@ -11,6 +13,15 @@ import { sharedDefine, sharedResolve } from './vite.shared'; * against a fake Ghost Admin API (test-utils/acceptance/). Unit tests stay * in vite.config.ts (jsdom). */ + +/* + * Each worker drives its own Chromium page, so workers stay ~97% busy right up + * to the core count and then fall off a cliff (63% at 18 workers on an 18-core + * box, and worse wall-clock than 8). Leave a core for the Vite server and cap + * the top end; the floor keeps 2-core runners on their current two workers. + */ +const getWorkerCount = () => Math.min(8, Math.max(2, availableParallelism() - 1)); + export default defineConfig({ plugins: [tailwindcss() as PluginOption, react()], // Serves the MSW service worker script; scoped to the test config so it @@ -28,7 +39,7 @@ export default defineConfig({ test: { name: 'acceptance', include: ['src/**/*.acceptance.test.tsx'], - maxWorkers: process.env.CI ? 2 : undefined, + maxWorkers: getWorkerCount(), setupFiles: ['./test-utils/acceptance/setup.ts'], expect: { // Full-app renders are slower than unit renders; the harness's diff --git a/apps/ember-admin/app/templates/site.hbs b/apps/ember-admin/app/templates/site.hbs index c25b66ce757..2170561630d 100644 --- a/apps/ember-admin/app/templates/site.hbs +++ b/apps/ember-admin/app/templates/site.hbs @@ -1 +1 @@ - \ No newline at end of file + diff --git a/apps/ember-admin/tests/integration/components/gh-site-iframe-test.js b/apps/ember-admin/tests/integration/components/gh-site-iframe-test.js new file mode 100644 index 00000000000..4bb24b6ce33 --- /dev/null +++ b/apps/ember-admin/tests/integration/components/gh-site-iframe-test.js @@ -0,0 +1,23 @@ +import hbs from 'htmlbars-inline-precompile'; +import {describe, it} from 'mocha'; +import {expect} from 'chai'; +import {find, render} from '@ember/test-helpers'; +import {setupRenderingTest} from 'ember-mocha'; + +describe('Integration: Component: gh-site-iframe', function () { + setupRenderingTest(); + + beforeEach(function () { + this.owner.register('config:main', { + blogUrl: 'http://localhost:2368' + }, {instantiate: false}); + }); + + it('forwards the View site preview marker to the iframe element', async function () { + await render(hbs``); + + const iframe = find('iframe'); + expect(iframe).to.have.attribute('data-view-site-preview'); + expect(iframe).to.have.class('site-frame'); + }); +}); diff --git a/apps/shade/src/utils.ts b/apps/shade/src/utils.ts index d5ac751627d..cd5fdeaf25d 100644 --- a/apps/shade/src/utils.ts +++ b/apps/shade/src/utils.ts @@ -4,6 +4,7 @@ export * as LucideIcon from 'lucide-react'; export { default as useGlobalDirtyState } from './hooks/use-global-dirty-state'; export { useSimplePagination } from './hooks/use-simple-pagination'; +export { useIsMobile } from './hooks/use-mobile'; export { cn, diff --git a/compose.dev.analytics.yaml b/compose.dev.analytics.yaml index 79861848387..7548a6e8493 100644 --- a/compose.dev.analytics.yaml +++ b/compose.dev.analytics.yaml @@ -41,6 +41,9 @@ services: stop_grace_period: 2s ports: - '7181:7181' + volumes: + - tinybird-clickhouse:/var/lib/clickhouse + - tinybird-metadata:/redis-data healthcheck: test: ['CMD', 'curl', '-f', 'http://localhost:7181/v0/health'] interval: 30s @@ -90,3 +93,5 @@ services: volumes: shared-config: + tinybird-clickhouse: + tinybird-metadata: diff --git a/docker/tinybird-local-slim/Dockerfile b/docker/tinybird-local-slim/Dockerfile new file mode 100644 index 00000000000..691333138a0 --- /dev/null +++ b/docker/tinybird-local-slim/Dockerfile @@ -0,0 +1,46 @@ +# syntax=docker/dockerfile:1 +# +# Distilled tinybird-local for Ghost CI. See README.md in this directory. +# +# Stage 1 prunes the upstream image (see cleanup.sh); stage 2 flattens the +# result into a single layer via `COPY --from`. The flatten is where the size +# reduction comes from: upstream installs ClickHouse twice across separate +# layers, and a squashed rootfs keeps only the final copy. +# +# TINYBIRD_LOCAL_REF has no default on purpose — compose.dev.analytics.yaml is +# the single source of truth for the upstream digest, and the publish workflow +# reads it from there. Pass it explicitly for local builds. +ARG TINYBIRD_LOCAL_REF + +FROM ${TINYBIRD_LOCAL_REF} AS pruned +COPY docker/tinybird-local-slim/cleanup.sh /tmp/cleanup.sh +RUN /tmp/cleanup.sh && rm /tmp/cleanup.sh + +FROM scratch +COPY --from=pruned / / + +# A flatten drops the upstream image's config, so re-declare it here. The publish +# workflow diffs this against upstream and fails the build when they drift. +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + CLICKHOUSE_VERSION=25.3.7.194 \ + CLICKHOUSE_CLUSTER_HOST=localhost \ + CLICKHOUSE_CLUSTER_HTTP_PORT=8123 \ + CLICKHOUSE_CLUSTER_INTERNAL_HOST=localhost \ + CLICKHOUSE_CLUSTER_INTERNAL_HTTP_PORT=8123 \ + CLICKHOUSE_INTERNAL_PORT=8123 \ + USE_GATHERER=False \ + MINIO_ROOT_USER=admin \ + MINIO_ROOT_PASSWORD=password +WORKDIR /app +EXPOSE 7181 7182 +# Copied verbatim from upstream, notably the licence pointers — the flatten drops +# them, and they are proprietary notices we are not permitted to strip. Upstream's +# own `license` label says SSPL and contradicts the LICENSE.md the image ships +# (and license_url serves); it is reproduced as-is rather than corrected. +LABEL license="SSPL" \ + license_url="https://www.tinybird.co/docs/tb-local-license.txt" \ + org.opencontainers.image.ref.name="ubuntu" \ + org.opencontainers.image.version="22.04" +HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=10 \ + CMD tb --output json --cloud sql 'SELECT 1 AS healthcheck' | grep '"healthcheck": 1' || exit 1 +CMD ["/usr/bin/supervisord"] diff --git a/docker/tinybird-local-slim/README.md b/docker/tinybird-local-slim/README.md new file mode 100644 index 00000000000..7c92cb05e83 --- /dev/null +++ b/docker/tinybird-local-slim/README.md @@ -0,0 +1,74 @@ +# tinybird-local-slim + +A distilled build of `tinybirdco/tinybird-local` for CI. Published to +`ghcr.io/tryghost/tinybird-local-slim` by +[`publish-tinybird-local-slim.yml`](../../.github/workflows/publish-tinybird-local-slim.yml) +and used by the analytics E2E jobs via `GHOST_E2E_TINYBIRD_SLIM=true`. + +## Why + +The upstream image is ~2.1GB to pull and ~6.9GB once unpacked, which does not +fit alongside the rest of the E2E infra in a GitHub Actions runner's disk +budget. The slim build is ~0.7GB to pull and ~2.4GB unpacked, with the same boot +time (~25s to healthy in both cases). + +Most of that comes from flattening, not from deleting things. Upstream installs +ClickHouse twice and strips the binary in a separate layer, so the image carries +several gigabytes of superseded content in lower layers. `COPY --from` into a +`FROM scratch` stage keeps only the final rootfs. + +`cleanup.sh` removes the rest: the build toolchain, package installers, VCS and +transfer tools, docs/man/locale, apt metadata, and the supervisord programs Ghost +never exercises. It deliberately keeps every Python package and the shipped +`__pycache__` — the Tinybird server imports its whole feature surface eagerly at +boot, and dropping the bytecode cache made startup measurably slower. + +## Bumping the upstream version + +`compose.dev.analytics.yaml` is the single source of truth for the upstream +digest. Bump it there; the publish workflow reads the digest from that file, so +no change is needed here. + +If the new upstream release changes the image's runtime config (env vars, +command, ports, healthcheck), the workflow's config-parity check fails — the +`FROM scratch` flatten discards upstream's config, so the `ENV`/`CMD`/`EXPOSE` +block in the Dockerfile has to be updated to match. Reproduce locally with +`verify-config.sh `. + +## Building locally + +```bash +docker buildx build --platform linux/amd64 \ + --build-arg TINYBIRD_LOCAL_REF="$(grep -oE 'tinybirdco/tinybird-local:[^ ]+' compose.dev.analytics.yaml)" \ + -f docker/tinybird-local-slim/Dockerfile -t tinybird-local-slim:local --load . +``` + +Then point E2E at it: + +```bash +GHOST_E2E_TINYBIRD_SLIM=true GHOST_E2E_TINYBIRD_SLIM_IMAGE=tinybird-local-slim:local pnpm test:e2e:analytics +``` + +## Licensing + +`tinybird-local` is proprietary, under the Tinybird License (Self-Managed) — +`/LICENSE.md` in the image. Two clauses shape what this directory does: + +- **Derivative works are allowed** (§2c), provided they do not circumvent + technical limitations or remove license enforcement, auditing, or access + control. Distilling the image is fine; `cleanup.sh` touches none of that, and + leaves `/LICENSE.md` and the bundled third-party copyright files in place (§3e). +- **Distribution is limited to within our own organization** (§2d). The GHCR + package must therefore stay **internal** — publishing it publicly would be + distribution outside the licensee organization. Grant the private forks that + need it access through package settings rather than making it public. + +An internal package is unreadable from a PR opened from a public fork, whose +token is scoped to the fork. CI leaves `GHOST_E2E_TINYBIRD_SLIM` off for those +runs so they use upstream directly, and `e2e/scripts/infra-up.sh` falls back to +upstream on any failed pull regardless — so a missing access grant, or the +window before the package is first published, degrades to a slower, fatter run +rather than a broken one. + +Use here is testing, not a Production Environment, so the production usage limits +in §3a do not apply. diff --git a/docker/tinybird-local-slim/cleanup.sh b/docker/tinybird-local-slim/cleanup.sh new file mode 100755 index 00000000000..e8da3e265df --- /dev/null +++ b/docker/tinybird-local-slim/cleanup.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Distills the upstream tinybirdco/tinybird-local image for Ghost CI use. +# +# The image is proprietary (Tinybird License, Self-Managed — /LICENSE.md in the +# image). Derivative works are permitted, so long as they do not remove license +# enforcement, auditing, or proprietary notices; /LICENSE.md and the third-party +# copyright files are deliberately left in place. +# +# It only removes things Ghost provably does not use. Every Python runtime +# dependency is kept on purpose: the (closed-source) Tinybird server imports its +# entire feature surface eagerly at boot (LLM, GCP connectors, scipy, ...), so +# pruning "unused" packages breaks startup. The shipped __pycache__ is kept for +# the same reason: deleting it makes every boot recompile the tree. Most of the +# size win instead comes from flattening the result to a single layer (the +# `FROM scratch` stage in the Dockerfile), which drops the ClickHouse install the +# upstream image bakes in twice across separate layers. +set -euo pipefail + +SUPERVISORD=/etc/supervisor/conf.d/supervisord.conf + +# Drop the supervisord programs Ghost never exercises (Kafka ingestion + MCP/AI +# server). Editing the shipped config in-place keeps this robust to upstream +# version bumps rather than committing a copy that can drift. +awk 'BEGIN{RS="";ORS="\n\n"} !/\[program:tinybird-kafka\]/ && !/\[program:tinybird-mcp\]/' \ + "$SUPERVISORD" > "$SUPERVISORD.slim" +mv "$SUPERVISORD.slim" "$SUPERVISORD" + +# Build toolchain / linkers — runtime-unnecessary (the image ships prebuilt wheels). +rm -rf /usr/lib/gcc /usr/bin/gcc* /usr/bin/g++* /usr/bin/*-gcc* /usr/bin/*-g++* \ + /usr/bin/cpp* /usr/bin/*-cpp* /usr/bin/x86_64-linux-gnu-lto-dump* \ + /usr/bin/x86_64-linux-gnu-gcc* /usr/bin/x86_64-linux-gnu-g++* + +# Package installers + VCS/transfer tools not used at runtime. +rm -rf /usr/bin/uv /usr/bin/uvx \ + /usr/bin/git /usr/lib/git-core /usr/share/git-core \ + /usr/bin/rsync /usr/bin/ssh /usr/bin/scp + +# Apt metadata and caches. /usr/share/doc stays: it is only ~6MB, and it holds +# the Debian copyright files for the third-party packages the image bundles. +rm -rf /var/lib/apt/lists/* /var/cache/apt/* /root/.cache + +# Clear log *files* but keep the dirs supervisord + services expect to exist. +find /var/log -type f -delete 2>/dev/null || true +mkdir -p /var/log/supervisor /var/log/clickhouse-server /var/log/nginx /var/log/redis diff --git a/docker/tinybird-local-slim/verify-config.sh b/docker/tinybird-local-slim/verify-config.sh new file mode 100755 index 00000000000..162affa43d6 --- /dev/null +++ b/docker/tinybird-local-slim/verify-config.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Fails when the slim image's re-declared runtime config has drifted from upstream. +# +# The slim build flattens the rootfs into a `FROM scratch` stage, which discards +# the upstream image config, so the Dockerfile restates it by hand. An upstream +# release that adds an env var or changes the command would silently ship a +# broken image without this check. +# +# Usage: verify-config.sh +set -euo pipefail + +UPSTREAM="${1:?upstream image ref required}" +SLIM="${2:?slim image ref required}" + +config() { + local out + out=$(docker image inspect --platform linux/amd64 "$1" --format '{{json .Config}}' | jq -S '{ + Env: (.Env // []| sort), + Cmd, Entrypoint, WorkingDir, User, StopSignal, + ExposedPorts: (.ExposedPorts // {} | keys), + Volumes: (.Volumes // {} | keys), + Labels: (.Labels // {}), + Healthcheck: (.Healthcheck // null) + }') + # An index manifest with no local amd64 child inspects to an empty config — + # that is a missing `docker pull --platform linux/amd64`, not real drift. + if [[ "$(jq -r '.Env | length' <<<"$out")" == "0" ]]; then + echo "No linux/amd64 config for $1 — pull it for that platform first." >&2 + return 1 + fi + printf '%s\n' "$out" +} + +upstream_config=$(config "$UPSTREAM") +slim_config=$(config "$SLIM") + +if diff -u <(printf '%s\n' "$upstream_config") <(printf '%s\n' "$slim_config"); then + echo "Slim image config matches upstream." + exit 0 +fi + +cat >&2 < { + /** Creates a field of the named type. The modal closes itself on success. */ + async createField(name: string, type?: string): Promise { await this.addButton.waitFor(); await this.addButton.click(); await this.modal.getByLabel('Name').fill(name); + + if (type) { + await this.modal.getByTestId('custom-field-type').click(); + await this.page.getByRole('option', { name: type, exact: true }).click(); + } + await this.modal.getByRole('button', { name: 'Save' }).click(); await this.listItem(name).waitFor(); } + /** Short text is the default type, and keeps the member detail editor a plain input. */ + async createShortTextField(name: string): Promise { + await this.createField(name); + } + /** - * Create a composite field, whose value is several parts rather than one string. - * The type can only be chosen at creation — the picker is disabled thereafter. + * An address is a composite: one field storing several named parts, each filtered as a + * field in its own right. See `MemberDetailsPage.setCompositeCustomFieldValue`. */ async createAddressField(name: string): Promise { - await this.addButton.waitFor(); - await this.addButton.click(); - await this.modal.getByLabel('Name').fill(name); - await this.modal.getByLabel('Type').click(); - await this.page.getByRole('option', { name: 'Address' }).click(); - await this.modal.getByRole('button', { name: 'Save' }).click(); - await this.listItem(name).waitFor(); + await this.createField(name, 'Address'); } } diff --git a/e2e/scripts/infra-up.sh b/e2e/scripts/infra-up.sh index 9b61469ac89..78d4b3e9eff 100755 --- a/e2e/scripts/infra-up.sh +++ b/e2e/scripts/infra-up.sh @@ -11,6 +11,7 @@ MODE="$(resolve_e2e_mode)" export GHOST_E2E_MODE="$MODE" ANALYTICS_ENABLED="${GHOST_E2E_ANALYTICS:-true}" MYSQL_TMPFS_ENABLED="${GHOST_E2E_MYSQL_TMPFS:-true}" +TINYBIRD_SLIM_ENABLED="${GHOST_E2E_TINYBIRD_SLIM:-false}" if [[ "$MODE" != "build" ]]; then DEV_COMPOSE_PROJECT="${COMPOSE_PROJECT_NAME:-ghost-dev}" @@ -33,6 +34,27 @@ fi if [[ "$ANALYTICS_ENABLED" == "true" ]]; then compose_files+=(-f compose.dev.analytics.yaml) services+=(tinybird-local analytics) + + # Opt-in override to the distilled slim Tinybird image, which is a fraction of + # upstream's size on disk. Must be layered after compose.dev.analytics.yaml to + # override its image. + # + # The image is licensed for distribution within our organization only, so its + # GHCR package is internal and unreadable from a fork PR's scoped token. Fall + # back to the upstream image whenever the pull fails, rather than failing the + # run: it also covers the window before the package is first published, and a + # revoked or not-yet-granted access grant. + if [[ "$TINYBIRD_SLIM_ENABLED" == "true" ]]; then + export GHOST_E2E_TINYBIRD_SLIM_IMAGE="${GHOST_E2E_TINYBIRD_SLIM_IMAGE:-ghcr.io/tryghost/tinybird-local-slim:latest}" + + if docker image inspect "$GHOST_E2E_TINYBIRD_SLIM_IMAGE" >/dev/null 2>&1 \ + || docker pull "$GHOST_E2E_TINYBIRD_SLIM_IMAGE"; then + compose_files+=(-f e2e/compose.e2e.tinybird-slim.yaml) + else + echo "WARNING: could not pull ${GHOST_E2E_TINYBIRD_SLIM_IMAGE} — falling back to the upstream Tinybird image." + echo "WARNING: the upstream image needs several more GB of runner disk; expect a disk-space failure on a constrained runner." + fi + fi fi docker compose "${compose_files[@]}" up -d --wait "${services[@]}" diff --git a/e2e/scripts/prepare-ci-e2e-build-mode.sh b/e2e/scripts/prepare-ci-e2e-build-mode.sh index 2af4f5ce067..66fd538b6ee 100755 --- a/e2e/scripts/prepare-ci-e2e-build-mode.sh +++ b/e2e/scripts/prepare-ci-e2e-build-mode.sh @@ -4,11 +4,13 @@ set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/load-playwright-container-env.sh" GATEWAY_IMAGE="${GHOST_E2E_GATEWAY_IMAGE:-caddy:2-alpine}" ANALYTICS_ENABLED="${GHOST_E2E_ANALYTICS:-true}" +TINYBIRD_SLIM_ENABLED="${GHOST_E2E_TINYBIRD_SLIM:-false}" echo "Preparing E2E build-mode runtime" echo "Playwright image: ${PLAYWRIGHT_IMAGE}" echo "Gateway image: ${GATEWAY_IMAGE}" echo "Analytics enabled: ${ANALYTICS_ENABLED}" +echo "Tinybird slim image: ${TINYBIRD_SLIM_ENABLED}" pids=() labels=() @@ -27,7 +29,7 @@ run_bg() { run_bg "pull-gateway-image" docker pull "$GATEWAY_IMAGE" run_bg "pull-playwright-image" ensure_playwright_image -run_bg "start-infra" env GHOST_E2E_MODE=build GHOST_E2E_ANALYTICS="$ANALYTICS_ENABLED" bash "$REPO_ROOT/e2e/scripts/infra-up.sh" +run_bg "start-infra" env GHOST_E2E_MODE=build GHOST_E2E_ANALYTICS="$ANALYTICS_ENABLED" GHOST_E2E_TINYBIRD_SLIM="$TINYBIRD_SLIM_ENABLED" bash "$REPO_ROOT/e2e/scripts/infra-up.sh" for i in "${!pids[@]}"; do if ! wait "${pids[$i]}"; then diff --git a/e2e/scripts/sync-tinybird-state.mjs b/e2e/scripts/sync-tinybird-state.mjs index f9243e6b0f9..5d968b4f99d 100644 --- a/e2e/scripts/sync-tinybird-state.mjs +++ b/e2e/scripts/sync-tinybird-state.mjs @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; @@ -9,14 +10,8 @@ const repoRoot = path.resolve(__dirname, '../..'); const stateDir = path.resolve(repoRoot, 'e2e/data/state'); const configPath = path.resolve(stateDir, 'tinybird.json'); -const composeArgs = [ - 'compose', - '-f', - path.resolve(repoRoot, 'compose.dev.yaml'), - '-f', - path.resolve(repoRoot, 'compose.dev.analytics.yaml'), -]; const composeProject = process.env.COMPOSE_PROJECT_NAME || 'ghost-dev'; +const tinybirdConfigPath = '/mnt/shared-config/.env.tinybird'; function log(message) { process.stdout.write(`${message}\n`); @@ -49,25 +44,16 @@ function clearConfigIfPresent() { } } -function runCompose(args) { - return execFileSync('docker', [...composeArgs, ...args], { - cwd: repoRoot, - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'pipe'], - }); -} - -function isTinybirdRunning() { +function findContainer(service, extraFilters = []) { const output = execFileSync( 'docker', [ 'ps', + ...extraFilters, '--filter', `label=com.docker.compose.project=${composeProject}`, '--filter', - 'label=com.docker.compose.service=tinybird-local', - '--filter', - 'status=running', + `label=com.docker.compose.service=${service}`, '--format', '{{.Names}}', ], @@ -78,11 +64,40 @@ function isTinybirdRunning() { }, ); - return Boolean(output.trim()); + return output.trim().split('\n')[0] || null; } +function isTinybirdRunning() { + return Boolean(findContainer('tinybird-local', ['--filter', 'status=running'])); +} + +// Copied out of the tb-cli container rather than read through `docker compose +// run`. That would re-run the tb-cli entrypoint (a full datafile deploy), and, +// worse, compose would recreate any dependency whose config differs from the +// compose files listed here — silently replacing a tinybird-local started from +// an override (see e2e/compose.e2e.tinybird-slim.yaml). `docker cp` reads the +// same file straight from the exited container and cannot diverge. function fetchConfigFromTbCli() { - return runCompose(['run', '--rm', '-T', 'tb-cli', 'cat', '/mnt/shared-config/.env.tinybird']); + const container = findContainer('tb-cli', ['-a']); + + if (!container) { + throw new Error(`No tb-cli container found for compose project ${composeProject}`); + } + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ghost-e2e-tinybird-')); + const tmpFile = path.join(tmpDir, '.env.tinybird'); + + try { + execFileSync('docker', ['cp', `${container}:${tinybirdConfigPath}`, tmpFile], { + cwd: repoRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + + return fs.readFileSync(tmpFile, 'utf8'); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } } function writeConfig(env) { @@ -116,7 +131,7 @@ try { if (!env.TINYBIRD_WORKSPACE_ID || !env.TINYBIRD_ADMIN_TOKEN) { clearConfigIfPresent(); throw new Error( - 'Tinybird is running but required config values are missing in /mnt/shared-config/.env.tinybird', + `Tinybird is running but required config values are missing in ${tinybirdConfigPath}`, ); } diff --git a/e2e/tests/admin/members/custom-field-filter-round-trip.test.ts b/e2e/tests/admin/members/custom-field-filter-round-trip.test.ts index b92bf27dda2..ca7442083a0 100644 --- a/e2e/tests/admin/members/custom-field-filter-round-trip.test.ts +++ b/e2e/tests/admin/members/custom-field-filter-round-trip.test.ts @@ -68,7 +68,7 @@ test.describe('Ghost Admin - Filter members by custom fields', () => { await expect(sidebar.getNavLink(viewName)).not.toHaveAttribute('aria-current', 'page'); // Reopen it: the filter round-trips (the view goes active again only if the - // reopened NQL re-serialises to the exact saved string), the custom-field + // reopened NQL re-serializes to the exact saved string), the custom-field // filter is present, and the same member is matched. await sidebar.getNavLink(viewName).click(); @@ -78,6 +78,64 @@ test.describe('Ghost Admin - Filter members by custom fields', () => { await expect(membersPage.getMemberByName(`Acme Employee ${stamp}`)).toHaveCount(0); }); + /** + * A composite stores one row per part, and each part filters as a field in its own right, + * so the pill carries a part alongside the value. Filtering on one part must not match a + * member whose other parts happen to hold that value. + */ + test('an address filter matches on the chosen part only', async ({ page }) => { + test.slow(); + + const stamp = Date.now(); + const fieldName = `Shipping ${stamp}`; + const memberFactory = createMemberFactory(page.request); + + const inLondon = await memberFactory.create({ + name: `London Buyer ${stamp}`, + email: `london-${stamp}@example.com`, + }); + const inBoston = await memberFactory.create({ + name: `Boston Buyer ${stamp}`, + email: `boston-${stamp}@example.com`, + }); + + const settingsPage = new SettingsPage(page); + const memberDetailsPage = new MemberDetailsPage(page); + const membersPage = new MembersListPage(page); + + await settingsPage.goto(); + await settingsPage.customFieldsSection.createAddressField(fieldName); + + await page.goto(`/ghost/#/members/${inLondon.id}`); + await memberDetailsPage.setCompositeCustomFieldValue(fieldName, { + 'Address line 1': '1 King St', + City: 'London', + // The country part validates as a 2-letter code. + Country: 'GB', + }); + + // 'London' sits in this member's Address line 1, so a filter on City must not match it. + await page.goto(`/ghost/#/members/${inBoston.id}`); + await memberDetailsPage.setCompositeCustomFieldValue(fieldName, { + 'Address line 1': 'London House', + City: 'Boston', + Country: 'US', + }); + + await page.goto('/ghost/#/members'); + // A composite defaults to the presence operator, which takes no value, so the + // operator is chosen explicitly here. + await membersPage.addCustomFieldFilter({ + field: fieldName, + subfield: 'City', + operator: 'is', + value: 'London', + }); + + await expect(membersPage.getMemberByName(`London Buyer ${stamp}`)).toBeVisible(); + await expect(membersPage.getMemberByName(`Boston Buyer ${stamp}`)).toHaveCount(0); + }); + test('an is-set custom field filter matches members that have a value', async ({ page }) => { test.slow(); diff --git a/e2e/tests/admin/members/member-detail.test.ts b/e2e/tests/admin/members/member-detail.test.ts index 1cdf178ae63..5545be2005c 100644 --- a/e2e/tests/admin/members/member-detail.test.ts +++ b/e2e/tests/admin/members/member-detail.test.ts @@ -499,6 +499,63 @@ test.describe('Ghost Admin - Member Detail', () => { await expect(page.getByText(/Renews 15 Feb 2026/).first()).toBeVisible(); }); + test('subscription attribution with an unsafe url - shows the page as text, not a link', async ({ + page, + }) => { + // Subscription attribution URLs come from public signup/checkout URL + // history, which the server does not scheme-validate, so a member can get + // a `javascript:` URL stored as their own subscription attribution. It + // should still be shown, but never as a clickable link a staff user could + // trigger — the same rule the sidebar and activity feed already apply. + const member = await memberFactory.create({ + name: 'Unsafe Attribution', + email: 'unsafe-attribution@ghost.org', + }); + await seedSubscriptions(page, member.id, 'paid', [ + paidSubscription({ + attribution: { + title: 'Signup page', + url: "javascript:localStorage.setItem('attribution_xss', 'executed')", + referrer_source: 'Twitter', + }, + }), + ]); + + await page.goto(memberPath(member.id)); + await page.getByTestId('member-subscription-details-toggle').first().click(); + + await expect(page.getByText('Signup page')).toBeVisible(); + await expect(page.getByRole('link', { name: 'Signup page' })).toHaveCount(0); + }); + + test('subscription attribution with a safe url - shows the page as a working link', async ({ + page, + }) => { + // The guard must not swallow legitimate attribution: a site-relative or + // http(s) URL still renders as a link the staff user can follow. + const member = await memberFactory.create({ + name: 'Safe Attribution', + email: 'safe-attribution@ghost.org', + }); + await seedSubscriptions(page, member.id, 'paid', [ + paidSubscription({ + attribution: { title: 'Welcome post', url: '/welcome/', referrer_source: 'Twitter' }, + }), + ]); + + await page.goto(memberPath(member.id)); + await page.getByTestId('member-subscription-details-toggle').first().click(); + + const link = page.getByRole('link', { name: 'Welcome post' }); + await expect(link).toBeVisible(); + + // Following it opens the page in a new tab — proof the affordance is real. + const popupPromise = page.context().waitForEvent('page'); + await link.click(); + const popup = await popupPromise; + await expect(popup).toHaveURL(/\/welcome/); + }); + test('subscription set to cancel - shows remaining access rather than a renewal', async ({ page, }) => { diff --git a/ghost/core/core/server/services/stripe/services/webhook/checkout-session-event-service.js b/ghost/core/core/server/services/stripe/services/webhook/checkout-session-event-service.js index bcba8cdbcb7..77b1e587437 100644 --- a/ghost/core/core/server/services/stripe/services/webhook/checkout-session-event-service.js +++ b/ghost/core/core/server/services/stripe/services/webhook/checkout-session-event-service.js @@ -1,7 +1,10 @@ const _ = require('lodash'); const errors = require('@tryghost/errors'); const logging = require('@tryghost/logging'); -const { canWelcomeEmailReplaceSignupPaidEmail } = require('../../../lib/member-signup-contexts'); +const { + SIGNUP_CONTEXTS, + canWelcomeEmailReplaceSignupPaidEmail, +} = require('../../../lib/member-signup-contexts'); const { collectedByPort } = require('../checkout/completed-session'); /** @typedef {import('../../../lib/member-signup-contexts').SignupContext} SignupContext */ @@ -335,6 +338,7 @@ module.exports = class CheckoutSessionEventService { email: customer.email, }); + const memberPreexisted = Boolean(member); const checkoutType = _.get(session, 'metadata.checkoutType'); if (!member) { @@ -426,7 +430,7 @@ module.exports = class CheckoutSessionEventService { // After the subscription work, and deliberately not part of it: a value the member // gave us for free must never be able to fail the webhook. A throw here would make // Stripe retry the event and risk doing the payment work twice. - await this.writeCollectedFields(member.id, session); + await this.writeCollectedFields(member.id, session, { memberPreexisted }); if (checkoutType !== 'upgrade') { const ghostSignupContext = /** @type {SignupContext | undefined} */ ( @@ -457,8 +461,10 @@ module.exports = class CheckoutSessionEventService { * * @param {string} memberId * @param {import('stripe').Stripe.Checkout.Session} session + * @param {object} options + * @param {boolean} options.memberPreexisted Whether the member's record existed before this event resolved it */ - async writeCollectedFields(memberId, session) { + async writeCollectedFields(memberId, session, { memberPreexisted }) { // Stamped at create time. A session predating this feature carries none. Read // outside the try so a failure below can name the tier whose answers were lost. const tierId = session.metadata?.ghostTierId; @@ -472,6 +478,25 @@ module.exports = class CheckoutSessionEventService { return; } + // A checkout can be started with nothing but an email address, and typing an + // email is not proof of owning it. Values may land on the member this event just + // created — that record holds nothing the buyer didn't supply — but a record + // that existed before the checkout belongs to whoever verified that email, so it + // is only written when the session was created by a signed-in member. + const wasAuthenticated = + session.metadata?.ghostSignupContext === SIGNUP_CONTEXTS.ALREADY_AUTHENTICATED; + if (memberPreexisted && !wasAuthenticated) { + logging.warn( + { + event: { name: 'stripe_checkout.collected_fields.write_skipped' }, + memberId, + tierId, + }, + 'Skipped storing the fields an unverified checkout collected for an existing member', + ); + return; + } + await this.deps.customFieldBindings.writeCollected( memberId, tierId, diff --git a/ghost/core/core/shared/labs.js b/ghost/core/core/shared/labs.js index 74767ccb4c1..4999cb1a962 100644 --- a/ghost/core/core/shared/labs.js +++ b/ghost/core/core/shared/labs.js @@ -46,6 +46,7 @@ const PRIVATE_FEATURES = [ 'importMemberTier', 'csvContentImporter', 'adminUIRefresh', + 'admin7PageChrome', 'tagsX', 'emailUniqueid', 'themeTranslation', diff --git a/ghost/core/test/e2e-api/admin/config.test.js b/ghost/core/test/e2e-api/admin/config.test.js index b8a6123f207..af079ee514e 100644 --- a/ghost/core/test/e2e-api/admin/config.test.js +++ b/ghost/core/test/e2e-api/admin/config.test.js @@ -61,6 +61,9 @@ describe('Config API', function () { labsValues.every((value) => typeof value === 'boolean'), 'expected all labs flags to be booleans', ); + // Fixture setup enables every registered writable flag. Keep an + // explicit assertion while this private rollout uses dynamic snapshots. + assert.equal(labs.admin7PageChrome, true); }) .matchHeaderSnapshot({ 'content-version': anyContentVersion, diff --git a/ghost/core/test/e2e-api/members/webhooks.test.js b/ghost/core/test/e2e-api/members/webhooks.test.js index ca9b78a7048..c17a919fcdf 100644 --- a/ghost/core/test/e2e-api/members/webhooks.test.js +++ b/ghost/core/test/e2e-api/members/webhooks.test.js @@ -1692,6 +1692,58 @@ describe('Members API', function () { assert.equal(member.status, 'paid'); assert.deepEqual(member.custom_fields, {}); }); + + // A checkout session can be started with nothing but an email address, and typing + // an email is not proof of owning it. The tests above all write onto the member + // the webhook itself created, which is safe: that record holds nothing the buyer + // didn't supply. A record that existed before the checkout is only written when + // the session was started by a signed-in member. + it('does not write onto a member that existed before an unverified checkout', async function () { + const email = 'checkout-collected-preexisting@email.com'; + const { body: created } = await adminAgent + .post('/members/') + .body({ members: [{ email }] }) + .expectStatus(201); + await adminAgent + .put(`/members/${created.members[0].id}/`) + .body({ members: [{ custom_fields: { [fieldKeys.question]: 'Small' } }] }) + .expectStatus(200); + + const member = await sendCheckoutWebhook(email, { + custom_fields: [{ key: fieldKeys.question, type: 'text', text: { value: 'Large' } }], + shipping: { + name: 'Someone Else', + address: { line1: '1 High Street', country: 'GB' }, + }, + }); + + assert.equal(member.status, 'paid', 'the payment work still happened'); + assert.equal( + member.custom_fields[fieldKeys.question], + 'Small', + 'the stored answer was not overwritten', + ); + assert.equal(member.custom_fields[fieldKeys.recipient], undefined); + assert.equal(member.custom_fields[fieldKeys.address], undefined); + }); + + it('writes onto an existing member when the checkout was started signed in', async function () { + const email = 'checkout-collected-signed-in@email.com'; + await adminAgent + .post('/members/') + .body({ members: [{ email }] }) + .expectStatus(201); + + const member = await sendCheckoutWebhook(email, { + metadata: { + ghostTierId: (await getPaidProduct()).id, + ghostSignupContext: 'already_authenticated', + }, + custom_fields: [{ key: fieldKeys.question, type: 'text', text: { value: 'Large' } }], + }); + + assert.equal(member.custom_fields[fieldKeys.question], 'Large'); + }); }); it('Will create a member with default newsletter subscriptions', async function () { diff --git a/ghost/core/test/unit/server/services/stripe/services/webhooks/checkout-session-event-service.test.js b/ghost/core/test/unit/server/services/stripe/services/webhooks/checkout-session-event-service.test.js index b3411964b09..5f2710d0773 100644 --- a/ghost/core/test/unit/server/services/stripe/services/webhooks/checkout-session-event-service.test.js +++ b/ghost/core/test/unit/server/services/stripe/services/webhooks/checkout-session-event-service.test.js @@ -1,5 +1,6 @@ const assert = require('node:assert/strict'); const errors = require('@tryghost/errors'); +const logging = require('@tryghost/logging'); const sinon = require('sinon'); const CheckoutSessionEventService = require('../../../../../../../core/server/services/stripe/services/webhook/checkout-session-event-service'); @@ -1157,5 +1158,103 @@ describe('CheckoutSessionEventService', function () { sinon.assert.notCalled(sendSignupEmail); }); }); + + // A checkout session can be created for any email address without proof the buyer + // owns it, so what gates the write is whether the target record predates the + // checkout, and whether the session was started by a signed-in member. + describe('collected fields writeback', function () { + let labsService; + let customFieldBindings; + + beforeEach(function () { + labsService = { isSet: sinon.stub().returns(true) }; + customFieldBindings = { writeCollected: sinon.stub().resolves() }; + service = createService({ labsService, customFieldBindings }); + session.metadata.ghostTierId = 'tier_123'; + api.getCustomer.resolves(customer); + sinon.stub(logging, 'warn'); + sinon.stub(logging, 'error'); + }); + + afterEach(function () { + sinon.restore(); + }); + + it('writes onto the member this event created, even unverified', async function () { + memberRepository.get.resolves(null); + session.metadata.ghostSignupContext = 'needs_magic_link_email'; + + await service.handleSubscriptionEvent(session); + + sinon.assert.calledOnce(customFieldBindings.writeCollected); + sinon.assert.calledWith( + customFieldBindings.writeCollected, + 'created_member', + 'tier_123', + sinon.match.array, + ); + }); + + it('does not write onto a member that existed before an unverified checkout', async function () { + memberRepository.get.resolves(member); + session.metadata.ghostSignupContext = 'needs_magic_link_email'; + + await service.handleSubscriptionEvent(session); + + sinon.assert.notCalled(customFieldBindings.writeCollected); + }); + + it('treats a session carrying no signup context as unverified', async function () { + memberRepository.get.resolves(member); + delete session.metadata.ghostSignupContext; + + await service.handleSubscriptionEvent(session); + + sinon.assert.notCalled(customFieldBindings.writeCollected); + }); + + it('writes onto an existing member when the checkout was started signed in', async function () { + memberRepository.get.resolves(member); + session.metadata.ghostSignupContext = 'already_authenticated'; + + await service.handleSubscriptionEvent(session); + + sinon.assert.calledOnce(customFieldBindings.writeCollected); + sinon.assert.calledWith( + customFieldBindings.writeCollected, + 'member_123', + 'tier_123', + sinon.match.array, + ); + }); + + it('does not write when the flag is off', async function () { + labsService.isSet.returns(false); + memberRepository.get.resolves(null); + session.metadata.ghostSignupContext = 'already_authenticated'; + + await service.handleSubscriptionEvent(session); + + sinon.assert.notCalled(customFieldBindings.writeCollected); + }); + + it('does not write when the session names no tier', async function () { + memberRepository.get.resolves(null); + delete session.metadata.ghostTierId; + + await service.handleSubscriptionEvent(session); + + sinon.assert.notCalled(customFieldBindings.writeCollected); + }); + + it('does not fail the webhook when the write is rejected', async function () { + memberRepository.get.resolves(null); + customFieldBindings.writeCollected.rejects(new Error('storage broke')); + + await service.handleSubscriptionEvent(session); + + sinon.assert.calledOnce(customFieldBindings.writeCollected); + }); + }); }); }); diff --git a/ghost/core/test/unit/shared/labs.test.js b/ghost/core/test/unit/shared/labs.test.js index f33ec2ec6ed..2841b077411 100644 --- a/ghost/core/test/unit/shared/labs.test.js +++ b/ghost/core/test/unit/shared/labs.test.js @@ -32,6 +32,18 @@ describe('Labs Service', function () { await configUtils.restore(); }); + it('keeps page chrome opt-in and respects an explicit rollback override', function () { + const getStub = sinon.stub(settingsCache, 'get'); + getStub.withArgs('labs').returns({}); + assert.equal(labs.isSet('admin7PageChrome'), false); + + getStub.withArgs('labs').returns({ admin7PageChrome: true }); + assert.equal(labs.isSet('admin7PageChrome'), true); + + configUtils.set('labs', { admin7PageChrome: false }); + assert.equal(labs.isSet('admin7PageChrome'), false); + }); + it('can getAll, even if empty with enabled members', function () { assert.deepEqual( labs.getAll(), diff --git a/ghost/core/vitest.config.db.ts b/ghost/core/vitest.config.db.ts index 062586dbca4..2c0ad07aca3 100644 --- a/ghost/core/vitest.config.db.ts +++ b/ghost/core/vitest.config.db.ts @@ -1,4 +1,5 @@ import path from 'node:path'; +import { availableParallelism } from 'node:os'; import { defineConfig } from 'vitest/config'; // DB-backed suite runner (integration / e2e / legacy) — separate from the unit @@ -45,6 +46,15 @@ const sharedSsrConfig = { resolve: { conditions: ['source', 'node'] }, }; +/* + * Vitest defaults maxWorkers to availableParallelism() - 1, which is 1 on a + * 2-core runner — the whole DB suite then runs serially. Floor it at 2: a + * 4-core runner measured 2.08x (e2e) and 1.79x (integration) against a 2-core + * one purely on worker count. `legacy` runs on the threads pool, so the + * "forks 2 hangs" wedge below is not in play here. + */ +const getWorkerCount = () => Math.max(2, availableParallelism() - 1); + // Shared by every DB-backed project — the execution model is identical for all // of them; only the include globs and per-suite timeouts differ. const sharedDbConfig = { @@ -57,6 +67,7 @@ const sharedDbConfig = { // here except `legacy`, which sets pool: 'threads' (see its note below). pool: 'forks' as const, isolate: false, + maxWorkers: getWorkerCount(), sequence: { shuffle: { files: !!process.env.CI } }, setupFiles: ['./test/utils/vitest-setup-db.ts'], resolveSnapshotPath, diff --git a/packages/custom-field-types/src/index.ts b/packages/custom-field-types/src/index.ts index 54efe50c3fc..d8305ec8ad6 100644 --- a/packages/custom-field-types/src/index.ts +++ b/packages/custom-field-types/src/index.ts @@ -21,7 +21,7 @@ import { z } from 'zod'; * cleared. That is why every part accepts empty regardless of its own rule — emptying is * a statement about the write, not about the part. * - * A name nobody recognises is an error rather than a silent drop, at both depths. A + * A name nobody recognizes is an error rather than a silent drop, at both depths. A * misspelled field key is refused by the values service, which alone knows which fields a * site has defined; a misspelled part is refused here, because a type's parts are declared * in this file and nowhere else. Each is enforced where the names are known. @@ -60,6 +60,20 @@ export const FIELD_TYPE_IDS = ['short_text', 'long_text', 'address'] as const; export type FieldType = (typeof FIELD_TYPE_IDS)[number]; export const FieldTypeSchema = z.enum(FIELD_TYPE_IDS); +/** + * What kind of thing a type's value is, as anything comparing values needs to know. + * + * Coarser than the type: `short_text` and `long_text` are both text, and differ only in how + * much of it. This is the level at which a value can be ordered, matched or grouped, so it + * is what a filter, a sort or an export reads to decide how to treat a value — without + * either of them enumerating the types themselves. + * + * Deliberately not presentation: it says a value is a date, not that its operator is called + * "is before". Naming the operators stays with whoever renders them. + */ +export const FIELD_KINDS = ['text', 'date', 'number', 'record'] as const; +export type FieldKind = (typeof FIELD_KINDS)[number]; + /** * Bytes, not characters, because MySQL TEXT holds 65,535 of them: a character bound would * accept a multibyte value the column cannot hold, and 65,535 emoji is four times over. @@ -100,7 +114,7 @@ const postalCode = () => text().max(32, { error: 'Use 32 characters or fewer.' } * arbiter of it for every member of every site. The collection form can offer countries to * pick from without this deciding which ones exist. * - * Case is normalised so that `gb` and `GB` are not two values for one place, which a + * Case is normalized so that `gb` and `GB` are not two values for one place, which a * filter for either would silently half-miss. * * Checked as two ASCII letters on the way in rather than by length on the way out, because @@ -130,6 +144,8 @@ const clearable = (part: T) => .optional(); export interface FieldTypeDefinition { + /** What kind of value this is, for anything that has to compare one. */ + kind: FieldKind; value: z.ZodType; /** * A record type's parts, in declaration order. Each part's own rule, and nothing @@ -138,22 +154,13 @@ export interface FieldTypeDefinition { fields?: Record; } -type FieldTypeDeclaration = PartSchema | FieldTypeDefinition; - -type Defined = D extends z.ZodType ? { value: D } : D; - -/** A type that is simply a value is declared as one; a record announces itself. */ -function defineFieldTypes>( - declarations: D, -): { [K in keyof D]: Defined } { - // Restated for the type system, which cannot follow a conditional through - // `Object.fromEntries`. - return Object.fromEntries( - Object.entries(declarations).map(([type, declared]) => [ - type, - declared instanceof z.ZodType ? { value: declared } : declared, - ]), - ) as { [K in keyof D]: Defined }; +/** + * Every type states its kind alongside its schema, so no type can exist that nothing knows + * how to compare. The `Record` is what makes that exhaustive: an id added to + * `FIELD_TYPE_IDS` fails to compile until it is declared here. + */ +function defineFieldTypes>(declarations: D): D { + return declarations; } /** @@ -179,14 +186,14 @@ function record>(fields: F, { error }: { er .strictObject(shape) .refine((parts) => Object.values(parts).some((part) => typeof part === 'string'), { error }); - return { value, fields }; + return { kind: 'record' as const, value, fields }; } export const FIELD_TYPES = defineFieldTypes({ - short_text: shortText(), - long_text: longText(), + short_text: { kind: 'text', value: shortText() }, + long_text: { kind: 'text', value: longText() }, // An address is a delivery address, so its bounds are what a courier will accept - // rather than what the column could hold. Modelled on Stripe's Address object. + // rather than what the column could hold. Modeled on Stripe's Address object. // // Who the parcel is addressed to is not here. A parcel needs a name as well as an // address, but that is a fact about posting parcels rather than about either type, diff --git a/skills-lock.json b/skills-lock.json new file mode 100644 index 00000000000..5d0f92a17b0 --- /dev/null +++ b/skills-lock.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "skills": { + "tinybird": { + "source": "tinybirdco/tinybird-agent-skills", + "sourceType": "github", + "skillPath": "skills/tinybird-best-practices/SKILL.md", + "computedHash": "3f5d961a3953220b77a91137a019c82a857a43b658ee5a417f6dadcd3b6b5fac" + }, + "tinybird-cli-guidelines": { + "source": "tinybirdco/tinybird-agent-skills", + "sourceType": "github", + "skillPath": "skills/tinybird-cli-guidelines/SKILL.md", + "computedHash": "fdd265aa6a502220b17f5d608b87c30beca0eadb9c88c368f3f3b2cebba2ad8a" + } + } +}