From 8d38b94486c7aee9df6a3d36831f3bb899071e40 Mon Sep 17 00:00:00 2001 From: Radovan Jorgic Date: Thu, 15 Oct 2026 15:00:08 +0200 Subject: [PATCH 1/5] feat: add runnable Tasks API connector example --- .github/templates/README.md | 72 ------ .github/workflows/setup.yaml | 67 ------ README.md | 96 ++++++-- TEMPLATE_SOURCE.md | 6 + VALIDATION.md | 18 ++ code/.env.example | 12 +- code/babel.config.js | 1 - code/eslint.config.mts | 16 +- .../start_extracting_attachments/event.json | 16 ++ .../start_extracting_attachments/state.json | 5 + .../fixtures/start_extracting_data/event.json | 10 +- .../fixtures/start_extracting_data/state.json | 6 +- .../event.json | 17 ++ .../state.json | 5 + .../event.json | 10 +- .../extraction_scope.json | 6 +- .../state.json | 6 +- .../event.json | 13 +- .../state.json | 5 + .../start_extracting_metadata/event.json | 16 ++ .../start_extracting_metadata/state.json | 5 + code/fixtures/start_loading_data/event.json | 14 -- code/fixtures/start_loading_data/state.json | 1 - code/integration/connector.test.cjs | 132 +++++++++++ code/jest.config.js | 2 +- code/package-lock.json | 10 +- code/package.json | 16 +- code/scripts/configure-source.mjs | 11 + code/src/function-factory.ts | 6 +- .../external-system/data-denormalization.ts | 24 -- .../external-system/data-normalization.ts | 61 +----- .../external_domain_metadata.json | 51 +---- .../functions/external-system/http-client.ts | 206 +++++++----------- .../initial_domain_mapping.json | 129 +---------- .../external-system/source-config.json | 3 + code/src/functions/external-system/types.ts | 37 +--- .../__tests__/tasks-extraction.test.ts | 69 ++++++ code/src/functions/extraction/index.ts | 30 +-- .../functions/extraction/tasks-extraction.ts | 35 +++ .../workers/attachments-extraction.ts | 70 +----- .../extraction/workers/data-extraction.ts | 79 +------ .../workers/external-sync-units-extraction.ts | 39 +--- .../extraction/workers/metadata-extraction.ts | 24 +- code/src/functions/loading/index.ts | 31 --- .../loading/workers/load-attachments.ts | 31 --- .../functions/loading/workers/load-data.ts | 71 ------ code/src/test-runner/test-runner.ts | 2 - code/tsconfig.json | 14 +- manifest.yaml | 98 ++------- source-api/server.mjs | 96 ++++++++ source-api/server.test.mjs | 52 +++++ 51 files changed, 792 insertions(+), 1060 deletions(-) delete mode 100644 .github/templates/README.md delete mode 100644 .github/workflows/setup.yaml create mode 100644 TEMPLATE_SOURCE.md create mode 100644 VALIDATION.md create mode 100644 code/fixtures/start_extracting_attachments/event.json create mode 100644 code/fixtures/start_extracting_attachments/state.json create mode 100644 code/fixtures/start_extracting_data_incremental/event.json create mode 100644 code/fixtures/start_extracting_data_incremental/state.json create mode 100644 code/fixtures/start_extracting_external_sync_units/state.json create mode 100644 code/fixtures/start_extracting_metadata/event.json create mode 100644 code/fixtures/start_extracting_metadata/state.json delete mode 100644 code/fixtures/start_loading_data/event.json delete mode 100644 code/fixtures/start_loading_data/state.json create mode 100644 code/integration/connector.test.cjs create mode 100644 code/scripts/configure-source.mjs delete mode 100644 code/src/functions/external-system/data-denormalization.ts create mode 100644 code/src/functions/external-system/source-config.json create mode 100644 code/src/functions/extraction/__tests__/tasks-extraction.test.ts create mode 100644 code/src/functions/extraction/tasks-extraction.ts delete mode 100644 code/src/functions/loading/index.ts delete mode 100644 code/src/functions/loading/workers/load-attachments.ts delete mode 100644 code/src/functions/loading/workers/load-data.ts create mode 100644 source-api/server.mjs create mode 100644 source-api/server.test.mjs diff --git a/.github/templates/README.md b/.github/templates/README.md deleted file mode 100644 index f315156..0000000 --- a/.github/templates/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# AirSync Todo snap-in - -Syncs data between Todo and DevRev using DevRev's AirSync platform. - -## Prerequisites - -Install the following tools: - -- [Node.js 20 or newer](https://nodejs.org/en/download/) -- [DevRev CLI](https://developer.devrev.ai/snapin-development/references/cli-install) -- [jq](https://jqlang.github.io/jq/download/) -- [ngrok](https://ngrok.com/download) (for local development) - -## Setup - -1. Navigate to the `code` directory: - -```sh -cd code -``` - -2. Install dependencies: - -```sh -npm ci -``` - -3. (Optional) Create a `.env` file to set default values for the prompts: - -```sh -cp .env.example .env -``` - -Edit `.env` with your DevRev organization slug and email: - -```ini -DEV_ORG=my-org -USER_EMAIL=my@email.com -``` - -These values will be used as defaults when the scripts prompt you for credentials. - -The connector uses `@devrev/airsync-sdk`. Its local test server sets -`AIRSYNC_LOCAL_DEVELOPMENT=1` to enable local logging and artifact copies. - -## Deploy - -Run the deployment script: - -```sh -npm run deploy -``` - -The script will prompt you for: - -1. **Deployment mode**: Local (ngrok) or Lambda -2. **Organization**: Your DevRev org slug (defaults to `.env` value if set) -3. **Email**: Your DevRev email (defaults to `.env` value if set) - -## Cleanup - -Remove all snap-in packages and versions from your organization: - -```sh -npm run cleanup -``` - -The script will prompt you for organization and email credentials. - -## Start an Import - -After deploying, go to DevRev UI: `AirSync` > `Start AirSync` > `` diff --git a/.github/workflows/setup.yaml b/.github/workflows/setup.yaml deleted file mode 100644 index 56d0189..0000000 --- a/.github/workflows/setup.yaml +++ /dev/null @@ -1,67 +0,0 @@ -name: Set up the repository -on: - push: - branches: [main] -jobs: - cleanup: - name: Set up the repository - runs-on: ubuntu-latest - if: github.event.repository.name != 'airdrop-template' - permissions: - contents: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Replace templates - run: | - shopt -s dotglob - - export LC_CTYPE=C - export LANG=C - - # Project name is lowercase repository name (used in configs) - REPO_NAME="${GITHUB_REPOSITORY,,}" # Of the form owner/repo - REPO_NAME="${REPO_NAME##*/}" # Just the repo name - - # Extract EXTERNAL_SYSTEM from the project name - # Expected format: airdrop-{EXTERNAL_SYSTEM}-snap-in where {EXTERNAL_SYSTEM} can be multiple words with dashes - EXTERNAL_SYSTEM="TODO" - if [[ "$REPO_NAME" =~ ^airdrop-(.+)-snap-in$ ]]; then - EXTERNAL_SYSTEM="${BASH_REMATCH[1]}" - fi - - # Create pretty version of external system name with proper capitalization - PRETTY_EXTERNAL_SYSTEM="$(echo "$EXTERNAL_SYSTEM" | sed 's/-/ /g; s/\b\(.\)/\u\1/g')" - ESCAPED_PRETTY_EXTERNAL_SYSTEM="$(echo "$PRETTY_EXTERNAL_SYSTEM" | sed 's/ /\\ /g')" - - echo "Project name: '$REPO_NAME'" - echo "External system name: '$EXTERNAL_SYSTEM'" - echo "Pretty external system name: '$PRETTY_EXTERNAL_SYSTEM'" - - echo "Replacing TODO placeholders in manifest.yaml and README.md" - sed -i "s/Todo/$ESCAPED_PRETTY_EXTERNAL_SYSTEM/g" manifest.yaml - sed -i "s/Todo/$ESCAPED_PRETTY_EXTERNAL_SYSTEM/g" .github/templates/README.md - sed -i "s/airdrop-todo-snap-in/airdrop-$EXTERNAL_SYSTEM-snap-in/g" manifest.yaml - - echo "Moving template files to the root of the repository" - mv -f .github/templates/* . - rm -rf .github - cp .cursor/mcp.json .mcp.json - - # Create CLAUDE.md from initial-mapping.mdc, removing the header - sed '1,/^---$/d' initial-mapping.mdc | sed '1,/^$/d' > CLAUDE.md - - - name: Commit - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add . - git commit --amend --no-edit - - - name: Push - uses: ad-m/github-push-action@v0.8.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: main - force: true diff --git a/README.md b/README.md index d00ef3c..29d29ea 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,92 @@ -# AirSync snap-in template +# Tasks Demo AirSync connector -This repository provides a template to implement an AirSync snap-in that imports data from an external system into DevRev. +A one-way connector from a small Tasks API to DevRev issues, built from `devrev/airsync-template` and the supplied data extraction guide. See [TEMPLATE_SOURCE.md](TEMPLATE_SOURCE.md) for the exact template revision and source link. -## Usage +## Scope -1. Click "Use this template" > "Create a new repository" -2. Name your repository `airdrop--snap-in` (e.g., `airdrop-github-snap-in`) -3. Clone and follow the setup instructions in the generated README +- One project, `demo-project`, discovered as an external sync unit. +- Ten seeded tasks, imported as issues with their source title and timestamps. +- Three tasks per worker invocation, with persisted continuation state. +- Incremental extraction using the source update time and the platform time window. +- Selection, source validation, HTTP timeout, and rate-limit delay handling. +- Static metadata and initial issue mapping, plus a bearer-token connection. -## Documentation +The connector exposes extraction only. Attachment extraction completes without records because this source has no attachments. Reverse sync, users, comments, and deletions are outside this example. -See the [AirSync snap-in documentation](https://developer.devrev.ai/airsync) for development guidance. +## Run locally -The template uses `@devrev/airsync-sdk` and requires Node.js 20 or newer. +Requires Node.js 22 or newer and npm. From the repository root: -## Local Fixture Testing +```sh +cd code +npm ci +cp .env.example .env +npm run source +``` -From the `code/` directory, you can run the built-in fixture runner to test the -template locally: +Keep that terminal running. In a second terminal, from `code/`: -```bash +```sh npm start -- --fixturePath start_extracting_external_sync_units +npm start -- --fixturePath start_extracting_metadata npm start -- --fixturePath start_extracting_data npm start -- --fixturePath start_extracting_data_selective -npm start -- --fixturePath start_loading_data +npm start -- --fixturePath start_extracting_attachments +``` + +These use the template fixture runner and a local mock of DevRev platform endpoints. SDK output files are copied to `code/extracted_files/`. Each fixture invokes one worker; the data fixture extracts the first three tasks. The selective fixture excludes tasks and produces no task artifact. The integration test below drives all continuations automatically. + +The source API listens on `127.0.0.1:3100`. It stores records in memory and resets when restarted. `.env.example` contains a local demonstration token; use the same token on the source and its connection. + +## Change source data + +```sh +curl -X PATCH -H 'Authorization: Bearer local-demo-token' \ + -H 'Content-Type: application/json' \ + -d '{"title":"Edited example task"}' \ + http://127.0.0.1:3100/tasks/1 +``` + +After editing, the `start_extracting_data_incremental` fixture selects updates since February 1, 2026; the seeded task timestamps precede that date. The API's `GET /tasks` accepts `project_id`, `limit`, `cursor`, `updated_from`, and `updated_to`. Time boundaries are inclusive. Its offset pagination assumes the source is not edited during a paginated run. + +## Verify + +From `code/`: + +```sh +npm run build +npm run lint +npm test +npm run test:source +npm run test:integration +npm run package ``` -The runner reads `code/.env` and resolves `${TODO_API_KEY}` in fixture files, -so copy `code/.env.example` before running the fixtures. -The fixture runner and local test server set `AIRSYNC_LOCAL_DEVELOPMENT=1` -to enable local logging and artifact copies in `code/extracted_files/`. +The integration test starts both HTTP servers itself. It executes the actual SDK worker threads, installs the mapping against a simulated platform endpoint, saves and reloads state, receives callbacks, uploads and decompresses JSONL artifacts, and checks discovery, metadata, all ten tasks across four invocations, one incremental update, selection exclusion, and attachment completion. + +Unit checks cover rate-limit delay, upload failure without cursor advancement, malformed data, repeated cursors, and numeric/HTTP-date retry hints. These checks do not run DevRev's transformer or create real issues. + +## Configure for a live DevRev import + +1. Host `source-api/server.mjs` at an HTTPS origin reachable from DevRev. Set `EXAMPLE_API_TOKEN` to a new token and set `HOST=0.0.0.0` if your hosting platform needs it. `PORT` defaults to 3100. The included source is an in-memory demonstration service. +2. From `code/`, set the source origin before building: + + ```sh + npm run configure:source -- https://your-tasks-host.example + npm run package + ``` + + This writes the origin to the bundled `source-config.json`. `EXAMPLE_API_ORIGIN` overrides it for local runs. Deployed credentials come from the connection key, not local environment variables. +3. Validate the manifest against the intended authenticated DevRev profile: + + ```sh + devrev snap_in_version validate-manifest ../manifest.yaml + ``` + +4. The template deployment script is retained as `npm run deploy`. It prompts for your environment, organization, email, deployment mode, and package slug. Deployment has not been run here. +5. In DevRev, create the **Tasks API Connection** using the source bearer token, start **Tasks Demo**, and select **Demo Project**. Credentials are checked by the authenticated project-discovery request; the manifest does not define a separate token-verification endpoint. +6. Review the Tasks-to-issue mapping. The template's `applies_to_part_id` mapping needs an existing DevRev product selected for the issues; no organization-specific target ID is hardcoded. + +## Validation status + +Build, lint, five unit tests, one source API test, one complete SDK lifecycle test, four fixture runs, manifest YAML/wiring checks, and package creation passed locally. The DevRev CLI manifest check could not run because its selected profile is not authenticated. Live manifest acceptance, mapping acceptance, and real DevRev issue creation remain unverified. diff --git a/TEMPLATE_SOURCE.md b/TEMPLATE_SOURCE.md new file mode 100644 index 0000000..0093371 --- /dev/null +++ b/TEMPLATE_SOURCE.md @@ -0,0 +1,6 @@ +Template: https://github.com/devrev/airsync-template +Commit: 050349c2e3308507cfcb63cf974883565c72a63c + +Extraction guide: https://devrev-marketing-developer-git-docs-iss-383917-a-68bdfc-dev-rev.vercel.app/airsync/data-extraction-example + +The connector uses the template runtime, manifest structure, metadata/mapping format, fixture runner, and SDK dependency. The extraction loop follows the supplied guide. No other connector documentation was consulted. diff --git a/VALIDATION.md b/VALIDATION.md new file mode 100644 index 0000000..9399cd0 --- /dev/null +++ b/VALIDATION.md @@ -0,0 +1,18 @@ +# Local validation + +Date: October 15, 2026 + +| Check | Result | +| --- | --- | +| TypeScript build | Passed | +| ESLint | Passed | +| Worker/client unit tests | 5 passed | +| Source API integration test | 1 passed | +| Actual SDK worker lifecycle test | 1 passed | +| Template fixture runner | Discovery, metadata, initial data page, and selection passed | +| Manifest YAML and local function wiring | Passed | +| Code package creation | Passed | +| Live manifest validation | Not run: selected CLI profile is not authenticated | +| Deployment and DevRev issue creation | Not run | + +The SDK lifecycle test uses the real `@devrev/airsync-sdk` 2.0.0 worker threads and a local HTTP platform simulator. It verifies project discovery, metadata artifacts, ten normalized tasks across four invocations, persisted cursor state, a new incremental run importing one edited task, selection exclusion, and empty attachment completion. It decompresses uploaded artifact bytes to check their records. It does not validate DevRev transformer behavior or live mapping semantics. diff --git a/code/.env.example b/code/.env.example index 4c16bae..10de4f4 100644 --- a/code/.env.example +++ b/code/.env.example @@ -1,11 +1,5 @@ -# Your DevRev organization slug (the part after app.devrev.ai/ in your DevRev URL) +EXAMPLE_API_ORIGIN=http://127.0.0.1:3100 +EXAMPLE_API_TOKEN=local-demo-token DEV_ORG=my-org - -# Your email address USER_EMAIL=my@email.com - -# Environment -ENV=dev|qa|prod - -# TODO: add External system specific variables that will be replaced in the Fixture tests that you want to run. -TODO_API_KEY=test-api-key +ENV=dev diff --git a/code/babel.config.js b/code/babel.config.js index 8b64f8d..8165fe4 100644 --- a/code/babel.config.js +++ b/code/babel.config.js @@ -1,4 +1,3 @@ -// eslint-disable-next-line no-undef module.exports = { presets: [ ['@babel/preset-env', { targets: { node: 'current' } }], diff --git a/code/eslint.config.mts b/code/eslint.config.mts index 15f44a7..e9aa08f 100644 --- a/code/eslint.config.mts +++ b/code/eslint.config.mts @@ -1,12 +1,14 @@ -import js from "@eslint/js"; -import globals from "globals"; -import tseslint from "typescript-eslint"; -import { defineConfig } from "eslint/config"; -import prettierConfig from "eslint-config-prettier"; +import js from '@eslint/js'; +import globals from 'globals'; +import tseslint from 'typescript-eslint'; +import { defineConfig } from 'eslint/config'; +import prettierConfig from 'eslint-config-prettier'; export default defineConfig([ - { ignores: ["dist/**", "test/**"] }, - { files: ["**/*.{js,mjs,cjs,ts,mts,cts}"], plugins: { js }, extends: ["js/recommended"], languageOptions: { globals: globals.browser } }, + { ignores: ['dist/**', 'test/**', 'extracted_files/**'] }, + { files: ['**/*.{js,mjs,cjs,ts,mts,cts}'], languageOptions: { globals: globals.node } }, + js.configs.recommended, ...tseslint.configs.recommended, + { files: ['integration/**/*.cjs'], rules: { '@typescript-eslint/no-require-imports': 'off' } }, prettierConfig, ]); diff --git a/code/fixtures/start_extracting_attachments/event.json b/code/fixtures/start_extracting_attachments/event.json new file mode 100644 index 0000000..7ea05c8 --- /dev/null +++ b/code/fixtures/start_extracting_attachments/event.json @@ -0,0 +1,16 @@ +{ + "payload": { + "connection_data": { + "org_id": "tasks-demo", + "org_name": "Tasks Demo", + "key": "${EXAMPLE_API_TOKEN}", + "key_type": "pat" + }, + "event_type": "START_EXTRACTING_ATTACHMENTS", + "event_context": { + "external_sync_unit_id": "demo-project", + "external_sync_unit_name": "Demo Project", + "run_id": "demo-initial-run" + } + } +} diff --git a/code/fixtures/start_extracting_attachments/state.json b/code/fixtures/start_extracting_attachments/state.json new file mode 100644 index 0000000..e41718b --- /dev/null +++ b/code/fixtures/start_extracting_attachments/state.json @@ -0,0 +1,5 @@ +{ + "runId": "", + "cursor": null, + "completed": false +} diff --git a/code/fixtures/start_extracting_data/event.json b/code/fixtures/start_extracting_data/event.json index d076c25..a11132f 100644 --- a/code/fixtures/start_extracting_data/event.json +++ b/code/fixtures/start_extracting_data/event.json @@ -1,14 +1,16 @@ { "payload": { "connection_data": { - "org_id": "test_org", - "org_name": "Test Organization", - "key": "${TODO_API_KEY}", + "org_id": "tasks-demo", + "org_name": "Tasks Demo", + "key": "${EXAMPLE_API_TOKEN}", "key_type": "pat" }, "event_type": "START_EXTRACTING_DATA", "event_context": { - "external_sync_unit_id": "test_external_sync_unit_id" + "external_sync_unit_id": "demo-project", + "external_sync_unit_name": "Demo Project", + "run_id": "demo-initial-run" } } } diff --git a/code/fixtures/start_extracting_data/state.json b/code/fixtures/start_extracting_data/state.json index 3d5b4c0..e41718b 100644 --- a/code/fixtures/start_extracting_data/state.json +++ b/code/fixtures/start_extracting_data/state.json @@ -1,5 +1,5 @@ { - "todos": { "completed": false }, - "users": { "completed": false }, - "attachments": { "completed": false } + "runId": "", + "cursor": null, + "completed": false } diff --git a/code/fixtures/start_extracting_data_incremental/event.json b/code/fixtures/start_extracting_data_incremental/event.json new file mode 100644 index 0000000..18650b9 --- /dev/null +++ b/code/fixtures/start_extracting_data_incremental/event.json @@ -0,0 +1,17 @@ +{ + "payload": { + "connection_data": { + "org_id": "tasks-demo", + "org_name": "Tasks Demo", + "key": "${EXAMPLE_API_TOKEN}", + "key_type": "pat" + }, + "event_type": "START_EXTRACTING_DATA", + "event_context": { + "external_sync_unit_id": "demo-project", + "external_sync_unit_name": "Demo Project", + "run_id": "demo-incremental-run", + "extract_from": "2026-02-01T00:00:00.000Z" + } + } +} diff --git a/code/fixtures/start_extracting_data_incremental/state.json b/code/fixtures/start_extracting_data_incremental/state.json new file mode 100644 index 0000000..e41718b --- /dev/null +++ b/code/fixtures/start_extracting_data_incremental/state.json @@ -0,0 +1,5 @@ +{ + "runId": "", + "cursor": null, + "completed": false +} diff --git a/code/fixtures/start_extracting_data_selective/event.json b/code/fixtures/start_extracting_data_selective/event.json index d076c25..a11132f 100644 --- a/code/fixtures/start_extracting_data_selective/event.json +++ b/code/fixtures/start_extracting_data_selective/event.json @@ -1,14 +1,16 @@ { "payload": { "connection_data": { - "org_id": "test_org", - "org_name": "Test Organization", - "key": "${TODO_API_KEY}", + "org_id": "tasks-demo", + "org_name": "Tasks Demo", + "key": "${EXAMPLE_API_TOKEN}", "key_type": "pat" }, "event_type": "START_EXTRACTING_DATA", "event_context": { - "external_sync_unit_id": "test_external_sync_unit_id" + "external_sync_unit_id": "demo-project", + "external_sync_unit_name": "Demo Project", + "run_id": "demo-initial-run" } } } diff --git a/code/fixtures/start_extracting_data_selective/extraction_scope.json b/code/fixtures/start_extracting_data_selective/extraction_scope.json index f4d21c6..3134b85 100644 --- a/code/fixtures/start_extracting_data_selective/extraction_scope.json +++ b/code/fixtures/start_extracting_data_selective/extraction_scope.json @@ -1,5 +1,5 @@ { - "todos": { "extract": true }, - "users": { "extract": false }, - "attachments": { "extract": false } + "tasks": { + "extract": false + } } diff --git a/code/fixtures/start_extracting_data_selective/state.json b/code/fixtures/start_extracting_data_selective/state.json index 3d5b4c0..e41718b 100644 --- a/code/fixtures/start_extracting_data_selective/state.json +++ b/code/fixtures/start_extracting_data_selective/state.json @@ -1,5 +1,5 @@ { - "todos": { "completed": false }, - "users": { "completed": false }, - "attachments": { "completed": false } + "runId": "", + "cursor": null, + "completed": false } diff --git a/code/fixtures/start_extracting_external_sync_units/event.json b/code/fixtures/start_extracting_external_sync_units/event.json index f8adaa9..dcfeb73 100644 --- a/code/fixtures/start_extracting_external_sync_units/event.json +++ b/code/fixtures/start_extracting_external_sync_units/event.json @@ -1,11 +1,16 @@ { "payload": { - "event_type": "START_EXTRACTING_EXTERNAL_SYNC_UNITS", "connection_data": { - "org_id": "test_org", - "org_name": "Test Organization", - "key": "${TODO_API_KEY}", + "org_id": "tasks-demo", + "org_name": "Tasks Demo", + "key": "${EXAMPLE_API_TOKEN}", "key_type": "pat" + }, + "event_type": "START_EXTRACTING_EXTERNAL_SYNC_UNITS", + "event_context": { + "external_sync_unit_id": "demo-project", + "external_sync_unit_name": "Demo Project", + "run_id": "demo-initial-run" } } } diff --git a/code/fixtures/start_extracting_external_sync_units/state.json b/code/fixtures/start_extracting_external_sync_units/state.json new file mode 100644 index 0000000..e41718b --- /dev/null +++ b/code/fixtures/start_extracting_external_sync_units/state.json @@ -0,0 +1,5 @@ +{ + "runId": "", + "cursor": null, + "completed": false +} diff --git a/code/fixtures/start_extracting_metadata/event.json b/code/fixtures/start_extracting_metadata/event.json new file mode 100644 index 0000000..9380d32 --- /dev/null +++ b/code/fixtures/start_extracting_metadata/event.json @@ -0,0 +1,16 @@ +{ + "payload": { + "connection_data": { + "org_id": "tasks-demo", + "org_name": "Tasks Demo", + "key": "${EXAMPLE_API_TOKEN}", + "key_type": "pat" + }, + "event_type": "START_EXTRACTING_METADATA", + "event_context": { + "external_sync_unit_id": "demo-project", + "external_sync_unit_name": "Demo Project", + "run_id": "demo-initial-run" + } + } +} diff --git a/code/fixtures/start_extracting_metadata/state.json b/code/fixtures/start_extracting_metadata/state.json new file mode 100644 index 0000000..e41718b --- /dev/null +++ b/code/fixtures/start_extracting_metadata/state.json @@ -0,0 +1,5 @@ +{ + "runId": "", + "cursor": null, + "completed": false +} diff --git a/code/fixtures/start_loading_data/event.json b/code/fixtures/start_loading_data/event.json deleted file mode 100644 index 2397ca5..0000000 --- a/code/fixtures/start_loading_data/event.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "payload": { - "connection_data": { - "org_id": "test_org", - "org_name": "Test Organization", - "key": "${TODO_API_KEY}", - "key_type": "pat" - }, - "event_type": "START_LOADING_DATA", - "event_context": { - "external_sync_unit_id": "test_external_sync_unit_id" - } - } -} diff --git a/code/fixtures/start_loading_data/state.json b/code/fixtures/start_loading_data/state.json deleted file mode 100644 index 0967ef4..0000000 --- a/code/fixtures/start_loading_data/state.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/code/integration/connector.test.cjs b/code/integration/connector.test.cjs new file mode 100644 index 0000000..98a560a --- /dev/null +++ b/code/integration/connector.test.cjs @@ -0,0 +1,132 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { createServer } = require('node:http'); +const { once } = require('node:events'); +const { gunzipSync } = require('node:zlib'); +const { createMockEvent, EventType } = require('@devrev/airsync-sdk'); +const { functionFactory } = require('../dist/function-factory'); + +async function startPlatform() { + const callbacks = []; + const artifacts = []; + let state = {}; + let scope = {}; + let origin; + const server = createServer(async (req, res) => { + try { + const url = new URL(req.url, origin); + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const raw = Buffer.concat(chunks); + const send = body => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }; + if (req.method === 'GET' && url.pathname === '/worker_data_url.get') { + send({ state: JSON.stringify(state), objects: JSON.stringify(scope) }); + } else if (req.method === 'GET' && url.pathname === '/internal/snap-ins.get') { + send({ snap_in: { imports: [{ name: 'test_import_slug' }], snap_in_version: { slug: 'test_snap_in_slug' } } }); + } else if (req.method === 'POST' && url.pathname === '/worker_data_url.update') { + state = JSON.parse(JSON.parse(raw.toString()).state); + send({ success: true }); + } else if (req.method === 'GET' && url.pathname === '/internal/airdrop.artifacts.upload-url') { + send({ upload_url: `${origin}/upload?name=${encodeURIComponent(url.searchParams.get('file_name'))}`, + artifact_id: `artifact-${artifacts.length}`, form_data: {} }); + } else if (req.method === 'POST' && url.pathname === '/upload') { + const boundary = req.headers['content-type'].split('boundary=')[1]; + const start = raw.indexOf('\r\n\r\n') + 4; + const end = raw.lastIndexOf(`\r\n--${boundary}`); + const lines = gunzipSync(raw.subarray(start, end)).toString().trim(); + artifacts.push({ name: url.searchParams.get('name'), items: lines ? lines.split('\n').map(JSON.parse) : [] }); + send({ success: true }); + } else if (req.method === 'POST' && url.pathname === '/callback_url') { + callbacks.push(JSON.parse(raw.toString())); + send({ success: true }); + } else { + send({ success: true }); + } + } catch (error) { + res.writeHead(500); + res.end(JSON.stringify({ error: error.message })); + } + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + origin = `http://127.0.0.1:${server.address().port}`; + return { server, origin, callbacks, artifacts, + reset: () => { state = {}; scope = {}; }, + setScope: value => { scope = value; }, + get state() { return state; }, + }; +} + +test('complete import lifecycle using actual SDK workers and HTTP artifacts', { timeout: 60000 }, async t => { + const { createTasksServer } = await import('../../source-api/server.mjs'); + const source = createTasksServer({ token: 'integration-token', clock: () => new Date('2026-02-01T12:00:00Z') }); + source.listen(0, '127.0.0.1'); + await once(source, 'listening'); + t.after(() => new Promise(resolve => source.close(resolve))); + const sourceOrigin = `http://127.0.0.1:${source.address().port}`; + const platform = await startPlatform(); + t.after(() => new Promise(resolve => platform.server.close(resolve))); + const previousOrigin = process.env.EXAMPLE_API_ORIGIN; + process.env.EXAMPLE_API_ORIGIN = sourceOrigin; + t.after(() => { + if (previousOrigin === undefined) delete process.env.EXAMPLE_API_ORIGIN; + else process.env.EXAMPLE_API_ORIGIN = previousOrigin; + }); + const invoke = async (eventType, context = {}) => { + const before = platform.callbacks.length; + await functionFactory.extraction([createMockEvent(platform.origin, { + payload: { + event_type: eventType, + connection_data: { key: 'integration-token' }, + event_context: { external_sync_unit_id: 'demo-project', run_id: 'run-1', ...context }, + }, + })]); + assert.equal(platform.callbacks.length, before + 1, 'one callback per invocation'); + return platform.callbacks.at(-1); + }; + + const discovery = await invoke(EventType.StartExtractingExternalSyncUnits); + assert.equal(discovery.event_type, 'EXTERNAL_SYNC_UNIT_EXTRACTION_DONE'); + assert.deepEqual(platform.artifacts.at(-1).items, [{ + id: 'demo-project', name: 'Demo Project', description: 'Ten example tasks', item_count: 10, item_type: 'tasks', + }]); + + assert.equal((await invoke(EventType.StartExtractingMetadata)).event_type, 'METADATA_EXTRACTION_DONE'); + const metadata = platform.artifacts.at(-1).items[0]; + assert.deepEqual(Object.keys(metadata.record_types), ['tasks']); + assert.equal(metadata.record_types.tasks.is_loadable, false); + + const statuses = []; + for (let index = 0; index < 4; index++) { + const callback = await invoke(index === 0 ? EventType.StartExtractingData : EventType.ContinueExtractingData); + statuses.push(callback.event_type); + } + assert.deepEqual(statuses, ['DATA_EXTRACTION_PROGRESS', 'DATA_EXTRACTION_PROGRESS', 'DATA_EXTRACTION_PROGRESS', 'DATA_EXTRACTION_DONE']); + const tasks = platform.artifacts.filter(file => file.name === 'tasks.jsonl.gz').flatMap(file => file.items); + assert.equal(tasks.length, 10); + assert.equal(new Set(tasks.map(task => task.id)).size, 10); + assert.equal(tasks[0].data.title, 'Example task 1'); + assert.equal(platform.state.connectorState.completed, true); + + await fetch(`${sourceOrigin}/tasks/1`, { + method: 'PATCH', headers: { authorization: 'Bearer integration-token' }, + body: JSON.stringify({ title: 'Edited for incremental sync' }), + }); + const beforeIncremental = platform.artifacts.length; + const incremental = await invoke(EventType.StartExtractingData, { + run_id: 'run-2', extract_from: '2026-02-01T00:00:00Z', extract_to: '2026-02-02T00:00:00Z', + }); + assert.equal(incremental.event_type, 'DATA_EXTRACTION_DONE'); + const changed = platform.artifacts.slice(beforeIncremental).flatMap(file => file.items); + assert.deepEqual(changed.map(task => task.id), ['1']); + assert.equal(changed[0].data.title, 'Edited for incremental sync'); + + platform.setScope({ tasks: { extract: false } }); + const beforeExcluded = platform.artifacts.length; + assert.equal((await invoke(EventType.StartExtractingData, { run_id: 'run-3' })).event_type, 'DATA_EXTRACTION_DONE'); + assert.equal(platform.artifacts.length, beforeExcluded); + assert.equal((await invoke(EventType.StartExtractingAttachments)).event_type, 'ATTACHMENT_EXTRACTION_DONE'); +}); diff --git a/code/jest.config.js b/code/jest.config.js index 57b110f..0b889af 100644 --- a/code/jest.config.js +++ b/code/jest.config.js @@ -1,5 +1,5 @@ -// eslint-disable-next-line no-undef module.exports = { preset: 'ts-jest', testEnvironment: 'node', + testMatch: ['**/src/**/*.test.ts'], }; diff --git a/code/package-lock.json b/code/package-lock.json index b0fdb47..5f0f253 100644 --- a/code/package-lock.json +++ b/code/package-lock.json @@ -1,12 +1,12 @@ { - "name": "adaas-connector", - "version": "1.1.6", + "name": "airsync-tasks-connector", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "adaas-connector", - "version": "1.1.6", + "name": "airsync-tasks-connector", + "version": "0.1.0", "license": "ISC", "dependencies": { "@devrev/airsync-sdk": "2.0.0", @@ -53,7 +53,7 @@ "yargs": "^17.6.2" }, "engines": { - "node": ">=20" + "node": ">=22" } }, "node_modules/@babel/code-frame": { diff --git a/code/package.json b/code/package.json index 451f86a..6c97729 100644 --- a/code/package.json +++ b/code/package.json @@ -1,7 +1,7 @@ { - "name": "adaas-connector", - "version": "1.1.6", - "description": "DevRev ADaaS (Airdrop-as-a-service) Connector", + "name": "airsync-tasks-connector", + "version": "0.1.0", + "description": "One-way Tasks API import into DevRev issues", "main": "./dist/index.js", "scripts": { "deploy": "./scripts/deploy.sh", @@ -16,8 +16,12 @@ "start:watch": "nodemon ./src/main.ts", "start:production": "node dist/main.js", "test:server": "nodemon --watch src --watch test test/main.ts", - "test": "jest", - "test:watch": "jest --watch" + "test": "jest --runInBand", + "test:watch": "jest --watch", + "source": "node --env-file-if-exists=.env ../source-api/server.mjs", + "test:source": "node --test ../source-api/server.test.mjs", + "test:integration": "npm run build && node --test integration/*.test.cjs", + "configure:source": "node scripts/configure-source.mjs" }, "keywords": [], "author": "", @@ -67,6 +71,6 @@ "yargs": "^17.6.2" }, "engines": { - "node": ">=20" + "node": ">=22" } } diff --git a/code/scripts/configure-source.mjs b/code/scripts/configure-source.mjs new file mode 100644 index 0000000..3ddd069 --- /dev/null +++ b/code/scripts/configure-source.mjs @@ -0,0 +1,11 @@ +import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const origin = new URL(process.argv[2]); +if (origin.protocol !== 'https:' || origin.username || origin.password || + origin.pathname !== '/' || origin.search || origin.hash) { + throw new Error('Provide an HTTPS origin, for example https://tasks.example.com'); +} +const target = fileURLToPath(new URL('../src/functions/external-system/source-config.json', import.meta.url)); +fs.writeFileSync(target, JSON.stringify({ origin: origin.origin }, null, 2) + '\n'); +console.log(`Connector source set to ${origin.origin}. Rebuild before packaging.`); diff --git a/code/src/function-factory.ts b/code/src/function-factory.ts index 2a09b7f..2b481fa 100644 --- a/code/src/function-factory.ts +++ b/code/src/function-factory.ts @@ -1,9 +1,5 @@ import extraction from './functions/extraction'; -import loading from './functions/loading'; -export const functionFactory = { - extraction, - loading, -} as const; +export const functionFactory = { extraction } as const; export type FunctionFactoryType = keyof typeof functionFactory; diff --git a/code/src/functions/external-system/data-denormalization.ts b/code/src/functions/external-system/data-denormalization.ts deleted file mode 100644 index 7949f8d..0000000 --- a/code/src/functions/external-system/data-denormalization.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { ExternalSystemAttachment, ExternalSystemItem } from '@devrev/airsync-sdk'; -import { ExternalAttachment, ExternalTodo } from './types'; - -export function denormalizeTodo(item: ExternalSystemItem): ExternalTodo { - return { - id: item.id.devrev, - body: item.data.body, - creator: item.data.creator, - owner: item.data.owner, - title: item.data.title, - created_date: item.created_date, - modified_date: item.modified_date, - }; -} - -export function denormalizeAttachment(item: ExternalSystemAttachment): ExternalAttachment { - return { - id: item.reference_id, - url: item.url, - file_name: item.file_name, - author_id: item.created_by_id, - parent_id: item.parent_id ?? item.parent_reference_id, - }; -} diff --git a/code/src/functions/external-system/data-normalization.ts b/code/src/functions/external-system/data-normalization.ts index 06a8281..9df7adb 100644 --- a/code/src/functions/external-system/data-normalization.ts +++ b/code/src/functions/external-system/data-normalization.ts @@ -1,57 +1,18 @@ -import { ExternalSyncUnit, NormalizedAttachment, NormalizedItem } from '@devrev/airsync-sdk'; -import { ExternalAttachment, ExternalTodo, ExternalTodoList, ExternalUser } from './types'; +import { ExternalSyncUnit, NormalizedItem } from '@devrev/airsync-sdk'; -// TODO: Replace with your actual normalization functions that will be used to -// normalize the data received from the external system. You can modify the -// normalization functions to suit your needs. For example, you might want to -// include additional fields or change the structure of the normalized item. -export function normalizeTodoList(item: ExternalTodoList): ExternalSyncUnit { - return { - id: item.id, - name: item.name, - description: item.description, - item_count: item.item_count, - item_type: item.item_type, - }; -} - -export function normalizeTodo(item: ExternalTodo): NormalizedItem { - // createItemUrl function returns the url that points to this item in the external system. - // TODO: Adjust this function to your external system. - const createItemUrl = (id: string) => `https://external-system.com/todos/${id}`; - - return { - id: item.id, - created_date: item.created_date, - modified_date: item.modified_date, - data: { - body: item.body, - creator: item.creator, - owner: item.owner, - title: item.title, - item_url_field: createItemUrl(item.id), // Url that points to the item in the external system. - }, - }; -} +import { validateTask } from './http-client'; +import { SourceProject } from './types'; -export function normalizeUser(item: ExternalUser): NormalizedItem { - return { - id: item.id, - created_date: item.created_date, - modified_date: item.modified_date, - data: { - email: item.email, - name: item.name, - }, - }; +export function normalizeProject(project: SourceProject): ExternalSyncUnit { + return { ...project, item_type: 'tasks' }; } -export function normalizeAttachment(item: ExternalAttachment): NormalizedAttachment { +export function normalizeTask(record: object): NormalizedItem { + validateTask(record); return { - url: item.url, - id: item.id, - file_name: item.file_name, - author_id: item.author_id, - parent_id: item.parent_id, + id: record.id, + created_date: new Date(record.created_at).toISOString(), + modified_date: new Date(record.updated_at).toISOString(), + data: { title: record.title }, }; } diff --git a/code/src/functions/external-system/external_domain_metadata.json b/code/src/functions/external-system/external_domain_metadata.json index 822b2c6..b5eb2ad 100644 --- a/code/src/functions/external-system/external_domain_metadata.json +++ b/code/src/functions/external-system/external_domain_metadata.json @@ -1,33 +1,10 @@ { "schema_version": "v0.2.0", "record_types": { - "todos": { - "name": "Todos", - "is_loadable": true, + "tasks": { + "name": "Tasks", + "is_loadable": false, "fields": { - "body": { - "name": "body", - "is_required": true, - "type": "rich_text" - }, - "creator": { - "is_required": true, - "type": "reference", - "reference": { - "refers_to": { - "#record:users": {} - } - } - }, - "owner": { - "is_required": true, - "type": "reference", - "reference": { - "refers_to": { - "#record:users": {} - } - } - }, "title": { "name": "Title", "is_required": true, @@ -35,28 +12,6 @@ "text": { "min_length": 1 } - }, - "item_url_field": { - "name": "Link to item", - "type": "text" - } - } - }, - "users": { - "name": "Users", - "fields": { - "email": { - "name": "Email", - "is_required": true, - "type": "text" - }, - "name": { - "name": "Name", - "is_required": true, - "type": "text", - "text": { - "min_length": 1 - } } } } diff --git a/code/src/functions/external-system/http-client.ts b/code/src/functions/external-system/http-client.ts index bbcd937..7cc73c1 100644 --- a/code/src/functions/external-system/http-client.ts +++ b/code/src/functions/external-system/http-client.ts @@ -1,145 +1,93 @@ -import { AirSyncEvent, ExternalSystemItemLoadingResponse } from '@devrev/airsync-sdk'; -import { ExternalAttachment, ExternalTodo, ExternalTodoList, ExternalUser } from './types'; +import { AirSyncEvent } from '@devrev/airsync-sdk'; -// --------------------------------------------------------------------------- -// In-memory data store -// -// This simulates an external system's database. In a real connector you would -// replace the methods below with actual HTTP calls to your external API. -// --------------------------------------------------------------------------- +import sourceConfig from './source-config.json'; +import { SourceProject, SourceTask, TaskPage } from './types'; -const USERS: ExternalUser[] = [ - { id: 'user-1', created_date: '2018-03-10T08:00:00Z', modified_date: '2018-03-10T08:00:00Z', email: 'alice@example.com', name: 'Alice Johnson' }, - { id: 'user-2', created_date: '2019-07-22T14:30:00Z', modified_date: '2019-07-22T14:30:00Z', email: 'bob@example.com', name: 'Bob Smith' }, - { id: 'user-3', created_date: '2020-11-05T09:15:00Z', modified_date: '2021-02-18T11:00:00Z', email: 'carol@example.com', name: 'Carol White' }, - { id: 'user-4', created_date: '2021-04-12T16:45:00Z', modified_date: '2022-08-30T10:20:00Z', email: 'dave@example.com', name: 'Dave Brown' }, - { id: 'user-5', created_date: '2022-09-01T07:00:00Z', modified_date: '2023-01-15T13:00:00Z', email: 'eve@example.com', name: 'Eve Davis' }, - { id: 'user-6', created_date: '2023-03-20T12:00:00Z', modified_date: '2023-06-10T09:30:00Z', email: 'frank@example.com', name: 'Frank Miller' }, - { id: 'user-7', created_date: '2023-10-01T08:00:00Z', modified_date: '2024-01-05T14:00:00Z', email: 'grace@example.com', name: 'Grace Lee' }, - { id: 'user-8', created_date: '2024-02-14T10:00:00Z', modified_date: '2024-05-20T16:00:00Z', email: 'henry@example.com', name: 'Henry Zhang' }, - { id: 'user-9', created_date: '2024-08-01T09:00:00Z', modified_date: '2025-01-10T11:00:00Z', email: 'irene@example.com', name: 'Irene Park' }, - { id: 'user-10', created_date: '2025-03-15T08:30:00Z', modified_date: '2026-01-20T10:00:00Z', email: 'jack@example.com', name: 'Jack Wilson' }, -]; - -const TODO_LISTS: ExternalTodoList[] = [ - { id: 'list-1', name: 'Work Tasks', description: 'Tasks related to work', item_count: 5, item_type: 'todos' }, - { id: 'list-2', name: 'Personal Tasks', description: 'Personal errands and goals', item_count: 5, item_type: 'todos' }, - { id: 'list-3', name: 'Project Tasks', description: 'Tasks for the current project', item_count: 5, item_type: 'todos' }, -]; - -const TODOS: ExternalTodo[] = [ - { id: 'todo-1', created_date: '2018-06-01T08:00:00Z', modified_date: '2018-06-01T08:00:00Z', title: 'Set up development environment', body: '

Install Node.js, configure editor, clone repos.

', creator: 'user-1', owner: 'user-1' }, - { id: 'todo-2', created_date: '2019-09-15T10:00:00Z', modified_date: '2019-09-15T10:00:00Z', title: 'Write project proposal', body: '

Draft the initial project proposal document.

', creator: 'user-2', owner: 'user-2' }, - { id: 'todo-3', created_date: '2020-02-20T09:00:00Z', modified_date: '2020-02-20T09:00:00Z', title: 'Design database schema', body: '

Define entities, relationships and indexes.

', creator: 'user-1', owner: 'user-3' }, - { id: 'todo-4', created_date: '2021-05-10T14:00:00Z', modified_date: '2021-05-10T14:00:00Z', title: 'Implement authentication', body: '

Add JWT-based authentication to the API.

', creator: 'user-3', owner: 'user-4' }, - { id: 'todo-5', created_date: '2022-01-18T11:30:00Z', modified_date: '2022-03-05T16:00:00Z', title: 'Add unit tests', body: '

Achieve 80% test coverage for the core module.

', creator: 'user-2', owner: 'user-5' }, - { id: 'todo-6', created_date: '2022-08-22T08:00:00Z', modified_date: '2023-02-14T09:00:00Z', title: 'Performance profiling', body: '

Profile API endpoints and fix bottlenecks.

', creator: 'user-4', owner: 'user-4' }, - { id: 'todo-7', created_date: '2023-04-05T10:00:00Z', modified_date: '2023-07-01T13:00:00Z', title: 'Integrate third-party payment provider', body: '

Connect Stripe for billing.

', creator: 'user-5', owner: 'user-6' }, - { id: 'todo-8', created_date: '2023-11-01T08:30:00Z', modified_date: '2024-02-10T10:00:00Z', title: 'Migrate to TypeScript', body: '

Convert remaining JS files to TypeScript.

', creator: 'user-6', owner: 'user-7' }, - { id: 'todo-9', created_date: '2024-03-12T09:00:00Z', modified_date: '2024-06-20T15:00:00Z', title: 'Implement dark mode', body: '

Add dark mode toggle and persist preference.

', creator: 'user-7', owner: 'user-8' }, - { id: 'todo-10', created_date: '2024-07-01T11:00:00Z', modified_date: '2024-09-30T12:00:00Z', title: 'Set up CI/CD pipeline', body: '

Configure GitHub Actions for build and deploy.

', creator: 'user-8', owner: 'user-9' }, - { id: 'todo-11', created_date: '2024-10-15T08:00:00Z', modified_date: '2025-01-08T09:30:00Z', title: 'Refactor API error handling', body: '

Standardise error responses across all endpoints.

', creator: 'user-9', owner: 'user-10' }, - { id: 'todo-12', created_date: '2025-02-01T10:00:00Z', modified_date: '2025-03-15T11:00:00Z', title: 'Add search functionality', body: '

Implement full-text search with filters.

', creator: 'user-10', owner: 'user-1' }, - { id: 'todo-13', created_date: '2025-05-20T09:00:00Z', modified_date: '2025-07-10T14:00:00Z', title: 'Optimise database queries', body: '

Add missing indexes and rewrite slow queries.

', creator: 'user-1', owner: 'user-2' }, - { id: 'todo-14', created_date: '2025-09-01T08:00:00Z', modified_date: '2025-11-20T16:30:00Z', title: 'Write API documentation', body: '

Document all public endpoints using OpenAPI spec.

', creator: 'user-2', owner: 'user-3' }, - { id: 'todo-15', created_date: '2026-01-10T09:00:00Z', modified_date: '2026-03-01T10:00:00Z', title: 'Launch v2.0', body: '

Final QA pass and production deployment.

', creator: 'user-3', owner: 'user-4' }, -]; - -const ATTACHMENTS: ExternalAttachment[] = [ - { id: 'att-1', url: 'https://www.devrev.ai/favicon.ico', file_name: 'devrev-icon.ico', author_id: 'user-1', parent_id: 'todo-1' }, - { id: 'att-2', url: 'https://www.devrev.ai/favicon.ico', file_name: 'devrev-icon.ico', author_id: 'user-2', parent_id: 'todo-5' }, -]; - -// --------------------------------------------------------------------------- -// In-memory mutable maps for create/update operations (loading direction) -// --------------------------------------------------------------------------- -const todosStore = new Map(TODOS.map((t) => [t.id, { ...t }])); - -// --------------------------------------------------------------------------- -// Pagination helper -// -// In a real connector, pagination would be driven by cursor or offset/limit -// parameters on the external API. Here we simulate it with a fixed page size -// so the pattern is visible even though the data fits in a single page. -// --------------------------------------------------------------------------- -const PAGE_SIZE = 5; - -function paginate(items: T[]): T[] { - // Simulate paginated retrieval. In a real connector you would loop here, - // calling the API with limit/offset (or a cursor) until hasMore is false. - const result: T[] = []; - for (let offset = 0; offset < items.length; offset += PAGE_SIZE) { - const page = items.slice(offset, offset + PAGE_SIZE); - result.push(...page); - // In production: if (page.length < PAGE_SIZE) break; // no more pages +export class SourceHttpError extends Error { + constructor(public readonly status: number, public readonly delaySeconds = 30) { + super(`Source HTTP ${status}`); } - - return result; } -// --------------------------------------------------------------------------- -// HttpClient -// --------------------------------------------------------------------------- - -export class HttpClient { - private apiEndpoint: string; - private apiToken: string; - - constructor(event: AirSyncEvent) { - // TODO: Replace with the API endpoint of the external system. This is - // passed through the event payload (e.g. event.payload.connection_data.org_id - // or a keyring subdomain field). - this.apiEndpoint = ''; - - // TODO: Replace with the API token of the external system. This is passed - // through the event payload. The configuration for the token is defined in - // manifest.yaml under keyring_types. - this.apiToken = event.payload.connection_data.key; - } - - // TODO: Replace with actual API calls that fetch external sync units (e.g. - // repos, projects, boards) from the external system. - async getTodoLists(): Promise { - return paginate(TODO_LISTS); - } - - // TODO: Replace with actual API calls that fetch todos from the external system. - async getTodos(): Promise { - return paginate([...todosStore.values()]); - } - - // TODO: Replace with actual API calls that fetch users from the external system. - async getUsers(): Promise { - return paginate(USERS); +export function validateTask(value: unknown): asserts value is SourceTask { + if (!value || typeof value !== 'object') throw new Error('Invalid source task'); + const task = value as Partial; + if (typeof task.id !== 'string' || !task.id || + typeof task.title !== 'string' || !task.title.trim() || + typeof task.created_at !== 'string' || !Number.isFinite(Date.parse(task.created_at)) || + typeof task.updated_at !== 'string' || !Number.isFinite(Date.parse(task.updated_at))) { + throw new Error('Invalid source task'); } +} - // TODO: Replace with actual API calls that fetch attachments from the - // external system. - async getAttachments(): Promise { - return paginate(ATTACHMENTS); +export class HttpClient { + private readonly origin: string; + private readonly token: string; + + constructor(private readonly event: AirSyncEvent) { + this.origin = process.env.EXAMPLE_API_ORIGIN || sourceConfig.origin; + this.token = event.payload.connection_data.key; + const url = new URL(this.origin); + const isLocal = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname); + if (url.protocol !== 'https:' && !(url.protocol === 'http:' && isLocal)) { + throw new Error('Source origin must use HTTPS or local HTTP'); + } + if (url.username || url.password || url.pathname !== '/' || url.search || url.hash) { + throw new Error('Source origin must contain only a scheme, hostname, and optional port'); + } + if (!this.token) throw new Error('Source API token is missing from the connection'); } - // TODO: Replace with an actual API call that creates a todo in the external system. - // Return a status with an id on success or an error with a message on failure. - async createTodo(todo: ExternalTodo): Promise { - const id = `todo-${Date.now()}`; - todosStore.set(id, { ...todo, id, modified_date: new Date().toISOString() }); - return { status: 'success', id }; + private async get(url: URL): Promise { + const response = await fetch(url, { + headers: { Authorization: `Bearer ${this.token}` }, + signal: AbortSignal.timeout(30_000), + redirect: 'error', + }); + if (!response.ok) { + const header = response.headers.get('retry-after'); + const numeric = header === null || !header.trim() ? NaN : Number(header); + const delay = Number.isFinite(numeric) + ? numeric + : Math.ceil((Date.parse(header ?? '') - Date.now()) / 1000); + throw new SourceHttpError(response.status, Number.isFinite(delay) && delay > 0 ? delay : 30); + } + return response.json(); } - // TODO: Replace with an actual API call that updates a todo in the external system. - // Return a status with an id on success or an error with a message on failure. - async updateTodo(todo: ExternalTodo): Promise { - if (!todosStore.has(todo.id)) { - return { status: 'error', error: { message: `Todo with id "${todo.id}" not found in external system.` } }; + async getProjects(): Promise { + const response = await this.get(new URL('/projects', this.origin)); + if (!Array.isArray(response) || response.some((project: Partial | null) => + !project || typeof project.id !== 'string' || !project.id || + typeof project.name !== 'string' || !project.name.trim() || + typeof project.description !== 'string' || + !Number.isSafeInteger(project.item_count) || (project.item_count ?? -1) < 0)) { + throw new Error('Invalid source projects'); } - todosStore.set(todo.id, { ...todo, modified_date: new Date().toISOString() }); - return { status: 'success', id: todo.id }; + return response; } - // TODO: Replace with an actual API call that creates an attachment in the external system. - // Return a status with an id on success or an error with a message on failure. - async createAttachment(attachment: ExternalAttachment): Promise { - return { status: 'success', id: attachment.id }; + async getTasks(cursor: string | null): Promise { + const context = this.event.payload.event_context; + if (!context.external_sync_unit_id) throw new Error('Source project ID is missing'); + const url = new URL('/tasks', this.origin); + url.searchParams.set('project_id', context.external_sync_unit_id); + url.searchParams.set('limit', '3'); + if (cursor) url.searchParams.set('cursor', cursor); + if (context.extract_from) url.searchParams.set('updated_from', context.extract_from); + if (context.extract_to) url.searchParams.set('updated_to', context.extract_to); + const page = await this.get(url) as Partial | null; + if (!page || !Array.isArray(page.items) || + !(page.next_cursor === null || (typeof page.next_cursor === 'string' && page.next_cursor.length > 0))) { + throw new Error('Invalid source page'); + } + if (page.next_cursor !== null && page.next_cursor === cursor) { + throw new Error('Source pagination cursor did not advance'); + } + if (page.next_cursor !== null && page.items.length === 0) { + throw new Error('Source pagination returned an empty continuation page'); + } + page.items.forEach(validateTask); + return page as TaskPage; } } diff --git a/code/src/functions/external-system/initial_domain_mapping.json b/code/src/functions/external-system/initial_domain_mapping.json index e6642c9..6218345 100644 --- a/code/src/functions/external-system/initial_domain_mapping.json +++ b/code/src/functions/external-system/initial_domain_mapping.json @@ -1,7 +1,7 @@ { "additional_mappings": { "record_type_mappings": { - "todos": { + "tasks": { "default_mapping": { "object_category": "stock", "object_type": "issue" @@ -10,47 +10,9 @@ { "devrev_leaf_type": "issue", "forward": true, - "reverse": true, + "reverse": false, "shard": { - "constructed_custom_fields": { - "ext_object_type": { - "field_descriptor": { - "allowed_values": ["Todos"], - "data_name": "ext_object_type", - "db_name": "ext_object_type", - "default_value": "Todos", - "description": "The source object type/subtype that the item was created from.", - "field_type": "enum", - "is_filterable": true, - "is_required": true, - "name": "ext_object_type", - "oasis": { - "name": "ext_object_type" - }, - "ui": { - "display_name": "External Object Type", - "is_hidden_during_create": true, - "is_read_only": true - } - }, - "transformation_method": { - "can_apply_to_null": false, - "custom_field_type": { - "allowed_values": ["Todos"], - "data_name": "dummy name", - "db_name": "dummy name", - "field_type": "enum", - "name": "dummy name", - "oasis": { - "name": "dummy name" - } - }, - "forward_jq": "\"Todos\"", - "transformation_method": "use_raw_jq", - "use_primary_input": false - } - } - }, + "constructed_custom_fields": {}, "devrev_leaf_type": { "object_category": "stock", "object_type": "issue" @@ -69,56 +31,6 @@ "transformation_method": "use_devrev_record" } }, - "body": { - "forward": true, - "primary_external_field": "body", - "reverse": true, - "transformation_method_for_set": { - "transformation_method": "use_directly" - } - }, - "created_by_id": { - "forward": true, - "primary_external_field": "creator", - "reverse": true, - "transformation_method_for_set": { - "transformation_method": "use_directly" - } - }, - "item_url_field": { - "forward": true, - "primary_external_field": "item_url_field", - "reverse": false, - "transformation_method_for_set": { - "transformation_method": "use_directly" - } - }, - "owned_by_ids": { - "forward": true, - "primary_external_field": "owner", - "reverse": true, - "transformation_method_for_set": { - "transformation_method": "use_as_array_value" - } - }, - "priority": { - "forward": true, - "reverse": false, - "transformation_method_for_set": { - "enum": "P1", - "transformation_method": "use_fixed_value", - "value": "enum_value" - } - }, - "stage": { - "forward": true, - "reverse": false, - "transformation_method_for_set": { - "enum": "in_development", - "transformation_method": "use_fixed_value", - "value": "enum_value" - } - }, "title": { "fallback": { "type": "text", @@ -126,41 +38,6 @@ }, "forward": true, "primary_external_field": "title", - "reverse": true, - "transformation_method_for_set": { - "transformation_method": "use_directly" - } - } - } - } - } - ] - }, - "users": { - "default_mapping": { - "object_category": "stock", - "object_type": "devu" - }, - "possible_record_type_mappings": [ - { - "devrev_leaf_type": "devu", - "forward": true, - "reverse": false, - "shard": { - "constructed_custom_fields": {}, - "devrev_leaf_type": { - "object_category": "stock", - "object_type": "devu" - }, - "mode": "create_shard", - "stock_field_mappings": { - "display_name": { - "fallback": { - "type": "text", - "value": "Fallback name" - }, - "forward": true, - "primary_external_field": "name", "reverse": false, "transformation_method_for_set": { "transformation_method": "use_directly" diff --git a/code/src/functions/external-system/source-config.json b/code/src/functions/external-system/source-config.json new file mode 100644 index 0000000..4bd0e65 --- /dev/null +++ b/code/src/functions/external-system/source-config.json @@ -0,0 +1,3 @@ +{ + "origin": "http://127.0.0.1:3100" +} diff --git a/code/src/functions/external-system/types.ts b/code/src/functions/external-system/types.ts index 97cbdfe..83e9941 100644 --- a/code/src/functions/external-system/types.ts +++ b/code/src/functions/external-system/types.ts @@ -1,35 +1,18 @@ -// Custom system data types based on the structures returned by HttpClient - -export interface ExternalTodoList { - id: string; - name: string; - description: string; - item_count: number; - item_type: string; -} - -export interface ExternalTodo { +export interface SourceTask { id: string; - created_date: string; - modified_date: string; - body: string; - creator: string; - owner: string; title: string; + created_at: string; + updated_at: string; } -export interface ExternalUser { - id: string; - created_date: string; - modified_date: string; - email: string; - name: string; +export interface TaskPage { + items: SourceTask[]; + next_cursor: string | null; } -export interface ExternalAttachment { - url: string; +export interface SourceProject { id: string; - file_name: string; - author_id?: string; - parent_id: string; + name: string; + description: string; + item_count: number; } diff --git a/code/src/functions/extraction/__tests__/tasks-extraction.test.ts b/code/src/functions/extraction/__tests__/tasks-extraction.test.ts new file mode 100644 index 0000000..e4d8fc3 --- /dev/null +++ b/code/src/functions/extraction/__tests__/tasks-extraction.test.ts @@ -0,0 +1,69 @@ +import { createMockEvent, createTestExtractionAdapter } from '@devrev/airsync-sdk'; + +import { HttpClient, SourceHttpError } from '../../external-system/http-client'; +import { normalizeTask } from '../../external-system/data-normalization'; +import { initialExtractorState } from '../index'; +import { extractTasks } from '../tasks-extraction'; + +const item = { id: '1', title: 'Example', created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-02T00:00:00Z' }; + +function adapter() { + return createTestExtractionAdapter({ + event: createMockEvent('http://127.0.0.1:1', { payload: { event_context: { run_id: 'run-1' } } }), + initialState: { ...initialExtractorState, runId: 'run-1', cursor: '3' }, + }); +} + +afterEach(() => jest.restoreAllMocks()); + +test('rate limit delays without changing the source cursor', async () => { + const current = adapter(); + jest.spyOn(HttpClient.prototype, 'getTasks').mockRejectedValue(new SourceHttpError(429, 7)); + await expect(extractTasks(current)).resolves.toEqual({ status: 'delay', delaySeconds: 7 }); + expect(current.state.cursor).toBe('3'); + expect(current.state.completed).toBe(false); +}); + +test('failed upload does not advance the cursor', async () => { + const current = adapter(); + jest.spyOn(HttpClient.prototype, 'getTasks').mockResolvedValue({ items: [item], next_cursor: '6' }); + current.initializeRepos([{ itemType: 'tasks', normalize: normalizeTask }]); + const repo = current.getRepo('tasks'); + if (!repo) throw new Error('Missing test repo'); + jest.spyOn(current, 'initializeRepos').mockImplementation(() => {}); + jest.spyOn(repo, 'upload').mockRejectedValue(new Error('Upload failed')); + await expect(extractTasks(current)).rejects.toThrow('Upload failed'); + expect(current.state.cursor).toBe('3'); + expect(current.state.completed).toBe(false); +}); + +test('invalid timestamps and empty titles are rejected', () => { + expect(() => normalizeTask({ ...item, updated_at: 'bad' })).toThrow('Invalid source task'); + expect(() => normalizeTask({ ...item, title: ' ' })).toThrow('Invalid source task'); +}); + +test('client rejects a repeated cursor and malformed page', async () => { + const current = adapter(); + jest.spyOn(global, 'fetch').mockResolvedValue(new Response(JSON.stringify({ items: [item], next_cursor: '3' }))); + const client = new HttpClient(current.event); + await expect(client.getTasks('3')).rejects.toThrow('did not advance'); + jest.mocked(global.fetch).mockResolvedValue(new Response(JSON.stringify({ items: [{ ...item, title: null }], next_cursor: null }))); + await expect(client.getTasks('3')).rejects.toThrow('Invalid source task'); +}); + +test('client accepts numeric and HTTP date retry hints', async () => { + const current = adapter(); + const request = jest.spyOn(global, 'fetch'); + request.mockResolvedValue(new Response(null, { status: 429, headers: { 'retry-after': '9' } })); + await expect(new HttpClient(current.event).getTasks(null)).rejects.toMatchObject({ status: 429, delaySeconds: 9 }); + request.mockResolvedValue(new Response(null, { status: 429, headers: { + 'retry-after': new Date(Date.now() + 60_000).toUTCString(), + } })); + try { + await new HttpClient(current.event).getTasks(null); + throw new Error('Expected rate limit'); + } catch (error) { + expect(error).toBeInstanceOf(SourceHttpError); + expect((error as SourceHttpError).delaySeconds).toBeGreaterThan(55); + } +}); diff --git a/code/src/functions/extraction/index.ts b/code/src/functions/extraction/index.ts index 6694f6f..beb4f03 100644 --- a/code/src/functions/extraction/index.ts +++ b/code/src/functions/extraction/index.ts @@ -2,39 +2,25 @@ import { AirSyncEvent, spawn } from '@devrev/airsync-sdk'; import initialDomainMapping from '../external-system/initial_domain_mapping.json'; -// TODO: Replace with your state interface that will keep track of the -// extraction progress. For example, the page number, the number of items -// processed, if the extraction is completed, etc. export interface ExtractorState { - todos: { completed: boolean }; - users: { completed: boolean }; - attachments: { completed: boolean }; + runId: string; + cursor: string | null; + completed: boolean; } -// TODO: Replace with your initial state that will be passed to the worker. -// This state will be used as a starting point for the extraction process. export const initialExtractorState: ExtractorState = { - todos: { completed: false }, - users: { completed: false }, - attachments: { completed: false }, + runId: '', + cursor: null, + completed: false, }; -const run = async (events: AirSyncEvent[]) => { +export default async function run(events: AirSyncEvent[]) { for (const event of events) { await spawn({ event, initialState: initialExtractorState, initialDomainMapping, baseWorkerPath: __dirname, - - // TODO: If needed you can pass additional options to the spawn function. - // For example timeout of the lambda, batch size, etc. - // options: { - // timeout: 1 * 1000 * 60, // 1 minute - // batchSize: 50, // 50 items per batch - // }, }); } -}; - -export default run; +} diff --git a/code/src/functions/extraction/tasks-extraction.ts b/code/src/functions/extraction/tasks-extraction.ts new file mode 100644 index 0000000..c2f86a7 --- /dev/null +++ b/code/src/functions/extraction/tasks-extraction.ts @@ -0,0 +1,35 @@ +import { ExtractionAdapter, TaskResult } from '@devrev/airsync-sdk'; + +import { normalizeTask } from '../external-system/data-normalization'; +import { HttpClient, SourceHttpError } from '../external-system/http-client'; +import type { ExtractorState } from './index'; + +export async function extractTasks(adapter: ExtractionAdapter): Promise { + const context = adapter.event.payload.event_context; + if (!context.run_id) throw new Error('Extraction run ID is missing'); + if (adapter.state.runId !== context.run_id) { + adapter.state = { runId: context.run_id, cursor: null, completed: false }; + } + if (!adapter.shouldExtract('tasks') || adapter.state.completed) return { status: 'success' }; + if (adapter.isTimeout) return { status: 'progress' }; + + let page; + try { + page = await new HttpClient(adapter.event).getTasks(adapter.state.cursor); + } catch (error) { + if (error instanceof SourceHttpError) { + if (error.status === 429) return { status: 'delay', delaySeconds: error.delaySeconds }; + return { status: 'error', error: { message: error.message } }; + } + throw error; + } + + adapter.initializeRepos([{ itemType: 'tasks', normalize: normalizeTask }]); + const repo = adapter.getRepo('tasks'); + if (!repo) throw new Error('Tasks repo is missing'); + await repo.push(page.items); + await repo.upload(); + adapter.state.cursor = page.next_cursor; + adapter.state.completed = page.next_cursor === null; + return { status: adapter.state.completed ? 'success' : 'progress' }; +} diff --git a/code/src/functions/extraction/workers/attachments-extraction.ts b/code/src/functions/extraction/workers/attachments-extraction.ts index 26d1a0c..9ae64bc 100644 --- a/code/src/functions/extraction/workers/attachments-extraction.ts +++ b/code/src/functions/extraction/workers/attachments-extraction.ts @@ -1,69 +1,3 @@ -import axios from 'axios'; +import { processExtractionTask } from '@devrev/airsync-sdk'; -import { - ExternalSystemAttachmentStreamingParams, - ExternalSystemAttachmentStreamingResponse, - processExtractionTask, -} from '@devrev/airsync-sdk'; - -// TODO: Replace with function for fetching attachment streams from the -// external system. This function should return either a stream of the -// attachment data, a delay or an error. -async function getFileStream({ - item, -}: ExternalSystemAttachmentStreamingParams): Promise { - const { id, url } = item; - - try { - const fileStreamResponse = await axios.get(url, { - responseType: 'stream', - timeout: 30000, - headers: { - 'Accept-Encoding': 'identity', - }, - }); - - return { status: 'success', httpStream: fileStreamResponse }; - } catch (error) { - if (axios.isAxiosError(error)) { - if (error?.response?.status === 429) { - const retryAfter = String(error.response?.headers['retry-after'] ?? '').trim(); - const delaySeconds = Number(retryAfter); - if (retryAfter && Number.isFinite(delaySeconds) && delaySeconds >= 0) { - return { status: 'delay', delaySeconds }; - } - } - - return { - status: 'error', - error: { - message: `Error while fetching attachment ${id} from URL. Error code: ${error.response?.status}. Error message: ${error.response?.data?.message || error.message}.`, - }, - }; - } - - return { - status: 'error', - error: { - message: `Unknown error while fetching attachment ${id} from URL. Error: ${error}.`, - }, - }; - } -} - -processExtractionTask({ - task: async ({ adapter }) => { - try { - return await adapter.streamAttachments({ - stream: getFileStream, - - // TODO: If needed you can specify how many attachments to stream at - // once. Minimum is 1 and maximum is 50. - // batchSize: 10, - }); - } catch (error) { - console.error('An error occured while processing a task.', error); - throw error; - } - }, -}); +processExtractionTask({ task: async () => ({ status: 'success' }) }); diff --git a/code/src/functions/extraction/workers/data-extraction.ts b/code/src/functions/extraction/workers/data-extraction.ts index 5c4b88d..a02c093 100644 --- a/code/src/functions/extraction/workers/data-extraction.ts +++ b/code/src/functions/extraction/workers/data-extraction.ts @@ -1,79 +1,6 @@ import { processExtractionTask } from '@devrev/airsync-sdk'; -import { normalizeAttachment, normalizeTodo, normalizeUser } from '../../external-system/data-normalization'; -import { HttpClient } from '../../external-system/http-client'; -import { ExtractorState } from '../index'; -import { ExternalTodo, ExternalUser, ExternalAttachment } from '../../external-system/types'; +import type { ExtractorState } from '../index'; +import { extractTasks } from '../tasks-extraction'; -// TODO: Replace with actual repos that will be used to store the -// data extracted from the external system. For example, you might want to -// create repos for todos, users, and attachments. Also replace and modify -// the normalization functions which are used to normalize the data. -const repos = [ - { - itemType: 'todos', - normalize: (item: object) => normalizeTodo(item as ExternalTodo), - }, - { - itemType: 'users', - normalize: (item: object) => normalizeUser(item as ExternalUser), - }, - { - itemType: 'attachments', - normalize: (item: object) => normalizeAttachment(item as ExternalAttachment), - }, -]; - -// TODO: Replace with item types you want to extract from the external system. -// Also replace the extract functions with the actual functions that will be -// used to extract the data. You can use this to easier iterate over the item -// types and extract them. -interface ItemTypeToExtract { - name: 'todos' | 'users' | 'attachments'; - extractFunction: (client: HttpClient) => Promise; -} - -const itemTypesToExtract: ItemTypeToExtract[] = [ - { - name: 'todos', - extractFunction: (client: HttpClient) => client.getTodos(), - }, - { - name: 'users', - extractFunction: (client: HttpClient) => client.getUsers(), - }, - { - name: 'attachments', - extractFunction: (client: HttpClient) => client.getAttachments(), - }, -]; - -processExtractionTask({ - task: async ({ adapter }) => { - adapter.initializeRepos(repos); - - // TODO: Replace with the HTTP client that will be used to make API calls - // to the external system. - const httpClient = new HttpClient(adapter.event); - - // TODO: Replace with your implementation to extract data from the external - // system. This is just an example how you can iterate over the item types, - // extract them, push them to the repo, and save the state. - for (const itemTypeToExtract of itemTypesToExtract) { - if (adapter.isTimeout) { - return { status: 'progress' }; - } - - if (!adapter.shouldExtract(itemTypeToExtract.name)) { - adapter.state[itemTypeToExtract.name].completed = true; - continue; - } - - const items = await itemTypeToExtract.extractFunction(httpClient); - await adapter.getRepo(itemTypeToExtract.name)?.push(items); - adapter.state[itemTypeToExtract.name].completed = true; - } - - return { status: 'success' }; - }, -}); +processExtractionTask({ task: ({ adapter }) => extractTasks(adapter) }); diff --git a/code/src/functions/extraction/workers/external-sync-units-extraction.ts b/code/src/functions/extraction/workers/external-sync-units-extraction.ts index db743d9..a5bacca 100644 --- a/code/src/functions/extraction/workers/external-sync-units-extraction.ts +++ b/code/src/functions/extraction/workers/external-sync-units-extraction.ts @@ -1,39 +1,16 @@ -import { - AirSyncDefaultItemTypes, - ExternalSyncUnit, - processExtractionTask, -} from '@devrev/airsync-sdk'; +import { AirSyncDefaultItemTypes, processExtractionTask } from '@devrev/airsync-sdk'; -import { normalizeTodoList } from '../../external-system/data-normalization'; +import { normalizeProject } from '../../external-system/data-normalization'; import { HttpClient } from '../../external-system/http-client'; processExtractionTask({ task: async ({ adapter }) => { - adapter.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overriddenOptions: { - batchSize: 25000, - }, - }, - ]); - - // TODO: Replace with HTTP client that will be used to make API calls - // to the external system. - const httpClient = new HttpClient(adapter.event); - - // TODO: Replace with actual API call to fetch external sync units. - const todoLists = await httpClient.getTodoLists(); - - // TODO: Normalize the data received from the API call to match the - // ExternalSyncUnit interface. Modify the normalization function to suit - // your needs. - const externalSyncUnits: ExternalSyncUnit[] = todoLists.map((todoList) => normalizeTodoList(todoList)); - - await adapter - .getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS) - ?.push(externalSyncUnits); - + const projects = await new HttpClient(adapter.event).getProjects(); + adapter.initializeRepos([{ itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS }]); + const repo = adapter.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS); + if (!repo) throw new Error('External sync units repo is missing'); + await repo.push(projects.map(normalizeProject)); + await repo.upload(); return { status: 'success' }; }, }); diff --git a/code/src/functions/extraction/workers/metadata-extraction.ts b/code/src/functions/extraction/workers/metadata-extraction.ts index 1791495..34232e9 100644 --- a/code/src/functions/extraction/workers/metadata-extraction.ts +++ b/code/src/functions/extraction/workers/metadata-extraction.ts @@ -1,26 +1,14 @@ import { processExtractionTask } from '@devrev/airsync-sdk'; -import staticExternalDomainMetadata from '../../external-system/external_domain_metadata.json'; - -const repos = [ - { - itemType: 'external_domain_metadata', - }, -]; +import externalDomainMetadata from '../../external-system/external_domain_metadata.json'; processExtractionTask({ task: async ({ adapter }) => { - adapter.initializeRepos(repos); - - // TODO: If needed, add handling for dynamic external domain metadata - // extraction here. For example, you might want to call an external system API to - // get custom fields and their values and append them to the static - // external domain metadata. - const externalDomainMetadata = { - ...staticExternalDomainMetadata, - }; - - await adapter.getRepo('external_domain_metadata')?.push([externalDomainMetadata]); + adapter.initializeRepos([{ itemType: 'external_domain_metadata' }]); + const repo = adapter.getRepo('external_domain_metadata'); + if (!repo) throw new Error('Metadata repo is missing'); + await repo.push([externalDomainMetadata]); + await repo.upload(); return { status: 'success' }; }, }); diff --git a/code/src/functions/loading/index.ts b/code/src/functions/loading/index.ts deleted file mode 100644 index 4a65edf..0000000 --- a/code/src/functions/loading/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { AirSyncEvent, spawn } from '@devrev/airsync-sdk'; - -import initialDomainMapping from '../external-system/initial_domain_mapping.json'; - -// TODO: If needed, you can replace this with state interface that will keep -// track of the loading progress. -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -export interface LoaderState {} - -// TODO: Replace with your initial state that will be passed to the worker. -// This state will be used as a starting point for the loading process. -export const initialLoaderState: LoaderState = {}; - -const run = async (events: AirSyncEvent[]) => { - for (const event of events) { - await spawn({ - event, - initialState: initialLoaderState, - initialDomainMapping, - baseWorkerPath: __dirname, - - // TODO: If needed you can pass additional options to the spawn function. - // For example timeout of the lambda, batch size, etc. - // options: { - // timeout: 1 * 1000 * 60, // 1 minute - // }, - }); - } -}; - -export default run; diff --git a/code/src/functions/loading/workers/load-attachments.ts b/code/src/functions/loading/workers/load-attachments.ts deleted file mode 100644 index 357d5d2..0000000 --- a/code/src/functions/loading/workers/load-attachments.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { - ExternalSystemAttachment, - ExternalSystemItemLoadingParams, - processLoadingTask, -} from '@devrev/airsync-sdk'; - -import { denormalizeAttachment } from '../../external-system/data-denormalization'; -import { HttpClient } from '../../external-system/http-client'; -import { LoaderState } from '../index'; - -/* eslint-disable @typescript-eslint/no-unused-vars */ - -// TODO: Replace with your create function that will be used to make API calls -// to the external system to create a new attachment. Function must return -// a status with an id or error depending on the response from the external system. -async function createAttachment({ item, mappers, event }: ExternalSystemItemLoadingParams) { - // TODO: Replace with your HTTP client that will be used to make API calls - // to the external system. - const httpClient = new HttpClient(event); - const attachment = denormalizeAttachment(item); - const createAttachmentResponse = await httpClient.createAttachment(attachment); - return createAttachmentResponse; -} - -processLoadingTask({ - task: async ({ adapter }) => { - return adapter.loadAttachments({ - create: createAttachment, - }); - }, -}); diff --git a/code/src/functions/loading/workers/load-data.ts b/code/src/functions/loading/workers/load-data.ts deleted file mode 100644 index 8c88629..0000000 --- a/code/src/functions/loading/workers/load-data.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { - ExternalSystemItem, - ExternalSystemItemLoadingParams, - ExternalSystemItemLoadingResponse, - processLoadingTask, -} from '@devrev/airsync-sdk'; - -import { denormalizeTodo } from '../../external-system/data-denormalization'; -import { HttpClient } from '../../external-system/http-client'; -import { LoaderState } from '../index'; - -/* eslint-disable @typescript-eslint/no-unused-vars */ - -// TODO: Replace with your create function that will be used to make API calls -// to the external system to create a new item. Return a status with an id, -// error or delaySeconds depending on the response from the external system. -async function createTodo({ - item, - mappers, - event, -}: ExternalSystemItemLoadingParams): Promise { - // TODO: Replace with your HTTP client that will be used to make API calls - // to the external system. - const httpClient = new HttpClient(event); - const todo = denormalizeTodo(item); - - const createTodoResponse = await httpClient.createTodo(todo); - return createTodoResponse; -} - -// TODO: Replace with your update function that will be used to make API calls -// to the external system to update an existing item. Return a status with an -// id, error or delaySeconds depending on the response from the external system. -async function updateTodo({ - item, - mappers, - event, -}: ExternalSystemItemLoadingParams): Promise { - // TODO: Replace with your HTTP client that will be used to make API calls - // to the external system. - const httpClient = new HttpClient(event); - - // TODO: In case you need to get the external id of the item, you can use - // the mappers.getByTargetId function to get the sync mapper record - // for the item. The sync mapper record will contain the external id of - // the item in the external system. - // const syncMapperRecordResponse = await mappers.getByTargetId({ - // sync_unit: event.payload.event_context.sync_unit, - // target: item.id.devrev, - // }); - // const todoExternalId = syncMapperRecordResponse?.sync_mapper_record.external_ids[0]; - - const todo = denormalizeTodo(item); - - const updateTodoResponse = await httpClient.updateTodo(todo); - return updateTodoResponse; -} - -processLoadingTask({ - task: async ({ adapter }) => { - return adapter.loadItemTypes({ - itemTypesToLoad: [ - { - itemType: 'todos', - create: createTodo, - update: updateTodo, - }, - ], - }); - }, -}); diff --git a/code/src/test-runner/test-runner.ts b/code/src/test-runner/test-runner.ts index 69ef529..3685da2 100644 --- a/code/src/test-runner/test-runner.ts +++ b/code/src/test-runner/test-runner.ts @@ -60,8 +60,6 @@ export const testRunner = async ({ fixturePath, functionName }: LocalRunnerProps function getFunctionName(event_type: string): FunctionFactoryType { if (event_type.indexOf('EXTRACT') != -1) { return 'extraction'; - } else if (event_type.indexOf('LOAD') != -1) { - return 'loading'; } throw new Error(`No functionName found for event ${event_type}. Specify functionName using '--functionName' parameter.`); diff --git a/code/tsconfig.json b/code/tsconfig.json index 4a7a802..cf25c4d 100644 --- a/code/tsconfig.json +++ b/code/tsconfig.json @@ -4,7 +4,9 @@ "module": "commonjs", "baseUrl": "./", "paths": { - "*": ["./src/*"] + "*": [ + "./src/*" + ] }, "outDir": "./dist", "esModuleInterop": true, @@ -14,6 +16,12 @@ "declaration": true, "resolveJsonModule": true }, - "include": ["src"], - "exclude": ["node_modules", "dist"] + "include": [ + "src" + ], + "exclude": [ + "node_modules", + "dist", + "src/**/*.test.ts" + ] } diff --git a/manifest.yaml b/manifest.yaml index fc6ff08..a6ada09 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -1,101 +1,47 @@ version: "2" - -# This manifest defines your snap-in. -# The comments below will tell you which fields you can change and which you should keep as they are. -# For more information about manifests, see https://developer.devrev.ai/public/snapin-development/references/manifest. - -# General information about the snap-in, which you can change to your liking. -name: Todo -description: An Airdrop snap-in for syncing data between Todo and DevRev - -# The name of the service account that your snap-in will use in DevRev. -# You can change the display name value to your liking. +name: Tasks Demo +description: Import tasks from a simple Tasks API into DevRev issues service_account: - display_name: Todo Bot - # Scopes needed for the Airdrop snap-in to work properly. + display_name: Tasks Demo Bot scopes: self: - scope: sync_mapper_record:write optional: false - reason: "Allow write access to sync mapper records" + reason: Allow write access to sync mapper records - scope: sync_mapper_record:read optional: false - reason: "Allow read access to sync mapper records" + reason: Allow read access to sync mapper records - scope: sync_unit:read optional: false - reason: "Allow read access to sync units" + reason: Allow read access to sync units - scope: sync_snap_in:all optional: false - reason: "Allow all access to sync snap ins" + reason: Allow AirSync operations - scope: sync_history:read optional: false - reason: "Allow read access to sync history" - -# Functions that the snap-in exposes. -# The names of the functions should be in sync with their references in the 'imports' and -# 'hooks' sections below and with your code, if you happen to change the names. -# You can freely change the descriptions of the functions. + reason: Allow read access to sync history functions: - name: extraction - description: Airdrop extraction function for Todo - - name: loading - description: Airdrop loading function for Todo - -# The types of connections that the snap-in supports. -# If an external system supports OAuth, then you should use that. - # Here we have just one example of a keyring, you can see more advanced examples - # at https://github.com/devrev/snap-in-examples/tree/main/13-keyring-type. + description: Discover projects and extract Tasks API records keyring_types: - # The keyring type ID can be whatever you want, but it has to match one of the allowed - # connection types in the 'imports' section below. - - id: example-pat-connection - - # Connection details that are shown when creating a connection in the DevRev app. - # You can freely change them, but the external system name should have the first letter capitalized. - name: Todo Connection - description: Connection details for authenticating with Todo - external_system_name: Todo - - # Configuration for passing the secret to the external system, if necessary. - # See https://github.com/devrev/snap-in-examples/blob/main/13-keyring-type/custom-keyring-type-basic.yaml - # for more info on how the secret config should look like and what these fields mean. - kind: "Secret" - is_subdomain: true + - id: tasks-api-token + name: Tasks API Connection + description: Bearer token for the Tasks API + external_system_name: Tasks Demo + kind: Secret + is_subdomain: false secret_config: secret_transform: ".token" fields: - id: token name: Token - description: Example API token - - # Airdrop will call the URL you set with the token to verify its validity. - # TODO: Update the token verification URL to use an authenticated endpoint. - token_verification: - url: https://app.devrev.ai/favicon.ico - method: GET - -# Imports define the external systems that the snap-in will sync data from. + description: Tasks API bearer token imports: - # The slug is the machine-readable name for the import. - # In the example below, it's automatically taken from the repository name if you used the template. - - slug: airdrop-todo-snap-in - display_name: Todo - description: Import data from Todo using Airdrop - - # The extractor and loader functions specify which functions from the section - # 'functions' above should be called when extracting data from an external - # system, or loading data into it. - # Make sure these are in sync with the function names in the 'functions' section above - # and with your code, if you changed the names. + - slug: airsync-tasks-demo + display_name: Tasks Demo + description: Import tasks as DevRev issues extractor_function: extraction - - loader_function: loading - - # The list of connection types that are available for the external system. - # Make sure these are in sync with the keyring types in the 'keyring_types' section above. allowed_connection_types: - - example-pat-connection - - # The list of capabilities that the snap-in supports. - # capabilities: - # - TIME_SCOPED_SYNCS + - tasks-api-token + capabilities: + - TIME_SCOPED_SYNCS diff --git a/source-api/server.mjs b/source-api/server.mjs new file mode 100644 index 0000000..c2d642c --- /dev/null +++ b/source-api/server.mjs @@ -0,0 +1,96 @@ +import { createServer } from 'node:http'; +import { pathToFileURL } from 'node:url'; + +export function createTasksServer({ token, clock = () => new Date() }) { + if (!token) throw new Error('EXAMPLE_API_TOKEN is required'); + const tasks = Array.from({ length: 10 }, (_, index) => ({ + id: String(index + 1), + project_id: 'demo-project', + title: `Example task ${index + 1}`, + created_at: '2026-01-01T09:00:00.000Z', + updated_at: '2026-01-01T09:00:00.000Z', + })); + return createServer(async (request, response) => { + const send = (status, body) => { + response.writeHead(status, { 'content-type': 'application/json' }); + response.end(JSON.stringify(body)); + }; + try { + if (request.headers.authorization !== `Bearer ${token}`) { + send(401, { error: 'Unauthorized' }); + return; + } + const url = new URL(request.url, 'http://localhost'); + if (request.method === 'GET' && url.pathname === '/projects') { + send(200, [{ + id: 'demo-project', name: 'Demo Project', description: 'Ten example tasks', + item_count: tasks.length, + }]); + return; + } + if (request.method === 'GET' && url.pathname === '/tasks') { + const project = url.searchParams.get('project_id'); + const limit = Number(url.searchParams.get('limit') ?? 3); + const cursor = url.searchParams.get('cursor') ?? '0'; + const offset = Number(cursor); + const from = url.searchParams.get('updated_from'); + const to = url.searchParams.get('updated_to'); + if (!project || !Number.isInteger(limit) || limit < 1 || limit > 100 || + !/^\d+$/.test(cursor) || !Number.isSafeInteger(offset) || + (from !== null && !Number.isFinite(Date.parse(from))) || + (to !== null && !Number.isFinite(Date.parse(to))) || + (from !== null && to !== null && Date.parse(from) > Date.parse(to))) { + send(400, { error: 'Invalid project, pagination, or time window' }); + return; + } + const filtered = tasks.filter(task => task.project_id === project && + (from === null || Date.parse(task.updated_at) >= Date.parse(from)) && + (to === null || Date.parse(task.updated_at) <= Date.parse(to))); + const items = filtered.slice(offset, offset + limit); + const nextOffset = offset + items.length; + send(200, { items, next_cursor: nextOffset < filtered.length ? String(nextOffset) : null }); + return; + } + const match = /^\/tasks\/(\d+)$/.exec(url.pathname); + if (request.method === 'PATCH' && match) { + const task = tasks.find(task => task.id === match[1]); + if (!task) { + send(404, { error: 'Task not found' }); + return; + } + let body = ''; + for await (const chunk of request) { + body += chunk; + if (Buffer.byteLength(body) > 4096) { + send(413, { error: 'Request too large' }); + return; + } + } + let payload; + try { payload = JSON.parse(body); } catch { + send(400, { error: 'Invalid JSON' }); + return; + } + if (!payload || typeof payload.title !== 'string' || !payload.title.trim()) { + send(400, { error: 'A nonempty title is required' }); + return; + } + task.title = payload.title.trim(); + task.updated_at = clock().toISOString(); + send(200, task); + return; + } + send(404, { error: 'Not found' }); + } catch { + if (!response.headersSent) send(500, { error: 'Source API failure' }); + else response.end(); + } + }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const server = createTasksServer({ token: process.env.EXAMPLE_API_TOKEN }); + server.listen(Number(process.env.PORT ?? 3100), process.env.HOST ?? '127.0.0.1', () => { + console.log(`Tasks API: http://127.0.0.1:${server.address().port}`); + }); +} diff --git a/source-api/server.test.mjs b/source-api/server.test.mjs new file mode 100644 index 0000000..ae840eb --- /dev/null +++ b/source-api/server.test.mjs @@ -0,0 +1,52 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { createTasksServer } from './server.mjs'; + +test('source API supports authenticated pagination and incremental windows', async t => { + const server = createTasksServer({ token: 'test-token', clock: () => new Date('2026-02-01T12:00:00Z') }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => new Promise(resolve => server.close(resolve))); + const origin = `http://127.0.0.1:${server.address().port}`; + const headers = { authorization: 'Bearer test-token' }; + assert.equal((await fetch(`${origin}/tasks?project_id=demo-project`)).status, 401); + const ids = []; + let cursor = null; + let pages = 0; + do { + const url = new URL('/tasks', origin); + url.searchParams.set('project_id', 'demo-project'); + url.searchParams.set('limit', '3'); + if (cursor) url.searchParams.set('cursor', cursor); + const response = await fetch(url, { headers }); + assert.equal(response.status, 200); + const page = await response.json(); + ids.push(...page.items.map(task => task.id)); + cursor = page.next_cursor; + pages++; + } while (cursor); + assert.equal(pages, 4); + assert.deepEqual(ids, Array.from({ length: 10 }, (_, i) => String(i + 1))); + const update = await fetch(`${origin}/tasks/1`, { + method: 'PATCH', headers: { ...headers, 'content-type': 'application/json' }, + body: JSON.stringify({ title: 'Updated task' }), + }); + assert.equal(update.status, 200); + const incremental = new URL('/tasks', origin); + incremental.searchParams.set('project_id', 'demo-project'); + incremental.searchParams.set('updated_from', '2026-02-01T00:00:00Z'); + incremental.searchParams.set('updated_to', '2026-02-02T00:00:00Z'); + const result = await (await fetch(incremental, { headers })).json(); + assert.deepEqual(result.items.map(task => task.id), ['1']); + assert.equal(result.items[0].title, 'Updated task'); + assert.equal(result.next_cursor, null); + const empty = await (await fetch(`${origin}/tasks?project_id=unknown`, { headers })).json(); + assert.deepEqual(empty, { items: [], next_cursor: null }); + for (const query of ['limit=0', 'cursor=bad', 'updated_from=bad']) { + assert.equal((await fetch(`${origin}/tasks?project_id=demo-project&${query}`, { headers })).status, 400); + } + assert.equal((await fetch(`${origin}/tasks/1`, { + method: 'PATCH', headers, body: '{', + })).status, 400); +}); From 353f6cb1bb99761aec737bd5deb47d21bdaaa164 Mon Sep 17 00:00:00 2001 From: Radovan Jorgic Date: Thu, 15 Oct 2026 15:07:48 +0200 Subject: [PATCH 2/5] fix: verify Tasks API credentials during connection setup --- README.md | 4 ++-- TEMPLATE_SOURCE.md | 2 ++ code/scripts/configure-source.mjs | 4 ++++ manifest.yaml | 6 ++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 29d29ea..7a6d6ca 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Unit checks cover rate-limit delay, upload failure without cursor advancement, m npm run package ``` - This writes the origin to the bundled `source-config.json`. `EXAMPLE_API_ORIGIN` overrides it for local runs. Deployed credentials come from the connection key, not local environment variables. + This writes the origin to the bundled `source-config.json` and the manifest token-verification URL. `EXAMPLE_API_ORIGIN` overrides it for local runs. Deployed credentials come from the connection key, not local environment variables. 3. Validate the manifest against the intended authenticated DevRev profile: ```sh @@ -84,7 +84,7 @@ Unit checks cover rate-limit delay, upload failure without cursor advancement, m ``` 4. The template deployment script is retained as `npm run deploy`. It prompts for your environment, organization, email, deployment mode, and package slug. Deployment has not been run here. -5. In DevRev, create the **Tasks API Connection** using the source bearer token, start **Tasks Demo**, and select **Demo Project**. Credentials are checked by the authenticated project-discovery request; the manifest does not define a separate token-verification endpoint. +5. In DevRev, create the **Tasks API Connection** using the source bearer token, start **Tasks Demo**, and select **Demo Project**. Credentials are verified against the authenticated `/projects` endpoint when creating the connection. 6. Review the Tasks-to-issue mapping. The template's `applies_to_part_id` mapping needs an existing DevRev product selected for the issues; no organization-specific target ID is hardcoded. ## Validation status diff --git a/TEMPLATE_SOURCE.md b/TEMPLATE_SOURCE.md index 0093371..481f6c7 100644 --- a/TEMPLATE_SOURCE.md +++ b/TEMPLATE_SOURCE.md @@ -4,3 +4,5 @@ Commit: 050349c2e3308507cfcb63cf974883565c72a63c Extraction guide: https://devrev-marketing-developer-git-docs-iss-383917-a-68bdfc-dev-rev.vercel.app/airsync/data-extraction-example The connector uses the template runtime, manifest structure, metadata/mapping format, fixture runner, and SDK dependency. The extraction loop follows the supplied guide. No other connector documentation was consulted. + +Live deployment exposed a missing required secret token-verification configuration. The follow-up fix uses the connector deployment plugin's documented bearer-token manifest pattern and verifies against the source `/projects` endpoint. diff --git a/code/scripts/configure-source.mjs b/code/scripts/configure-source.mjs index 3ddd069..508d283 100644 --- a/code/scripts/configure-source.mjs +++ b/code/scripts/configure-source.mjs @@ -6,6 +6,10 @@ if (origin.protocol !== 'https:' || origin.username || origin.password || origin.pathname !== '/' || origin.search || origin.hash) { throw new Error('Provide an HTTPS origin, for example https://tasks.example.com'); } +const manifestPath = fileURLToPath(new URL('../../manifest.yaml', import.meta.url)); +const manifest = fs.readFileSync(manifestPath, 'utf8'); +if (!/^ {8}url: .+$/m.test(manifest)) throw new Error('Token verification URL is missing'); const target = fileURLToPath(new URL('../src/functions/external-system/source-config.json', import.meta.url)); fs.writeFileSync(target, JSON.stringify({ origin: origin.origin }, null, 2) + '\n'); +fs.writeFileSync(manifestPath, manifest.replace(/^ {8}url: .+$/m, ` url: ${origin.origin}/projects`)); console.log(`Connector source set to ${origin.origin}. Rebuild before packaging.`); diff --git a/manifest.yaml b/manifest.yaml index a6ada09..93a6271 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -36,6 +36,12 @@ keyring_types: - id: token name: Token description: Tasks API bearer token + input_type: password + token_verification: + url: http://127.0.0.1:3100/projects + method: GET + headers: + Authorization: "Bearer [API_KEY]" imports: - slug: airsync-tasks-demo display_name: Tasks Demo From 2fd20566f4d5323f85657284c7c2d35c534666e2 Mon Sep 17 00:00:00 2001 From: Radovan Jorgic Date: Thu, 15 Oct 2026 15:25:28 +0200 Subject: [PATCH 3/5] fix: populate Tasks connection organization metadata --- README.md | 4 ++-- TEMPLATE_SOURCE.md | 2 ++ code/scripts/configure-source.mjs | 2 +- manifest.yaml | 7 +++++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7a6d6ca..996d1d8 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Unit checks cover rate-limit delay, upload failure without cursor advancement, m npm run package ``` - This writes the origin to the bundled `source-config.json` and the manifest token-verification URL. `EXAMPLE_API_ORIGIN` overrides it for local runs. Deployed credentials come from the connection key, not local environment variables. + This writes the origin to the bundled `source-config.json` and the manifest verification and organization URLs. `EXAMPLE_API_ORIGIN` overrides it for local runs. Deployed credentials come from the connection key, not local environment variables. 3. Validate the manifest against the intended authenticated DevRev profile: ```sh @@ -84,7 +84,7 @@ Unit checks cover rate-limit delay, upload failure without cursor advancement, m ``` 4. The template deployment script is retained as `npm run deploy`. It prompts for your environment, organization, email, deployment mode, and package slug. Deployment has not been run here. -5. In DevRev, create the **Tasks API Connection** using the source bearer token, start **Tasks Demo**, and select **Demo Project**. Credentials are verified against the authenticated `/projects` endpoint when creating the connection. +5. In DevRev, create the **Tasks API Connection** using the source bearer token, start **Tasks Demo**, and select **Demo Project**. Credentials are verified against the authenticated `/projects` endpoint when creating the connection. Organization discovery uses that same endpoint and identifies this single-account demo as `tasks-demo` / **Tasks Demo**. 6. Review the Tasks-to-issue mapping. The template's `applies_to_part_id` mapping needs an existing DevRev product selected for the issues; no organization-specific target ID is hardcoded. ## Validation status diff --git a/TEMPLATE_SOURCE.md b/TEMPLATE_SOURCE.md index 481f6c7..d48b347 100644 --- a/TEMPLATE_SOURCE.md +++ b/TEMPLATE_SOURCE.md @@ -6,3 +6,5 @@ Extraction guide: https://devrev-marketing-developer-git-docs-iss-383917-a-68bdf The connector uses the template runtime, manifest structure, metadata/mapping format, fixture runner, and SDK dependency. The extraction loop follows the supplied guide. No other connector documentation was consulted. Live deployment exposed a missing required secret token-verification configuration. The follow-up fix uses the connector deployment plugin's documented bearer-token manifest pattern and verifies against the source `/projects` endpoint. + +Live discovery also requires the connection's external organization ID and name. The manifest uses the plugin's documented static organization pattern for this single-account demo, verified against `/projects`. The requirement was confirmed in the platform's keyring validation source. diff --git a/code/scripts/configure-source.mjs b/code/scripts/configure-source.mjs index 508d283..e1d50e2 100644 --- a/code/scripts/configure-source.mjs +++ b/code/scripts/configure-source.mjs @@ -11,5 +11,5 @@ const manifest = fs.readFileSync(manifestPath, 'utf8'); if (!/^ {8}url: .+$/m.test(manifest)) throw new Error('Token verification URL is missing'); const target = fileURLToPath(new URL('../src/functions/external-system/source-config.json', import.meta.url)); fs.writeFileSync(target, JSON.stringify({ origin: origin.origin }, null, 2) + '\n'); -fs.writeFileSync(manifestPath, manifest.replace(/^ {8}url: .+$/m, ` url: ${origin.origin}/projects`)); +fs.writeFileSync(manifestPath, manifest.replace(/^( {6,8})url: .+$/gm, (_, indent) => `${indent}url: ${origin.origin}/projects`)); console.log(`Connector source set to ${origin.origin}. Rebuild before packaging.`); diff --git a/manifest.yaml b/manifest.yaml index 93a6271..28ed798 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -42,6 +42,13 @@ keyring_types: method: GET headers: Authorization: "Bearer [API_KEY]" + organization_data: + type: config + url: http://127.0.0.1:3100/projects + method: GET + headers: + Authorization: "Bearer [API_KEY]" + response_jq: '{id: "tasks-demo", name: "Tasks Demo"}' imports: - slug: airsync-tasks-demo display_name: Tasks Demo From 96f1b3d7f06528677db68822909d0501e9c9dbc3 Mon Sep 17 00:00:00 2001 From: Radovan Jorgic Date: Thu, 15 Oct 2026 15:31:50 +0200 Subject: [PATCH 4/5] fix: supply required issue stage priority and owner mappings --- .../initial_domain_mapping.json | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/code/src/functions/external-system/initial_domain_mapping.json b/code/src/functions/external-system/initial_domain_mapping.json index 6218345..3e8ef38 100644 --- a/code/src/functions/external-system/initial_domain_mapping.json +++ b/code/src/functions/external-system/initial_domain_mapping.json @@ -31,6 +31,36 @@ "transformation_method": "use_devrev_record" } }, + "owned_by_ids": { + "forward": true, + "reverse": false, + "transformation_method": { + "is_array": true, + "leaf_type": { + "object_category": "stock", + "object_type": "devu" + }, + "transformation_method": "use_devrev_record" + } + }, + "priority": { + "forward": true, + "reverse": false, + "transformation_method": { + "enum": "P2", + "transformation_method": "use_fixed_value", + "value": "enum_value" + } + }, + "stage": { + "forward": true, + "reverse": false, + "transformation_method": { + "enum": "backlog", + "transformation_method": "use_fixed_value", + "value": "enum_value" + } + }, "title": { "fallback": { "type": "text", @@ -51,5 +81,6 @@ }, "format_version": "v0.2.0", "devrev_metadata_version": 13 - } + }, + "_dev_external_system_type": "" } From 296a13ecc457ad98938cc924f353aae2eb91594a Mon Sep 17 00:00:00 2001 From: Radovan Jorgic Date: Thu, 15 Oct 2026 15:36:21 +0200 Subject: [PATCH 5/5] fix: configure demo issue attribution to prevent staged imports --- README.md | 2 +- TEMPLATE_SOURCE.md | 2 ++ .../initial_domain_mapping.json | 24 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 996d1d8..842498c 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Unit checks cover rate-limit delay, upload failure without cursor advancement, m 4. The template deployment script is retained as `npm run deploy`. It prompts for your environment, organization, email, deployment mode, and package slug. Deployment has not been run here. 5. In DevRev, create the **Tasks API Connection** using the source bearer token, start **Tasks Demo**, and select **Demo Project**. Credentials are verified against the authenticated `/projects` endpoint when creating the connection. Organization discovery uses that same endpoint and identifies this single-account demo as `tasks-demo` / **Tasks Demo**. -6. Review the Tasks-to-issue mapping. The template's `applies_to_part_id` mapping needs an existing DevRev product selected for the issues; no organization-specific target ID is hardcoded. +6. Review the Tasks-to-issue mapping and select a DevRev product and default issue owner, creator, and modifier. Issues default to Backlog and priority P2 because this source only provides titles and timestamps. No organization-specific record IDs are hardcoded. ## Validation status diff --git a/TEMPLATE_SOURCE.md b/TEMPLATE_SOURCE.md index d48b347..064a64a 100644 --- a/TEMPLATE_SOURCE.md +++ b/TEMPLATE_SOURCE.md @@ -8,3 +8,5 @@ The connector uses the template runtime, manifest structure, metadata/mapping fo Live deployment exposed a missing required secret token-verification configuration. The follow-up fix uses the connector deployment plugin's documented bearer-token manifest pattern and verifies against the source `/projects` endpoint. Live discovery also requires the connection's external organization ID and name. The manifest uses the plugin's documented static organization pattern for this single-account demo, verified against `/projects`. The requirement was confirmed in the platform's keyring validation source. + +Live mapping installation requires issue stage, priority, and owner mappings. The domain-mapping skill and AirSync mapping tools supplied Backlog, P2, and a configurable default owner; both metadata and initial mapping pass chef-cli validation. diff --git a/code/src/functions/external-system/initial_domain_mapping.json b/code/src/functions/external-system/initial_domain_mapping.json index 3e8ef38..5165910 100644 --- a/code/src/functions/external-system/initial_domain_mapping.json +++ b/code/src/functions/external-system/initial_domain_mapping.json @@ -31,6 +31,30 @@ "transformation_method": "use_devrev_record" } }, + "created_by_id": { + "forward": true, + "reverse": false, + "transformation_method": { + "is_array": false, + "leaf_type": { + "object_category": "stock", + "object_type": "devu" + }, + "transformation_method": "use_devrev_record" + } + }, + "modified_by_id": { + "forward": true, + "reverse": false, + "transformation_method": { + "is_array": false, + "leaf_type": { + "object_category": "stock", + "object_type": "devu" + }, + "transformation_method": "use_devrev_record" + } + }, "owned_by_ids": { "forward": true, "reverse": false,