diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0da1924..020b033 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1 @@ @radovanjorgic @gasperzgonec @devrev/airdrop - -# Test folder ownership -code/test/ @navneel99 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index aa88a06..3e45ff9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: "weekly" allow: - - dependency-name: "@devrev/ts-adaas" + - dependency-name: "@devrev/airsync-sdk" labels: - "dependencies" - "marketplace-submission" diff --git a/.github/templates/README.md b/.github/templates/README.md index 6250838..f315156 100644 --- a/.github/templates/README.md +++ b/.github/templates/README.md @@ -6,7 +6,7 @@ Syncs data between Todo and DevRev using DevRev's AirSync platform. Install the following tools: -- [Node.js](https://nodejs.org/en/download/) +- [Node.js 20 or newer](https://nodejs.org/en/download/) - [DevRev CLI](https://developer.devrev.ai/snapin-development/references/cli-install) - [jq](https://jqlang.github.io/jq/download/) - [ngrok](https://ngrok.com/download) (for local development) @@ -40,6 +40,9 @@ USER_EMAIL=my@email.com These values will be used as defaults when the scripts prompt you for credentials. +The connector uses `@devrev/airsync-sdk`. Its local test server sets +`AIRSYNC_LOCAL_DEVELOPMENT=1` to enable local logging and artifact copies. + ## Deploy Run the deployment script: diff --git a/.github/workflows/dependabot-labeler.yml b/.github/workflows/dependabot-labeler.yml index 28d76fb..285bada 100644 --- a/.github/workflows/dependabot-labeler.yml +++ b/.github/workflows/dependabot-labeler.yml @@ -20,7 +20,7 @@ jobs: const { owner, repo, number } = context.issue; const prTitle = context.payload.pull_request.title; - if (!prTitle.includes('@devrev/ts-adaas')) { + if (!prTitle.includes('@devrev/airsync-sdk')) { console.log('Skipping PR: Not a DevRev SDK update'); return; } diff --git a/.gitignore b/.gitignore index c7b88a4..0647ebd 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ # Local deploy logs (written by code/scripts/deploy.sh) code/logs + +.airsync/ diff --git a/README.md b/README.md index 1553410..d00ef3c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ This repository provides a template to implement an AirSync snap-in that imports See the [AirSync snap-in documentation](https://developer.devrev.ai/airsync) for development guidance. +The template uses `@devrev/airsync-sdk` and requires Node.js 20 or newer. + ## Local Fixture Testing From the `code/` directory, you can run the built-in fixture runner to test the @@ -26,3 +28,5 @@ npm start -- --fixturePath start_loading_data The runner reads `code/.env` and resolves `${TODO_API_KEY}` in fixture files, so copy `code/.env.example` before running the fixtures. +The fixture runner and local test server set `AIRSYNC_LOCAL_DEVELOPMENT=1` +to enable local logging and artifact copies in `code/extracted_files/`. diff --git a/code/package-lock.json b/code/package-lock.json index 44199ab..b0fdb47 100644 --- a/code/package-lock.json +++ b/code/package-lock.json @@ -9,7 +9,7 @@ "version": "1.1.6", "license": "ISC", "dependencies": { - "@devrev/ts-adaas": "1.20.2", + "@devrev/airsync-sdk": "2.0.0", "@devrev/typescript-sdk": "1.1.78", "axios": "^1.20.0", "dotenv": "^16.0.3", @@ -51,6 +51,9 @@ "typescript": "^4.9.5", "typescript-eslint": "8.39.0", "yargs": "^17.6.2" + }, + "engines": { + "node": ">=20" } }, "node_modules/@babel/code-frame": { @@ -1859,6 +1862,7 @@ "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "devOptional": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "0.3.9" @@ -1871,25 +1875,34 @@ "version": "0.3.9", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "devOptional": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, - "node_modules/@devrev/ts-adaas": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/@devrev/ts-adaas/-/ts-adaas-1.20.2.tgz", - "integrity": "sha512-GoS/S1QLFxaMF12U+Fci/NBrNv1M0xNNotNqUkhfoBHN9oWcLeYeMuNpph/tXb/O0NS5jRC2syZmIPqt24Zffw==", + "node_modules/@devrev/airsync-sdk": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@devrev/airsync-sdk/-/airsync-sdk-2.0.0.tgz", + "integrity": "sha512-w7VQ5zefo1vzhHeVS5aOHajDPH8r44nuqfwgAf69G0lUY7mU+zhVCZjhPiY4W8SPrtxkid/j2LrzoUj+GmkKdQ==", "license": "ISC", "dependencies": { - "@devrev/typescript-sdk": "^1.1.78", - "axios": "^1.18.0", + "axios": "^1.20.0", "axios-retry": "^4.5.0", "form-data": "^4.0.4", - "js-jsonl": "^1.1.1", - "ts-node": "^10.9.2", - "yargs": "^17.7.3" + "js-jsonl": "^1.1.1" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "ts-node": "^10.9.2" + }, + "peerDependenciesMeta": { + "ts-node": { + "optional": true + } } }, "node_modules/@devrev/typescript-sdk": { @@ -2589,6 +2602,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "devOptional": true, "license": "MIT", "engines": { "node": ">=6.0.0" @@ -2598,6 +2612,7 @@ "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "devOptional": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { @@ -2756,24 +2771,28 @@ "version": "1.0.12", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "devOptional": true, "license": "MIT" }, "node_modules/@tsconfig/node12": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "devOptional": true, "license": "MIT" }, "node_modules/@tsconfig/node14": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "devOptional": true, "license": "MIT" }, "node_modules/@tsconfig/node16": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "devOptional": true, "license": "MIT" }, "node_modules/@types/babel__core": { @@ -3293,6 +3312,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "devOptional": true, "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -3315,6 +3335,7 @@ "version": "8.3.5", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", + "devOptional": true, "license": "MIT", "dependencies": { "acorn": "^8.11.0" @@ -3410,6 +3431,7 @@ "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "devOptional": true, "license": "MIT" }, "node_modules/argparse": { @@ -4260,6 +4282,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "devOptional": true, "license": "MIT" }, "node_modules/cross-spawn": { @@ -4460,6 +4483,7 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", + "devOptional": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -7652,6 +7676,7 @@ "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "devOptional": true, "license": "ISC" }, "node_modules/makeerror": { @@ -9758,6 +9783,7 @@ "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "devOptional": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "^0.8.0", @@ -9965,6 +9991,7 @@ "version": "4.9.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -10434,6 +10461,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "devOptional": true, "license": "MIT" }, "node_modules/v8-to-istanbul": { @@ -10678,6 +10706,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "devOptional": true, "license": "MIT", "engines": { "node": ">=6" diff --git a/code/package.json b/code/package.json index 07b6cc3..451f86a 100644 --- a/code/package.json +++ b/code/package.json @@ -58,12 +58,15 @@ "yargs": "^17.6.2" }, "dependencies": { - "@devrev/ts-adaas": "1.20.2", + "@devrev/airsync-sdk": "2.0.0", "@devrev/typescript-sdk": "1.1.78", "axios": "^1.20.0", "dotenv": "^16.0.3", "jiti": "2.5.1", "js-jsonl": "^1.1.1", "yargs": "^17.6.2" + }, + "engines": { + "node": ">=20" } } diff --git a/code/scripts/deploy.sh b/code/scripts/deploy.sh index ad7ba75..752b467 100755 --- a/code/scripts/deploy.sh +++ b/code/scripts/deploy.sh @@ -413,7 +413,7 @@ if [ "$DEPLOY_MODE" = "local" ]; then echo "" cd "$CODE_DIR" - npm run test:server -- local 2>&1 | tee "$LOG_FILE" + AIRSYNC_LOCAL_DEVELOPMENT=1 npm run test:server 2>&1 | tee "$LOG_FILE" fi # LAMBDA DEPLOYMENT diff --git a/code/src/functions/external-system/data-denormalization.ts b/code/src/functions/external-system/data-denormalization.ts index 3ea0cfb..7949f8d 100644 --- a/code/src/functions/external-system/data-denormalization.ts +++ b/code/src/functions/external-system/data-denormalization.ts @@ -1,4 +1,4 @@ -import { ExternalSystemAttachment, ExternalSystemItem } from '@devrev/ts-adaas'; +import { ExternalSystemAttachment, ExternalSystemItem } from '@devrev/airsync-sdk'; import { ExternalAttachment, ExternalTodo } from './types'; export function denormalizeTodo(item: ExternalSystemItem): ExternalTodo { diff --git a/code/src/functions/external-system/data-normalization.ts b/code/src/functions/external-system/data-normalization.ts index 6ea7d54..06a8281 100644 --- a/code/src/functions/external-system/data-normalization.ts +++ b/code/src/functions/external-system/data-normalization.ts @@ -1,4 +1,4 @@ -import { ExternalSyncUnit, NormalizedAttachment, NormalizedItem } from '@devrev/ts-adaas'; +import { ExternalSyncUnit, NormalizedAttachment, NormalizedItem } from '@devrev/airsync-sdk'; import { ExternalAttachment, ExternalTodo, ExternalTodoList, ExternalUser } from './types'; // TODO: Replace with your actual normalization functions that will be used to diff --git a/code/src/functions/external-system/http-client.ts b/code/src/functions/external-system/http-client.ts index c729d68..bbcd937 100644 --- a/code/src/functions/external-system/http-client.ts +++ b/code/src/functions/external-system/http-client.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, ExternalSystemItemLoadingResponse } from '@devrev/ts-adaas'; +import { AirSyncEvent, ExternalSystemItemLoadingResponse } from '@devrev/airsync-sdk'; import { ExternalAttachment, ExternalTodo, ExternalTodoList, ExternalUser } from './types'; // --------------------------------------------------------------------------- @@ -85,7 +85,7 @@ export class HttpClient { private apiEndpoint: string; private apiToken: string; - constructor(event: AirdropEvent) { + constructor(event: AirSyncEvent) { // TODO: Replace with the API endpoint of the external system. This is // passed through the event payload (e.g. event.payload.connection_data.org_id // or a keyring subdomain field). @@ -120,26 +120,26 @@ export class HttpClient { } // TODO: Replace with an actual API call that creates a todo in the external system. - // The function must return { id } on success or { error } on failure. + // Return a status with an id on success or an error with a message on failure. async createTodo(todo: ExternalTodo): Promise { const id = `todo-${Date.now()}`; todosStore.set(id, { ...todo, id, modified_date: new Date().toISOString() }); - return { id }; + return { status: 'success', id }; } // TODO: Replace with an actual API call that updates a todo in the external system. - // The function must return { id } on success or { error } on failure. + // Return a status with an id on success or an error with a message on failure. async updateTodo(todo: ExternalTodo): Promise { if (!todosStore.has(todo.id)) { - return { error: `Todo with id "${todo.id}" not found in external system.` }; + return { status: 'error', error: { message: `Todo with id "${todo.id}" not found in external system.` } }; } todosStore.set(todo.id, { ...todo, modified_date: new Date().toISOString() }); - return { id: todo.id }; + return { status: 'success', id: todo.id }; } // TODO: Replace with an actual API call that creates an attachment in the external system. - // The function must return { id } on success or { error } on failure. + // Return a status with an id on success or an error with a message on failure. async createAttachment(attachment: ExternalAttachment): Promise { - return { id: attachment.id }; + return { status: 'success', id: attachment.id }; } } diff --git a/code/src/functions/extraction/index.ts b/code/src/functions/extraction/index.ts index 4bfa21d..6694f6f 100644 --- a/code/src/functions/extraction/index.ts +++ b/code/src/functions/extraction/index.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '@devrev/ts-adaas'; +import { AirSyncEvent, spawn } from '@devrev/airsync-sdk'; import initialDomainMapping from '../external-system/initial_domain_mapping.json'; @@ -19,7 +19,7 @@ export const initialExtractorState: ExtractorState = { attachments: { completed: false }, }; -const run = async (events: AirdropEvent[]) => { +const run = async (events: AirSyncEvent[]) => { for (const event of events) { await spawn({ event, diff --git a/code/src/functions/extraction/workers/attachments-extraction.ts b/code/src/functions/extraction/workers/attachments-extraction.ts index c7a3448..26d1a0c 100644 --- a/code/src/functions/extraction/workers/attachments-extraction.ts +++ b/code/src/functions/extraction/workers/attachments-extraction.ts @@ -3,9 +3,8 @@ import axios from 'axios'; import { ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, - ExtractorEventType, - processTask, -} from '@devrev/ts-adaas'; + processExtractionTask, +} from '@devrev/airsync-sdk'; // TODO: Replace with function for fetching attachment streams from the // external system. This function should return either a stream of the @@ -18,28 +17,33 @@ async function getFileStream({ try { const fileStreamResponse = await axios.get(url, { responseType: 'stream', + timeout: 30000, headers: { 'Accept-Encoding': 'identity', - timeout: 30000, }, }); - return { httpStream: fileStreamResponse }; + return { status: 'success', httpStream: fileStreamResponse }; } catch (error) { if (axios.isAxiosError(error)) { if (error?.response?.status === 429) { - const retryAfter = error.response?.headers['retry-after']; - return { delay: retryAfter }; - } else { - return { - error: { - message: `Error while fetching attachment ${id} from URL. Error code: ${error.response?.status}. Error message: ${error.response?.data.message}.`, - }, - }; + const retryAfter = String(error.response?.headers['retry-after'] ?? '').trim(); + const delaySeconds = Number(retryAfter); + if (retryAfter && Number.isFinite(delaySeconds) && delaySeconds >= 0) { + return { status: 'delay', delaySeconds }; + } } + + return { + status: 'error', + error: { + message: `Error while fetching attachment ${id} from URL. Error code: ${error.response?.status}. Error message: ${error.response?.data?.message || error.message}.`, + }, + }; } return { + status: 'error', error: { message: `Unknown error while fetching attachment ${id} from URL. Error: ${error}.`, }, @@ -47,33 +51,19 @@ async function getFileStream({ } } -processTask({ +processExtractionTask({ task: async ({ adapter }) => { try { - const response = await adapter.streamAttachments({ + return await adapter.streamAttachments({ stream: getFileStream, // TODO: If needed you can specify how many attachments to stream at // once. Minimum is 1 and maximum is 50. // batchSize: 10, }); - - if (response?.delay) { - await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { - delay: response.delay, - }); - } else if (response?.error) { - await adapter.emit(ExtractorEventType.AttachmentExtractionError, { - error: response.error, - }); - } else { - await adapter.emit(ExtractorEventType.AttachmentExtractionDone); - } } catch (error) { console.error('An error occured while processing a task.', error); + throw error; } }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.AttachmentExtractionProgress); - }, }); diff --git a/code/src/functions/extraction/workers/data-extraction.ts b/code/src/functions/extraction/workers/data-extraction.ts index 1f0d055..5c4b88d 100644 --- a/code/src/functions/extraction/workers/data-extraction.ts +++ b/code/src/functions/extraction/workers/data-extraction.ts @@ -1,4 +1,4 @@ -import { ExtractorEventType, processTask } from '@devrev/ts-adaas'; +import { processExtractionTask } from '@devrev/airsync-sdk'; import { normalizeAttachment, normalizeTodo, normalizeUser } from '../../external-system/data-normalization'; import { HttpClient } from '../../external-system/http-client'; @@ -48,7 +48,7 @@ const itemTypesToExtract: ItemTypeToExtract[] = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -60,9 +60,8 @@ processTask({ // system. This is just an example how you can iterate over the item types, // extract them, push them to the repo, and save the state. for (const itemTypeToExtract of itemTypesToExtract) { - // If the worker is about to time out, exit early so that `onTimeout` can run and emit progress. if (adapter.isTimeout) { - return; + return { status: 'progress' }; } if (!adapter.shouldExtract(itemTypeToExtract.name)) { @@ -75,9 +74,6 @@ processTask({ adapter.state[itemTypeToExtract.name].completed = true; } - await adapter.emit(ExtractorEventType.DataExtractionDone); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + return { status: 'success' }; }, }); diff --git a/code/src/functions/extraction/workers/external-sync-units-extraction.ts b/code/src/functions/extraction/workers/external-sync-units-extraction.ts index a3c8280..db743d9 100644 --- a/code/src/functions/extraction/workers/external-sync-units-extraction.ts +++ b/code/src/functions/extraction/workers/external-sync-units-extraction.ts @@ -1,21 +1,19 @@ import { AirSyncDefaultItemTypes, ExternalSyncUnit, - ExtractorEventType, - processTask, -} from '@devrev/ts-adaas'; + processExtractionTask, +} from '@devrev/airsync-sdk'; import { normalizeTodoList } from '../../external-system/data-normalization'; import { HttpClient } from '../../external-system/http-client'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos([ { itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { + overriddenOptions: { batchSize: 25000, - skipConfirmation: true, }, }, ]); @@ -36,13 +34,6 @@ processTask({ .getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS) ?.push(externalSyncUnits); - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionError, { - error: { - message: 'Failed to extract external sync units. Lambda timeout.', - }, - }); + return { status: 'success' }; }, }); diff --git a/code/src/functions/extraction/workers/metadata-extraction.ts b/code/src/functions/extraction/workers/metadata-extraction.ts index 2d3b0b4..1791495 100644 --- a/code/src/functions/extraction/workers/metadata-extraction.ts +++ b/code/src/functions/extraction/workers/metadata-extraction.ts @@ -1,4 +1,4 @@ -import { ExtractorEventType, processTask } from '@devrev/ts-adaas'; +import { processExtractionTask } from '@devrev/airsync-sdk'; import staticExternalDomainMetadata from '../../external-system/external_domain_metadata.json'; @@ -8,7 +8,7 @@ const repos = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -21,11 +21,6 @@ processTask({ }; await adapter.getRepo('external_domain_metadata')?.push([externalDomainMetadata]); - await adapter.emit(ExtractorEventType.MetadataExtractionDone); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - error: { message: 'Failed to extract metadata. Lambda timeout.' }, - }); + return { status: 'success' }; }, }); diff --git a/code/src/functions/loading/index.ts b/code/src/functions/loading/index.ts index 040c6dd..4a65edf 100644 --- a/code/src/functions/loading/index.ts +++ b/code/src/functions/loading/index.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '@devrev/ts-adaas'; +import { AirSyncEvent, spawn } from '@devrev/airsync-sdk'; import initialDomainMapping from '../external-system/initial_domain_mapping.json'; @@ -11,7 +11,7 @@ export interface LoaderState {} // This state will be used as a starting point for the loading process. export const initialLoaderState: LoaderState = {}; -const run = async (events: AirdropEvent[]) => { +const run = async (events: AirSyncEvent[]) => { for (const event of events) { await spawn({ event, diff --git a/code/src/functions/loading/workers/load-attachments.ts b/code/src/functions/loading/workers/load-attachments.ts index b8e988e..357d5d2 100644 --- a/code/src/functions/loading/workers/load-attachments.ts +++ b/code/src/functions/loading/workers/load-attachments.ts @@ -1,9 +1,8 @@ import { ExternalSystemAttachment, ExternalSystemItemLoadingParams, - LoaderEventType, - processTask, -} from '@devrev/ts-adaas'; + processLoadingTask, +} from '@devrev/airsync-sdk'; import { denormalizeAttachment } from '../../external-system/data-denormalization'; import { HttpClient } from '../../external-system/http-client'; @@ -13,7 +12,7 @@ import { LoaderState } from '../index'; // TODO: Replace with your create function that will be used to make API calls // to the external system to create a new attachment. Function must return -// object with id or error depending on the response from the external system. +// a status with an id or error depending on the response from the external system. async function createAttachment({ item, mappers, event }: ExternalSystemItemLoadingParams) { // TODO: Replace with your HTTP client that will be used to make API calls // to the external system. @@ -23,21 +22,10 @@ async function createAttachment({ item, mappers, event }: ExternalSystemItemLoad return createAttachmentResponse; } -processTask({ +processLoadingTask({ task: async ({ adapter }) => { - const { reports, processed_files } = await adapter.loadAttachments({ + return adapter.loadAttachments({ create: createAttachment, }); - - await adapter.emit(LoaderEventType.AttachmentLoadingDone, { - reports, - processed_files, - }); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(LoaderEventType.AttachmentLoadingProgress, { - reports: adapter.reports, - processed_files: adapter.processedFiles, - }); }, }); diff --git a/code/src/functions/loading/workers/load-data.ts b/code/src/functions/loading/workers/load-data.ts index a7849fe..8c88629 100644 --- a/code/src/functions/loading/workers/load-data.ts +++ b/code/src/functions/loading/workers/load-data.ts @@ -2,9 +2,8 @@ import { ExternalSystemItem, ExternalSystemItemLoadingParams, ExternalSystemItemLoadingResponse, - LoaderEventType, - processTask, -} from '@devrev/ts-adaas'; + processLoadingTask, +} from '@devrev/airsync-sdk'; import { denormalizeTodo } from '../../external-system/data-denormalization'; import { HttpClient } from '../../external-system/http-client'; @@ -13,8 +12,8 @@ import { LoaderState } from '../index'; /* eslint-disable @typescript-eslint/no-unused-vars */ // TODO: Replace with your create function that will be used to make API calls -// to the external system to create a new item. Function must return object with -// id, error or delay depending on the response from the external system. +// to the external system to create a new item. Return a status with an id, +// error or delaySeconds depending on the response from the external system. async function createTodo({ item, mappers, @@ -30,9 +29,8 @@ async function createTodo({ } // TODO: Replace with your update function that will be used to make API calls -// to the external system to update an existing item. Function must return -// object with id, error or delay depending on the response from the external -// system. +// to the external system to update an existing item. Return a status with an +// id, error or delaySeconds depending on the response from the external system. async function updateTodo({ item, mappers, @@ -50,7 +48,7 @@ async function updateTodo({ // sync_unit: event.payload.event_context.sync_unit, // target: item.id.devrev, // }); - // const todoExternalId = syncMapperRecordResponse.data.sync_mapper_record.external_ids[0]; + // const todoExternalId = syncMapperRecordResponse?.sync_mapper_record.external_ids[0]; const todo = denormalizeTodo(item); @@ -58,9 +56,9 @@ async function updateTodo({ return updateTodoResponse; } -processTask({ +processLoadingTask({ task: async ({ adapter }) => { - const { reports, processed_files } = await adapter.loadItemTypes({ + return adapter.loadItemTypes({ itemTypesToLoad: [ { itemType: 'todos', @@ -69,16 +67,5 @@ processTask({ }, ], }); - - await adapter.emit(LoaderEventType.DataLoadingDone, { - reports, - processed_files, - }); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(LoaderEventType.DataLoadingProgress, { - reports: adapter.reports, - processed_files: adapter.processedFiles, - }); }, }); diff --git a/code/src/test-runner/test-runner.ts b/code/src/test-runner/test-runner.ts index 5182dd3..69ef529 100644 --- a/code/src/test-runner/test-runner.ts +++ b/code/src/test-runner/test-runner.ts @@ -2,7 +2,7 @@ import * as dotenv from 'dotenv'; import * as fs from 'fs'; import * as path from 'path'; -import { AirdropEvent, createMockEvent, DeepPartial, EventType, MockServer } from '@devrev/ts-adaas'; +import { AirSyncEvent, createMockEvent, DeepPartial, EventType, MockServer } from '@devrev/airsync-sdk'; import { functionFactory, FunctionFactoryType } from '../function-factory'; @@ -72,7 +72,7 @@ async function runWithFixtureDir(fixturesDir: string, functionName?: FunctionFac const statePath = path.join(fixturesDir, 'state.json'); const extractionScopePath = path.join(fixturesDir, 'extraction_scope.json'); - const fixtureEvent = readFixtureFile>(eventPath); + const fixtureEvent = readFixtureFile>(eventPath); const fixtureState = readFixtureFile>(statePath); const fixtureExtractionScope = readFixtureFile>(extractionScopePath); @@ -140,7 +140,7 @@ async function runWithFixtureDir(fixturesDir: string, functionName?: FunctionFac const event = createMockEvent(mockServer.baseUrl, fixtureEvent); - process.argv.push('--local'); + process.env.AIRSYNC_LOCAL_DEVELOPMENT = '1'; const run = functionFactory[resolvedFunctionName]; diff --git a/code/test/main.ts b/code/test/main.ts index 8ff9ef3..eb41301 100644 --- a/code/test/main.ts +++ b/code/test/main.ts @@ -25,5 +25,6 @@ import { startServer } from './runner'; }).argv; const port = argv.port || 8000; + process.env.AIRSYNC_LOCAL_DEVELOPMENT = '1'; startServer(port); })();