diff --git a/.depcheckrc.json b/.depcheckrc.json new file mode 100644 index 00000000..99146461 --- /dev/null +++ b/.depcheckrc.json @@ -0,0 +1,3 @@ +{ + "ignorePatterns": ["e2e"] +} diff --git a/e2e/helpers/.gitattributes b/e2e/helpers/.gitattributes new file mode 100644 index 00000000..ab59db08 --- /dev/null +++ b/e2e/helpers/.gitattributes @@ -0,0 +1,5 @@ +# Synced from serverless-ci/e2e/shared -- generated by the e2e-shared sync. Collapsed in diffs. +exec.ts linguist-generated=true +lambda-telemetry-checker.ts linguist-generated=true +lambda-verifier.ts linguist-generated=true +naming.ts linguist-generated=true diff --git a/e2e/helpers/exec.ts b/e2e/helpers/exec.ts new file mode 100644 index 00000000..58941747 --- /dev/null +++ b/e2e/helpers/exec.ts @@ -0,0 +1,90 @@ +// Code generated by serverless-ci e2e-shared sync. DO NOT EDIT. +// Source of truth lives in serverless-ci/e2e/shared -- edit there; local changes are overwritten. + +/* + * Unless explicitly stated otherwise all files in this repository are licensed + * under the Apache License Version 2.0. + * + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026 Datadog, Inc. + */ + +import child_process from 'node:child_process'; + +// Runner-agnostic exec + bounded-retry helper: no jest/vitest imports, so it backs any +// test runner. "Retry the cloud, not the assertions": only transient provider errors are +// retried, on a bounded budget; a real failure surfaces immediately. + +export interface ExecResult { + exitCode: number; + stdout: string; + stderr: string; +} + +export interface ExecOptions { + env?: Record; + cwd?: string; + // Serverless / AWS / CFN calls can emit large output; default generous but bounded. + // CFN deploys need ~16MB; sls/cdk need more, so callers raise it as needed. + maxBuffer?: number; + // Transient cloud-provider error substrings safe to retry. Supplied by the caller so + // each cloud/tool contributes its own patterns; empty means never retry. + retryPatterns?: string[]; + maxAttempts?: number; + delaySeconds?: number; +} + +const DEFAULT_MAX_BUFFER = 50 * 1024 * 1024; +const DEFAULT_MAX_ATTEMPTS = 3; +const DEFAULT_DELAY_SECONDS = 10; + +export const execPromise = async (command: string, options: ExecOptions = {}): Promise => { + const {env, cwd, maxBuffer = DEFAULT_MAX_BUFFER} = options; + + return new Promise((resolve) => { + child_process.exec(command, {env: {...process.env, ...env}, cwd, maxBuffer}, (error, stdout, stderr) => { + resolve({ + exitCode: error ? (typeof error.code === 'number' ? error.code : 1) : 0, + stdout: stdout.trim(), + stderr: stderr.trim(), + }); + }); + }); +}; + +const isRetryable = (result: ExecResult, patterns: string[]): boolean => { + const output = `${result.stdout} ${result.stderr}`; + + return patterns.some((pattern) => output.includes(pattern)); +}; + +const waitFor = (seconds: number): Promise => new Promise((resolve) => setTimeout(resolve, seconds * 1000)); + +export const execPromiseWithRetries = async (command: string, options: ExecOptions = {}): Promise => { + const { + retryPatterns = [], + maxAttempts = DEFAULT_MAX_ATTEMPTS, + delaySeconds = DEFAULT_DELAY_SECONDS, + } = options; + + let result: ExecResult = {exitCode: 1, stdout: '', stderr: 'not run'}; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + result = await execPromise(command, options); + if (result.exitCode === 0) { + return result; + } + if (attempt < maxAttempts && isRetryable(result, retryPatterns)) { + // eslint-disable-next-line no-console + console.log(`Command failed with retryable error (attempt ${attempt}/${maxAttempts}), retrying in ${delaySeconds}s...`); + // eslint-disable-next-line no-console + console.log(`stdout: ${result.stdout}`); + // eslint-disable-next-line no-console + console.log(`stderr: ${result.stderr}`); + await waitFor(delaySeconds); + } else { + return result; + } + } + + return result; +}; diff --git a/e2e/helpers/lambda-telemetry-checker.ts b/e2e/helpers/lambda-telemetry-checker.ts new file mode 100644 index 00000000..0954909e --- /dev/null +++ b/e2e/helpers/lambda-telemetry-checker.ts @@ -0,0 +1,218 @@ +// Code generated by serverless-ci e2e-shared sync. DO NOT EDIT. +// Source of truth lives in serverless-ci/e2e/shared -- edit there; local changes are overwritten. + +/* + * Unless explicitly stated otherwise all files in this repository are licensed + * under the Apache License Version 2.0. + * + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026 Datadog, Inc. + */ + +import {client, v2} from '@datadog/datadog-api-client'; + +import {RUN_ID_TAG_KEY} from './naming'; + +// Runner-agnostic telemetry poller. Mirrors the datadog-ci reference +// (cloud-run-telemetry-checker.ts): poll spans + logs on a bounded budget, then assert +// *identity* on the matched records, not mere existence. + +const POLL_INTERVAL_SECONDS = 15; +const MAX_ATTEMPTS = 20; + +// A single ingested span or log, flattened to the fields we assert on -- mirrors the Go +// shared Event: top-level string attributes plus "key:value" tag strings. +interface ParsedEvent { + attrs: Record; + tags: string[]; +} + +interface IdentityTag { + key: string; + value: string; +} + +// Flatten a raw span/log record into attrs + tags. Reserved fields (service/env/version) +// sit at the top of `attributes`; logs nest their structured attributes one level deeper; +// tags arrive as a "key:value" string array. +const parseEvent = (record: unknown): ParsedEvent => { + const attrs: Record = {}; + const tags: string[] = []; + const attributes = (record as {attributes?: Record})?.attributes; + if (attributes && typeof attributes === 'object') { + for (const key of ['service', 'env', 'version']) { + const value = attributes[key]; + if (typeof value === 'string' && value !== '') { + attrs[key] = value; + } + } + const nested = attributes.attributes; + if (nested && typeof nested === 'object') { + for (const [key, value] of Object.entries(nested)) { + if (typeof value === 'string') { + attrs[key] = value; + } + } + } + if (Array.isArray(attributes.tags)) { + for (const tag of attributes.tags) { + if (typeof tag === 'string') { + tags.push(tag); + } + } + } + } + + return {attrs, tags}; +}; + +// Assert key=value as a structured attribute or a "key:value" tag -- identity, not a +// substring match against the serialized blob. +const has = (event: ParsedEvent, key: string, value: string): boolean => + event.attrs[key] === value || event.tags.includes(`${key}:${value}`); + +const identityLabel = (identity: IdentityTag[]): string => identity.map(({key, value}) => `${key}:${value}`).join(', '); + +// Auth failures (bad/missing DATADOG_API_KEY/DATADOG_APP_KEY) are non-retryable: polling +// won't fix credentials, so surface them immediately instead of burning the full budget. +// The datadog-api-client throws ApiException with a numeric `code` carrying the status. +const authErrorCode = (error: unknown): number | undefined => { + const code = (error as {code?: unknown})?.code; + + return code === 401 || code === 403 ? code : undefined; +}; + +const waitFor = (seconds: number): Promise => new Promise((resolve) => setTimeout(resolve, seconds * 1000)); + +const buildConfiguration = (): client.Configuration => { + const configuration = client.createConfiguration({ + authMethods: { + apiKeyAuth: process.env.DATADOG_API_KEY ?? process.env.DD_API_KEY, + appKeyAuth: process.env.DATADOG_APP_KEY ?? process.env.DD_APP_KEY, + }, + }); + const site = process.env.DATADOG_SITE ?? process.env.DD_SITE; + if (site) { + configuration.setServerVariables({site}); + } + + return configuration; +}; + +// Poll until at least one returned record carries every identity marker. We filter +// in-process (rather than trusting the query alone) so a stray record that merely +// matches the service filter can't pass for one stamped with the full identity. +const pollUntilIdentity = async ( + label: string, + query: () => Promise, + identity: IdentityTag[], +): Promise => { + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + // eslint-disable-next-line no-console + console.log(`[${label}] attempt ${attempt}/${MAX_ATTEMPTS}`); + try { + const results = await query(); + const matching = results.filter((record) => { + const event = parseEvent(record); + + return identity.every(({key, value}) => has(event, key, value)); + }); + if (matching.length > 0) { + // eslint-disable-next-line no-console + console.log(`[${label}] found ${matching.length} record(s) with identity [${identityLabel(identity)}]`); + + return; + } + } catch (error) { + const code = authErrorCode(error); + if (code !== undefined) { + throw new Error( + `[${label}] authentication failed (HTTP ${code}) -- check DATADOG_API_KEY/DATADOG_APP_KEY ` + + `and DATADOG_SITE; not retrying`, + ); + } + // eslint-disable-next-line no-console + console.error(`[${label}] query error:`, error); + } + + if (attempt < MAX_ATTEMPTS) { + await waitFor(POLL_INTERVAL_SECONDS); + } + } + throw new Error( + `[${label}] timed out after ${MAX_ATTEMPTS} attempts (${MAX_ATTEMPTS * POLL_INTERVAL_SECONDS}s) ` + + `waiting for telemetry with identity [${identityLabel(identity)}]`, + ); +}; + +const recentWindow = (): {from: string; to: string} => { + const now = new Date(); + const from = new Date(now.getTime() - 15 * 60 * 1000); + + return {from: from.toISOString(), to: now.toISOString()}; +}; + +const querySpans = async (configuration: client.Configuration, serviceName: string): Promise => { + const api = new v2.SpansApi(configuration); + const {from, to} = recentWindow(); + const response = await api.listSpans({ + body: { + data: { + attributes: { + filter: {query: `@service:${serviceName}`, from, to}, + page: {limit: 25}, + }, + type: 'search_request', + }, + }, + }); + + return response.data ?? []; +}; + +const queryLogs = async (configuration: client.Configuration, serviceName: string): Promise => { + const api = new v2.LogsApi(configuration); + const {from, to} = recentWindow(); + const response = await api.listLogs({ + body: { + filter: {query: `service:${serviceName}`, from, to}, + page: {limit: 25}, + }, + }); + + return response.data ?? []; +}; + +export interface TelemetryIdentity { + serviceName: string; + env: string; + version: string; + runId: string; + // Run-id tag key; defaults to the shared convention. Override only if a repo diverges. + runIdTagKey?: string; +} + +export const checkTelemetryFlowing = async ({ + serviceName, + env, + version, + runId, + runIdTagKey = RUN_ID_TAG_KEY, +}: TelemetryIdentity): Promise => { + const configuration = buildConfiguration(); + await Promise.all([ + // Traces carry service + env + version + run-id identity. + pollUntilIdentity('spans', () => querySpans(configuration, serviceName), [ + {key: 'service', value: serviceName}, + {key: 'env', value: env}, + {key: 'version', value: version}, + {key: runIdTagKey, value: runId}, + ]), + // Logs carry service + env + run-id identity. + pollUntilIdentity('logs', () => queryLogs(configuration, serviceName), [ + {key: 'service', value: serviceName}, + {key: 'env', value: env}, + {key: runIdTagKey, value: runId}, + ]), + ]); +}; diff --git a/e2e/helpers/lambda-verifier.ts b/e2e/helpers/lambda-verifier.ts new file mode 100644 index 00000000..15692f3d --- /dev/null +++ b/e2e/helpers/lambda-verifier.ts @@ -0,0 +1,182 @@ +// Code generated by serverless-ci e2e-shared sync. DO NOT EDIT. +// Source of truth lives in serverless-ci/e2e/shared -- edit there; local changes are overwritten. + +/* + * Unless explicitly stated otherwise all files in this repository are licensed + * under the Apache License Version 2.0. + * + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026 Datadog, Inc. + */ + +import assert from 'node:assert/strict'; + +import {execPromise} from './exec'; +import {FRESHNESS_TAG_KEY} from './naming'; + +// Runner-agnostic Lambda config verifier: drives the AWS CLI and asserts the deployed +// function's identity with node:assert. "Config present" for Lambda (per spec) = DD +// layers + extension layer + DD_* env vars + tags. The parts that vary per tool (the +// CDK construct vs CFN macro vs sls plugin) are supplied through LambdaVerifierConfig, +// so this file is identical across the Lambda TS repos. + +interface LambdaLayer { + Arn: string; +} +interface LambdaConfiguration { + FunctionArn: string; + Handler: string; + Runtime: string; + Layers?: LambdaLayer[]; + Environment?: {Variables?: Record}; +} + +export interface FunctionSnapshot { + handler: string; + layerArns: string[]; + ddEnv: Record; +} + +export interface ExpectedLayers { + node: string; + extension: string; +} + +export interface LambdaVerifierConfig { + // functionName maps the run-unique service name to the deployed function name, which + // differs per tool (e.g. sls deploy => `-e2e-hello`). + functionName: (serviceName: string) => string; + // expectedLayerArns resolves the pinned node + extension layer ARNs for a region, + // read from whatever source the tool pins them in (e.g. the plugin's src/layers.json). + expectedLayerArns: (region: string) => ExpectedLayers; + // redirectHandler is the wrapper handler the tool redirects to. + redirectHandler: string; + // originalHandler, when set, is the expected DD_LAMBDA_HANDLER value (the preserved + // user handler). Omit to skip the double-wrap check. + originalHandler?: string; + // toolTag is the marker tag the tool applies; pattern asserts its shape (e.g. version). + toolTag: {key: string; pattern: RegExp}; + // env identity. apiKeyVars: at least one must be set. present: must be set (any value). + // values: must equal exactly (service/env/version identity). + env: { + apiKeyVars: string[]; + present: string[]; + values: (serviceName: string) => Record; + }; +} + +const getConfiguration = async (fnName: string, region: string): Promise => { + const result = await execPromise( + `aws lambda get-function-configuration --function-name "${fnName}" --region "${region}" --output json`, + ); + assert.equal(result.exitCode, 0, `get-function-configuration failed: ${result.stderr}`); + + return JSON.parse(result.stdout) as LambdaConfiguration; +}; + +const getTags = async (functionArn: string, region: string): Promise> => { + const result = await execPromise( + `aws lambda list-tags --resource "${functionArn}" --region "${region}" --output json`, + ); + assert.equal(result.exitCode, 0, `list-tags failed: ${result.stderr}`); + + return (JSON.parse(result.stdout).Tags ?? {}) as Record; +}; + +// A normalized view used to assert idempotency (re-apply produces no diff). +export const functionSnapshot = async ( + config: LambdaVerifierConfig, + serviceName: string, + region: string, +): Promise => { + const lambda = await getConfiguration(config.functionName(serviceName), region); + const vars = lambda.Environment?.Variables ?? {}; + const ddEnv: Record = {}; + for (const key of Object.keys(vars).sort()) { + if (key.startsWith('DD_')) { + ddEnv[key] = vars[key]; + } + } + + return { + handler: lambda.Handler, + layerArns: (lambda.Layers ?? []).map((l) => l.Arn).sort(), + ddEnv, + }; +}; + +export const verifyInstrumented = async ( + config: LambdaVerifierConfig, + serviceName: string, + region: string, +): Promise => { + const fnName = config.functionName(serviceName); + // eslint-disable-next-line no-console + console.log(`Verifying instrumented state of "${fnName}"...`); + const lambda = await getConfiguration(fnName, region); + const vars = lambda.Environment?.Variables ?? {}; + const layerArns = (lambda.Layers ?? []).map((l) => l.Arn); + const expected = config.expectedLayerArns(region); + + // Handler is redirected to the Datadog wrapper; original preserved (no double-wrap). + assert.equal(lambda.Handler, config.redirectHandler, 'handler not redirected to the Datadog wrapper'); + if (config.originalHandler !== undefined) { + assert.equal( + vars.DD_LAMBDA_HANDLER, + config.originalHandler, + 'DD_LAMBDA_HANDLER should hold the original handler (a different value means a double-wrap)', + ); + } + + // Layers: library layer + extension layer, each present exactly once, pinned. + assert.ok(layerArns.includes(expected.node), `missing pinned Node layer ${expected.node}; got ${JSON.stringify(layerArns)}`); + assert.ok( + layerArns.includes(expected.extension), + `missing pinned extension layer ${expected.extension}; got ${JSON.stringify(layerArns)}`, + ); + assert.equal(layerArns.filter((a) => a === expected.node).length, 1, 'Node layer attached more than once'); + assert.equal(layerArns.filter((a) => a === expected.extension).length, 1, 'extension layer attached more than once'); + + // Env: API key wiring + required DD_* vars + identity. + assert.ok(config.env.apiKeyVars.some((k) => vars[k]), `no API key wiring env var set (one of ${config.env.apiKeyVars.join(', ')})`); + for (const key of config.env.present) { + assert.ok(vars[key], `missing required env var ${key}`); + } + for (const [key, want] of Object.entries(config.env.values(serviceName))) { + assert.equal(vars[key], want, `env var ${key} = ${vars[key]}, want ${want}`); + } + + // Tags: tool marker (proof the tool tagged the function) + freshness tag (set + // atomically at creation for the sweeper). Service/env/version identity is carried by + // the DD_* env vars above and on ingested telemetry. + const tags = await getTags(lambda.FunctionArn, region); + assert.match(tags[config.toolTag.key] ?? '', config.toolTag.pattern, `${config.toolTag.key} tag has unexpected shape`); + assert.ok(tags[FRESHNESS_TAG_KEY], `missing freshness tag ${FRESHNESS_TAG_KEY}`); + + // eslint-disable-next-line no-console + console.log('All instrumented checks passed.'); +}; + +// After remove the whole stack is torn down -- the function itself is gone, which is the +// clean end-state for this mechanism (no per-resource un-instrument). Assert absence. +export const verifyUninstrumented = async ( + config: LambdaVerifierConfig, + serviceName: string, + region: string, +): Promise => { + const fnName = config.functionName(serviceName); + // eslint-disable-next-line no-console + console.log(`Verifying clean (removed) state of "${fnName}"...`); + const result = await execPromise( + `aws lambda get-function-configuration --function-name "${fnName}" --region "${region}" --output json`, + ); + assert.notEqual(result.exitCode, 0, 'function still exists after remove'); + assert.match( + `${result.stdout} ${result.stderr}`, + /ResourceNotFoundException|Function not found/, + `expected ResourceNotFoundException, got: ${result.stderr || result.stdout}`, + ); + + // eslint-disable-next-line no-console + console.log('Clean-state check passed (function and its DD config are gone).'); +}; diff --git a/e2e/helpers/naming.ts b/e2e/helpers/naming.ts new file mode 100644 index 00000000..ea35fa27 --- /dev/null +++ b/e2e/helpers/naming.ts @@ -0,0 +1,35 @@ +// Code generated by serverless-ci e2e-shared sync. DO NOT EDIT. +// Source of truth lives in serverless-ci/e2e/shared -- edit there; local changes are overwritten. + +/* + * Unless explicitly stated otherwise all files in this repository are licensed + * under the Apache License Version 2.0. + * + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026 Datadog, Inc. + */ + +import crypto from 'node:crypto'; + +// Resource-hygiene convention shared across the e2e suites (see spec "Resource +// Hygiene"). The name prefix is the identity + blast-radius guard the sweeper keys on; +// the freshness tag lets it age resources out safely. The tool + platform vary per +// repo, so they are supplied through a config object rather than hardcoded. + +export interface E2ENaming { + tool: string; + platform: string; +} + +// `one` = team marker (`dd-` implied). Prefix is set atomically at creation. +export const namePrefix = (config: E2ENaming, runId: string): string => + `one-e2e-${config.tool}-${config.platform}-${runId}`; + +export const newRunId = (): string => crypto.randomBytes(4).toString('hex'); + +// Freshness tag value. Native creation time isn't usable cross-cloud, so we stamp it +// ourselves at create time. +export const FRESHNESS_TAG_KEY = 'one_e2e_created'; +// Run-id marker. Standardized across every repo so telemetry and resources cross-reference. +export const RUN_ID_TAG_KEY = 'one_e2e_run_id'; +export const freshnessTimestamp = (): string => `${Math.floor(Date.now() / 1000)}`;