diff --git a/.buildkite/ensure-cloud-fixtures.sh b/.buildkite/ensure-cloud-fixtures.sh new file mode 100755 index 00000000..b7f5e138 --- /dev/null +++ b/.buildkite/ensure-cloud-fixtures.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Copyright Elasticsearch B.V. and contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Idempotent QA fixtures so Cloud item GET tests have something to fetch. +# Creates one serverless project per type, one serverless traffic filter, +# one hosted traffic-filter ruleset, and one hosted deployment when the +# matching list is empty. Skips extensions (need a plugin zip). Does not +# print create responses (they can contain creds). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELASTIC=(node "$REPO_ROOT/dist/cli.js" --json) +REGION="${CLOUD_FIXTURE_REGION:-gcp-us-central1}" +RULES='[{"source":"192.0.2.1"}]' + +first_id () { + jq -r ' + if type == "array" then .[0].id // empty + else + .items[0].id + // .projects[0].id + // .filters[0].id + // .rulesets[0].id + // .deployments[0].id + // empty + end + ' +} + +ensure () { + local label="$1" + shift + local list_args=() + while [ "$1" != "--" ]; do + list_args+=("$1") + shift + done + shift + local id + id="$("${ELASTIC[@]}" "${list_args[@]}" | first_id)" + if [ -n "$id" ] && [ "$id" != "null" ]; then + echo "fixture exists: $label $id" + return 0 + fi + echo "creating fixture: $label" + # stdout discarded so create responses never land in CI logs + "${ELASTIC[@]}" "$@" >/dev/null +} + +ensure "search project" \ + cloud serverless projects search list -- \ + cloud serverless projects search create \ + --name cli-functional-search \ + --region-id "$REGION" \ + --optimized-for general_purpose \ + --yes + +ensure "observability project" \ + cloud serverless projects observability list -- \ + cloud serverless projects observability create \ + --name cli-functional-o11y \ + --region-id "$REGION" \ + --product-tier logs_essentials \ + --yes + +ensure "security project" \ + cloud serverless projects security list -- \ + cloud serverless projects security create \ + --name cli-functional-security \ + --region-id "$REGION" \ + --yes + +ensure "serverless traffic filter" \ + cloud serverless traffic-filters list-traffic-filters -- \ + cloud serverless traffic-filters create-traffic-filter \ + --name cli-functional-filter \ + --type ip \ + --region "$REGION" \ + --rules "$RULES" \ + --yes + +ensure "hosted traffic filter ruleset" \ + cloud hosted traffic-filters get-traffic-filter-rulesets -- \ + cloud hosted traffic-filters create-traffic-filter-ruleset \ + --name cli-functional-ruleset \ + --type ip \ + --include-by-default false \ + --region "$REGION" \ + --rules "$RULES" + +ensure "hosted deployment" \ + cloud hosted deployments list-deployments -- \ + cloud hosted deployments create-deployment \ + --name cli-functional \ + --region "$REGION" \ + --template-id gcp-general-purpose diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0583561b..d3998deb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -65,4 +65,3 @@ steps: command: ".buildkite/run-cloud-tests.sh" artifact_paths: - "test/functional/cloud/*.log" - soft_fail: true diff --git a/.buildkite/run-cloud-tests.sh b/.buildkite/run-cloud-tests.sh index 9890b9df..14017057 100755 --- a/.buildkite/run-cloud-tests.sh +++ b/.buildkite/run-cloud-tests.sh @@ -35,6 +35,9 @@ npm run build echo "--- Setting up Cloud credentials" source .buildkite/setup-env.sh +echo "--- Ensuring Cloud fixtures" +.buildkite/ensure-cloud-fixtures.sh + echo "--- Generating Cloud functional tests" npm run codegen:functional:cloud diff --git a/codegen/functional/cloud.ts b/codegen/functional/cloud.ts index 9c07d966..e829b633 100644 --- a/codegen/functional/cloud.ts +++ b/codegen/functional/cloud.ts @@ -67,7 +67,9 @@ for (const file of yamlFiles) { const result = generateScript(testFile, apis, { clientArgs: ['cloud'], - preamble: CLOUD_PREAMBLE + preamble: CLOUD_PREAMBLE, + skipEmptySet: true, + skipNotFound: true }) for (const action of result.skippedActions) allSkippedActions.add(action) diff --git a/codegen/functional/generator.ts b/codegen/functional/generator.ts index 0591fcf0..c4005740 100644 --- a/codegen/functional/generator.ts +++ b/codegen/functional/generator.ts @@ -46,6 +46,18 @@ export interface GenerateOptions { clientArgs?: string[] /** Preamble lines defining the `$ELASTIC` invocation and `$RESPONSE` (default: the `elastic` binary). */ preamble?: string[] + /** + * If a `set` extraction is empty or `null`, try `.[0].id` (bare-array list + * responses) and skip the script when still empty. Used by Cloud tests + * against orgs that have no deployments or projects. + */ + skipEmptySet?: boolean + /** + * If a do-step exits non-zero and the JSON error mentions 404 (or a + * hosted deployment with no version yet), skip the script. Used by Cloud + * tests for endpoints the org or public QA API does not expose. + */ + skipNotFound?: boolean } const DEFAULT_PREAMBLE = ['exec < /dev/null', 'ELASTIC="elastic --json"', 'RESPONSE=""'] @@ -57,6 +69,8 @@ export function generateScript ( ): GenerateResult { const clientArgs = opts.clientArgs ?? ['stack', 'es'] const preamble = opts.preamble ?? DEFAULT_PREAMBLE + const skipEmptySet = opts.skipEmptySet === true + const skipNotFound = opts.skipNotFound === true const actionMap = buildActionMap(definitions) const skippedActions: string[] = [] const lines: string[] = [] @@ -70,7 +84,7 @@ export function generateScript ( if (testFile.teardown.length > 0) { const teardownBody: string[] = [] - renderSteps(testFile.teardown, actionMap, clientArgs, teardownBody, skippedActions, ' ') + renderSteps(testFile.teardown, actionMap, clientArgs, teardownBody, skippedActions, ' ', false, skipEmptySet, skipNotFound) if (!hasExecutableLine(teardownBody)) { teardownBody.push(' :') } @@ -98,13 +112,13 @@ export function generateScript ( if (testFile.setup.length > 0) { lines.push('# --- Setup ---') - hadSkippedDo = renderSteps(testFile.setup, actionMap, clientArgs, lines, skippedActions, '') + hadSkippedDo = renderSteps(testFile.setup, actionMap, clientArgs, lines, skippedActions, '', false, skipEmptySet, skipNotFound) lines.push('') } for (const section of testFile.tests) { lines.push(`# --- Test: ${section.name} ---`) - renderSteps(section.steps, actionMap, clientArgs, lines, skippedActions, '', hadSkippedDo) + renderSteps(section.steps, actionMap, clientArgs, lines, skippedActions, '', hadSkippedDo, skipEmptySet, skipNotFound) lines.push('') } @@ -255,7 +269,9 @@ function renderSteps ( lines: string[], skippedActions: string[], indent: string, - initialHadSkippedDo = false + initialHadSkippedDo = false, + skipEmptySet = false, + skipNotFound = false ): boolean { // Assertions and set-steps read $RESPONSE, which is written by the most // recent successful `do`. If the last `do` was skipped (unmapped action, @@ -279,7 +295,7 @@ function renderSteps ( } // Only pass allowFailure from the setup→test propagation, not from // prior skipped steps within the same section (those are unrelated). - const result = renderDo(step, actionMap, clientArgs, lines, skippedActions, indent, initialHadSkippedDo) + const result = renderDo(step, actionMap, clientArgs, lines, skippedActions, indent, initialHadSkippedDo, skipNotFound) if (result === 'skipped') { responseFromLastDo = false hadSkippedDo = true @@ -310,7 +326,7 @@ function renderSteps ( switch (step.kind) { case 'set': - renderSet(step, lines, indent) + renderSet(step, lines, indent, skipEmptySet) // If a variable was previously unset, it's now set for (const varName of Object.values(step.assignments)) { unsetVars.delete(varName.toUpperCase().replace(/[^A-Z0-9]/g, '_')) @@ -360,7 +376,8 @@ function renderDo ( lines: string[], skippedActions: string[], indent: string, - allowFailure = false + allowFailure = false, + skipNotFound = false ): 'executed' | 'optional' | 'skipped' { if (step.catch != null) { lines.push(`${indent}# SKIPPED: catch not supported in MVP (catch: ${step.catch})`) @@ -384,9 +401,25 @@ function renderDo ( if (optional) { lines.push(`${indent}RESPONSE=$(${cmd}) || true`) return 'optional' - } else { - lines.push(`${indent}RESPONSE=$(${cmd})`) } + if (skipNotFound) { + // Handler errors go to stderr (`writeErr`); keep them out of $RESPONSE. + const errFile = '"${TMPDIR:-/tmp}/elastic-cli-do-err.$$"' + lines.push(`${indent}set +e`) + lines.push(`${indent}RESPONSE=$(${cmd} 2>${errFile})`) + lines.push(`${indent}_ec=$?`) + lines.push(`${indent}set -e`) + lines.push(`${indent}if [ "$_ec" -ne 0 ]; then`) + lines.push(`${indent} if jq -e '(.error.message | tostring | test("404|Could not determine the version"))' ${errFile} >/dev/null 2>&1; then`) + lines.push(`${indent} echo "SKIP: ${step.action} not available"`) + lines.push(`${indent} exit 0`) + lines.push(`${indent} fi`) + lines.push(`${indent} cat ${errFile} >&2`) + lines.push(`${indent} exit $_ec`) + lines.push(`${indent}fi`) + return 'executed' + } + lines.push(`${indent}RESPONSE=$(${cmd})`) return 'executed' } @@ -510,11 +543,26 @@ function buildCommand (mapped: MappedAction, step: DoStep): string { return base } -function renderSet (step: SetStep, lines: string[], indent: string): void { +function renderSet (step: SetStep, lines: string[], indent: string, skipEmptySet = false): void { for (const [responsePath, varName] of Object.entries(step.assignments)) { const bashVar = varName.toUpperCase().replace(/[^A-Z0-9]/g, '_') const jqPath = toJqPath(responsePath) - lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r '${jqPath}')`) + if (skipEmptySet) { + // `try` so a nested path like `.regions[0].id` against a bare array + // yields empty instead of aborting the script (jq cannot index an + // array with a string). + lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r 'try (${jqPath} // empty) catch empty')`) + lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`) + // Serverless lists wrap items in `.items`; regions return a bare array. + lines.push(`${indent} ${bashVar}=$(echo "$RESPONSE" | jq -r 'if type=="array" then .[0].id // empty else .items[0].id // empty end')`) + lines.push(`${indent}fi`) + lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`) + lines.push(`${indent} echo "SKIP: no ${varName} in list response"`) + lines.push(`${indent} exit 0`) + lines.push(`${indent}fi`) + } else { + lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r '${jqPath}')`) + } } } diff --git a/codegen/functional/test/generator.test.ts b/codegen/functional/test/generator.test.ts index 28d42444..abe5b5d6 100644 --- a/codegen/functional/test/generator.test.ts +++ b/codegen/functional/test/generator.test.ts @@ -5,6 +5,7 @@ import { describe, it } from 'node:test' import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { join } from 'node:path' import { parseTestFile } from '../parser.ts' @@ -226,6 +227,87 @@ describe('generateScript', () => { assert.ok(result.script.includes('echo "PASS: get.yml"')) }) + it('skipEmptySet retries bare array id then skips when empty', () => { + const testFile: TestFile = { + sourceFile: 'list.yml', + requires: { serverless: true, stack: true }, + setup: [], + teardown: [], + tests: [{ + name: 'get item', + steps: [ + { kind: 'do', action: 'count', params: { index: 'x' }, body: undefined }, + { kind: 'set', assignments: { 'regions.0.id': 'id' } } + ] + }] + } + const result = generateScript(testFile, testDefs, { skipEmptySet: true }) + assert.ok(result.script.includes('try (.regions[0].id // empty) catch empty')) + assert.ok(result.script.includes('if type=="array" then .[0].id // empty else .items[0].id // empty end')) + assert.ok(result.script.includes('SKIP: no id in list response')) + assert.ok(result.script.includes('exit 0')) + assert.equal( + execFileSync('jq', ['-r', 'try (.regions[0].id // empty) catch empty'], { + input: '[{"id":"r1"}]', + encoding: 'utf8' + }).trim(), + '' + ) + const fallback = 'if type=="array" then .[0].id // empty else .items[0].id // empty end' + assert.equal( + execFileSync('jq', ['-r', fallback], { input: '[{"id":"r1"}]', encoding: 'utf8' }).trim(), + 'r1' + ) + assert.equal( + execFileSync('jq', ['-r', fallback], { input: '{"items":[{"id":"p1"}]}', encoding: 'utf8' }).trim(), + 'p1' + ) + assert.equal( + execFileSync('jq', ['-r', 'try (.deployments[0].id // empty) catch empty'], { + input: '{"deployments":[]}', + encoding: 'utf8' + }).trim(), + '' + ) + }) + + it('does not skip empty set extractions by default', () => { + const testFile: TestFile = { + sourceFile: 'list.yml', + requires: { serverless: true, stack: true }, + setup: [], + teardown: [], + tests: [{ + name: 'get item', + steps: [ + { kind: 'do', action: 'count', params: { index: 'x' }, body: undefined }, + { kind: 'set', assignments: { 'regions.0.id': 'id' } } + ] + }] + } + const result = generateScript(testFile, testDefs) + assert.equal(result.script.includes('SKIP: no id in list response'), false) + assert.equal(result.script.includes('.[0].id // empty'), false) + }) + + it('skipNotFound wraps do-steps to skip 404 errors', () => { + const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8') + const testFile = parseTestFile(content, 'get.yml') + const result = generateScript(testFile, testDefs, { skipNotFound: true }) + assert.ok(result.script.includes('set +e')) + assert.ok(result.script.includes('elastic-cli-do-err.$$')) + assert.ok(result.script.includes('not available')) + assert.ok(result.script.includes('Could not determine the version')) + }) + + it('does not wrap do-steps for 404 by default', () => { + const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8') + const testFile = parseTestFile(content, 'get.yml') + const result = generateScript(testFile, testDefs) + assert.equal(result.script.includes('not available'), false) + assert.equal(result.script.includes('elastic-cli-do-err.$$'), false) + }) + it('tracks skipped actions for unregistered APIs', () => { const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8') const testFile = parseTestFile(content, 'get.yml')