From 0ba2b858c2aed697343236abf07a97ce489d6204 Mon Sep 17 00:00:00 2001 From: margaretjgu Date: Tue, 1 Sep 2026 12:54:21 -0400 Subject: [PATCH 1/6] test: skip missing cloud functional items --- codegen/functional/cloud.ts | 4 +- codegen/functional/generator.ts | 69 +++++++++++++++---- codegen/functional/test/generator.test.ts | 80 +++++++++++++++++++++++ 3 files changed, 141 insertions(+), 12 deletions(-) diff --git a/codegen/functional/cloud.ts b/codegen/functional/cloud.ts index 9c07d966..e829b633 100644 --- a/codegen/functional/cloud.ts +++ b/codegen/functional/cloud.ts @@ -67,7 +67,9 @@ for (const file of yamlFiles) { const result = generateScript(testFile, apis, { clientArgs: ['cloud'], - preamble: CLOUD_PREAMBLE + preamble: CLOUD_PREAMBLE, + skipEmptySet: true, + skipNotFound: true }) for (const action of result.skippedActions) allSkippedActions.add(action) diff --git a/codegen/functional/generator.ts b/codegen/functional/generator.ts index 0591fcf0..c91fdac1 100644 --- a/codegen/functional/generator.ts +++ b/codegen/functional/generator.ts @@ -46,6 +46,18 @@ export interface GenerateOptions { clientArgs?: string[] /** Preamble lines defining the `$ELASTIC` invocation and `$RESPONSE` (default: the `elastic` binary). */ preamble?: string[] + /** + * If a `set` extraction is empty or `null`, try `.[0].id` (bare-array list + * responses) and skip the script when still empty. Used by Cloud tests + * against orgs that have no deployments or projects. + */ + skipEmptySet?: boolean + /** + * If a do-step exits non-zero and the JSON error mentions 404, skip the + * script. Used by Cloud tests for endpoints the org or public QA API + * does not expose (stack versions, org IdP). + */ + skipNotFound?: boolean } const DEFAULT_PREAMBLE = ['exec < /dev/null', 'ELASTIC="elastic --json"', 'RESPONSE=""'] @@ -57,6 +69,8 @@ export function generateScript ( ): GenerateResult { const clientArgs = opts.clientArgs ?? ['stack', 'es'] const preamble = opts.preamble ?? DEFAULT_PREAMBLE + const skipEmptySet = opts.skipEmptySet === true + const skipNotFound = opts.skipNotFound === true const actionMap = buildActionMap(definitions) const skippedActions: string[] = [] const lines: string[] = [] @@ -70,7 +84,7 @@ export function generateScript ( if (testFile.teardown.length > 0) { const teardownBody: string[] = [] - renderSteps(testFile.teardown, actionMap, clientArgs, teardownBody, skippedActions, ' ') + renderSteps(testFile.teardown, actionMap, clientArgs, teardownBody, skippedActions, ' ', false, skipEmptySet, skipNotFound) if (!hasExecutableLine(teardownBody)) { teardownBody.push(' :') } @@ -98,13 +112,13 @@ export function generateScript ( if (testFile.setup.length > 0) { lines.push('# --- Setup ---') - hadSkippedDo = renderSteps(testFile.setup, actionMap, clientArgs, lines, skippedActions, '') + hadSkippedDo = renderSteps(testFile.setup, actionMap, clientArgs, lines, skippedActions, '', false, skipEmptySet, skipNotFound) lines.push('') } for (const section of testFile.tests) { lines.push(`# --- Test: ${section.name} ---`) - renderSteps(section.steps, actionMap, clientArgs, lines, skippedActions, '', hadSkippedDo) + renderSteps(section.steps, actionMap, clientArgs, lines, skippedActions, '', hadSkippedDo, skipEmptySet, skipNotFound) lines.push('') } @@ -255,7 +269,9 @@ function renderSteps ( lines: string[], skippedActions: string[], indent: string, - initialHadSkippedDo = false + initialHadSkippedDo = false, + skipEmptySet = false, + skipNotFound = false ): boolean { // Assertions and set-steps read $RESPONSE, which is written by the most // recent successful `do`. If the last `do` was skipped (unmapped action, @@ -279,7 +295,7 @@ function renderSteps ( } // Only pass allowFailure from the setup→test propagation, not from // prior skipped steps within the same section (those are unrelated). - const result = renderDo(step, actionMap, clientArgs, lines, skippedActions, indent, initialHadSkippedDo) + const result = renderDo(step, actionMap, clientArgs, lines, skippedActions, indent, initialHadSkippedDo, skipNotFound) if (result === 'skipped') { responseFromLastDo = false hadSkippedDo = true @@ -310,7 +326,7 @@ function renderSteps ( switch (step.kind) { case 'set': - renderSet(step, lines, indent) + renderSet(step, lines, indent, skipEmptySet) // If a variable was previously unset, it's now set for (const varName of Object.values(step.assignments)) { unsetVars.delete(varName.toUpperCase().replace(/[^A-Z0-9]/g, '_')) @@ -360,7 +376,8 @@ function renderDo ( lines: string[], skippedActions: string[], indent: string, - allowFailure = false + allowFailure = false, + skipNotFound = false ): 'executed' | 'optional' | 'skipped' { if (step.catch != null) { lines.push(`${indent}# SKIPPED: catch not supported in MVP (catch: ${step.catch})`) @@ -384,9 +401,25 @@ function renderDo ( if (optional) { lines.push(`${indent}RESPONSE=$(${cmd}) || true`) return 'optional' - } else { - lines.push(`${indent}RESPONSE=$(${cmd})`) } + if (skipNotFound) { + // Handler errors go to stderr (`writeErr`); keep them out of $RESPONSE. + const errFile = '"${TMPDIR:-/tmp}/elastic-cli-do-err.$$"' + lines.push(`${indent}set +e`) + lines.push(`${indent}RESPONSE=$(${cmd} 2>${errFile})`) + lines.push(`${indent}_ec=$?`) + lines.push(`${indent}set -e`) + lines.push(`${indent}if [ "$_ec" -ne 0 ]; then`) + lines.push(`${indent} if jq -e '(.error.message | tostring | test("404"))' ${errFile} >/dev/null 2>&1; then`) + lines.push(`${indent} echo "SKIP: ${step.action} returned 404"`) + lines.push(`${indent} exit 0`) + lines.push(`${indent} fi`) + lines.push(`${indent} cat ${errFile} >&2`) + lines.push(`${indent} exit $_ec`) + lines.push(`${indent}fi`) + return 'executed' + } + lines.push(`${indent}RESPONSE=$(${cmd})`) return 'executed' } @@ -510,11 +543,25 @@ function buildCommand (mapped: MappedAction, step: DoStep): string { return base } -function renderSet (step: SetStep, lines: string[], indent: string): void { +function renderSet (step: SetStep, lines: string[], indent: string, skipEmptySet = false): void { for (const [responsePath, varName] of Object.entries(step.assignments)) { const bashVar = varName.toUpperCase().replace(/[^A-Z0-9]/g, '_') const jqPath = toJqPath(responsePath) - lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r '${jqPath}')`) + if (skipEmptySet) { + // `try` so a nested path like `.regions[0].id` against a bare array + // yields empty instead of aborting the script (jq cannot index an + // array with a string). + lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r 'try (${jqPath} // empty) catch empty')`) + lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`) + lines.push(`${indent} ${bashVar}=$(echo "$RESPONSE" | jq -r 'if type=="array" then .[0].id // empty else empty end')`) + lines.push(`${indent}fi`) + lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`) + lines.push(`${indent} echo "SKIP: no ${varName} in list response"`) + lines.push(`${indent} exit 0`) + lines.push(`${indent}fi`) + } else { + lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r '${jqPath}')`) + } } } diff --git a/codegen/functional/test/generator.test.ts b/codegen/functional/test/generator.test.ts index 28d42444..d72981b1 100644 --- a/codegen/functional/test/generator.test.ts +++ b/codegen/functional/test/generator.test.ts @@ -5,6 +5,7 @@ import { describe, it } from 'node:test' import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { join } from 'node:path' import { parseTestFile } from '../parser.ts' @@ -226,6 +227,85 @@ describe('generateScript', () => { assert.ok(result.script.includes('echo "PASS: get.yml"')) }) + it('skipEmptySet retries bare array id then skips when empty', () => { + const testFile: TestFile = { + sourceFile: 'list.yml', + requires: { serverless: true, stack: true }, + setup: [], + teardown: [], + tests: [{ + name: 'get item', + steps: [ + { kind: 'do', action: 'count', params: { index: 'x' }, body: undefined }, + { kind: 'set', assignments: { 'regions.0.id': 'id' } } + ] + }] + } + const result = generateScript(testFile, testDefs, { skipEmptySet: true }) + assert.ok(result.script.includes('try (.regions[0].id // empty) catch empty')) + assert.ok(result.script.includes('if type=="array" then .[0].id // empty else empty end')) + assert.ok(result.script.includes('SKIP: no id in list response')) + assert.ok(result.script.includes('exit 0')) + assert.equal( + execFileSync('jq', ['-r', 'try (.regions[0].id // empty) catch empty'], { + input: '[{"id":"r1"}]', + encoding: 'utf8' + }).trim(), + '' + ) + assert.equal( + execFileSync('jq', ['-r', 'if type=="array" then .[0].id // empty else empty end'], { + input: '[{"id":"r1"}]', + encoding: 'utf8' + }).trim(), + 'r1' + ) + assert.equal( + execFileSync('jq', ['-r', 'try (.deployments[0].id // empty) catch empty'], { + input: '{"deployments":[]}', + encoding: 'utf8' + }).trim(), + '' + ) + }) + + it('does not skip empty set extractions by default', () => { + const testFile: TestFile = { + sourceFile: 'list.yml', + requires: { serverless: true, stack: true }, + setup: [], + teardown: [], + tests: [{ + name: 'get item', + steps: [ + { kind: 'do', action: 'count', params: { index: 'x' }, body: undefined }, + { kind: 'set', assignments: { 'regions.0.id': 'id' } } + ] + }] + } + const result = generateScript(testFile, testDefs) + assert.equal(result.script.includes('SKIP: no id in list response'), false) + assert.equal(result.script.includes('.[0].id // empty'), false) + }) + + it('skipNotFound wraps do-steps to skip 404 errors', () => { + const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8') + const testFile = parseTestFile(content, 'get.yml') + const result = generateScript(testFile, testDefs, { skipNotFound: true }) + assert.ok(result.script.includes('set +e')) + assert.ok(result.script.includes('elastic-cli-do-err.$$')) + assert.ok(result.script.includes('returned 404')) + assert.ok(result.script.includes('test("404")')) + }) + + it('does not wrap do-steps for 404 by default', () => { + const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8') + const testFile = parseTestFile(content, 'get.yml') + const result = generateScript(testFile, testDefs) + assert.equal(result.script.includes('returned 404'), false) + assert.equal(result.script.includes('elastic-cli-do-err.$$'), false) + }) + it('tracks skipped actions for unregistered APIs', () => { const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8') const testFile = parseTestFile(content, 'get.yml') From 8ba6d5d77e63175796221e3905cb7d1a4eea7283 Mon Sep 17 00:00:00 2001 From: margaretjgu Date: Tue, 1 Sep 2026 12:54:21 -0400 Subject: [PATCH 2/6] ci: drop cloud functional soft fail --- .buildkite/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0583561b..d3998deb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -65,4 +65,3 @@ steps: command: ".buildkite/run-cloud-tests.sh" artifact_paths: - "test/functional/cloud/*.log" - soft_fail: true From 95ee869a0b7842d798782689c3a7da856472c5d4 Mon Sep 17 00:00:00 2001 From: margaretjgu Date: Tue, 1 Sep 2026 13:24:38 -0400 Subject: [PATCH 3/6] test: read serverless list items for ids --- codegen/functional/generator.ts | 3 ++- codegen/functional/test/generator.test.ts | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/codegen/functional/generator.ts b/codegen/functional/generator.ts index c91fdac1..8eca1953 100644 --- a/codegen/functional/generator.ts +++ b/codegen/functional/generator.ts @@ -553,7 +553,8 @@ function renderSet (step: SetStep, lines: string[], indent: string, skipEmptySet // array with a string). lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r 'try (${jqPath} // empty) catch empty')`) lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`) - lines.push(`${indent} ${bashVar}=$(echo "$RESPONSE" | jq -r 'if type=="array" then .[0].id // empty else empty end')`) + // Serverless lists wrap items in `.items`; regions return a bare array. + lines.push(`${indent} ${bashVar}=$(echo "$RESPONSE" | jq -r 'if type=="array" then .[0].id // empty else .items[0].id // empty end')`) lines.push(`${indent}fi`) lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`) lines.push(`${indent} echo "SKIP: no ${varName} in list response"`) diff --git a/codegen/functional/test/generator.test.ts b/codegen/functional/test/generator.test.ts index d72981b1..4cab0fb8 100644 --- a/codegen/functional/test/generator.test.ts +++ b/codegen/functional/test/generator.test.ts @@ -243,7 +243,7 @@ describe('generateScript', () => { } const result = generateScript(testFile, testDefs, { skipEmptySet: true }) assert.ok(result.script.includes('try (.regions[0].id // empty) catch empty')) - assert.ok(result.script.includes('if type=="array" then .[0].id // empty else empty end')) + assert.ok(result.script.includes('if type=="array" then .[0].id // empty else .items[0].id // empty end')) assert.ok(result.script.includes('SKIP: no id in list response')) assert.ok(result.script.includes('exit 0')) assert.equal( @@ -253,13 +253,15 @@ describe('generateScript', () => { }).trim(), '' ) + const fallback = 'if type=="array" then .[0].id // empty else .items[0].id // empty end' assert.equal( - execFileSync('jq', ['-r', 'if type=="array" then .[0].id // empty else empty end'], { - input: '[{"id":"r1"}]', - encoding: 'utf8' - }).trim(), + execFileSync('jq', ['-r', fallback], { input: '[{"id":"r1"}]', encoding: 'utf8' }).trim(), 'r1' ) + assert.equal( + execFileSync('jq', ['-r', fallback], { input: '{"items":[{"id":"p1"}]}', encoding: 'utf8' }).trim(), + 'p1' + ) assert.equal( execFileSync('jq', ['-r', 'try (.deployments[0].id // empty) catch empty'], { input: '{"deployments":[]}', From d157dd10a45f4fc1a827a3d7164318230502851b Mon Sep 17 00:00:00 2001 From: margaretjgu Date: Tue, 1 Sep 2026 13:24:38 -0400 Subject: [PATCH 4/6] ci: ensure cloud qa fixtures exist --- .buildkite/ensure-cloud-fixtures.sh | 91 +++++++++++++++++++++++++++++ .buildkite/run-cloud-tests.sh | 3 + 2 files changed, 94 insertions(+) create mode 100755 .buildkite/ensure-cloud-fixtures.sh diff --git a/.buildkite/ensure-cloud-fixtures.sh b/.buildkite/ensure-cloud-fixtures.sh new file mode 100755 index 00000000..9792821c --- /dev/null +++ b/.buildkite/ensure-cloud-fixtures.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Copyright Elasticsearch B.V. and contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Idempotent QA fixtures so Cloud item GET tests have something to fetch. +# Creates one serverless project per type, one serverless traffic filter, +# and one hosted traffic-filter ruleset when the matching list is empty. +# Does not create hosted deployments or extensions (those need a template +# or a plugin zip). Does not print create responses (they can contain creds). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELASTIC=(node "$REPO_ROOT/dist/cli.js" --json) +REGION="${CLOUD_FIXTURE_REGION:-gcp-us-central1}" +RULES='[{"source":"192.0.2.1"}]' + +first_id () { + jq -r ' + if type == "array" then .[0].id // empty + else + .items[0].id + // .projects[0].id + // .filters[0].id + // .rulesets[0].id + // .deployments[0].id + // empty + end + ' +} + +ensure () { + local label="$1" + shift + local list_args=() + while [ "$1" != "--" ]; do + list_args+=("$1") + shift + done + shift + local id + id="$("${ELASTIC[@]}" "${list_args[@]}" | first_id)" + if [ -n "$id" ] && [ "$id" != "null" ]; then + echo "fixture exists: $label $id" + return 0 + fi + echo "creating fixture: $label" + # stdout discarded so create responses never land in CI logs + "${ELASTIC[@]}" "$@" >/dev/null +} + +ensure "search project" \ + cloud serverless projects search list -- \ + cloud serverless projects search create \ + --name cli-functional-search \ + --region-id "$REGION" \ + --optimized-for general_purpose \ + --yes + +ensure "observability project" \ + cloud serverless projects observability list -- \ + cloud serverless projects observability create \ + --name cli-functional-o11y \ + --region-id "$REGION" \ + --product-tier logs_essentials \ + --yes + +ensure "security project" \ + cloud serverless projects security list -- \ + cloud serverless projects security create \ + --name cli-functional-security \ + --region-id "$REGION" \ + --yes + +ensure "serverless traffic filter" \ + cloud serverless traffic-filters list-traffic-filters -- \ + cloud serverless traffic-filters create-traffic-filter \ + --name cli-functional-filter \ + --type ip \ + --region "$REGION" \ + --rules "$RULES" \ + --yes + +ensure "hosted traffic filter ruleset" \ + cloud hosted traffic-filters get-traffic-filter-rulesets -- \ + cloud hosted traffic-filters create-traffic-filter-ruleset \ + --name cli-functional-ruleset \ + --type ip \ + --include-by-default false \ + --region "$REGION" \ + --rules "$RULES" diff --git a/.buildkite/run-cloud-tests.sh b/.buildkite/run-cloud-tests.sh index 9890b9df..14017057 100755 --- a/.buildkite/run-cloud-tests.sh +++ b/.buildkite/run-cloud-tests.sh @@ -35,6 +35,9 @@ npm run build echo "--- Setting up Cloud credentials" source .buildkite/setup-env.sh +echo "--- Ensuring Cloud fixtures" +.buildkite/ensure-cloud-fixtures.sh + echo "--- Generating Cloud functional tests" npm run codegen:functional:cloud From dd360d5eeb710605d76c8bebf8e76aac525d69a3 Mon Sep 17 00:00:00 2001 From: margaretjgu Date: Tue, 1 Sep 2026 14:20:19 -0400 Subject: [PATCH 5/6] ci: create hosted deployment fixture --- .buildkite/ensure-cloud-fixtures.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.buildkite/ensure-cloud-fixtures.sh b/.buildkite/ensure-cloud-fixtures.sh index 9792821c..b7f5e138 100755 --- a/.buildkite/ensure-cloud-fixtures.sh +++ b/.buildkite/ensure-cloud-fixtures.sh @@ -4,9 +4,9 @@ # # Idempotent QA fixtures so Cloud item GET tests have something to fetch. # Creates one serverless project per type, one serverless traffic filter, -# and one hosted traffic-filter ruleset when the matching list is empty. -# Does not create hosted deployments or extensions (those need a template -# or a plugin zip). Does not print create responses (they can contain creds). +# one hosted traffic-filter ruleset, and one hosted deployment when the +# matching list is empty. Skips extensions (need a plugin zip). Does not +# print create responses (they can contain creds). set -euo pipefail @@ -89,3 +89,10 @@ ensure "hosted traffic filter ruleset" \ --include-by-default false \ --region "$REGION" \ --rules "$RULES" + +ensure "hosted deployment" \ + cloud hosted deployments list-deployments -- \ + cloud hosted deployments create-deployment \ + --name cli-functional \ + --region "$REGION" \ + --template-id gcp-general-purpose From 8629b6e7fc4099dcdae37643adaaacfa57a0112b Mon Sep 17 00:00:00 2001 From: margaretjgu Date: Tue, 1 Sep 2026 14:20:19 -0400 Subject: [PATCH 6/6] test: skip upgrade assistant before version --- codegen/functional/generator.ts | 10 +++++----- codegen/functional/test/generator.test.ts | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/codegen/functional/generator.ts b/codegen/functional/generator.ts index 8eca1953..c4005740 100644 --- a/codegen/functional/generator.ts +++ b/codegen/functional/generator.ts @@ -53,9 +53,9 @@ export interface GenerateOptions { */ skipEmptySet?: boolean /** - * If a do-step exits non-zero and the JSON error mentions 404, skip the - * script. Used by Cloud tests for endpoints the org or public QA API - * does not expose (stack versions, org IdP). + * If a do-step exits non-zero and the JSON error mentions 404 (or a + * hosted deployment with no version yet), skip the script. Used by Cloud + * tests for endpoints the org or public QA API does not expose. */ skipNotFound?: boolean } @@ -410,8 +410,8 @@ function renderDo ( lines.push(`${indent}_ec=$?`) lines.push(`${indent}set -e`) lines.push(`${indent}if [ "$_ec" -ne 0 ]; then`) - lines.push(`${indent} if jq -e '(.error.message | tostring | test("404"))' ${errFile} >/dev/null 2>&1; then`) - lines.push(`${indent} echo "SKIP: ${step.action} returned 404"`) + lines.push(`${indent} if jq -e '(.error.message | tostring | test("404|Could not determine the version"))' ${errFile} >/dev/null 2>&1; then`) + lines.push(`${indent} echo "SKIP: ${step.action} not available"`) lines.push(`${indent} exit 0`) lines.push(`${indent} fi`) lines.push(`${indent} cat ${errFile} >&2`) diff --git a/codegen/functional/test/generator.test.ts b/codegen/functional/test/generator.test.ts index 4cab0fb8..abe5b5d6 100644 --- a/codegen/functional/test/generator.test.ts +++ b/codegen/functional/test/generator.test.ts @@ -296,15 +296,15 @@ describe('generateScript', () => { const result = generateScript(testFile, testDefs, { skipNotFound: true }) assert.ok(result.script.includes('set +e')) assert.ok(result.script.includes('elastic-cli-do-err.$$')) - assert.ok(result.script.includes('returned 404')) - assert.ok(result.script.includes('test("404")')) + assert.ok(result.script.includes('not available')) + assert.ok(result.script.includes('Could not determine the version')) }) it('does not wrap do-steps for 404 by default', () => { const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8') const testFile = parseTestFile(content, 'get.yml') const result = generateScript(testFile, testDefs) - assert.equal(result.script.includes('returned 404'), false) + assert.equal(result.script.includes('not available'), false) assert.equal(result.script.includes('elastic-cli-do-err.$$'), false) })