From 747d79c46bffc37184db344535dc1512f1168244 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 14 Oct 2026 15:25:53 +0300 Subject: [PATCH] feat(standard-tests): fail on an empty test-scenario set unless acceptance-test-config.yml declares a bypass_reason An acceptance-test-config.yml that yielded no runnable config made pytest skip every scenario test as an empty parameter set, and the run passed. A connector whose configs were never collected, whose file was missing, or whose layout the reader did not understand looked the same as a tested one. Collection now fails when no scenario can run and no bypass_reason says why. A bypass is accepted at the top level of the file (the whole connector, and the opt-out for a connector without test configs), on a section, or on an entry without config_path. Bypasses are reported as skipped tests carrying the declared reason, grouped by reason, plus one line after collection. The reader also: - parses legacy list-format sections as the section's tests list; - fails with a migration message on a legacy top-level `tests:` key; - treats a missing file as an error for Python and manifest-only connectors, and as a bypass for Java connectors; - wraps invalid entries in an error that names the entry and field but never prints the offending values. Co-Authored-By: Claude Opus 5.5 --- airbyte_cdk/test/models/scenario.py | 3 + .../standard_tests/_acceptance_test_config.py | 331 ++++++++++++ .../standard_tests/_spec_compatibility.py | 7 +- .../test/standard_tests/docker_base.py | 111 ++-- .../test/standard_tests/pytest_hooks.py | 104 +++- cdk-migrations.md | 35 ++ unit_tests/test/test_empty_scenarios.py | 509 ++++++++++++++++++ 7 files changed, 1036 insertions(+), 64 deletions(-) create mode 100644 airbyte_cdk/test/standard_tests/_acceptance_test_config.py create mode 100644 unit_tests/test/test_empty_scenarios.py diff --git a/airbyte_cdk/test/models/scenario.py b/airbyte_cdk/test/models/scenario.py index 217c285491..51f646f78c 100644 --- a/airbyte_cdk/test/models/scenario.py +++ b/airbyte_cdk/test/models/scenario.py @@ -103,6 +103,9 @@ def __hash__(self) -> int: expect_records: AcceptanceTestExpectRecords | None = None file_types: AcceptanceTestFileTypes | None = None status: Literal["succeed", "failed", "exception"] | None = None + # Why this scenario is not run: set on the placeholder scenarios that report a declared + # `bypass_reason` of `acceptance-test-config.yml` as skipped tests. + bypass_reason: str | None = None # CAT's `basic_read` opt-out from validating records against the stream schemas. validate_schema: bool = True # The connector's top-level `test_strictness_level` in `acceptance-test-config.yml`. Populated diff --git a/airbyte_cdk/test/standard_tests/_acceptance_test_config.py b/airbyte_cdk/test/standard_tests/_acceptance_test_config.py new file mode 100644 index 0000000000..5b432e0d6d --- /dev/null +++ b/airbyte_cdk/test/standard_tests/_acceptance_test_config.py @@ -0,0 +1,331 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +"""Read `acceptance-test-config.yml` into test scenarios and declared bypasses. + +Every connector that runs the standard tests (Python and manifest-only sources and destinations) +needs an `acceptance-test-config.yml`, and the file must yield at least one runnable scenario (a +config listed under a collected section) or declare why it has none: + +- a top-level `bypass_reason` bypasses the whole connector; a file holding only that key is the + opt-out for a connector without test configs; +- a `bypass_reason` on a collected section (`connection: {bypass_reason: ...}`) bypasses that + section, the layout the Connector Acceptance Tests used; +- a `bypass_reason` on an entry without `config_path` bypasses that entry. + +Only Java connectors, which are tested by their own Gradle suites, may have no file at all. + +Failure messages name sections, entries and keys, never values, as configs may hold secrets. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, TypeGuard + +import yaml +from pydantic import ValidationError + +from airbyte_cdk.test.models.scenario import SCENARIO_SECTIONS, ConnectorTestScenario +from airbyte_cdk.utils.connector_paths import ACCEPTANCE_TEST_CONFIG, METADATA_YAML + +JAVA_LANGUAGE_TAGS = frozenset({"language:java"}) +"""The `metadata.yaml` tags of connectors that may have no `acceptance-test-config.yml`.""" + +JAVA_BYPASS_REASON = ( + f"Java connector without {ACCEPTANCE_TEST_CONFIG}: Java connectors are tested by their " + "Gradle suites, not by the standard tests." +) + +BYPASS_ONLY_EXAMPLE = 'bypass_reason: "No test config in GSM for this connector"' + + +class AcceptanceTestConfigError(ValueError): + """`acceptance-test-config.yml` cannot be turned into test scenarios.""" + + +class MissingAcceptanceTestConfigError(AcceptanceTestConfigError, FileNotFoundError): + """A connector that runs the standard tests has no `acceptance-test-config.yml`.""" + + +@dataclass(frozen=True) +class ScenarioBypass: + """A declared reason why a part of `acceptance-test-config.yml` has no runnable config.""" + + reason: str + location: str + """Where the reason is declared, e.g. `acceptance_tests.connection.bypass_reason`.""" + sections: tuple[str, ...] = () + """The collected sections it bypasses; empty for a connector-wide bypass.""" + + +@dataclass +class CollectedScenarios: + """What `acceptance-test-config.yml` yields: runnable scenarios, bypasses, and why.""" + + config_path: Path + scenarios: list[ConnectorTestScenario] = field(default_factory=list) + """One scenario per runnable entry, before deduplication by config.""" + bypasses: list[ScenarioBypass] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + """Why each section or entry that is not a runnable config or a bypass produced nothing.""" + + def describe_empty(self, connector_name: str) -> str: + """The collection error for a connector with no runnable scenario and no bypass.""" + found = self.notes or ["(no sections)"] + return "\n".join( + [ + f"No test scenarios for connector '{connector_name}': {self.config_path} lists " + "no config the standard tests can run, and declares no `bypass_reason`. " + "Found:", + *(f" - {note}" for note in found), + "Add a `config_path` entry under one of " + f"{', '.join(f'`{section}`' for section in SCENARIO_SECTIONS)}, or declare why " + "the connector has none: a `bypass_reason` on each section " + "(`connection: {bypass_reason: ...}`), or a top-level `bypass_reason` for the " + "whole connector.", + ] + ) + + +def section_entries(section: Any) -> list[Any] | None: + """Return the entries of an `acceptance_tests` section, in either layout. + + The current layout lists them under `tests` (`spec: {tests: [...]}`); the legacy one makes the + section the list itself (`spec: [...]`). A section of any other shape has no entries. + """ + if isinstance(section, Mapping): + tests = section.get("tests") + return tests if isinstance(tests, list) else None + if isinstance(section, list): + return section + return None + + +def is_java_connector(connector_root: Path) -> bool: + """Whether `metadata.yaml` tags the connector as Java (`language:java`).""" + metadata_path = connector_root / METADATA_YAML + try: + metadata = yaml.safe_load(metadata_path.read_text()) + except (OSError, yaml.YAMLError): + return False + data = metadata.get("data") if isinstance(metadata, Mapping) else None + tags = data.get("tags") if isinstance(data, Mapping) else None + return isinstance(tags, list) and any(tag in JAVA_LANGUAGE_TAGS for tag in tags) + + +def load_acceptance_test_config(config_path: Path) -> dict[str, Any]: + """Load and validate the layout of `acceptance-test-config.yml`. + + Raises: + MissingAcceptanceTestConfigError: The file does not exist. + AcceptanceTestConfigError: The file is not valid YAML, is empty, uses the legacy + top-level `tests:` layout, or has neither `acceptance_tests` nor a top-level + `bypass_reason`. + """ + config_path = config_path.absolute() + if not config_path.exists(): + raise MissingAcceptanceTestConfigError( + f"Acceptance test config file not found at: {config_path}. Every Python and " + "manifest-only connector needs one for the standard tests (only Java connectors may " + "omit it). List the connector's test configs in it, or, if it has none, create the " + f"file with only a top-level `bypass_reason` that says why, e.g. " + f"`{BYPASS_ONLY_EXAMPLE}`." + ) + + try: + config = yaml.safe_load(config_path.read_text()) + except yaml.YAMLError as error: + raise AcceptanceTestConfigError(f"{config_path} is not valid YAML: {error}") from error + + if not isinstance(config, Mapping): + found = "is empty" if config is None else f"holds a {type(config).__name__}" + raise AcceptanceTestConfigError( + f"{config_path} {found}; expected a mapping with an `acceptance_tests` key, or only a " + f"top-level `bypass_reason` (`{BYPASS_ONLY_EXAMPLE}`)." + ) + + top_level_reason = config.get("bypass_reason") + if "bypass_reason" in config and not _is_reason(top_level_reason): + raise AcceptanceTestConfigError( + f"The top-level `bypass_reason` of {config_path} must be a non-empty string." + ) + + if "acceptance_tests" not in config: + if "tests" in config: + raise AcceptanceTestConfigError( + f"{config_path} uses the legacy layout with a top-level `tests:` key, which the " + "standard tests do not read. Migrate it: rename `tests:` to `acceptance_tests:` " + "and nest each section's entries under `tests` " + "(`connection: {tests: [{config_path: ...}]}`), keeping the other top-level keys." + ) + if top_level_reason is None: + raise AcceptanceTestConfigError( + f"{config_path} has no `acceptance_tests` key and no top-level `bypass_reason`. " + f"Found top-level keys: {', '.join(f'`{key}`' for key in config) or 'none'}." + ) + elif not isinstance(config["acceptance_tests"], Mapping): + raise AcceptanceTestConfigError( + f"`acceptance_tests` in {config_path} must be a mapping of sections " + f"(`connection: {{tests: [...]}}`), not a {type(config['acceptance_tests']).__name__}." + ) + + return dict(config) + + +def collect_scenarios(config: Mapping[str, Any], config_path: Path) -> CollectedScenarios: + """Collect the runnable scenarios and the declared bypasses of a loaded config. + + Every section of `SCENARIO_SECTIONS` is read, in that order, in the current layout + (`section: {tests: [...]}`) and the legacy list layout (`section: [...]`). An entry with a + `config_path` is a runnable scenario, except an IAM-role config, which the standard tests do + not support. An entry without `config_path` contributes its `bypass_reason`, if it has one. + A section `bypass_reason` counts only when the section lists no runnable config. + + Raises: + AcceptanceTestConfigError: An entry is not a valid scenario (e.g. `empty_streams` as plain + strings), or a top-level `bypass_reason` is declared next to runnable configs, which + would leave it unclear whether the connector is tested. + """ + collected = CollectedScenarios(config_path=config_path) + top_level_reason = config.get("bypass_reason") + if _is_reason(top_level_reason): + collected.bypasses.append(ScenarioBypass(reason=top_level_reason, location="bypass_reason")) + + acceptance_tests = config.get("acceptance_tests") or {} + strictness_level = config.get("test_strictness_level") + for section_name in SCENARIO_SECTIONS: + if section_name in acceptance_tests: + _collect_section( + collected, + section_name, + acceptance_tests[section_name], + strictness_level=strictness_level, + ) + for section_name in acceptance_tests: + if section_name not in SCENARIO_SECTIONS: + collected.notes.append( + f"`acceptance_tests.{section_name}`: not a section the standard tests collect." + ) + + if collected.scenarios and _is_reason(top_level_reason): + raise AcceptanceTestConfigError( + f"{config_path} declares a top-level `bypass_reason`, which bypasses the whole " + f"connector, but also lists {len(collected.scenarios)} runnable config entries. Remove " + "the top-level `bypass_reason` to run them, or remove the entries." + ) + return collected + + +def _collect_section( + collected: CollectedScenarios, + section_name: str, + section: Any, + *, + strictness_level: Any, +) -> None: + where = f"acceptance_tests.{section_name}" + entries = section_entries(section) + section_reason = section.get("bypass_reason") if isinstance(section, Mapping) else None + + runnable = 0 + if entries is None: + if section is None: + collected.notes.append(f"`{where}`: empty.") + elif not isinstance(section, Mapping): + collected.notes.append( + f"`{where}`: a {type(section).__name__}, not a mapping with a `tests` list." + ) + elif section_reason is None: + keys = ", ".join(f"`{key}`" for key in section) or "none" + collected.notes.append( + f"`{where}`: no `tests` list and no `bypass_reason` (keys: {keys})." + ) + else: + prefix = f"{where}.tests" if isinstance(section, Mapping) else where + if not entries: + collected.notes.append(f"`{prefix}`: empty list.") + for index, entry in enumerate(entries): + location = f"{prefix}[{index}]" + if _collect_entry(collected, section_name, location, entry, strictness_level): + runnable += 1 + + if section_reason is None: + return + if not _is_reason(section_reason): + collected.notes.append(f"`{where}.bypass_reason`: not a non-empty string, so not counted.") + elif runnable: + collected.notes.append( + f"`{where}.bypass_reason`: ignored, as the section lists {runnable} runnable config " + "entries." + ) + else: + collected.bypasses.append( + ScenarioBypass( + reason=section_reason, + location=f"{where}.bypass_reason", + sections=(section_name,), + ) + ) + + +def _collect_entry( + collected: CollectedScenarios, + section_name: str, + location: str, + entry: Any, + strictness_level: Any, +) -> bool: + """Collect one section entry; return whether it is a runnable scenario.""" + if not isinstance(entry, Mapping): + collected.notes.append(f"`{location}`: a {type(entry).__name__}, not a mapping.") + return False + + config_path = entry.get("config_path") + if not config_path: + reason = entry.get("bypass_reason") + if _is_reason(reason): + collected.bypasses.append( + ScenarioBypass( + reason=reason, + location=f"{location}.bypass_reason", + sections=(section_name,), + ) + ) + else: + keys = ", ".join(f"`{key}`" for key in entry) or "none" + collected.notes.append( + f"`{location}`: no `config_path` and no `bypass_reason` (keys: {keys})." + ) + return False + + if "iam_role" in str(config_path): + collected.notes.append( + f"`{location}`: an IAM-role config, which the standard tests do not support." + ) + return False + + fields = {key: value for key, value in entry.items() if key != "bypass_reason"} + try: + scenario = ConnectorTestScenario.model_validate( + { + **fields, + "sections": (section_name,), + "test_strictness_level": strictness_level, + } + ) + except ValidationError as error: + # Pydantic's own message repeats the offending input; name the field and rule only. + problems = "; ".join( + f"`{'.'.join(str(part) for part in problem['loc']) or '(entry)'}`: {problem['msg']}" + for problem in error.errors(include_input=False, include_url=False) + ) + raise AcceptanceTestConfigError( + f"`{location}` in {collected.config_path} is not a valid test entry: {problems}." + ) from None + collected.scenarios.append(scenario) + return True + + +def _is_reason(value: Any) -> TypeGuard[str]: + return isinstance(value, str) and bool(value.strip()) diff --git a/airbyte_cdk/test/standard_tests/_spec_compatibility.py b/airbyte_cdk/test/standard_tests/_spec_compatibility.py index c62f34f5e9..10ec136593 100644 --- a/airbyte_cdk/test/standard_tests/_spec_compatibility.py +++ b/airbyte_cdk/test/standard_tests/_spec_compatibility.py @@ -26,6 +26,8 @@ from packaging.version import InvalidVersion, Version from requests.adapters import HTTPAdapter, Retry +from airbyte_cdk.test.standard_tests._acceptance_test_config import section_entries + DeploymentMode = Literal["oss", "cloud"] REGISTRY_ENTRY_URL_TEMPLATE = ( @@ -1119,9 +1121,8 @@ def disabled_for_version(acceptance_test_config: Any) -> list[str]: acceptance_tests = acceptance_test_config.get("acceptance_tests") if not isinstance(acceptance_tests, Mapping): return [] - spec_section = acceptance_tests.get("spec") - tests = spec_section.get("tests") if isinstance(spec_section, Mapping) else spec_section - if not isinstance(tests, list): + tests = section_entries(acceptance_tests.get("spec")) + if tests is None: return [] versions: list[str] = [] diff --git a/airbyte_cdk/test/standard_tests/docker_base.py b/airbyte_cdk/test/standard_tests/docker_base.py index dd8b365dba..79e09f3034 100644 --- a/airbyte_cdk/test/standard_tests/docker_base.py +++ b/airbyte_cdk/test/standard_tests/docker_base.py @@ -29,7 +29,15 @@ from airbyte_cdk.models.connector_metadata import MetadataFile from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.models import ConnectorTestScenario -from airbyte_cdk.test.models.scenario import SCENARIO_SECTIONS, ScenarioCommand +from airbyte_cdk.test.models.scenario import ScenarioCommand +from airbyte_cdk.test.standard_tests._acceptance_test_config import ( + JAVA_BYPASS_REASON, + CollectedScenarios, + ScenarioBypass, + collect_scenarios, + is_java_connector, + load_acceptance_test_config, +) from airbyte_cdk.test.standard_tests._assertions import ( assert_check_outcome, assert_config_matches_spec, @@ -292,22 +300,15 @@ def is_destination_connector(cls) -> bool: def acceptance_test_config(cls) -> Any: """Get the contents of acceptance test config file. - Also perform some basic validation that the file has the expected structure. - """ - acceptance_test_config_path = cls.get_connector_root_dir() / ACCEPTANCE_TEST_CONFIG - if not acceptance_test_config_path.exists(): - raise FileNotFoundError( - f"Acceptance test config file not found at: {str(acceptance_test_config_path)}" - ) - - tests_config = yaml.safe_load(acceptance_test_config_path.read_text()) + Also perform some basic validation that the file has the expected structure: an + `acceptance_tests` mapping, or only a top-level `bypass_reason`. - if "acceptance_tests" not in tests_config: - raise ValueError( - f"Acceptance tests config not found in {acceptance_test_config_path}." - f" Found only: {str(tests_config)}." - ) - return tests_config + Raises: + MissingAcceptanceTestConfigError: The file does not exist (a `FileNotFoundError`). + AcceptanceTestConfigError: The file has an unusable layout, such as the legacy + top-level `tests:` key (a `ValueError`). + """ + return load_acceptance_test_config(cls.get_connector_root_dir() / ACCEPTANCE_TEST_CONFIG) @staticmethod def _dedup_scenarios(scenarios: list[ConnectorTestScenario]) -> list[ConnectorTestScenario]: @@ -391,6 +392,23 @@ def _dedup_scenarios(scenarios: list[ConnectorTestScenario]) -> list[ConnectorTe ) return deduped_scenarios + @classmethod + def _collect_scenarios(cls) -> CollectedScenarios: + """Read the runnable scenarios and the declared bypasses of `acceptance-test-config.yml`. + + A Java connector without the file is reported as bypassed: Java connectors are tested by + their Gradle suites. Every other connector without it fails, with + `MissingAcceptanceTestConfigError`. + """ + connector_root = cls.get_connector_root_dir() + config_path = (connector_root / ACCEPTANCE_TEST_CONFIG).absolute() + if not config_path.exists() and is_java_connector(connector_root): + return CollectedScenarios( + config_path=config_path, + bypasses=[ScenarioBypass(reason=JAVA_BYPASS_REASON, location="metadata.yaml")], + ) + return collect_scenarios(cls.acceptance_test_config, config_path) + @classmethod def get_scenarios( cls, @@ -402,53 +420,34 @@ def get_scenarios( against the spec, `connection` runs `check` against the declared `status`, `discovery` runs `discover`, and `basic_read` and `full_refresh` run a full-refresh `read` of the streams of the config's `configured_catalog_path`, or of every discovered stream when it - declares none (`SECTION_COMMANDS`). The `incremental` section is not collected. + declares none (`SECTION_COMMANDS`). The `incremental` section is not collected. A section + in the legacy list layout (`spec: [...]`) is read as its `tests` list. + + The file is required (except for Java connectors), and a legacy top-level `tests:` key + fails; see `load_acceptance_test_config`. An empty result is not an error here: + `pytest_generate_tests` fails collection when there is no scenario and no declared + bypass (`get_scenario_bypasses`). This has to be a separate function because pytest does not allow parametrization of fixtures with arguments from the test class itself. """ - try: - all_tests_config = cls.acceptance_test_config - except FileNotFoundError as e: - # Destinations sometimes do not have an acceptance tests file. - warnings.warn( - f"Acceptance test config file not found: {e!s}. No scenarios will be loaded.", - category=UserWarning, - stacklevel=1, - ) - return [] + return cls._dedup_scenarios(cls._collect_scenarios().scenarios) - test_strictness_level = all_tests_config.get("test_strictness_level") - test_scenarios: list[ConnectorTestScenario] = [] - for category in SCENARIO_SECTIONS: - if ( - category not in all_tests_config["acceptance_tests"] - or "tests" not in all_tests_config["acceptance_tests"][category] - ): - continue - - for test in all_tests_config["acceptance_tests"][category]["tests"]: - if "config_path" not in test: - # Skip tests without a config_path - continue - - if "iam_role" in test["config_path"]: - # We skip iam_role tests for now, as they are not supported in the test suite. - continue - - scenario = ConnectorTestScenario.model_validate( - { - **test, - "sections": (category,), - "test_strictness_level": test_strictness_level, - } - ) - - test_scenarios.append(scenario) + @classmethod + def get_scenario_bypasses(cls) -> list[ScenarioBypass]: + """The `bypass_reason`s that `acceptance-test-config.yml` declares for collected parts. - deduped_test_scenarios = cls._dedup_scenarios(test_scenarios) + A top-level `bypass_reason` bypasses the connector; one on a collected section with no + runnable config, or on an entry without `config_path`, bypasses that section or entry. + `pytest_generate_tests` reports them as skipped tests, and accepts a connector with no + scenario only when it declares at least one. + """ + return cls._collect_scenarios().bypasses - return deduped_test_scenarios + @classmethod + def describe_empty_scenarios(cls) -> str: + """Why `acceptance-test-config.yml` yields neither a scenario nor a bypass.""" + return cls._collect_scenarios().describe_empty(cast(str, cls.connector_name)) @classmethod def _connector_image( diff --git a/airbyte_cdk/test/standard_tests/pytest_hooks.py b/airbyte_cdk/test/standard_tests/pytest_hooks.py index fa905e7157..51567f1f52 100644 --- a/airbyte_cdk/test/standard_tests/pytest_hooks.py +++ b/airbyte_cdk/test/standard_tests/pytest_hooks.py @@ -13,11 +13,17 @@ ``` """ +from dataclasses import dataclass from typing import Literal, cast import pytest from airbyte_cdk.test.models.scenario import ConnectorTestScenario +from airbyte_cdk.test.standard_tests._acceptance_test_config import ( + AcceptanceTestConfigError, + ScenarioBypass, +) +from airbyte_cdk.utils.connector_paths import ACCEPTANCE_TEST_CONFIG @pytest.fixture @@ -195,8 +201,36 @@ class TestMyConnector(ConnectorTestSuiteBase): "Please define the 'scenarios' attribute in the test class." ) - # Get the scenarios defined or discovered in the test class - scenarios = test_class.get_scenarios() + # Get the scenarios defined or discovered in the test class, and the declared bypasses. + config_error: str | None = None + try: + scenarios = test_class.get_scenarios() + get_bypasses = getattr(test_class, "get_scenario_bypasses", None) + bypasses: list[ScenarioBypass] = get_bypasses() if get_bypasses else [] + except AcceptanceTestConfigError as error: + config_error = str(error) + if config_error is not None: + # Failed outside the `except` block, so pytest does not print the error twice. + pytest.fail(config_error, pytrace=False) + + if not scenarios and not bypasses: + # Parametrizing with an empty list would skip every scenario test and exit 0. + describe = getattr(test_class, "describe_empty_scenarios", None) + pytest.fail( + describe() + if describe + else f"Test class {test_class.__name__} has no test scenarios.", + pytrace=False, + ) + + bypass_groups = _group_bypasses(bypasses) + if bypass_groups: + connector_name = getattr(test_class, "connector_name", test_class.__name__) + metafunc.config.stash.setdefault(_BYPASS_REPORT_KEY, {})[str(connector_name)] = ( + f"{connector_name}: {ACCEPTANCE_TEST_CONFIG} declares {len(bypasses)} bypass(es), " + "reported as skipped tests: " + + "; ".join(f"{group.scope}: {group.reason}" for group in bypass_groups) + ) # Create pytest.param objects with special marks as needed parametrized_scenarios = [ @@ -206,13 +240,73 @@ class TestMyConnector(ConnectorTestSuiteBase): ) for scenario in scenarios ] + # One skipped param per distinct bypass reason, so the CI summary shows the reason. + bypass_params = [ + pytest.param( + ConnectorTestScenario(bypass_reason=group.reason, sections=group.sections), + marks=pytest.mark.skip( + reason=f"{ACCEPTANCE_TEST_CONFIG} bypasses {group.scope}: {group.reason}" + ), + ) + for group in bypass_groups + ] - # Parametrize the 'scenario' argument with the scenarios + # Parametrize the 'scenario' argument with the scenarios, then the bypasses metafunc.parametrize( "scenario", - parametrized_scenarios, - ids=_scenario_test_ids(scenarios), + parametrized_scenarios + bypass_params, + ids=_scenario_test_ids(scenarios) + [group.test_id for group in bypass_groups], + ) + + +_BYPASS_REPORT_KEY = pytest.StashKey[dict[str, str]]() +"""The collection report line of each connector whose config declares bypasses.""" + + +def pytest_report_collectionfinish(config: pytest.Config) -> list[str]: + """Print one line per connector that declares bypasses, after the collection summary.""" + return [line for _, line in sorted(config.stash.get(_BYPASS_REPORT_KEY, {}).items())] + + +@dataclass(frozen=True) +class _BypassGroup: + """The bypasses that share one reason, reported as one skipped test per scenario test.""" + + reason: str + scope: str + """What is bypassed: `the connector`, or the sections and entries, e.g. `connection.tests[1]`.""" + sections: tuple[str, ...] + """The bypassed sections; empty when the whole connector is bypassed.""" + test_id: str + + +def _group_bypasses(bypasses: list[ScenarioBypass]) -> list[_BypassGroup]: + """Group bypasses by reason: the usual config repeats one reason on every section.""" + by_reason: dict[str, list[ScenarioBypass]] = {} + for bypass in bypasses: + by_reason.setdefault(bypass.reason.strip(), []).append(bypass) + + groups: list[_BypassGroup] = [] + for reason, group in by_reason.items(): + connector_wide = any(not bypass.sections for bypass in group) + sections = ( + () + if connector_wide + else tuple(dict.fromkeys(section for bypass in group for section in bypass.sections)) + ) + scope = ( + "the connector" + if connector_wide + else ", ".join( + bypass.location.removeprefix("acceptance_tests.").removesuffix(".bypass_reason") + for bypass in group + ) ) + test_id = "bypassed-" + ("connector" if connector_wide else "-".join(sections)) + if any(existing.test_id == test_id for existing in groups): + test_id = f"{test_id}-{len(groups) + 1}" + groups.append(_BypassGroup(reason=reason, scope=scope, sections=sections, test_id=test_id)) + return groups def _scenario_test_ids(scenarios: list[ConnectorTestScenario]) -> list[str]: diff --git a/cdk-migrations.md b/cdk-migrations.md index ae3781c9b6..80df1435ef 100644 --- a/cdk-migrations.md +++ b/cdk-migrations.md @@ -1,5 +1,40 @@ # CDK Migration Guide +## Upgrading to the next minor release (unreleased) + +### A connector without test scenarios fails unless it says why + +This release changes how the Standard Tests (`airbyte-cdk connector test` and `airbyte-cdk image test`) read `acceptance-test-config.yml`. There is no change to how connectors run. Previously a connector whose file listed no config the tests could run, or that had no file at all, collected an empty scenario list: every test that takes a scenario (`test_check`, `test_discover`, `test_basic_read` and their Docker counterparts) was skipped as "got empty parameter set" and the run passed. Now: + +| `acceptance-test-config.yml` | Before | Now | +| ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | +| Lists at least one config under `spec`, `connection`, `discovery`, `basic_read` or `full_refresh` | Scenarios run | Scenarios run. Declared bypasses (below) are also reported as skipped tests. | +| Lists no runnable config, but declares a `bypass_reason` | Every scenario test skipped as an empty parameter set | One skipped test per scenario test and distinct reason, whose skip message is the declared reason, plus one line after collection. | +| Lists no runnable config and declares no `bypass_reason` | Every scenario test skipped as an empty parameter set | Collection fails, naming the file and why each section and entry produced nothing. | +| Missing (Python or manifest-only connector) | `UserWarning`, every scenario test skipped | Collection fails. | +| Missing (Java connector, `language:java` in `metadata.yaml`) | `UserWarning`, every scenario test skipped | Reported as a bypass: Java connectors are tested by their Gradle suites. | +| Legacy top-level `tests:` key instead of `acceptance_tests:` | Collection failed with "Acceptance tests config not found" and the whole parsed file | Collection fails with a message naming the file and the migration. | +| A section written as a plain list (`connection: [...]`, the legacy layout) | Silently ignored | Read as the section's `tests` list. | +| An entry that is not a valid scenario (e.g. `empty_streams` as plain strings) | Pydantic `ValidationError`, including the offending values | Collection fails, naming the entry, the field and the rule, without the values. | + +A `bypass_reason` counts in three places: + +- **Top level** (`bypass_reason: "..."` next to `acceptance_tests`): bypasses the whole connector. A file holding only this key is the opt-out for a connector without test configs. It cannot be combined with runnable configs, which fails collection. +- **On a section** (`connection: {bypass_reason: "..."}`), the layout the Connector Acceptance Tests used: bypasses that section, when it lists no runnable config. A section `bypass_reason` next to runnable configs is ignored, as before. +- **On an entry without `config_path`** (`- bypass_reason: "..."` in a section's `tests` list): bypasses that entry. An entry with a `config_path` always runs; its `bypass_reason` is ignored. + +Only the sections the Standard Tests collect count. A `bypass_reason` under `incremental`, which is not collected, does not, and neither does an IAM-role config (`config_path` containing `iam_role`), which the Standard Tests do not support. `DockerConnectorTestSuite.get_scenario_bypasses()` returns the bypasses, and `ConnectorTestScenario.bypass_reason` holds the reason on the placeholder scenario of each skipped test. `get_scenarios()` still returns only the runnable scenarios, and now raises `MissingAcceptanceTestConfigError` (a `FileNotFoundError`) or `AcceptanceTestConfigError` (a `ValueError`) instead of warning or failing with a generic `ValueError`. + +Migration steps: + +- A connector without `acceptance-test-config.yml`: list its test configs in a new file, or create the file with only a top-level `bypass_reason` that says why it has none, e.g. `bypass_reason: "No test config in GSM for this connector"`. Earlier CDK versions reject a file without `acceptance_tests`, so a file that must also load under them adds `acceptance_tests: {}`. +- A file whose sections list no `config_path`, such as one holding only a `spec_path` entry or only a `backward_compatibility_tests_config` waiver: add the configs, or declare a `bypass_reason` at the top level or on each section. +- A file with a top-level `tests:` key: rename it to `acceptance_tests:` and nest each section's entries under `tests` (`connection: {tests: [{config_path: ...}]}`). +- A section written as a plain list now runs its configs. Make sure they pass, or move the ones that cannot run out and declare a `bypass_reason` for them. +- `empty_streams` entries must be mappings (`- name: `), not plain strings. + +Rationale: an empty scenario list looked like a green test run, so a connector whose configs were never collected, or whose configuration silently degraded to nothing, reported coverage it did not have. The opt-out is explicit and machine-readable so that the connectors that legitimately have no test configs say so in the test report instead of being indistinguishable from a broken configuration. + ## Upgrading to 7.36.0 This release of the CDK adds checks to the Standard Tests (`airbyte-cdk connector test` and `airbyte-cdk image test`) and changes which commands they run for the configs listed in `acceptance-test-config.yml`. There is no change to how connectors run. The release is a minor one, since only the test harness changes, but the connector test suites of connectors that do not satisfy the new expectations start failing. The connector monorepo installs the `airbyte-cdk` CLI unpinned (`uv tool install --upgrade 'airbyte-cdk[dev]'`), so connector CI picks the new behaviour up with this release, without a per-connector CDK bump. diff --git a/unit_tests/test/test_empty_scenarios.py b/unit_tests/test/test_empty_scenarios.py new file mode 100644 index 0000000000..ed3f67f034 --- /dev/null +++ b/unit_tests/test/test_empty_scenarios.py @@ -0,0 +1,509 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +"""An `acceptance-test-config.yml` with no runnable scenario must fail, unless it says why. + +Collection fails when the config yields no runnable scenario and declares no `bypass_reason`; +declared bypasses are reported as skipped tests with their reason. A missing file fails except +for Java connectors, a legacy top-level `tests:` key fails with the migration, and a section in +the legacy list layout is read as its `tests` list. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from airbyte_cdk.test.standard_tests._acceptance_test_config import ( + JAVA_BYPASS_REASON, + AcceptanceTestConfigError, + MissingAcceptanceTestConfigError, + ScenarioBypass, + collect_scenarios, + load_acceptance_test_config, + section_entries, +) +from airbyte_cdk.test.standard_tests._spec_compatibility import disabled_for_version +from airbyte_cdk.test.standard_tests.docker_base import DockerConnectorTestSuite +from airbyte_cdk.utils.connector_paths import ACCEPTANCE_TEST_CONFIG + +pytest_plugins = ["pytester"] + +BUILDER_REASON = "This is a builder contribution, and we do not have secrets at this time" + +# The shape of the ~290 builder contributions without secrets: a `spec_path` entry, and a +# `bypass_reason` on every other section. +BYPASSED_SECTIONS_CONFIG: dict[str, Any] = { + "connector_image": "airbyte/source-test:dev", + "acceptance_tests": { + "spec": {"tests": [{"spec_path": "manifest.yaml"}]}, + "connection": {"bypass_reason": BUILDER_REASON}, + "discovery": {"bypass_reason": BUILDER_REASON}, + "basic_read": {"bypass_reason": BUILDER_REASON}, + "incremental": {"bypass_reason": BUILDER_REASON}, + "full_refresh": {"bypass_reason": BUILDER_REASON}, + }, +} + +# No runnable config and no bypass of a collected section: only `incremental` declares one. +ZERO_SCENARIOS_CONFIG: dict[str, Any] = { + "acceptance_tests": { + "spec": {"tests": [{"spec_path": "manifest.yaml"}]}, + "connection": {"tests": [{"config_path": "secrets/config_iam_role.json"}]}, + "basic_read": {"tests": []}, + "incremental": {"bypass_reason": "This connector does not implement incremental sync"}, + } +} + +RUNNABLE_CONFIG: dict[str, Any] = { + "acceptance_tests": { + "connection": {"tests": [{"config_path": "secrets/config.json", "status": "succeed"}]}, + "basic_read": {"bypass_reason": "The sandbox has no data"}, + } +} + + +def _write_connector( + root: Path, + config: dict[str, Any] | str | None, + *, + language: str = "manifest-only", +) -> Path: + root.mkdir(parents=True, exist_ok=True) + (root / "metadata.yaml").write_text( + yaml.safe_dump({"data": {"tags": [f"language:{language}"]}}) + ) + if config is not None: + text = config if isinstance(config, str) else yaml.safe_dump(config) + (root / ACCEPTANCE_TEST_CONFIG).write_text(text) + return root + + +def _suite(root: Path) -> type[DockerConnectorTestSuite]: + class _Suite(DockerConnectorTestSuite): + @classmethod + def get_connector_root_dir(cls) -> Path: + return root + + return _Suite + + +# --- Reading the config ----------------------------------------------------------------------- + + +def test_bypassed_sections_yield_no_scenario_and_one_bypass_per_collected_section( + tmp_path: Path, +) -> None: + suite = _suite(_write_connector(tmp_path / "source-test", BYPASSED_SECTIONS_CONFIG)) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [ + ScenarioBypass( + reason=BUILDER_REASON, + location=f"acceptance_tests.{section}.bypass_reason", + sections=(section,), + ) + for section in ("connection", "discovery", "basic_read", "full_refresh") + ], "the `incremental` bypass is not a bypass of anything the standard tests run" + + +def test_zero_scenarios_without_bypass_lists_why_each_part_produced_nothing( + tmp_path: Path, +) -> None: + suite = _suite(_write_connector(tmp_path / "source-test", ZERO_SCENARIOS_CONFIG)) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [] + message = suite.describe_empty_scenarios() + assert message.startswith("No test scenarios for connector 'source-test': ") + assert str(tmp_path / "source-test" / ACCEPTANCE_TEST_CONFIG) in message + for expected in [ + "`acceptance_tests.spec.tests[0]`: no `config_path` and no `bypass_reason` " + "(keys: `spec_path`).", + "`acceptance_tests.connection.tests[0]`: an IAM-role config", + "`acceptance_tests.basic_read.tests`: empty list.", + "`acceptance_tests.incremental`: not a section the standard tests collect.", + "a top-level `bypass_reason` for the whole connector", + ]: + assert expected in message + + +def test_entry_without_config_path_declares_a_bypass(tmp_path: Path) -> None: + config = { + "acceptance_tests": { + "basic_read": {"tests": [{"bypass_reason": "Read needs a paid plan", "timeout": 1}]} + } + } + suite = _suite(_write_connector(tmp_path / "source-test", config)) + + assert suite.get_scenario_bypasses() == [ + ScenarioBypass( + reason="Read needs a paid plan", + location="acceptance_tests.basic_read.tests[0].bypass_reason", + sections=("basic_read",), + ) + ] + + +def test_section_bypass_is_ignored_next_to_runnable_configs(tmp_path: Path) -> None: + config = { + "acceptance_tests": { + "connection": { + "bypass_reason": "stale", + "tests": [{"config_path": "secrets/config.json"}], + } + } + } + collected = collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG) + + assert [str(s.config_path) for s in collected.scenarios] == ["secrets/config.json"] + assert collected.bypasses == [] + assert collected.notes == [ + "`acceptance_tests.connection.bypass_reason`: ignored, as the section lists 1 runnable " + "config entries." + ] + + +def test_runnable_entry_does_not_keep_its_bypass_reason(tmp_path: Path) -> None: + config = { + "acceptance_tests": { + "connection": {"tests": [{"config_path": "secrets/config.json", "bypass_reason": "x"}]} + } + } + [scenario] = collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG).scenarios + assert scenario.bypass_reason is None, "a config with a `config_path` is run, not bypassed" + + +@pytest.mark.parametrize("reason", ["", " ", 1, True]) +def test_blank_or_non_string_section_bypass_does_not_count(tmp_path: Path, reason: Any) -> None: + config = {"acceptance_tests": {"connection": {"bypass_reason": reason}}} + collected = collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG) + + assert collected.bypasses == [] + assert collected.notes == [ + "`acceptance_tests.connection.bypass_reason`: not a non-empty string, so not counted." + ] + + +def test_top_level_bypass_only_file_bypasses_the_connector(tmp_path: Path) -> None: + root = _write_connector( + tmp_path / "destination-test", 'bypass_reason: "No test config in GSM for this connector"\n' + ) + suite = _suite(root) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [ + ScenarioBypass(reason="No test config in GSM for this connector", location="bypass_reason") + ] + assert disabled_for_version(suite.acceptance_test_config) == [] + + +def test_top_level_bypass_next_to_runnable_configs_fails(tmp_path: Path) -> None: + root = _write_connector(tmp_path / "source-test", {**RUNNABLE_CONFIG, "bypass_reason": "x"}) + + with pytest.raises(AcceptanceTestConfigError, match="top-level `bypass_reason`.*1 runnable"): + _suite(root).get_scenarios() + + +def test_top_level_bypass_with_bypassed_sections_is_accepted(tmp_path: Path) -> None: + """A spec waiver file can carry the connector-wide bypass next to its `spec` entry.""" + config = { + "bypass_reason": "No test config in GSM for this connector", + "acceptance_tests": { + "spec": { + "tests": [{"backward_compatibility_tests_config": {"disable_for_version": "1.0.0"}}] + } + }, + } + suite = _suite(_write_connector(tmp_path / "source-test", config)) + + assert [bypass.location for bypass in suite.get_scenario_bypasses()] == ["bypass_reason"] + assert disabled_for_version(suite.acceptance_test_config) == ["1.0.0"] + + +def test_invalid_entry_fails_with_its_location_and_without_values(tmp_path: Path) -> None: + """A legacy list section listing `empty_streams` as plain strings, as source-netsuite did.""" + config = { + "acceptance_tests": { + "basic_read": [ + {"config_path": "secrets/config.json", "empty_streams": ["secret_stream_name"]} + ] + } + } + with pytest.raises(AcceptanceTestConfigError) as error: + collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG) + + message = str(error.value) + assert message.startswith("`acceptance_tests.basic_read[0]` in ") + assert "`empty_streams.0`: Input should be a valid dictionary" in message + assert "secret_stream_name" not in message, "input values are not printed" + + +# --- Legacy layouts --------------------------------------------------------------------------- + + +def test_legacy_list_sections_are_read_as_tests_lists(tmp_path: Path) -> None: + config = { + "acceptance_tests": { + "spec": [{"spec_path": "manifest.yaml"}], + "connection": [ + {"config_path": "secrets/config.json", "status": "succeed"}, + {"config_path": "integration_tests/invalid_config.json", "status": "failed"}, + ], + "basic_read": [ + {"config_path": "secrets/config.json", "empty_streams": [{"name": "a"}]} + ], + "full_refresh": [{"bypass_reason": "Covered by basic_read"}], + } + } + suite = _suite(_write_connector(tmp_path / "source-test", config)) + + scenarios = {str(s.config_path): s for s in suite.get_scenarios()} + assert sorted(scenarios) == ["integration_tests/invalid_config.json", "secrets/config.json"] + assert scenarios["secrets/config.json"].sections == ("connection", "basic_read") + assert scenarios["integration_tests/invalid_config.json"].status == "failed" + assert suite.get_scenario_bypasses() == [ + ScenarioBypass( + reason="Covered by basic_read", + location="acceptance_tests.full_refresh[0].bypass_reason", + sections=("full_refresh",), + ) + ] + + +@pytest.mark.parametrize( + "section, expected", + [ + ({"tests": [{"a": 1}]}, [{"a": 1}]), + ([{"a": 1}], [{"a": 1}]), + ({"bypass_reason": "x"}, None), + ({"tests": None}, None), + (None, None), + ("tests", None), + ], +) +def test_section_entries_reads_both_layouts(section: Any, expected: Any) -> None: + assert section_entries(section) == expected + + +def test_legacy_top_level_tests_key_fails_with_the_migration(tmp_path: Path) -> None: + config = { + "connector_image": "airbyte/source-test:dev", + "tests": {"spec": [{"spec_path": "manifest.yaml"}]}, + } + root = _write_connector(tmp_path / "source-test", config) + + with pytest.raises(AcceptanceTestConfigError) as error: + _suite(root).get_scenarios() + message = str(error.value) + assert str(root / ACCEPTANCE_TEST_CONFIG) in message + assert "legacy layout with a top-level `tests:` key" in message + assert "rename `tests:` to `acceptance_tests:`" in message + assert "airbyte/source-test:dev" not in message, "values are not printed" + + +@pytest.mark.parametrize( + "text, match", + [ + ("", "is empty"), + ("# only comments\n", "is empty"), + ("- a\n", "holds a list"), + ("acceptance_tests: [a]\n", "must be a mapping of sections"), + ("connector_image: x\n", "no `acceptance_tests` key and no top-level `bypass_reason`"), + ("bypass_reason: ''\n", "must be a non-empty string"), + ("acceptance_tests: [\n", "is not valid YAML"), + ], +) +def test_unusable_config_layouts_fail(tmp_path: Path, text: str, match: str) -> None: + path = tmp_path / ACCEPTANCE_TEST_CONFIG + path.write_text(text) + + with pytest.raises(AcceptanceTestConfigError, match=match): + load_acceptance_test_config(path) + + +# --- Missing file ----------------------------------------------------------------------------- + + +@pytest.mark.parametrize("language", ["python", "manifest-only"]) +def test_missing_config_fails_for_connectors_that_run_the_standard_tests( + tmp_path: Path, language: str +) -> None: + root = _write_connector(tmp_path / "destination-test", None, language=language) + + with pytest.raises(MissingAcceptanceTestConfigError, match="only a top-level `bypass_reason`"): + _suite(root).get_scenarios() + + +def test_missing_config_error_is_a_file_not_found_error(tmp_path: Path) -> None: + """Callers that tolerated a missing file by catching `FileNotFoundError` keep working.""" + root = _write_connector(tmp_path / "source-test", None) + with pytest.raises(FileNotFoundError): + _suite(root).acceptance_test_config + + +def test_missing_config_is_a_declared_bypass_for_java_connectors(tmp_path: Path) -> None: + suite = _suite(_write_connector(tmp_path / "source-test", None, language="java")) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [ + ScenarioBypass(reason=JAVA_BYPASS_REASON, location="metadata.yaml") + ] + + +# --- Collection (pytest_generate_tests) ------------------------------------------------------- + + +def _run_collection(pytester: pytest.Pytester, root: Path, *args: str) -> pytest.RunResult: + pytester.makepyfile( + test_standard=textwrap.dedent( + f""" + from pathlib import Path + + from airbyte_cdk.test.standard_tests.docker_base import DockerConnectorTestSuite + + + class TestSuite(DockerConnectorTestSuite): + @classmethod + def get_connector_root_dir(cls) -> Path: + return Path({str(root)!r}) + + def test_with_scenario(self, scenario) -> None: + pass + """ + ) + ) + return pytester.runpytest_inprocess( + "-p", + "airbyte_cdk.test.standard_tests.pytest_hooks", + "-p", + "no:cacheprovider", + "-k", + "test_with_scenario", + "-rs", + *args, + ) + + +def test_collection_fails_when_no_scenario_and_no_bypass( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", ZERO_SCENARIOS_CONFIG) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines( + [ + "*No test scenarios for connector 'source-test': *acceptance-test-config.yml lists no " + "config the standard tests can run, and declares no `bypass_reason`. Found:", + "*`acceptance_tests.spec.tests[[]0[]]`: no `config_path` and no `bypass_reason`*", + "*1 error*", + ] + ) + assert "Traceback" not in result.stdout.str(), "the message is reported without a traceback" + + +def test_collection_fails_on_a_missing_config(pytester: pytest.Pytester, tmp_path: Path) -> None: + root = _write_connector(tmp_path / "source-test", None) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines(["*Acceptance test config file not found at: *", "*1 error*"]) + output = result.stdout.str() + assert output.count("Acceptance test config file not found at:") == 1, "reported once" + assert "During handling of the above exception" not in output + + +def test_collection_fails_on_the_legacy_top_level_tests_key( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", {"tests": {"spec": []}}) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines(["*legacy layout with a top-level `tests:` key*", "*1 error*"]) + + +def test_bypasses_are_reported_as_skipped_tests_and_at_collection( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", BYPASSED_SECTIONS_CONFIG) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.OK + result.assert_outcomes(skipped=1) + result.stdout.fnmatch_lines( + [ + "source-test: acceptance-test-config.yml declares 4 bypass(es), reported as skipped " + "tests: connection, discovery, basic_read, full_refresh: " + BUILDER_REASON, + "SKIPPED [[]1[]] *: acceptance-test-config.yml bypasses connection, discovery, " + "basic_read, full_refresh: " + BUILDER_REASON, + ] + ) + + +def test_bypass_params_have_readable_ids(pytester: pytest.Pytester, tmp_path: Path) -> None: + root = _write_connector(tmp_path / "source-test", RUNNABLE_CONFIG) + + result = _run_collection(pytester, root, "--collect-only", "-q") + + result.stdout.fnmatch_lines( + [ + "*test_with_scenario[[]'config' Test Scenario[]]", + "*test_with_scenario[[]bypassed-basic_read[]]", + ] + ) + + +def test_connector_wide_bypass_is_one_skipped_test( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", "bypass_reason: No sandbox account\n") + + result = _run_collection(pytester, root, "-v") + + result.assert_outcomes(skipped=1) + result.stdout.fnmatch_lines( + [ + "*test_with_scenario[[]bypassed-connector[]] SKIPPED*", + "SKIPPED [[]1[]] *: acceptance-test-config.yml bypasses the connector: " + "No sandbox account", + ] + ) + + +def test_bypasses_are_grouped_by_reason_and_name_their_entries( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + config = { + "acceptance_tests": { + "connection": { + "tests": [ + {"config_path": "integration_tests/invalid_config.json", "status": "failed"}, + {"bypass_reason": "The GSM config fails check"}, + ] + }, + "basic_read": {"bypass_reason": "The GSM config fails check"}, + "full_refresh": {"bypass_reason": "Covered by basic_read"}, + } + } + root = _write_connector(tmp_path / "source-test", config) + + result = _run_collection(pytester, root, "-v") + + result.assert_outcomes(passed=1, skipped=2) + result.stdout.fnmatch_lines( + [ + "source-test: acceptance-test-config.yml declares 3 bypass(es), reported as skipped " + "tests: connection.tests[[]1[]], basic_read: The GSM config fails check; " + "full_refresh: Covered by basic_read", + "*test_with_scenario[[]bypassed-connection-basic_read[]] SKIPPED*", + "*test_with_scenario[[]bypassed-full_refresh[]] SKIPPED*", + ] + )