From ac8ac9fdac3160005d4ee5555b3989a0d8dc4043 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:36:12 +0000 Subject: [PATCH 01/12] ci: test connector images against SDM base image built from current branch Co-Authored-By: Daryna Ishchenko --- .github/workflows/connector-tests.yml | 70 +++++++++++++++++++ airbyte_cdk/cli/airbyte_cdk/_image.py | 23 ++++++ .../test/standard_tests/docker_base.py | 28 +++++++- .../test/standard_tests/pytest_hooks.py | 22 ++++++ airbyte_cdk/utils/docker.py | 9 ++- unit_tests/utils/test_docker_build.py | 65 +++++++++++++++++ 6 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 unit_tests/utils/test_docker_build.py diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 29c4e04360..33cb86bde8 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -209,3 +209,73 @@ jobs: POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" run: | poetry run airbyte-cdk image test ${{ matrix.connector }} + + # Regression guard for https://github.com/airbytehq/airbyte-python-cdk/issues/1082: + # builds a `source-declarative-manifest` image FROM the current branch, then builds a bundled + # manifest-only connector (bundled `manifest.yaml` + `components.py`) FROM that image and runs + # `check`/`spec` inside the container with `AIRBYTE_ENABLE_UNSAFE_CODE` unset (the Cloud + # default). Unlike the matrix job above, this exercises the branch's CDK inside the connector + # image instead of the connector's already-published base image, so a regression that rejects + # trusted bundled custom components is caught before a CDK release. Uses an in-repo pokeapi + # fixture so it needs no connector secrets. + bundled_custom_components_image_test: + needs: cdk_changes + name: Bundled Custom Components Image Test + if: needs.cdk_changes.outputs['src'] == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Checkout CDK + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Build the Python package to create the wheel files needed for the SDM Docker build. + - name: Build Python Package + uses: hynek/build-and-inspect-python-package@v2 + env: + POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0dev0" + + - name: Copy wheel files to dist directory + run: | + mkdir -p dist + cp /tmp/baipp/dist/*.whl dist/ + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Build the source-declarative-manifest image FROM the current branch and load it into the + # local Docker image store so the connector image can be built `FROM` it. + - name: Build SDM image from current branch + uses: docker/build-push-action@v5 + with: + context: . + load: true + push: false + tags: airbyte/source-declarative-manifest:dev + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Set up Poetry + uses: snok/install-poetry@v1 + with: + version: "2.0.1" + + - name: Install CDK with Poetry + run: | + poetry install --all-extras + + # `AIRBYTE_ENABLE_UNSAFE_CODE` is intentionally NOT set here so the in-container `check` + # runs under the Cloud default. `--no-creds` is safe because the pokeapi fixture ships a + # local `valid_config.yaml` and needs no external secrets. + - name: Run bundled custom components image test (branch SDM base image) + env: + POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" + run: | + poetry run airbyte-cdk image test \ + unit_tests/resources/source_pokeapi_w_components_py \ + --base-image airbyte/source-declarative-manifest:dev \ + --no-creds diff --git a/airbyte_cdk/cli/airbyte_cdk/_image.py b/airbyte_cdk/cli/airbyte_cdk/_image.py index c89c05363b..903b89ca2a 100644 --- a/airbyte_cdk/cli/airbyte_cdk/_image.py +++ b/airbyte_cdk/cli/airbyte_cdk/_image.py @@ -42,12 +42,22 @@ def image_cli_group() -> None: type=click.Path(exists=True, file_okay=True, path_type=Path), help="Optional. Override the Dockerfile used for building the image.", ) +@click.option( + "--base-image", + default=None, + help=( + "Optional. Build the connector `FROM` this base image instead of the `baseImage` " + "declared in metadata.yaml. Useful for testing a connector against a locally-built " + "`source-declarative-manifest` image (e.g. built from the current CDK branch)." + ), +) def build( connector: str | None = None, *, tag: str = "dev", no_verify: bool = False, dockerfile: Path | None = None, + base_image: str | None = None, ) -> None: """Build a connector Docker image. @@ -80,6 +90,7 @@ def build( tag=tag, no_verify=no_verify, dockerfile_override=dockerfile or None, + base_image_override=base_image or None, ) except ConnectorImageBuildError as e: click.echo( @@ -106,11 +117,22 @@ def build( default=False, help="Skip tests that require credentials (marked with 'requires_creds').", ) +@click.option( + "--base-image", + default=None, + help=( + "Optional. When building the image to test, build it `FROM` this base image instead of " + "the `baseImage` declared in metadata.yaml. Useful for testing a manifest-only connector " + "against a locally-built `source-declarative-manifest` image (e.g. built from the current " + "CDK branch). Ignored when `--image` is provided." + ), +) def image_test( # "image test" command connector: str | None = None, *, image: str | None = None, no_creds: bool = False, + base_image: str | None = None, ) -> None: """Test a connector Docker image. @@ -157,6 +179,7 @@ def image_test( # "image test" command metadata=metadata, tag=tag, no_verify=True, + base_image_override=base_image or None, ) except ConnectorImageBuildError as e: click.echo( diff --git a/airbyte_cdk/test/standard_tests/docker_base.py b/airbyte_cdk/test/standard_tests/docker_base.py index e3e196ce1f..ace3bb7fe4 100644 --- a/airbyte_cdk/test/standard_tests/docker_base.py +++ b/airbyte_cdk/test/standard_tests/docker_base.py @@ -22,6 +22,7 @@ ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, + Status, SyncMode, ) from airbyte_cdk.models.connector_metadata import MetadataFile @@ -170,6 +171,7 @@ def get_scenarios( def test_docker_image_build_and_spec( self, connector_image_override: str | None, + connector_base_image_override: str | None, ) -> None: """Run `docker_image` acceptance tests.""" connector_root = self.get_connector_root_dir().absolute() @@ -184,6 +186,7 @@ def test_docker_image_build_and_spec( metadata=metadata, tag=tag, no_verify=False, + base_image_override=connector_base_image_override, ) _ = run_docker_airbyte_command( @@ -206,6 +209,7 @@ def test_docker_image_build_and_check( self, scenario: ConnectorTestScenario, connector_image_override: str | None, + connector_base_image_override: str | None, ) -> None: """Run `docker_image` acceptance tests. @@ -231,13 +235,14 @@ def test_docker_image_build_and_check( metadata=metadata, tag=tag, no_verify=False, + base_image_override=connector_base_image_override, ) container_config_path = "/secrets/config.json" with scenario.with_temp_config_file( connector_root=connector_root, ) as temp_config_file: - _ = run_docker_airbyte_command( + check_result = run_docker_airbyte_command( [ "docker", "run", @@ -252,6 +257,25 @@ def test_docker_image_build_and_check( raise_if_errors=True, ) + # A failing `check` reports `status: FAILED` in a `CONNECTION_STATUS` message and still + # exits 0, so `raise_if_errors` alone does not catch it. We therefore assert the reported + # status explicitly. This makes the image test exercise the connector's actual `check` + # outcome inside the container (e.g. it fails if bundled custom components are rejected by + # the CDK baked into the base image). + connection_statuses = [ + message.connectionStatus + for message in check_result.connection_status_messages + if message.connectionStatus is not None + ] + assert connection_statuses, ( + f"`check` for connector '{connector_root.name}' emitted no CONNECTION_STATUS message. " + f"Logs: {check_result.logs}" + ) + assert connection_statuses[-1].status == Status.SUCCEEDED, ( + f"`check` for connector '{connector_root.name}' did not succeed: " + f"{connection_statuses[-1]}" + ) + @pytest.mark.skipif( shutil.which("docker") is None, reason="docker CLI not found in PATH, skipping docker image tests", @@ -261,6 +285,7 @@ def test_docker_image_build_and_read( self, scenario: ConnectorTestScenario, connector_image_override: str | None, + connector_base_image_override: str | None, read_from_streams: Literal["all", "none", "default"] | list[str], read_scenarios: Literal["all", "none", "default"] | list[str], ) -> None: @@ -315,6 +340,7 @@ def test_docker_image_build_and_read( metadata=metadata, tag=tag, no_verify=False, + base_image_override=connector_base_image_override, ) container_config_path = "/secrets/config.json" diff --git a/airbyte_cdk/test/standard_tests/pytest_hooks.py b/airbyte_cdk/test/standard_tests/pytest_hooks.py index cae8f1abbf..1d38bea4ac 100644 --- a/airbyte_cdk/test/standard_tests/pytest_hooks.py +++ b/airbyte_cdk/test/standard_tests/pytest_hooks.py @@ -24,6 +24,18 @@ def connector_image_override(request: pytest.FixtureRequest) -> str | None: return cast(str | None, request.config.getoption("--connector-image")) +@pytest.fixture +def connector_base_image_override(request: pytest.FixtureRequest) -> str | None: + """Return the value of --base-image, or None if not set. + + When set, the connector image is built `FROM` this base image instead of the + `baseImage` declared in metadata.yaml. This lets the image tests exercise a + locally-built `source-declarative-manifest` image (e.g. one built from the current + CDK branch) rather than the published base image's CDK. + """ + return cast(str | None, request.config.getoption("--base-image")) + + @pytest.fixture def read_from_streams( request: pytest.FixtureRequest, @@ -116,6 +128,16 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Use this pre-built connector Docker image instead of building one.", ) + parser.addoption( + "--base-image", + action="store", + default=None, + help=( + "Build the connector image `FROM` this base image instead of the `baseImage` " + "declared in metadata.yaml. Useful for testing against a locally-built " + "`source-declarative-manifest` image built from the current CDK branch." + ), + ) parser.addoption( "--read-from-streams", action="store", diff --git a/airbyte_cdk/utils/docker.py b/airbyte_cdk/utils/docker.py index 24467ff3c0..7770c8a201 100644 --- a/airbyte_cdk/utils/docker.py +++ b/airbyte_cdk/utils/docker.py @@ -150,6 +150,7 @@ def build_connector_image( tag: str, no_verify: bool = False, dockerfile_override: Path | None = None, + base_image_override: str | None = None, ) -> str: """Build a connector Docker image. @@ -163,6 +164,12 @@ def build_connector_image( metadata: The metadata of the connector. tag: The tag to apply to the built image. no_verify: If True, skip verification of the built image. + dockerfile_override: Optional path to a Dockerfile to use instead of the connector's default. + base_image_override: Optional base image to build `FROM` instead of the `baseImage` + declared in `metadata.yaml`. This lets CI build a connector on top of a + locally-built `source-declarative-manifest` image (e.g. one built from the current + branch) so the image tests exercise the branch's CDK rather than the published base + image's CDK. Raises: ValueError: If the connector build options are not defined in metadata.yaml. @@ -228,7 +235,7 @@ def build_connector_image( "Please check the connector's metadata file." ) - base_image = metadata.data.connectorBuildOptions.baseImage + base_image = base_image_override or metadata.data.connectorBuildOptions.baseImage build_args: dict[str, str | None] = { "BASE_IMAGE": base_image, "CONNECTOR_NAME": connector_kebab_name, diff --git a/unit_tests/utils/test_docker_build.py b/unit_tests/utils/test_docker_build.py new file mode 100644 index 0000000000..6d5d889527 --- /dev/null +++ b/unit_tests/utils/test_docker_build.py @@ -0,0 +1,65 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""Tests for `build_connector_image` in `airbyte_cdk.utils.docker`.""" + +from pathlib import Path + +import pytest + +from airbyte_cdk.models.connector_metadata import MetadataFile +from airbyte_cdk.utils import docker + +POKEAPI_FIXTURE_DIR = Path(__file__).parent.parent / "resources" / "source_pokeapi_w_components_py" + + +@pytest.mark.parametrize( + "base_image_override, expected_base_image", + [ + pytest.param( + None, + "docker.io/airbyte/source-declarative-manifest:6.51.0@sha256:890b109f243b8b9406f23ea7522de41025f7b3e87f6fc9710bc1e521213a276f", + id="default_uses_metadata_base_image", + ), + pytest.param( + "airbyte/source-declarative-manifest:dev", + "airbyte/source-declarative-manifest:dev", + id="override_replaces_metadata_base_image", + ), + ], +) +def test_build_connector_image_base_image_override( + monkeypatch: pytest.MonkeyPatch, + base_image_override: str | None, + expected_base_image: str, +) -> None: + connector_directory = POKEAPI_FIXTURE_DIR + metadata = MetadataFile.from_file(connector_directory / "metadata.yaml") + + captured_build_args: list[dict[str, str | None]] = [] + + def fake_build_image( + *, + context_dir: Path, + dockerfile: Path, + metadata: MetadataFile, + tag: str, + arch: docker.ArchEnum, + build_args: dict[str, str | None], + ) -> str: + captured_build_args.append(build_args) + return tag + + monkeypatch.setattr(docker, "_build_image", fake_build_image) + monkeypatch.setattr(docker, "_tag_image", lambda *, tag, new_tags: None) + + docker.build_connector_image( + connector_name="source-pokeapi", + connector_directory=connector_directory, + metadata=metadata, + tag="test-tag", + no_verify=True, + base_image_override=base_image_override, + ) + + assert captured_build_args, "Expected at least one image build." + for build_args in captured_build_args: + assert build_args["BASE_IMAGE"] == expected_base_image From 909c48b9cdbef3452a435e9ddec165b04e5cdb5d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:43:19 +0000 Subject: [PATCH 02/12] ci: build SDM image into local image store so connector build can use it as base Co-Authored-By: Daryna Ishchenko --- .github/workflows/connector-tests.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 33cb86bde8..8654985bb0 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -241,18 +241,14 @@ jobs: mkdir -p dist cp /tmp/baipp/dist/*.whl dist/ - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # Build the source-declarative-manifest image FROM the current branch and load it into the - # local Docker image store so the connector image can be built `FROM` it. + # Build the source-declarative-manifest image FROM the current branch with a plain + # `docker build` (the default `docker` buildx driver) so it lands in the local image store. + # `airbyte-cdk image build` also uses the default builder, so it can resolve this image as + # its `FROM` base. A `docker-container` driver (e.g. from docker/setup-buildx-action) would + # NOT share the image store and the connector build would fail to find this base image. - name: Build SDM image from current branch - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - tags: airbyte/source-declarative-manifest:dev + run: | + docker build -t airbyte/source-declarative-manifest:dev . - name: Set up Python uses: actions/setup-python@v5 From 65d8c5fed17b4dd8c403b26ae96616b1cb42f589 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:49:14 +0000 Subject: [PATCH 03/12] ci: restrict GITHUB_TOKEN permissions and disable credential persistence in new job Co-Authored-By: Daryna Ishchenko --- .github/workflows/connector-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 8654985bb0..973b5cf68d 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -224,11 +224,14 @@ jobs: if: needs.cdk_changes.outputs['src'] == 'true' runs-on: ubuntu-24.04 timeout-minutes: 30 + permissions: + contents: read steps: - name: Checkout CDK uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false # Build the Python package to create the wheel files needed for the SDM Docker build. - name: Build Python Package From b946f4892358d58edfbe73a5f18d10ced6d12c1a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:05:04 +0000 Subject: [PATCH 04/12] ci: build manifest-only connector images in the test matrix FROM branch-built SDM Co-Authored-By: Daryna Ishchenko --- .github/workflows/connector-tests.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 973b5cf68d..80e6377ad5 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -201,6 +201,22 @@ jobs: run: | poetry run airbyte-cdk connector test ${{ matrix.connector }} + # Manifest-only connector images are built `FROM` the published `source-declarative-manifest` + # base image declared in metadata.yaml, so by default the container tests exercise the + # already-released CDK instead of this branch's CDK. Build an SDM image from the current + # branch (with a plain `docker build` so it lands in the local image store, where the + # connector image build can resolve it as its `FROM` base) and use it via `--base-image`. + # Python connectors don't need this: the "Bump to Dev Branch CDK" step pins the connector's + # CDK dependency to this branch, which the container image build then installs. + - name: Build SDM image from current branch [Manifest-Only Connectors] + if: env.CONNECTOR_LANGUAGE == 'manifest-only' + working-directory: airbyte-python-cdk + env: + POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0dev0" + run: | + poetry build -f wheel + docker build -t airbyte/source-declarative-manifest:dev . + - name: Container Build and Test [All Connectors] if: steps.no_changes.outputs.status != 'cancelled' working-directory: airbyte-python-cdk @@ -208,7 +224,12 @@ jobs: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" run: | - poetry run airbyte-cdk image test ${{ matrix.connector }} + if [ "$CONNECTOR_LANGUAGE" = "manifest-only" ]; then + poetry run airbyte-cdk image test ${{ matrix.connector }} \ + --base-image airbyte/source-declarative-manifest:dev + else + poetry run airbyte-cdk image test ${{ matrix.connector }} + fi # Regression guard for https://github.com/airbytehq/airbyte-python-cdk/issues/1082: # builds a `source-declarative-manifest` image FROM the current branch, then builds a bundled From abc518b4bbce11ae0a472a9fb6efa3709feac87f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:12:25 +0000 Subject: [PATCH 05/12] ci: remove standalone bundled custom components job (covered by matrix) Co-Authored-By: Daryna Ishchenko --- .github/workflows/connector-tests.yml | 69 --------------------------- 1 file changed, 69 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 80e6377ad5..ee57582c97 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -230,72 +230,3 @@ jobs: else poetry run airbyte-cdk image test ${{ matrix.connector }} fi - - # Regression guard for https://github.com/airbytehq/airbyte-python-cdk/issues/1082: - # builds a `source-declarative-manifest` image FROM the current branch, then builds a bundled - # manifest-only connector (bundled `manifest.yaml` + `components.py`) FROM that image and runs - # `check`/`spec` inside the container with `AIRBYTE_ENABLE_UNSAFE_CODE` unset (the Cloud - # default). Unlike the matrix job above, this exercises the branch's CDK inside the connector - # image instead of the connector's already-published base image, so a regression that rejects - # trusted bundled custom components is caught before a CDK release. Uses an in-repo pokeapi - # fixture so it needs no connector secrets. - bundled_custom_components_image_test: - needs: cdk_changes - name: Bundled Custom Components Image Test - if: needs.cdk_changes.outputs['src'] == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 30 - permissions: - contents: read - steps: - - name: Checkout CDK - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - # Build the Python package to create the wheel files needed for the SDM Docker build. - - name: Build Python Package - uses: hynek/build-and-inspect-python-package@v2 - env: - POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0dev0" - - - name: Copy wheel files to dist directory - run: | - mkdir -p dist - cp /tmp/baipp/dist/*.whl dist/ - - # Build the source-declarative-manifest image FROM the current branch with a plain - # `docker build` (the default `docker` buildx driver) so it lands in the local image store. - # `airbyte-cdk image build` also uses the default builder, so it can resolve this image as - # its `FROM` base. A `docker-container` driver (e.g. from docker/setup-buildx-action) would - # NOT share the image store and the connector build would fail to find this base image. - - name: Build SDM image from current branch - run: | - docker build -t airbyte/source-declarative-manifest:dev . - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Set up Poetry - uses: snok/install-poetry@v1 - with: - version: "2.0.1" - - - name: Install CDK with Poetry - run: | - poetry install --all-extras - - # `AIRBYTE_ENABLE_UNSAFE_CODE` is intentionally NOT set here so the in-container `check` - # runs under the Cloud default. `--no-creds` is safe because the pokeapi fixture ships a - # local `valid_config.yaml` and needs no external secrets. - - name: Run bundled custom components image test (branch SDM base image) - env: - POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" - run: | - poetry run airbyte-cdk image test \ - unit_tests/resources/source_pokeapi_w_components_py \ - --base-image airbyte/source-declarative-manifest:dev \ - --no-creds From 570bf8516022742dfd2b692e70e634e48c36305b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:05:50 +0000 Subject: [PATCH 06/12] fix: use absolute connector root name in check assertion messages Co-Authored-By: Daryna Ishchenko --- airbyte_cdk/test/standard_tests/docker_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/test/standard_tests/docker_base.py b/airbyte_cdk/test/standard_tests/docker_base.py index ace3bb7fe4..ebde80c049 100644 --- a/airbyte_cdk/test/standard_tests/docker_base.py +++ b/airbyte_cdk/test/standard_tests/docker_base.py @@ -268,11 +268,11 @@ def test_docker_image_build_and_check( if message.connectionStatus is not None ] assert connection_statuses, ( - f"`check` for connector '{connector_root.name}' emitted no CONNECTION_STATUS message. " + f"`check` for connector '{connector_root.absolute().name}' emitted no CONNECTION_STATUS message. " f"Logs: {check_result.logs}" ) assert connection_statuses[-1].status == Status.SUCCEEDED, ( - f"`check` for connector '{connector_root.name}' did not succeed: " + f"`check` for connector '{connector_root.absolute().name}' did not succeed: " f"{connection_statuses[-1]}" ) From d1c8a152cf722155e40fdfef3494f7c2e700a302 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:28:04 +0000 Subject: [PATCH 07/12] ci: re-run checks (transient ConnectionResetError in Pytest 3.13 job) Co-Authored-By: Daryna Ishchenko From 7568da870caf434c9a94265f8a1288a0608ae8d3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:09:01 +0000 Subject: [PATCH 08/12] ci: run FAST manifest-only standard tests against branch-built SDM base image Also fix 'connector test' crashing when --pytest-arg is combined with --collect-only or --no-creds (click multiple=True yields a tuple). Co-Authored-By: Daryna Ishchenko --- .github/workflows/connector-tests.yml | 24 ++++++++++++++--------- airbyte_cdk/cli/airbyte_cdk/_connector.py | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index ee57582c97..53310a6c7d 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -192,15 +192,6 @@ jobs: run: | poetry run airbyte-cdk connector test - - name: Run FAST Standard Tests [Manifest-Only Connectors] - if: env.CONNECTOR_LANGUAGE == 'manifest-only' - working-directory: airbyte-python-cdk - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" - run: | - poetry run airbyte-cdk connector test ${{ matrix.connector }} - # Manifest-only connector images are built `FROM` the published `source-declarative-manifest` # base image declared in metadata.yaml, so by default the container tests exercise the # already-released CDK instead of this branch's CDK. Build an SDM image from the current @@ -217,6 +208,21 @@ jobs: poetry build -f wheel docker build -t airbyte/source-declarative-manifest:dev . + # The FAST standard test suite also collects the docker image tests, so pass the + # branch-built SDM base image here too. Otherwise those tests build the connector image + # `FROM` the published metadata base image (running the released CDK instead of this + # branch's), and the later `image test` step can't reuse its layers. + - name: Run FAST Standard Tests [Manifest-Only Connectors] + if: env.CONNECTOR_LANGUAGE == 'manifest-only' + working-directory: airbyte-python-cdk + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" + run: | + poetry run airbyte-cdk connector test ${{ matrix.connector }} \ + --pytest-arg=--base-image \ + --pytest-arg=airbyte/source-declarative-manifest:dev + - name: Container Build and Test [All Connectors] if: steps.no_changes.outputs.status != 'cancelled' working-directory: airbyte-python-cdk diff --git a/airbyte_cdk/cli/airbyte_cdk/_connector.py b/airbyte_cdk/cli/airbyte_cdk/_connector.py index 249e58f308..accac708fa 100644 --- a/airbyte_cdk/cli/airbyte_cdk/_connector.py +++ b/airbyte_cdk/cli/airbyte_cdk/_connector.py @@ -148,7 +148,7 @@ def connector_test( click.echo("Connector test command executed.") connector_name, connector_directory = resolve_connector_name_and_directory(connector) - pytest_args = pytest_args or [] + pytest_args = list(pytest_args or []) if collect_only: pytest_args.append("--collect-only") From 4ca7af5ac2224a2d203123d148c95f4933af10d8 Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Wed, 29 Jul 2026 15:38:24 +0300 Subject: [PATCH 09/12] fix(tests): honor ALLOW_ANY in check assertion; make docker build test hermetic - Only assert the in-container `check` reports SUCCEEDED when the scenario explicitly expects success, so scenarios with no declared status (ExpectedOutcome.ALLOW_ANY) keep their "either outcome passes" contract. - Run `test_build_connector_image_base_image_override` against a tmp_path copy of the pokeapi fixture and stub `get_dockerfile_templates`, so the unit test no longer writes into the checked-in fixture dir or falls back to downloading Dockerfile templates from GitHub. Co-Authored-By: Claude Fable 5 --- airbyte_cdk/test/standard_tests/docker_base.py | 11 +++++++---- unit_tests/utils/test_docker_build.py | 14 +++++++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/airbyte_cdk/test/standard_tests/docker_base.py b/airbyte_cdk/test/standard_tests/docker_base.py index ebde80c049..9ef32fb3b1 100644 --- a/airbyte_cdk/test/standard_tests/docker_base.py +++ b/airbyte_cdk/test/standard_tests/docker_base.py @@ -271,10 +271,13 @@ def test_docker_image_build_and_check( f"`check` for connector '{connector_root.absolute().name}' emitted no CONNECTION_STATUS message. " f"Logs: {check_result.logs}" ) - assert connection_statuses[-1].status == Status.SUCCEEDED, ( - f"`check` for connector '{connector_root.absolute().name}' did not succeed: " - f"{connection_statuses[-1]}" - ) + # Scenarios with no declared status map to ALLOW_ANY, whose contract is that either + # outcome passes - only assert SUCCEEDED when the scenario explicitly expects success. + if scenario.expected_outcome.expect_success(): + assert connection_statuses[-1].status == Status.SUCCEEDED, ( + f"`check` for connector '{connector_root.absolute().name}' did not succeed: " + f"{connection_statuses[-1]}" + ) @pytest.mark.skipif( shutil.which("docker") is None, diff --git a/unit_tests/utils/test_docker_build.py b/unit_tests/utils/test_docker_build.py index 6d5d889527..295932d770 100644 --- a/unit_tests/utils/test_docker_build.py +++ b/unit_tests/utils/test_docker_build.py @@ -1,6 +1,7 @@ # Copyright (c) 2025 Airbyte, Inc., all rights reserved. """Tests for `build_connector_image` in `airbyte_cdk.utils.docker`.""" +import shutil from pathlib import Path import pytest @@ -28,10 +29,21 @@ ) def test_build_connector_image_base_image_override( monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, base_image_override: str | None, expected_base_image: str, ) -> None: - connector_directory = POKEAPI_FIXTURE_DIR + # Copy the fixture so the Dockerfile templates that `build_connector_image` writes into + # `/build/docker/` land in a temp dir instead of the checked-in fixture, and stub + # template resolution, which otherwise falls back to downloading templates from GitHub when + # no airbyte monorepo checkout is present. + connector_directory = tmp_path / POKEAPI_FIXTURE_DIR.name + shutil.copytree(POKEAPI_FIXTURE_DIR, connector_directory) + monkeypatch.setattr( + docker, + "get_dockerfile_templates", + lambda *, metadata, connector_directory: ("FROM ${BASE_IMAGE}\n", ""), + ) metadata = MetadataFile.from_file(connector_directory / "metadata.yaml") captured_build_args: list[dict[str, str | None]] = [] From d525674d1def558d12de4da4cbd28caa27d81ede Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Wed, 29 Jul 2026 15:51:02 +0300 Subject: [PATCH 10/12] fix(tests): rename pytest option to --connector-base-image; document buildx driver caveat - Rename the pytest plugin option `--base-image` to `--connector-base-image`, matching the `--connector-image` naming pairing and keeping the generic `--base-image` name out of the shared pytest option namespace (a duplicate registration by any other plugin would fail collection for all CDK test users). The `airbyte-cdk image build/test` click flags keep `--base-image` since each command owns its own namespace. - Note in the CLI help texts and `build_connector_image` docstring that the override image must be visible to the default buildx builder: locally-built images resolve only with the default `docker` driver, not `docker-container` builders. Co-Authored-By: Claude Fable 5 --- .github/workflows/connector-tests.yml | 2 +- airbyte_cdk/cli/airbyte_cdk/_image.py | 8 ++++++-- airbyte_cdk/test/standard_tests/pytest_hooks.py | 10 ++++++---- airbyte_cdk/utils/docker.py | 4 +++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 53310a6c7d..ebf1d3bc06 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -220,7 +220,7 @@ jobs: POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" run: | poetry run airbyte-cdk connector test ${{ matrix.connector }} \ - --pytest-arg=--base-image \ + --pytest-arg=--connector-base-image \ --pytest-arg=airbyte/source-declarative-manifest:dev - name: Container Build and Test [All Connectors] diff --git a/airbyte_cdk/cli/airbyte_cdk/_image.py b/airbyte_cdk/cli/airbyte_cdk/_image.py index 903b89ca2a..e4e7c7011e 100644 --- a/airbyte_cdk/cli/airbyte_cdk/_image.py +++ b/airbyte_cdk/cli/airbyte_cdk/_image.py @@ -48,7 +48,9 @@ def image_cli_group() -> None: help=( "Optional. Build the connector `FROM` this base image instead of the `baseImage` " "declared in metadata.yaml. Useful for testing a connector against a locally-built " - "`source-declarative-manifest` image (e.g. built from the current CDK branch)." + "`source-declarative-manifest` image (e.g. built from the current CDK branch). " + "The image must be visible to your default buildx builder; locally-built images " + "are only resolved by the default `docker` driver, not `docker-container` builders." ), ) def build( @@ -124,7 +126,9 @@ def build( "Optional. When building the image to test, build it `FROM` this base image instead of " "the `baseImage` declared in metadata.yaml. Useful for testing a manifest-only connector " "against a locally-built `source-declarative-manifest` image (e.g. built from the current " - "CDK branch). Ignored when `--image` is provided." + "CDK branch). The image must be visible to your default buildx builder; locally-built " + "images are only resolved by the default `docker` driver, not `docker-container` " + "builders. Ignored when `--image` is provided." ), ) def image_test( # "image test" command diff --git a/airbyte_cdk/test/standard_tests/pytest_hooks.py b/airbyte_cdk/test/standard_tests/pytest_hooks.py index 1d38bea4ac..9ee2f292a7 100644 --- a/airbyte_cdk/test/standard_tests/pytest_hooks.py +++ b/airbyte_cdk/test/standard_tests/pytest_hooks.py @@ -26,14 +26,14 @@ def connector_image_override(request: pytest.FixtureRequest) -> str | None: @pytest.fixture def connector_base_image_override(request: pytest.FixtureRequest) -> str | None: - """Return the value of --base-image, or None if not set. + """Return the value of --connector-base-image, or None if not set. When set, the connector image is built `FROM` this base image instead of the `baseImage` declared in metadata.yaml. This lets the image tests exercise a locally-built `source-declarative-manifest` image (e.g. one built from the current CDK branch) rather than the published base image's CDK. """ - return cast(str | None, request.config.getoption("--base-image")) + return cast(str | None, request.config.getoption("--connector-base-image")) @pytest.fixture @@ -129,13 +129,15 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Use this pre-built connector Docker image instead of building one.", ) parser.addoption( - "--base-image", + "--connector-base-image", action="store", default=None, help=( "Build the connector image `FROM` this base image instead of the `baseImage` " "declared in metadata.yaml. Useful for testing against a locally-built " - "`source-declarative-manifest` image built from the current CDK branch." + "`source-declarative-manifest` image built from the current CDK branch. " + "The image must be visible to your default buildx builder; locally-built images " + "are only resolved by the default `docker` driver, not `docker-container` builders." ), ) parser.addoption( diff --git a/airbyte_cdk/utils/docker.py b/airbyte_cdk/utils/docker.py index 7770c8a201..9e66728573 100644 --- a/airbyte_cdk/utils/docker.py +++ b/airbyte_cdk/utils/docker.py @@ -169,7 +169,9 @@ def build_connector_image( declared in `metadata.yaml`. This lets CI build a connector on top of a locally-built `source-declarative-manifest` image (e.g. one built from the current branch) so the image tests exercise the branch's CDK rather than the published base - image's CDK. + image's CDK. The image must be visible to the default buildx builder: locally-built + images are only resolved by the default `docker` driver, not by `docker-container` + builders (which would instead attempt to pull the image from the registry). Raises: ValueError: If the connector build options are not defined in metadata.yaml. From 3b01ff8b679d5f7b24a77dfef684e8a4996a3f05 Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Wed, 29 Jul 2026 16:55:53 +0300 Subject: [PATCH 11/12] feat(tests): assert expected-failure scenarios fail in-container check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously `test_docker_image_build_and_check` skipped scenarios that expect failure, so a `check` that was expected to fail but started succeeding inside the container went unnoticed. The non-docker `test_check` already covers this in-process (`_job_runner.py` asserts FAILED for expected-failure scenarios), but in-container behavior can diverge from in-process — which is the point of these image tests. - Run `check` for expected-failure scenarios too (without raising on container errors, since a non-zero exit is an acceptable failure mode) and assert the reported CONNECTION_STATUS is not SUCCEEDED. - Extract the outcome assertion into `_assert_check_outcome` and unit-test the full expected-outcome x reported-status matrix. Co-Authored-By: Claude Fable 5 --- .../test/standard_tests/docker_base.py | 75 ++++++++++++------ .../test/test_docker_base_check_outcome.py | 76 +++++++++++++++++++ 2 files changed, 126 insertions(+), 25 deletions(-) create mode 100644 unit_tests/test/test_docker_base_check_outcome.py diff --git a/airbyte_cdk/test/standard_tests/docker_base.py b/airbyte_cdk/test/standard_tests/docker_base.py index 9ef32fb3b1..6fe78eb314 100644 --- a/airbyte_cdk/test/standard_tests/docker_base.py +++ b/airbyte_cdk/test/standard_tests/docker_base.py @@ -27,7 +27,7 @@ ) from airbyte_cdk.models.connector_metadata import MetadataFile from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput -from airbyte_cdk.test.models import ConnectorTestScenario +from airbyte_cdk.test.models import ConnectorTestScenario, ExpectedOutcome from airbyte_cdk.utils.connector_paths import ( ACCEPTANCE_TEST_CONFIG, find_connector_root, @@ -38,6 +38,44 @@ ) +def _assert_check_outcome( + *, + check_result: EntrypointOutput, + expected_outcome: ExpectedOutcome, + connector_name: str, +) -> None: + """Assert that the reported CONNECTION_STATUS matches the scenario's expected outcome. + + A failing `check` reports `status: FAILED` in a `CONNECTION_STATUS` message and still + exits 0, so exit-code checks alone do not catch it. We therefore assert the reported + status explicitly, in both directions: + - A scenario expecting success must report `SUCCEEDED`. + - A scenario expecting failure must not report `SUCCEEDED` (a graceful `FAILED` status + or an uncaught error with no status message both count as the expected failure). + - `ALLOW_ANY` scenarios accept either outcome. + """ + connection_statuses = [ + message.connectionStatus + for message in check_result.connection_status_messages + if message.connectionStatus is not None + ] + if expected_outcome.expect_exception(): + assert not connection_statuses or connection_statuses[-1].status != Status.SUCCEEDED, ( + f"`check` for connector '{connector_name}' was expected to fail, but reported: " + f"{connection_statuses[-1]}" + ) + return + + assert connection_statuses, ( + f"`check` for connector '{connector_name}' emitted no CONNECTION_STATUS message. " + f"Logs: {check_result.logs}" + ) + if expected_outcome.expect_success(): + assert connection_statuses[-1].status == Status.SUCCEEDED, ( + f"`check` for connector '{connector_name}' did not succeed: {connection_statuses[-1]}" + ) + + class DockerConnectorTestSuite: """Base class for connector test suites.""" @@ -220,9 +258,6 @@ def test_docker_image_build_and_check( - In the rare case that image caches need to be cleared, please clear the local docker image cache using `docker image prune -a` command. """ - if scenario.expected_outcome.expect_exception(): - pytest.skip("Skipping test_docker_image_build_and_check (expected to fail).") - tag = "dev-latest" connector_root = self.get_connector_root_dir() metadata = MetadataFile.from_file(connector_root / "metadata.yaml") @@ -254,30 +289,20 @@ def test_docker_image_build_and_check( "--config", container_config_path, ], - raise_if_errors=True, + # For expected-failure scenarios, a non-zero exit or trace error is an + # acceptable way for `check` to fail; don't raise before we assert on it. + raise_if_errors=not scenario.expected_outcome.expect_exception(), ) - # A failing `check` reports `status: FAILED` in a `CONNECTION_STATUS` message and still - # exits 0, so `raise_if_errors` alone does not catch it. We therefore assert the reported - # status explicitly. This makes the image test exercise the connector's actual `check` - # outcome inside the container (e.g. it fails if bundled custom components are rejected by - # the CDK baked into the base image). - connection_statuses = [ - message.connectionStatus - for message in check_result.connection_status_messages - if message.connectionStatus is not None - ] - assert connection_statuses, ( - f"`check` for connector '{connector_root.absolute().name}' emitted no CONNECTION_STATUS message. " - f"Logs: {check_result.logs}" + # This makes the image test exercise the connector's actual `check` outcome inside the + # container, in both directions (e.g. it fails if bundled custom components are rejected + # by the CDK baked into the base image, and it fails if a `check` that is expected to + # fail starts succeeding). + _assert_check_outcome( + check_result=check_result, + expected_outcome=scenario.expected_outcome, + connector_name=connector_root.absolute().name, ) - # Scenarios with no declared status map to ALLOW_ANY, whose contract is that either - # outcome passes - only assert SUCCEEDED when the scenario explicitly expects success. - if scenario.expected_outcome.expect_success(): - assert connection_statuses[-1].status == Status.SUCCEEDED, ( - f"`check` for connector '{connector_root.absolute().name}' did not succeed: " - f"{connection_statuses[-1]}" - ) @pytest.mark.skipif( shutil.which("docker") is None, diff --git a/unit_tests/test/test_docker_base_check_outcome.py b/unit_tests/test/test_docker_base_check_outcome.py new file mode 100644 index 0000000000..8874881e88 --- /dev/null +++ b/unit_tests/test/test_docker_base_check_outcome.py @@ -0,0 +1,76 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""Unit tests for `_assert_check_outcome` in `airbyte_cdk.test.standard_tests.docker_base`.""" + +import json + +import pytest + +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.models import ExpectedOutcome +from airbyte_cdk.test.standard_tests.docker_base import _assert_check_outcome + + +def _check_output(*statuses: str) -> EntrypointOutput: + """Build an EntrypointOutput with one CONNECTION_STATUS message per given status.""" + return EntrypointOutput( + messages=[ + json.dumps({"type": "CONNECTION_STATUS", "connectionStatus": {"status": status}}) + for status in statuses + ], + command=["docker", "run", "..."], + ) + + +@pytest.mark.parametrize( + "expected_outcome, statuses, should_pass", + [ + # Scenarios expecting success: only SUCCEEDED passes. + pytest.param(ExpectedOutcome.EXPECT_SUCCESS, ["SUCCEEDED"], True, id="success_succeeded"), + pytest.param(ExpectedOutcome.EXPECT_SUCCESS, ["FAILED"], False, id="success_failed"), + pytest.param(ExpectedOutcome.EXPECT_SUCCESS, [], False, id="success_no_status"), + # Scenarios expecting failure: a SUCCEEDED status is a regression and must fail; + # a FAILED status or no status at all (uncaught error) both count as failing as expected. + pytest.param(ExpectedOutcome.EXPECT_EXCEPTION, ["FAILED"], True, id="failure_failed"), + pytest.param( + ExpectedOutcome.EXPECT_EXCEPTION, ["SUCCEEDED"], False, id="failure_succeeded" + ), + pytest.param(ExpectedOutcome.EXPECT_EXCEPTION, [], True, id="failure_no_status"), + # ALLOW_ANY scenarios: either reported outcome passes, but `check` must still + # emit a CONNECTION_STATUS message. + pytest.param(ExpectedOutcome.ALLOW_ANY, ["SUCCEEDED"], True, id="allow_any_succeeded"), + pytest.param(ExpectedOutcome.ALLOW_ANY, ["FAILED"], True, id="allow_any_failed"), + pytest.param(ExpectedOutcome.ALLOW_ANY, [], False, id="allow_any_no_status"), + # The last CONNECTION_STATUS message wins. + pytest.param( + ExpectedOutcome.EXPECT_SUCCESS, + ["FAILED", "SUCCEEDED"], + True, + id="success_last_status_wins", + ), + pytest.param( + ExpectedOutcome.EXPECT_EXCEPTION, + ["FAILED", "SUCCEEDED"], + False, + id="failure_last_status_wins", + ), + ], +) +def test_assert_check_outcome( + expected_outcome: ExpectedOutcome, + statuses: list[str], + should_pass: bool, +) -> None: + check_result = _check_output(*statuses) + if should_pass: + _assert_check_outcome( + check_result=check_result, + expected_outcome=expected_outcome, + connector_name="source-test", + ) + else: + with pytest.raises(AssertionError): + _assert_check_outcome( + check_result=check_result, + expected_outcome=expected_outcome, + connector_name="source-test", + ) From d58c80accec352f1684789aa07a976052956153d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:14:16 +0000 Subject: [PATCH 12/12] fix(tests): merge declared status in scenario dedup; unify --connector-base-image flag; disambiguate colliding scenario test IDs Co-Authored-By: Daryna Ishchenko --- .github/workflows/connector-tests.yml | 4 +- airbyte_cdk/cli/airbyte_cdk/_image.py | 6 +- .../test/standard_tests/docker_base.py | 18 ++++- .../test/standard_tests/pytest_hooks.py | 23 +++++- unit_tests/test/test_standard_tests.py | 77 +++++++++++++++++++ 5 files changed, 122 insertions(+), 6 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index ebf1d3bc06..3eff71380d 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -196,7 +196,7 @@ jobs: # base image declared in metadata.yaml, so by default the container tests exercise the # already-released CDK instead of this branch's CDK. Build an SDM image from the current # branch (with a plain `docker build` so it lands in the local image store, where the - # connector image build can resolve it as its `FROM` base) and use it via `--base-image`. + # connector image build can resolve it as its `FROM` base) and use it via `--connector-base-image`. # Python connectors don't need this: the "Bump to Dev Branch CDK" step pins the connector's # CDK dependency to this branch, which the container image build then installs. - name: Build SDM image from current branch [Manifest-Only Connectors] @@ -232,7 +232,7 @@ jobs: run: | if [ "$CONNECTOR_LANGUAGE" = "manifest-only" ]; then poetry run airbyte-cdk image test ${{ matrix.connector }} \ - --base-image airbyte/source-declarative-manifest:dev + --connector-base-image airbyte/source-declarative-manifest:dev else poetry run airbyte-cdk image test ${{ matrix.connector }} fi diff --git a/airbyte_cdk/cli/airbyte_cdk/_image.py b/airbyte_cdk/cli/airbyte_cdk/_image.py index e4e7c7011e..c15a3ebd38 100644 --- a/airbyte_cdk/cli/airbyte_cdk/_image.py +++ b/airbyte_cdk/cli/airbyte_cdk/_image.py @@ -43,7 +43,8 @@ def image_cli_group() -> None: help="Optional. Override the Dockerfile used for building the image.", ) @click.option( - "--base-image", + "--connector-base-image", + "base_image", default=None, help=( "Optional. Build the connector `FROM` this base image instead of the `baseImage` " @@ -120,7 +121,8 @@ def build( help="Skip tests that require credentials (marked with 'requires_creds').", ) @click.option( - "--base-image", + "--connector-base-image", + "base_image", default=None, help=( "Optional. When building the image to test, build it `FROM` this base image instead of " diff --git a/airbyte_cdk/test/standard_tests/docker_base.py b/airbyte_cdk/test/standard_tests/docker_base.py index 6fe78eb314..84cfab2b00 100644 --- a/airbyte_cdk/test/standard_tests/docker_base.py +++ b/airbyte_cdk/test/standard_tests/docker_base.py @@ -133,6 +133,9 @@ def _dedup_scenarios(scenarios: list[ConnectorTestScenario]) -> list[ConnectorTe take the union of any defined empty_streams, to have high confidence that runnning a read with the config will not error on the lack of data in the empty streams or lack of permissions to read them. + We also carry over any explicitly declared `status`. Only the `connection` section declares one, + so without this an entry from another section (e.g. `spec`) would win and silently downgrade the + scenario to `ALLOW_ANY`, which accepts a failing `check`. """ deduped_scenarios: list[ConnectorTestScenario] = [] @@ -144,8 +147,21 @@ def _dedup_scenarios(scenarios: list[ConnectorTestScenario]) -> list[ConnectorTe all_empty_streams = (existing_scenario.empty_streams or []) + ( scenario.empty_streams or [] ) + if ( + existing_scenario.status is not None + and scenario.status is not None + and existing_scenario.status != scenario.status + ): + raise ValueError( + f"Conflicting expected statuses declared for config " + f"'{scenario.config_path}': '{existing_scenario.status}' and " + f"'{scenario.status}'." + ) merged_scenario = existing_scenario.model_copy( - update={"empty_streams": list(set(all_empty_streams))} + update={ + "empty_streams": list(set(all_empty_streams)), + "status": existing_scenario.status or scenario.status, + } ) deduped_scenarios.remove(existing_scenario) deduped_scenarios.append(merged_scenario) diff --git a/airbyte_cdk/test/standard_tests/pytest_hooks.py b/airbyte_cdk/test/standard_tests/pytest_hooks.py index 9ee2f292a7..fa905e7157 100644 --- a/airbyte_cdk/test/standard_tests/pytest_hooks.py +++ b/airbyte_cdk/test/standard_tests/pytest_hooks.py @@ -17,6 +17,8 @@ import pytest +from airbyte_cdk.test.models.scenario import ConnectorTestScenario + @pytest.fixture def connector_image_override(request: pytest.FixtureRequest) -> str | None: @@ -209,5 +211,24 @@ class TestMyConnector(ConnectorTestSuiteBase): metafunc.parametrize( "scenario", parametrized_scenarios, - ids=[str(scenario) for scenario in scenarios], + ids=_scenario_test_ids(scenarios), ) + + +def _scenario_test_ids(scenarios: list[ConnectorTestScenario]) -> list[str]: + """Return unique, human-readable pytest IDs for the given scenarios. + + Scenario IDs are derived from the config file stem, so configs in different directories + (e.g. `integration_tests/config.json` and `secrets/config.json`) collide. Where they do, + we qualify the ID with the config's parent directory so failures name the config that + failed. Note this only affects the displayed test ID, not `ConnectorTestScenario.id`, + which scenario selection (`--read-scenarios`) matches against. + """ + labels = [str(scenario) for scenario in scenarios] + duplicated = {label for label in labels if labels.count(label) > 1} + return [ + f"{scenario.config_path.parent.name}/{label}" + if label in duplicated and scenario.config_path is not None + else label + for scenario, label in zip(scenarios, labels) + ] diff --git a/unit_tests/test/test_standard_tests.py b/unit_tests/test/test_standard_tests.py index d5dd28277f..6250d04ac7 100644 --- a/unit_tests/test/test_standard_tests.py +++ b/unit_tests/test/test_standard_tests.py @@ -1,6 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. """Unit tests for FAST Airbyte Standard Tests.""" +from pathlib import Path from typing import Any import pytest @@ -9,7 +10,10 @@ ConcurrentDeclarativeSource, ) from airbyte_cdk.sources.source import Source +from airbyte_cdk.test.models.scenario import ConnectorTestScenario from airbyte_cdk.test.standard_tests._job_runner import IConnector +from airbyte_cdk.test.standard_tests.docker_base import DockerConnectorTestSuite +from airbyte_cdk.test.standard_tests.pytest_hooks import _scenario_test_ids @pytest.mark.parametrize( @@ -31,3 +35,76 @@ def test_is_iconnector_check(input: Any, expected: bool) -> None: return assert isinstance(input, IConnector) == expected + + +@pytest.mark.parametrize( + "scenarios, expected_statuses", + [ + pytest.param( + [ + ConnectorTestScenario(config_path=Path("integration_tests/config.json")), + ConnectorTestScenario( + config_path=Path("integration_tests/config.json"), status="succeed" + ), + ], + ["succeed"], + id="statusless_spec_entry_inherits_connection_status", + ), + pytest.param( + [ + ConnectorTestScenario( + config_path=Path("integration_tests/config.json"), status="failed" + ), + ConnectorTestScenario(config_path=Path("integration_tests/config.json")), + ], + ["failed"], + id="declared_status_survives_statusless_duplicate", + ), + pytest.param( + [ + ConnectorTestScenario(config_path=Path("integration_tests/config.json")), + ConnectorTestScenario(config_path=Path("secrets/config.json"), status="succeed"), + ], + [None, "succeed"], + id="different_configs_not_merged", + ), + ], +) +def test_dedup_scenarios_merges_status( + scenarios: list[ConnectorTestScenario], + expected_statuses: list[str | None], +) -> None: + deduped = DockerConnectorTestSuite._dedup_scenarios(scenarios) + assert [scenario.status for scenario in deduped] == expected_statuses + + +def test_dedup_scenarios_conflicting_statuses_raise() -> None: + scenarios = [ + ConnectorTestScenario(config_path=Path("integration_tests/config.json"), status="succeed"), + ConnectorTestScenario(config_path=Path("integration_tests/config.json"), status="failed"), + ] + with pytest.raises(ValueError, match="Conflicting expected statuses"): + DockerConnectorTestSuite._dedup_scenarios(scenarios) + + +@pytest.mark.parametrize( + "config_paths, expected_ids", + [ + pytest.param( + [Path("integration_tests/config.json"), Path("secrets/config.json")], + [ + "integration_tests/'config' Test Scenario", + "secrets/'config' Test Scenario", + ], + id="colliding_stems_qualified_with_parent_dir", + ), + pytest.param( + [Path("secrets/config.json"), Path("secrets/valid_config.json")], + ["'config' Test Scenario", "'valid_config' Test Scenario"], + id="unique_stems_unchanged", + ), + ], +) +def test_scenario_test_ids(config_paths: list[Path], expected_ids: list[str]) -> None: + scenarios = [ConnectorTestScenario(config_path=path) for path in config_paths] + assert _scenario_test_ids(scenarios) == expected_ids