From dfc8407e98781bc51c06c47922b7926bccc93194 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:19:53 +0000 Subject: [PATCH] perf: defer pandas import in ResponseToFileExtractor Co-Authored-By: bot_apk --- .../extractors/response_to_file_extractor.py | 4 +- ...est_source_declarative_manifest_imports.py | 60 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py diff --git a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py index f9a655db0c..10896c57e5 100644 --- a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +++ b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py @@ -10,7 +10,6 @@ from math import nan from typing import Any, Dict, Iterable, Mapping, Optional, Tuple -import pandas as pd import requests from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor @@ -139,6 +138,9 @@ def _read_with_chunks( ValueError: If an IO/Error occurs while reading the temporary data. """ + # Imported lazily: pandas (and numpy) are slow to import and only needed for CSV parsing. + import pandas as pd + try: with open(path, "r", encoding=file_encoding) as data: # When preserving NA values, disable pandas' default NA parsing so strings like "NA"/"N/A" are kept as-is, diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py new file mode 100644 index 0000000000..f3bbab3cc3 --- /dev/null +++ b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. + +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT = """ +import json +import sys +from unittest.mock import patch + +import requests_mock +import yaml + +from airbyte_cdk.cli.source_declarative_manifest import _run + +manifest_path, config_path = sys.argv[1:] +with open(manifest_path) as manifest_file: + manifest = yaml.safe_load(manifest_file) + +with patch( + "airbyte_cdk.cli.source_declarative_manifest._run._is_local_manifest_command", + return_value=True, +): + with patch( + "airbyte_cdk.cli.source_declarative_manifest._run.YamlDeclarativeSource._read_and_parse_yaml_file", + return_value=manifest, + ): + with requests_mock.Mocker() as mocker: + mocker.get( + "https://pokeapi.co/api/v2/pokemon/blastoise", + json={"name": "blastoise"}, + ) + _run.handle_command(["spec"]) + _run.handle_command(["check", "--config", config_path]) + _run.handle_command(["discover", "--config", config_path]) + +print("LOADED_HEAVY_MODULES=" + json.dumps(sorted(m for m in ("pandas", "numpy") if m in sys.modules))) +""" + + +def test_spec_check_discover_do_not_import_pandas_or_numpy() -> None: + manifest_path = Path(__file__).parent.parent / "resources" / "valid_local_manifest.yaml" + config_path = Path(__file__).parent.parent / "resources" / "valid_local_pokeapi_config.json" + + result = subprocess.run( + [sys.executable, "-c", SCRIPT, str(manifest_path), str(config_path)], + capture_output=True, + text=True, + timeout=120, + ) + + assert result.returncode == 0, result.stderr + assert '"connectionStatus":{"status":"SUCCEEDED"}' in result.stdout + assert '"catalog"' in result.stdout + marker = next( + line for line in result.stdout.splitlines() if line.startswith("LOADED_HEAVY_MODULES=") + ) + assert json.loads(marker.split("=", 1)[1]) == []