From 8cee2e844f40677e853990a056b18d5ccc066975 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:24:16 +0530 Subject: [PATCH 01/15] Add standalone mssql-python-odbc package for ODBC driver binaries Introduce mssql_python_odbc, a pure-data sibling package (v18.6.0) that ships the ODBC driver libs/ tree, plus setup_odbc.py which builds platform-specific, Python-agnostic (py3-none-) wheels for the 7 supported platforms. ddbc_bindings.cpp now resolves the driver/libs base directory from mssql_python_odbc when installed, and falls back to the bundled mssql_python libs during the Phase-2 transition (bundled libs are retained). Add tests/test_024_odbc_package_split.py to verify Python/C++ driver-path parity, and .gitignore entries for local build copies. --- .gitignore | 7 + mssql_python/pybind/ddbc_bindings.cpp | 43 +++++- mssql_python_odbc/__init__.py | 123 +++++++++++++++ setup_odbc.py | 213 ++++++++++++++++++++++++++ tests/test_024_odbc_package_split.py | 142 +++++++++++++++++ 5 files changed, 526 insertions(+), 2 deletions(-) create mode 100644 mssql_python_odbc/__init__.py create mode 100644 setup_odbc.py create mode 100644 tests/test_024_odbc_package_split.py diff --git a/.gitignore b/.gitignore index 080fc9e5..cd64adb9 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,13 @@ build/ **/build/ mssql_python.egg-info/ +# ODBC package (mssql-python-odbc): transition-period build artifacts. +# During Phase 2, setup_odbc.py copies the current platform's driver binaries +# from mssql_python/libs into this tree so a wheel can be built locally. The +# release pipeline populates these per-platform; do not commit the copies. +mssql_python_odbc/libs/ +mssql_python_odbc.egg-info/ + # Python bytecode __pycache__/ *.py[cod] diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 3cb00814..b1713263 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1125,6 +1125,41 @@ std::string GetModuleDirectory() { return parentDir.string(); } +// Resolve the base directory that contains the ODBC driver `libs/` tree. +// +// Post-split, the driver binaries ship in the standalone `mssql_python_odbc` +// package (a pure-data sibling with no native extension). We import it and use +// its directory as the base that `GetDriverPathCpp` (and the Windows +// `mssql-auth.dll` lookup) append `libs` to. +// +// During the Phase-2 transition we fall back to the bundled `mssql_python` +// directory when the external package is not installed, so a wheel that still +// bundles `libs/` keeps working. Importing `mssql_python_odbc` here is +// Alpine/musl-safe precisely because it is a separate pure package: it cannot +// trigger the partially-initialized-module circular import that motivated +// resolving these paths in C++ in the first place. +std::string GetOdbcLibsBaseDir() { + namespace fs = std::filesystem; + try { + py::object module = py::module::import("mssql_python_odbc"); + py::object module_path = module.attr("__file__"); + std::string module_file = module_path.cast(); + + fs::path parentDir = fs::path(module_file).parent_path(); + LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", + parentDir.string().c_str()); + return parentDir.string(); + } catch (const py::error_already_set& e) { + // External package not installed. pybind11 has already fetched and + // cleared the CPython error indicator when constructing this + // exception, so it is safe to import `mssql_python` again below. + LOG("GetOdbcLibsBaseDir: mssql_python_odbc not available (%s); " + "falling back to bundled libs in mssql_python", + e.what()); + return GetModuleDirectory(); + } +} + // Platform-agnostic function to load the driver dynamic library DriverHandle LoadDriverLibrary(const std::string& driverPath) { LOG("LoadDriverLibrary: Attempting to load ODBC driver from path='%s'", driverPath.c_str()); @@ -1240,8 +1275,12 @@ std::string GetDriverPathCpp(const std::string& moduleDir) { DriverHandle LoadDriverOrThrowException() { namespace fs = std::filesystem; - std::string moduleDir = GetModuleDirectory(); - LOG("LoadDriverOrThrowException: Module directory resolved to '%s'", moduleDir.c_str()); + // Resolve the base dir from the standalone `mssql_python_odbc` package + // (falls back to the bundled `mssql_python` libs during the transition). + // Both the driver path and the Windows `mssql-auth.dll` path below are + // derived from this directory. + std::string moduleDir = GetOdbcLibsBaseDir(); + LOG("LoadDriverOrThrowException: ODBC libs base directory resolved to '%s'", moduleDir.c_str()); std::string archStr = ARCHITECTURE; LOG("LoadDriverOrThrowException: Architecture detected as '%s'", archStr.c_str()); diff --git a/mssql_python_odbc/__init__.py b/mssql_python_odbc/__init__.py new file mode 100644 index 00000000..f8c64e1a --- /dev/null +++ b/mssql_python_odbc/__init__.py @@ -0,0 +1,123 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +mssql_python_odbc — Microsoft ODBC Driver 18 for SQL Server binaries. + +Internal implementation package for ``mssql-python``. It ships the +platform-specific ODBC driver binaries (``msodbcsql18``) and their supporting +libraries so that ``mssql-python`` does not have to bundle them in its own +wheel. It is not meant for direct consumption — install ``mssql-python`` +instead, which depends on this package. + +The public surface is :func:`get_driver_path`, which returns the absolute path +to the ODBC driver shared library for the current platform/architecture. The +native ``mssql_python.ddbc_bindings`` extension resolves the same location in +C++ (see ``GetOdbcLibsBaseDir`` / ``GetDriverPathCpp``); this Python API mirrors +that logic for tooling, tests, and diagnostics. +""" + +import os +import platform +import sys + +__all__ = ["get_driver_path", "get_libs_dir", "__version__"] + +# Version tracks the bundled Microsoft ODBC Driver 18 for SQL Server release. +__version__ = "18.6.0" + +# Driver shared-library file names per platform (must match the names produced +# by the ODBC driver packaging and expected by the native loader). +_DRIVER_FILENAME = { + "windows": "msodbcsql18.dll", + "linux": "libmsodbcsql-18.6.so.2.1", + "macos": "libmsodbcsql.18.dylib", +} + + +def get_libs_dir() -> str: + """Return the absolute path to this package's ``libs/`` directory. + + This is the root under which the platform-specific ODBC binaries live + (``libs///...``). It is the base the native loader appends + ``libs`` to when resolving the driver. + """ + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs") + + +def _detect_arch() -> str: + """Return the architecture directory name for the current interpreter. + + Mirrors the compile-time detection in ``GetDriverPathCpp``: + * Windows uses ``x64`` / ``x86`` / ``arm64``. + * Linux and macOS use ``x86_64`` / ``arm64``. + """ + machine = platform.machine().lower() + + if sys.platform.startswith("win"): + if machine in ("amd64", "x86_64"): + return "x64" + if machine in ("arm64", "aarch64"): + return "arm64" + if machine in ("x86", "i386", "i686"): + return "x86" + raise OSError(f"Unsupported Windows architecture: {platform.machine()!r}") + + # Linux / macOS + if machine in ("x86_64", "amd64"): + return "x86_64" + if machine in ("arm64", "aarch64"): + return "arm64" + raise OSError(f"Unsupported architecture: {platform.machine()!r}") + + +def _detect_linux_distro_family() -> str: + """Return the Linux distro family directory name. + + Mirrors the ``/etc/*-release`` probing in ``GetDriverPathCpp`` so the Python + and C++ paths agree. + """ + if os.path.exists("/etc/alpine-release"): + return "alpine" + if os.path.exists("/etc/redhat-release") or os.path.exists("/etc/centos-release"): + return "rhel" + if os.path.exists("/etc/SuSE-release") or os.path.exists("/etc/SUSE-brand"): + return "suse" + return "debian_ubuntu" # default for Debian/Ubuntu and other glibc distros + + +def get_driver_path() -> str: + """Return the absolute path to the ODBC driver shared library. + + Resolves the platform/architecture (and, on Linux, the distro family) and + returns the full path to ``msodbcsql18`` inside this package. + + Raises: + OSError: if the platform/architecture is unsupported. + FileNotFoundError: if the resolved driver file is not present (e.g. the + package was built without this platform's binaries). + """ + libs_dir = get_libs_dir() + arch = _detect_arch() + + if sys.platform.startswith("win"): + driver_path = os.path.join(libs_dir, "windows", arch, _DRIVER_FILENAME["windows"]) + elif sys.platform.startswith("darwin"): + driver_path = os.path.join( + libs_dir, "macos", arch, "lib", _DRIVER_FILENAME["macos"] + ) + elif sys.platform.startswith("linux"): + distro = _detect_linux_distro_family() + driver_path = os.path.join( + libs_dir, "linux", distro, arch, "lib", _DRIVER_FILENAME["linux"] + ) + else: + raise OSError(f"Unsupported platform: {sys.platform!r}") + + if not os.path.isfile(driver_path): + raise FileNotFoundError( + f"ODBC driver not found at {driver_path!r}. The mssql-python-odbc " + f"package may not include binaries for this platform/architecture." + ) + + return driver_path diff --git a/setup_odbc.py b/setup_odbc.py new file mode 100644 index 00000000..e6bed156 --- /dev/null +++ b/setup_odbc.py @@ -0,0 +1,213 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +Build script for the ``mssql-python-odbc`` package. + +This packages the Microsoft ODBC Driver 18 for SQL Server binaries into a +standalone, platform-specific wheel that ``mssql-python`` depends on. Build it +with:: + + python setup_odbc.py bdist_wheel + +During the transition period the driver binaries still live under +``mssql_python/libs/``. This script copies the current platform's subtree into +``mssql_python_odbc/libs/`` so a wheel can be produced locally. The release +pipeline populates ``libs/`` per-platform and is the source of truth for the +full release matrix. +""" + +import os +import shutil +import sys +from pathlib import Path + +from setuptools import setup +from setuptools.dist import Distribution +from wheel.bdist_wheel import bdist_wheel + +PROJECT_ROOT = Path(__file__).resolve().parent +PACKAGE_NAME = "mssql_python_odbc" +PACKAGE_DIR = PROJECT_ROOT / PACKAGE_NAME +BUNDLED_LIBS_ROOT = PROJECT_ROOT / "mssql_python" / "libs" + + +class BinaryDistribution(Distribution): + """Force a platform-specific wheel (the package ships native binaries).""" + + def has_ext_modules(self): + return True + + +def get_platform_info(): + """Get platform-specific architecture and platform tag information. + + Kept in sync with ``setup.py`` so the ODBC wheel carries the same platform + tags as the main ``mssql-python`` wheel. + """ + if sys.platform.startswith("win"): + arch = os.environ.get("ARCHITECTURE", "x64") + if isinstance(arch, str): + arch = arch.strip("\"'") + if arch in ["x86", "win32"]: + return "x86", "win32" + elif arch == "arm64": + return "arm64", "win_arm64" + else: + return "x64", "win_amd64" + + elif sys.platform.startswith("darwin"): + return "universal2", "macosx_15_0_universal2" + + elif sys.platform.startswith("linux"): + import platform + + target_arch = os.environ.get("targetArch", platform.machine()) + libc_name, _ = platform.libc_ver() + is_musl = libc_name == "" or "musl" in libc_name.lower() + manylinux_tag = os.environ.get("MANYLINUX_TAG", "manylinux_2_28") + + if target_arch == "x86_64": + return "x86_64", "musllinux_1_2_x86_64" if is_musl else f"{manylinux_tag}_x86_64" + elif target_arch in ["aarch64", "arm64"]: + return "aarch64", "musllinux_1_2_aarch64" if is_musl else f"{manylinux_tag}_aarch64" + else: + raise OSError( + f"Unsupported architecture '{target_arch}' for Linux; " + f"expected 'x86_64' or 'aarch64'." + ) + + raise OSError(f"Unsupported platform: {sys.platform!r}") + + +def _libs_arch(build_arch: str) -> str: + """Map the build arch from ``get_platform_info`` to the ``libs/`` dir name. + + ``libs/`` uses ``x64``/``x86``/``arm64`` on Windows and + ``x86_64``/``arm64`` on Linux/macOS. + """ + if sys.platform.startswith("win"): + return build_arch # already x64 / x86 / arm64 + if build_arch in ("x86_64", "amd64"): + return "x86_64" + if build_arch in ("aarch64", "arm64"): + return "arm64" + return build_arch + + +def _copytree(src: Path, dst: Path) -> None: + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + print(f" Copied {src} -> {dst}") + + +def sync_libs() -> None: + """Copy the current platform's ODBC libs into ``mssql_python_odbc/libs/``. + + Convenience for local/transition builds while ``mssql_python`` still bundles + the binaries. If ``mssql_python/libs`` is absent (e.g. the pipeline places + binaries directly into the package), this is a no-op. + """ + target_root = PACKAGE_DIR / "libs" + if not BUNDLED_LIBS_ROOT.is_dir(): + print(f"sync_libs: bundled libs not found at {BUNDLED_LIBS_ROOT}; skipping copy") + return + + build_arch, _ = get_platform_info() + arch = _libs_arch(build_arch) + + # Always carry the licensing files. + _copytree(BUNDLED_LIBS_ROOT / "LICENSING", target_root / "LICENSING") + + if sys.platform.startswith("win"): + _copytree(BUNDLED_LIBS_ROOT / "windows" / arch, target_root / "windows" / arch) + + elif sys.platform.startswith("darwin"): + # universal2 wheel serves both architectures. + for mac_arch in ("arm64", "x86_64"): + _copytree(BUNDLED_LIBS_ROOT / "macos" / mac_arch, target_root / "macos" / mac_arch) + + elif sys.platform.startswith("linux"): + # A single Linux wheel serves all distro families for its libc/arch; + # the driver is selected at runtime via /etc/*-release detection. + for distro in ("alpine", "debian_ubuntu", "rhel", "suse"): + _copytree( + BUNDLED_LIBS_ROOT / "linux" / distro / arch, + target_root / "linux" / distro / arch, + ) + + +class CustomBdistWheel(bdist_wheel): + """Force a platform-specific but Python-agnostic tag and sync libs. + + The package ships only pre-built ODBC driver binaries (data), not a compiled + Python extension, so one ``py3-none-`` wheel serves every supported + Python version (3.10+). The wheel stays platform-specific + (``root_is_pure = False``) because the binaries differ per OS/arch/libc, but + the interpreter/ABI tags are forced to ``py3``/``none``. Without this the + ``BinaryDistribution`` (``has_ext_modules`` -> True) would produce a + ``cp3XX``-specific tag, forcing a needless per-Python-version build matrix. + """ + + def finalize_options(self): + bdist_wheel.finalize_options(self) + arch, platform_tag = get_platform_info() + self.plat_name = platform_tag + # Platform-specific (ships native binaries) but not tied to a CPython ABI. + self.root_is_pure = False + print(f"Setting wheel platform tag to: {self.plat_name} (arch: {arch})") + + def get_tag(self): + # Preserve the platform tag from the base implementation but relabel the + # interpreter/ABI tags as Python-agnostic ("py3"/"none"). + _python, _abi, plat = bdist_wheel.get_tag(self) + return "py3", "none", plat + + def run(self): + sync_libs() + bdist_wheel.run(self) + + +setup( + name="mssql-python-odbc", + version="18.6.0", + description=( + "Internal implementation package for mssql-python: Microsoft ODBC " + "Driver 18 for SQL Server binaries. Not intended for direct use." + ), + long_description=( + "Internal implementation package not meant for direct consumption. " + "Install `mssql-python`, which depends on this package." + ), + long_description_content_type="text/plain", + author="Microsoft Corporation", + author_email="mssql-python@microsoft.com", + url="https://github.com/microsoft/mssql-python", + license="MIT", + packages=[PACKAGE_NAME], + package_data={ + PACKAGE_NAME: [ + "libs/*", + "libs/**/*", + ], + }, + include_package_data=True, + python_requires=">=3.10", + classifiers=[ + "License :: OSI Approved :: MIT License", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + ], + zip_safe=False, + distclass=BinaryDistribution, + exclude_package_data={ + PACKAGE_NAME: [ + "libs/*/vcredist/*", + "libs/*/vcredist/**/*", + ], + }, + cmdclass={ + "bdist_wheel": CustomBdistWheel, + }, +) diff --git a/tests/test_024_odbc_package_split.py b/tests/test_024_odbc_package_split.py new file mode 100644 index 00000000..d337a21d --- /dev/null +++ b/tests/test_024_odbc_package_split.py @@ -0,0 +1,142 @@ +""" +Tests for the ODBC driver package split (mssql-python-odbc). + +Part of splitting the bundled ODBC driver binaries out of ``mssql-python`` into +the standalone ``mssql-python-odbc`` package. These tests validate the Python +driver-path API and, crucially, that it agrees with the native C++ resolver +(``ddbc_bindings.GetDriverPathCpp``) so the path Python reports is exactly the +one the driver is loaded from at connect time. + +Notes: +- ``mssql_python_odbc/libs`` is populated per-platform at build time (and copied + locally by ``setup_odbc.py`` during the transition). Tests that need the actual + driver file skip when the libs directory is absent (e.g. a fresh CI checkout + where the binaries have not been synced). +- The C++ fallback (``GetOdbcLibsBaseDir`` -> bundled ``mssql_python`` libs when + the external package is not installed) is verified manually rather than here, + since exercising it requires uninstalling the package mid-run. +""" + +import os +import sys + +import pytest + +# Skip the whole module cleanly if the package is not importable. +mssql_python_odbc = pytest.importorskip("mssql_python_odbc") + + +EXPECTED_DRIVER_FILENAME = { + "win32": "msodbcsql18.dll", + "darwin": "libmsodbcsql.18.dylib", + "linux": "libmsodbcsql-18.6.so.2.1", +} + + +def _platform_key(): + if sys.platform.startswith("win"): + return "win32" + if sys.platform.startswith("darwin"): + return "darwin" + if sys.platform.startswith("linux"): + return "linux" + return sys.platform + + +def _package_dir(): + """Base directory the native loader appends ``libs`` to.""" + return os.path.dirname(os.path.abspath(mssql_python_odbc.__file__)) + + +def _libs_present(): + return os.path.isdir(mssql_python_odbc.get_libs_dir()) + + +_NO_LIBS_REASON = ( + "ODBC libs not present in package (fresh checkout, or not built/synced for " + "this platform)" +) + + +class TestOdbcPackageMetadata: + def test_version_is_driver_version(self): + assert mssql_python_odbc.__version__ == "18.6.0" + + def test_public_api_present(self): + assert callable(mssql_python_odbc.get_driver_path) + assert callable(mssql_python_odbc.get_libs_dir) + + +class TestLibsDir: + def test_libs_dir_under_package(self): + libs_dir = mssql_python_odbc.get_libs_dir() + assert os.path.basename(libs_dir) == "libs" + assert libs_dir.startswith(_package_dir()) + + +class TestArchDetection: + def test_detect_arch_matches_platform(self): + arch = mssql_python_odbc._detect_arch() + if sys.platform.startswith("win"): + assert arch in ("x64", "x86", "arm64") + else: + assert arch in ("x86_64", "arm64") + + @pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="Linux-only distro-family detection", + ) + def test_detect_linux_distro_family(self): + distro = mssql_python_odbc._detect_linux_distro_family() + assert distro in ("alpine", "rhel", "suse", "debian_ubuntu") + + +class TestDriverPath: + @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) + def test_driver_path_layout_and_exists(self): + driver_path = mssql_python_odbc.get_driver_path() + # Correct driver filename for this platform. + assert os.path.basename(driver_path) == EXPECTED_DRIVER_FILENAME[_platform_key()] + # Resolved under the package's own libs directory. + assert mssql_python_odbc.get_libs_dir() in driver_path + # The resolved driver file actually exists on disk. + assert os.path.isfile(driver_path), f"driver not found at {driver_path}" + + @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) + def test_driver_path_contains_expected_platform_dir(self): + driver_path = mssql_python_odbc.get_driver_path().replace("\\", "/") + expected_segment = { + "win32": "/libs/windows/", + "darwin": "/libs/macos/", + "linux": "/libs/linux/", + }[_platform_key()] + assert expected_segment in driver_path + + +class TestPythonCppParity: + """The Python ``get_driver_path()`` must resolve to the exact same path the + native C++ loader (``GetDriverPathCpp``) computes for the same base dir, so + tooling/tests agree with the driver actually loaded at connect time.""" + + @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) + def test_get_driver_path_matches_cpp(self): + try: + from mssql_python import ddbc_bindings + except Exception as exc: # native extension not built / driver load failed + pytest.skip(f"ddbc_bindings unavailable: {exc}") + + get_cpp = getattr(ddbc_bindings, "GetDriverPathCpp", None) + if get_cpp is None: + pytest.skip("GetDriverPathCpp not exposed by this build") + + py_path = mssql_python_odbc.get_driver_path() + cpp_path = get_cpp(_package_dir()) + + def norm(p): + return os.path.normcase(os.path.normpath(p)) + + assert norm(py_path) == norm(cpp_path), ( + "Python/C++ driver path mismatch:\n" + f" python={py_path}\n" + f" cpp ={cpp_path}" + ) From 69ca1b7eb6cfbc059e3491f4ca0275220c67489b Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:57:56 +0530 Subject: [PATCH 02/15] FIX: Robust ODBC driver fallback and address review feedback for package split --- mssql_python/pybind/ddbc_bindings.cpp | 42 ++++++++++++++++++++++----- mssql_python_odbc/__init__.py | 5 ++-- setup_odbc.py | 4 +-- tests/test_024_odbc_package_split.py | 21 +++++++++----- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index b1713263..1412b1d5 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1138,6 +1138,11 @@ std::string GetModuleDirectory() { // Alpine/musl-safe precisely because it is a separate pure package: it cannot // trigger the partially-initialized-module circular import that motivated // resolving these paths in C++ in the first place. +// +// (`GetDriverPathCpp` is defined further below; forward-declared here so we can +// verify the external package actually ships this platform's driver binary.) +std::string GetDriverPathCpp(const std::string& moduleDir); + std::string GetOdbcLibsBaseDir() { namespace fs = std::filesystem; try { @@ -1146,17 +1151,38 @@ std::string GetOdbcLibsBaseDir() { std::string module_file = module_path.cast(); fs::path parentDir = fs::path(module_file).parent_path(); - LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", + + // Only treat the external package as authoritative if it actually ships + // this platform's driver binary. In a source/dev checkout (and in CI) + // the package is importable from the repo root but its `libs/` tree is + // gitignored and absent; in that case fall back to the bundled + // `mssql_python` libs so driver resolution still points at a real binary. + std::error_code ec; + if (fs::exists(GetDriverPathCpp(parentDir.string()), ec)) { + LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", + parentDir.string().c_str()); + return parentDir.string(); + } + LOG("GetOdbcLibsBaseDir: mssql_python_odbc present at '%s' but has no driver binary; " + "falling back to bundled libs in mssql_python", parentDir.string().c_str()); - return parentDir.string(); + return GetModuleDirectory(); } catch (const py::error_already_set& e) { - // External package not installed. pybind11 has already fetched and - // cleared the CPython error indicator when constructing this - // exception, so it is safe to import `mssql_python` again below. - LOG("GetOdbcLibsBaseDir: mssql_python_odbc not available (%s); " - "falling back to bundled libs in mssql_python", + if (e.matches(PyExc_ModuleNotFoundError)) { + // Expected in Phase 2 when the standalone package is not installed. + // pybind11 has already fetched and cleared the CPython error + // indicator, so re-importing `mssql_python` below is safe. + LOG("GetOdbcLibsBaseDir: mssql_python_odbc not installed (%s); " + "falling back to bundled libs in mssql_python", + e.what()); + return GetModuleDirectory(); + } + // A different import-time error means the package is installed but + // broken; surface it instead of silently masking the real problem. + LOG("GetOdbcLibsBaseDir: importing mssql_python_odbc failed unexpectedly (%s); " + "re-raising", e.what()); - return GetModuleDirectory(); + throw; } } diff --git a/mssql_python_odbc/__init__.py b/mssql_python_odbc/__init__.py index f8c64e1a..90219724 100644 --- a/mssql_python_odbc/__init__.py +++ b/mssql_python_odbc/__init__.py @@ -39,8 +39,9 @@ def get_libs_dir() -> str: """Return the absolute path to this package's ``libs/`` directory. This is the root under which the platform-specific ODBC binaries live - (``libs///...``). It is the base the native loader appends - ``libs`` to when resolving the driver. + (``libs///...``). The parent of this path (the package + directory) is the base the native loader appends ``libs`` to when resolving + the driver. """ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs") diff --git a/setup_odbc.py b/setup_odbc.py index e6bed156..31c83fdd 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -203,8 +203,8 @@ def run(self): distclass=BinaryDistribution, exclude_package_data={ PACKAGE_NAME: [ - "libs/*/vcredist/*", - "libs/*/vcredist/**/*", + "libs/windows/*/vcredist/*", + "libs/windows/*/vcredist/**/*", ], }, cmdclass={ diff --git a/tests/test_024_odbc_package_split.py b/tests/test_024_odbc_package_split.py index d337a21d..f937e225 100644 --- a/tests/test_024_odbc_package_split.py +++ b/tests/test_024_odbc_package_split.py @@ -49,12 +49,21 @@ def _package_dir(): def _libs_present(): - return os.path.isdir(mssql_python_odbc.get_libs_dir()) + """True only when this platform's driver binary is actually present. + + Checks for the driver file rather than just the ``libs/`` directory, so a + stray or empty ``libs/`` in a dev checkout makes the driver tests skip + cleanly instead of failing. Mirrors the C++ resolver, which falls back to + the bundled libs unless the external package ships a real driver binary. + """ + try: + return os.path.isfile(mssql_python_odbc.get_driver_path()) + except OSError: + return False _NO_LIBS_REASON = ( - "ODBC libs not present in package (fresh checkout, or not built/synced for " - "this platform)" + "ODBC libs not present in package (fresh checkout, or not built/synced for " "this platform)" ) @@ -71,7 +80,7 @@ class TestLibsDir: def test_libs_dir_under_package(self): libs_dir = mssql_python_odbc.get_libs_dir() assert os.path.basename(libs_dir) == "libs" - assert libs_dir.startswith(_package_dir()) + assert os.path.normcase(libs_dir).startswith(os.path.normcase(_package_dir())) class TestArchDetection: @@ -136,7 +145,5 @@ def norm(p): return os.path.normcase(os.path.normpath(p)) assert norm(py_path) == norm(cpp_path), ( - "Python/C++ driver path mismatch:\n" - f" python={py_path}\n" - f" cpp ={cpp_path}" + "Python/C++ driver path mismatch:\n" f" python={py_path}\n" f" cpp ={cpp_path}" ) From 362b1df4e66711be8a95f0896818aa6f1a1dfc46 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:16:31 +0530 Subject: [PATCH 03/15] FIX: Require complete odbc libs (driver + mssql-auth.dll) before using external package --- mssql_python/pybind/ddbc_bindings.cpp | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 1412b1d5..39dda2e4 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1153,18 +1153,34 @@ std::string GetOdbcLibsBaseDir() { fs::path parentDir = fs::path(module_file).parent_path(); // Only treat the external package as authoritative if it actually ships - // this platform's driver binary. In a source/dev checkout (and in CI) - // the package is importable from the repo root but its `libs/` tree is - // gitignored and absent; in that case fall back to the bundled - // `mssql_python` libs so driver resolution still points at a real binary. + // a COMPLETE set of this platform's driver binaries. In a source/dev + // checkout (and in CI) the package is importable from the repo root but + // its `libs/` tree is gitignored and either absent or only partially + // populated; in that case fall back to the bundled `mssql_python` libs + // so driver resolution points at a real, complete installation. + // + // "Complete" means the ODBC driver itself and, on Windows, the + // co-located `mssql-auth.dll` that LoadDriverOrThrowException loads + // unconditionally. Verifying both here keeps this resolver's notion of a + // usable base dir consistent with what the loader below actually needs, + // so we never select a directory that would later make the loader throw + // (e.g. a dir that has msodbcsql18.dll but is missing mssql-auth.dll). std::error_code ec; - if (fs::exists(GetDriverPathCpp(parentDir.string()), ec)) { + fs::path externalDriver(GetDriverPathCpp(parentDir.string())); + bool externalComplete = fs::exists(externalDriver, ec); +#ifdef _WIN32 + if (externalComplete) { + fs::path externalAuthDll = externalDriver.parent_path() / "mssql-auth.dll"; + externalComplete = fs::exists(externalAuthDll, ec); + } +#endif + if (externalComplete) { LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", parentDir.string().c_str()); return parentDir.string(); } - LOG("GetOdbcLibsBaseDir: mssql_python_odbc present at '%s' but has no driver binary; " - "falling back to bundled libs in mssql_python", + LOG("GetOdbcLibsBaseDir: mssql_python_odbc present at '%s' but its libs are missing or " + "incomplete; falling back to bundled libs in mssql_python", parentDir.string().c_str()); return GetModuleDirectory(); } catch (const py::error_already_set& e) { From 25ff315de8c003be34990ee0a2252f69a415e097 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:34:20 +0530 Subject: [PATCH 04/15] Include vcredist (msvcp140.dll) in odbc package for full parity with main libs --- setup_odbc.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/setup_odbc.py b/setup_odbc.py index 31c83fdd..91b10a9a 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -201,12 +201,6 @@ def run(self): ], zip_safe=False, distclass=BinaryDistribution, - exclude_package_data={ - PACKAGE_NAME: [ - "libs/windows/*/vcredist/*", - "libs/windows/*/vcredist/**/*", - ], - }, cmdclass={ "bdist_wheel": CustomBdistWheel, }, From c3c3001847d0419be9562810ec6479929fdfd570 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 22:06:05 +0530 Subject: [PATCH 05/15] Address review: acquire GIL in GetOdbcLibsBaseDir, document driver-version coupling, guard minimum setuptools --- mssql_python/pybind/ddbc_bindings.cpp | 17 +++++++++++++++++ setup_odbc.py | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 39dda2e4..c5099b8b 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1145,6 +1145,13 @@ std::string GetDriverPathCpp(const std::string& moduleDir); std::string GetOdbcLibsBaseDir() { namespace fs = std::filesystem; + // This function calls into the Python C-API (py::module::import, attribute + // access, casts), so it must run with the GIL held. It is a no-op when the + // GIL is already held — which it is at the sole current call site, during + // module initialization — but acquiring it here self-documents the C-API + // dependency and keeps a future GIL-released caller from turning this into a + // hard crash. + py::gil_scoped_acquire gil; try { py::object module = py::module::import("mssql_python_odbc"); py::object module_path = module.attr("__file__"); @@ -1292,6 +1299,16 @@ std::string GetDriverPathCpp(const std::string& moduleDir) { platform = "debian_ubuntu"; // Default to debian_ubuntu for other distros } + // NOTE (version coupling): the Linux driver filename is pinned to the exact + // msodbcsql minor version (18.6 -> libmsodbcsql-18.6.so.2.1), matching the + // binaries bundled under mssql_python/libs and shipped by the standalone + // mssql_python_odbc package. If those driver binaries are upgraded (e.g. to + // 18.7) this literal MUST be updated in lockstep: GetOdbcLibsBaseDir() calls + // fs::exists() on this exact path to decide whether the external package is + // "complete", so a stale name would make it silently fall back to the + // bundled libs. (The macOS/Windows paths below key off the major version + // only and are more forgiving.) Tracked for the Phase 3 driver-version- + // agnostic resolution work. fs::path driverPath = basePath / "libs" / "linux" / platform / arch / "lib" / "libmsodbcsql-18.6.so.2.1"; return driverPath.string(); diff --git a/setup_odbc.py b/setup_odbc.py index 91b10a9a..3e756df5 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -18,10 +18,12 @@ """ import os +import re import shutil import sys from pathlib import Path +import setuptools from setuptools import setup from setuptools.dist import Distribution from wheel.bdist_wheel import bdist_wheel @@ -31,6 +33,29 @@ PACKAGE_DIR = PROJECT_ROOT / PACKAGE_NAME BUNDLED_LIBS_ROOT = PROJECT_ROOT / "mssql_python" / "libs" +# The driver binaries are packaged via the recursive ``libs/**/*`` glob in +# ``package_data`` below. Support for ``**`` recursive globs in ``package_data`` +# landed in setuptools 62.3.0; with an older setuptools the glob silently +# matches nothing and the wheel ships WITHOUT any driver binaries. The ``libs/`` +# tree is gitignored, so ``include_package_data`` cannot recover the files from +# version control either -- a stale setuptools would produce a broken-but-quiet +# wheel. Fail fast instead. +MIN_SETUPTOOLS = (62, 3, 0) + + +def _require_min_setuptools() -> None: + raw = setuptools.__version__ + parts = tuple(int(m) for m in re.findall(r"\d+", raw)[:3]) + parts += (0,) * (3 - len(parts)) + if parts < MIN_SETUPTOOLS: + raise SystemExit( + "setup_odbc.py requires setuptools >= " + f"{'.'.join(map(str, MIN_SETUPTOOLS))} to package the ODBC driver " + "binaries via the recursive 'libs/**/*' glob; found setuptools " + f"{raw}. Upgrade with:\n" + ' python -m pip install --upgrade "setuptools>=62.3.0"' + ) + class BinaryDistribution(Distribution): """Force a platform-specific wheel (the package ships native binaries).""" @@ -168,6 +193,8 @@ def run(self): bdist_wheel.run(self) +_require_min_setuptools() + setup( name="mssql-python-odbc", version="18.6.0", From d69ce14416c7e5554918f38fd8d01dbbc961abfc Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 22:17:07 +0530 Subject: [PATCH 06/15] Document deliberate vcredist/msvcp140.dll inclusion in odbc wheel Corrects the earlier 'parity' framing. Main does not strip the VC++ runtime: build.bat relocates msvcp140.dll next to the .pyd and setup.py excludes the duplicate vcredist/ folder (de-duplication). The odbc package has no .pyd, so it keeps msvcp140.dll in vcredist/ with its license. No Phase-2 skew: the driver's runtime dependency is satisfied in-process by the /MD-built extension in both paths. --- setup_odbc.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/setup_odbc.py b/setup_odbc.py index 3e756df5..ba75f794 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -212,6 +212,27 @@ def run(self): url="https://github.com/microsoft/mssql-python", license="MIT", packages=[PACKAGE_NAME], + # vcredist / VC++ runtime (deliberate divergence from the main wheel's SHAPE, + # not a behavior change): unlike setup.py we do NOT exclude + # ``libs//vcredist/``. This is intentional and consistent with what the + # main wheel actually ships: + # * The main mssql-python wheel still ships the VC++ runtime + # (``msvcp140.dll``) -- build.bat copies it out of + # ``libs/windows//vcredist/`` to the package root next to the + # compiled ``ddbc_bindings`` extension, and setup.py then excludes the + # now-duplicate ``vcredist/`` folder. That exclusion is DE-DUPLICATION, + # not a licensing/size-driven removal of the runtime. + # * This package is pure driver-binary data with no compiled extension, so + # there is no package-root ``.pyd`` to relocate the runtime beside. We + # therefore keep ``msvcp140.dll`` in its original ``vcredist/`` folder + # (with its license text) so the driver binaries and the runtime they + # were built against travel together and the wheel stays self-contained. + # This does not create a Phase-2 runtime skew: the driver's ``msvcp140.dll`` + # dependency is satisfied in-process by the mssql-python extension module + # (built ``/MD``, which loads ``msvcp140.dll`` from beside the ``.pyd``), so + # the external-package and bundled-libs paths behave identically. This copy + # is completeness + attribution, which is also why GetOdbcLibsBaseDir()'s + # completeness check does not need to verify vcredist. package_data={ PACKAGE_NAME: [ "libs/*", From 40961b72014018dd4ba93c20e7e797dc52a3431f Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 10:35:18 +0530 Subject: [PATCH 07/15] DOCS: Add CHANGELOG entry for the mssql-python-odbc package split --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61b6e571..e6720691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), connection. The callback is invoked by mssql-tds mid-handshake (FedAuth workflow 0x02) so the tenant id can be resolved from the server-supplied STS URL. Requires `mssql-py-core` 0.1.5+. Partial fix for #534. +- **Standalone `mssql-python-odbc` package (PRs #663, #664):** the Microsoft + ODBC Driver 18 for SQL Server binaries are now also published as a separate, + platform-specific `mssql-python-odbc` package. When it is installed, the + native driver loader resolves the driver from it; when it is absent or + incomplete, mssql-python transparently falls back to its own bundled `libs/`. + This is a non-breaking step toward decoupling driver-binary updates from + mssql-python releases; a future major version will make the dependency + explicit and drop the bundled binaries. ### Changed - Improved error handling in the connection module. From 74b6d62053b989294f817f587d3a95a069ba5ca3 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 14:27:51 +0530 Subject: [PATCH 08/15] Fix inaccurate setup_odbc.py docstring on libs packaging The comment above MIN_SETUPTOOLS claimed the libs/ tree is gitignored and that include_package_data cannot recover the binaries from version control. Both are inaccurate: the driver binaries are committed, and package_data globs enumerate files from the on-disk mssql_python_odbc/libs/ subtree, not from git. Replace with an accurate description; the real rationale for the guard (setuptools 62.3.0 '**' recursive-glob support) is retained. --- setup_odbc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup_odbc.py b/setup_odbc.py index ba75f794..4f5d2424 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -34,12 +34,12 @@ BUNDLED_LIBS_ROOT = PROJECT_ROOT / "mssql_python" / "libs" # The driver binaries are packaged via the recursive ``libs/**/*`` glob in -# ``package_data`` below. Support for ``**`` recursive globs in ``package_data`` -# landed in setuptools 62.3.0; with an older setuptools the glob silently -# matches nothing and the wheel ships WITHOUT any driver binaries. The ``libs/`` -# tree is gitignored, so ``include_package_data`` cannot recover the files from -# version control either -- a stale setuptools would produce a broken-but-quiet -# wheel. Fail fast instead. +# ``package_data`` below, which enumerates files from the copied +# ``mssql_python_odbc/libs/`` subtree on disk (not from version control). +# Support for ``**`` recursive globs in ``package_data`` landed in setuptools +# 62.3.0; with an older setuptools the glob silently matches nothing and the +# wheel ships WITHOUT any driver binaries. Fail fast instead of producing a +# broken-but-quiet wheel. MIN_SETUPTOOLS = (62, 3, 0) From 088462002107492cf18c01ed3c7588d55abad684 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:30:26 +0530 Subject: [PATCH 09/15] odbc: bump mssql-python-odbc package version 18.6.0 -> 18.6.2 --- mssql_python_odbc/__init__.py | 2 +- setup_odbc.py | 2 +- tests/test_024_odbc_package_split.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mssql_python_odbc/__init__.py b/mssql_python_odbc/__init__.py index 90219724..f51c2333 100644 --- a/mssql_python_odbc/__init__.py +++ b/mssql_python_odbc/__init__.py @@ -24,7 +24,7 @@ __all__ = ["get_driver_path", "get_libs_dir", "__version__"] # Version tracks the bundled Microsoft ODBC Driver 18 for SQL Server release. -__version__ = "18.6.0" +__version__ = "18.6.2" # Driver shared-library file names per platform (must match the names produced # by the ODBC driver packaging and expected by the native loader). diff --git a/setup_odbc.py b/setup_odbc.py index 4f5d2424..0d83846d 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -197,7 +197,7 @@ def run(self): setup( name="mssql-python-odbc", - version="18.6.0", + version="18.6.2", description=( "Internal implementation package for mssql-python: Microsoft ODBC " "Driver 18 for SQL Server binaries. Not intended for direct use." diff --git a/tests/test_024_odbc_package_split.py b/tests/test_024_odbc_package_split.py index f937e225..8212c2c4 100644 --- a/tests/test_024_odbc_package_split.py +++ b/tests/test_024_odbc_package_split.py @@ -69,7 +69,7 @@ def _libs_present(): class TestOdbcPackageMetadata: def test_version_is_driver_version(self): - assert mssql_python_odbc.__version__ == "18.6.0" + assert mssql_python_odbc.__version__ == "18.6.2" def test_public_api_present(self): assert callable(mssql_python_odbc.get_driver_path) From ab91887faa9937aab845d411250c7d185bc27166 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 13:05:29 +0530 Subject: [PATCH 10/15] Centralize ODBC driver version via single source of truth --- eng/pipelines/pr-validation-pipeline.yml | 24 +++- mssql_python/pybind/CMakeLists.txt | 30 +++++ mssql_python/pybind/build.sh | 2 +- mssql_python/pybind/configure_dylibs.sh | 31 +++-- mssql_python/pybind/ddbc_bindings.cpp | 34 +++-- mssql_python_odbc/__init__.py | 108 ++------------- setup_odbc.py | 18 ++- tests/test_000_dependencies.py | 164 +++++++++++++++++++---- tests/test_024_odbc_package_split.py | 149 -------------------- 9 files changed, 263 insertions(+), 297 deletions(-) delete mode 100644 tests/test_024_odbc_package_split.py diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 8cc7ea8e..5c8115ea 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -811,7 +811,9 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 debian_ubuntu driver library signatures:' - ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-18.6.so.2.1 + DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi + ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in $(distroName) container' @@ -1136,7 +1138,9 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 debian_ubuntu driver library signatures:' - ldd mssql_python/libs/linux/debian_ubuntu/arm64/lib/libmsodbcsql-18.6.so.2.1 + DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi + ldd mssql_python/libs/linux/debian_ubuntu/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in $(distroName) ARM64 container' @@ -1349,7 +1353,9 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 rhel driver library signatures:' - ldd mssql_python/libs/linux/rhel/x86_64/lib/libmsodbcsql-18.6.so.2.1 + DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi + ldd mssql_python/libs/linux/rhel/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in RHEL 9 container' @@ -1574,7 +1580,9 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 rhel driver library signatures:' - ldd mssql_python/libs/linux/rhel/arm64/lib/libmsodbcsql-18.6.so.2.1 + DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi + ldd mssql_python/libs/linux/rhel/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in RHEL 9 ARM64 container' @@ -1808,7 +1816,9 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 alpine driver library signatures:' - ldd mssql_python/libs/linux/alpine/x86_64/lib/libmsodbcsql-18.6.so.2.1 || echo 'Driver library not found' + DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi + ldd mssql_python/libs/linux/alpine/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " displayName: 'Uninstall system ODBC Driver before running tests in Alpine x86_64 container' @@ -2059,7 +2069,9 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying arm64 alpine driver library signatures:' - ldd mssql_python/libs/linux/alpine/arm64/lib/libmsodbcsql-18.6.so.2.1 || echo 'Driver library not found' + DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi + ldd mssql_python/libs/linux/alpine/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " displayName: 'Uninstall system ODBC Driver before running tests in Alpine ARM64 container' diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index d1835b1a..d5b8c1be 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -73,6 +73,36 @@ endif() add_definitions(-DARCHITECTURE="${ARCHITECTURE}") add_definitions(-DPLATFORM_NAME="${PLATFORM_NAME}") +# --- msodbcsql driver version (single source of truth) ----------------------- +# The ODBC driver filename embeds the msodbcsql version (e.g. +# libmsodbcsql-18.6.so.2.1 on Linux, libmsodbcsql.18.dylib on macOS, +# msodbcsql18.dll on Windows). Derive that version from +# mssql_python_odbc.__version__ -- the single source of truth for the driver +# version -- and inject it into GetDriverPathCpp() via -D defines, so the native +# resolver and the Python package can never drift. The literal is read straight +# from the file (not by importing the package) to keep configure hermetic and +# independent of whatever packages the build interpreter happens to have. +set(MSODBCSQL_VERSION_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/../../mssql_python_odbc/__init__.py") +if(NOT EXISTS "${MSODBCSQL_VERSION_SOURCE}") + message(FATAL_ERROR + "Cannot resolve the msodbcsql driver version: ${MSODBCSQL_VERSION_SOURCE} not found. " + "The native driver filename is derived from mssql_python_odbc.__version__; " + "build from a full repository checkout.") +endif() +file(STRINGS "${MSODBCSQL_VERSION_SOURCE}" _msodbcsql_version_line REGEX "^__version__") +string(REGEX REPLACE ".*['\"]([0-9]+\\.[0-9]+\\.[0-9]+)['\"].*" "\\1" + MSODBCSQL_VERSION "${_msodbcsql_version_line}") +if(NOT MSODBCSQL_VERSION MATCHES "^[0-9]+\\.[0-9]+\\.[0-9]+$") + message(FATAL_ERROR + "Failed to parse __version__ from ${MSODBCSQL_VERSION_SOURCE} " + "(got '${MSODBCSQL_VERSION}').") +endif() +string(REGEX MATCH "^[0-9]+" MSODBCSQL_VERSION_MAJOR "${MSODBCSQL_VERSION}") +string(REGEX MATCH "^[0-9]+\\.[0-9]+" MSODBCSQL_VERSION_MAJOR_MINOR "${MSODBCSQL_VERSION}") +message(STATUS "msodbcsql driver version (from mssql_python_odbc.__version__): ${MSODBCSQL_VERSION} -> major=${MSODBCSQL_VERSION_MAJOR}, major.minor=${MSODBCSQL_VERSION_MAJOR_MINOR}") +add_definitions(-DMSODBCSQL_VERSION_MAJOR="${MSODBCSQL_VERSION_MAJOR}") +add_definitions(-DMSODBCSQL_VERSION_MAJOR_MINOR="${MSODBCSQL_VERSION_MAJOR_MINOR}") + # For macOS, always set universal build if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "Build architectures for macOS" FORCE) diff --git a/mssql_python/pybind/build.sh b/mssql_python/pybind/build.sh index 98c5cb6c..2afec206 100755 --- a/mssql_python/pybind/build.sh +++ b/mssql_python/pybind/build.sh @@ -181,7 +181,7 @@ fi # TODO: Linux-specific: use patchelf to set RPATH of the driver .so file # Currently added Driver SO files right now are already patched -# patchelf --set-rpath '$ORIGIN' libmsodbcsql-18.6.so.2.1 +# patchelf --set-rpath '$ORIGIN' libmsodbcsql-..so.2.1 # This command sets the RPATH of the specified .so file to the directory containing the file (similar to Windows) # Needed since libodbcinst.so.2 is located in the same directory and needs to be resolved diff --git a/mssql_python/pybind/configure_dylibs.sh b/mssql_python/pybind/configure_dylibs.sh index 00ec8502..6022d977 100755 --- a/mssql_python/pybind/configure_dylibs.sh +++ b/mssql_python/pybind/configure_dylibs.sh @@ -6,13 +6,28 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +ODBC_VERSION_FILE="$PROJECT_DIR/../mssql_python_odbc/__init__.py" +if [ ! -f "$ODBC_VERSION_FILE" ]; then + echo "Error: SSOT version file not found: $ODBC_VERSION_FILE" + exit 1 +fi + +DRIVER_MAJOR=$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+)\.[0-9]+\.[0-9]+['\"].*/\1/p" "$ODBC_VERSION_FILE" | head -1) +if [ -z "$DRIVER_MAJOR" ]; then + echo "Error: failed to parse __version__ from $ODBC_VERSION_FILE" + exit 1 +fi + +DRIVER_DYLIB_NAME="libmsodbcsql.${DRIVER_MAJOR}.dylib" + # The universal2 wheel bundles a per-arch copy of the driver dylibs, so every # bundled arch must be configured, not just the build host's ($(uname -m)). # install_name_tool and codesign both work cross-arch. Fixing only the host arch # is what shipped the broken arm64 driver in issue #656. for ARCH in arm64 x86_64; do LIB_DIR="$PROJECT_DIR/libs/macos/$ARCH/lib" -LIBMSODBCSQL_PATH="$LIB_DIR/libmsodbcsql.18.dylib" +LIBMSODBCSQL_PATH="$LIB_DIR/$DRIVER_DYLIB_NAME" +LIBMSODBCSQL_NAME="$DRIVER_DYLIB_NAME" LIBODBCINST_PATH="$LIB_DIR/libodbcinst.2.dylib" LIBLTDL_PATH="$LIB_DIR/libltdl.7.dylib" @@ -23,7 +38,7 @@ if [ ! -d "$LIB_DIR" ]; then fi if [ ! -f "$LIBMSODBCSQL_PATH" ]; then - echo "Error: libmsodbcsql.18.dylib not found at: $LIBMSODBCSQL_PATH" + echo "Error: $DRIVER_DYLIB_NAME not found at: $LIBMSODBCSQL_PATH" exit 1 fi @@ -42,7 +57,7 @@ fi echo "Configuring dylibs in: $LIB_DIR" # Get the existing library paths which are linked to the dylibs -echo "Reading dependencies from libmsodbcsql.18.dylib..." +echo "Reading dependencies from $LIBMSODBCSQL_NAME..." OTOOL_LIST=$(otool -L "$LIBMSODBCSQL_PATH") OLD_LIBODBCINST_PATH="" @@ -68,12 +83,12 @@ done <<< "$OTOOL_LIST" # Configure the library paths if dependencies were found if [ -n "$OLD_LIBODBCINST_PATH" ]; then - echo "Fixing libmsodbcsql.18.dylib dependency on libodbcinst.2.dylib..." + echo "Fixing $LIBMSODBCSQL_NAME dependency on libodbcinst.2.dylib..." echo " Changing: $OLD_LIBODBCINST_PATH" echo " To: @loader_path/libodbcinst.2.dylib" install_name_tool -change "$OLD_LIBODBCINST_PATH" "@loader_path/libodbcinst.2.dylib" "$LIBMSODBCSQL_PATH" else - echo "Warning: libodbcinst dependency not found in libmsodbcsql.18.dylib" + echo "Warning: libodbcinst dependency not found in $LIBMSODBCSQL_NAME" fi if [ -n "$OLD_LIBLTDL_PATH" ] && [ -f "$LIBLTDL_PATH" ]; then @@ -89,8 +104,8 @@ fi # First set the IDs of the libraries using @loader_path echo "Setting library IDs with @loader_path..." -echo "Setting ID for libmsodbcsql.18.dylib..." -install_name_tool -id "@loader_path/libmsodbcsql.18.dylib" "$LIBMSODBCSQL_PATH" +echo "Setting ID for $LIBMSODBCSQL_NAME..." +install_name_tool -id "@loader_path/$LIBMSODBCSQL_NAME" "$LIBMSODBCSQL_PATH" echo "Setting ID for libodbcinst.2.dylib..." install_name_tool -id "@loader_path/libodbcinst.2.dylib" "$LIBODBCINST_PATH" @@ -100,7 +115,7 @@ if [ -f "$LIBLTDL_PATH" ]; then install_name_tool -id "@loader_path/libltdl.7.dylib" "$LIBLTDL_PATH" fi -echo "Codesigning libmsodbcsql.18.dylib..." +echo "Codesigning $LIBMSODBCSQL_NAME..." codesign -s - -f "$LIBMSODBCSQL_PATH" 2>/dev/null echo "Codesigning libodbcinst.2.dylib..." diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index c5099b8b..7119e36e 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1272,6 +1272,12 @@ std::string GetLastErrorMessage() { * all supported platforms. */ std::string GetDriverPathCpp(const std::string& moduleDir) { +#if !defined(MSODBCSQL_VERSION_MAJOR) || !defined(MSODBCSQL_VERSION_MAJOR_MINOR) +#error \ + "MSODBCSQL_VERSION_MAJOR / MSODBCSQL_VERSION_MAJOR_MINOR must be defined at build time. " \ + "They are derived from mssql_python_odbc.__version__ in CMakeLists.txt so the driver " \ + "filename can never drift from the packaged driver version." +#endif namespace fs = std::filesystem; fs::path basePath(moduleDir); @@ -1299,23 +1305,22 @@ std::string GetDriverPathCpp(const std::string& moduleDir) { platform = "debian_ubuntu"; // Default to debian_ubuntu for other distros } - // NOTE (version coupling): the Linux driver filename is pinned to the exact - // msodbcsql minor version (18.6 -> libmsodbcsql-18.6.so.2.1), matching the - // binaries bundled under mssql_python/libs and shipped by the standalone - // mssql_python_odbc package. If those driver binaries are upgraded (e.g. to - // 18.7) this literal MUST be updated in lockstep: GetOdbcLibsBaseDir() calls - // fs::exists() on this exact path to decide whether the external package is - // "complete", so a stale name would make it silently fall back to the - // bundled libs. (The macOS/Windows paths below key off the major version - // only and are more forgiving.) Tracked for the Phase 3 driver-version- - // agnostic resolution work. - fs::path driverPath = - basePath / "libs" / "linux" / platform / arch / "lib" / "libmsodbcsql-18.6.so.2.1"; + // The msodbcsql version embedded in the driver filename is injected at build + // time from mssql_python_odbc.__version__ (the single source of truth for the + // driver version) via the MSODBCSQL_VERSION_* macros defined in + // CMakeLists.txt. This keeps the native resolver and the Python package + // version from ever drifting: GetOdbcLibsBaseDir() calls fs::exists() on this + // exact path to decide whether the external package is "complete", so a stale + // name would silently fall back to the bundled libs. (The ".so.2.1" suffix is + // the driver's ELF soname, which is independent of the product version.) + fs::path driverPath = basePath / "libs" / "linux" / platform / arch / "lib" / + ("libmsodbcsql-" MSODBCSQL_VERSION_MAJOR_MINOR ".so.2.1"); return driverPath.string(); #elif defined(__APPLE__) platform = "macos"; - fs::path driverPath = basePath / "libs" / platform / arch / "lib" / "libmsodbcsql.18.dylib"; + fs::path driverPath = basePath / "libs" / platform / arch / "lib" / + ("libmsodbcsql." MSODBCSQL_VERSION_MAJOR ".dylib"); return driverPath.string(); #elif defined(_WIN32) @@ -1323,7 +1328,8 @@ std::string GetDriverPathCpp(const std::string& moduleDir) { // Normalize x86_64 to x64 for Windows naming if (arch == "x86_64") arch = "x64"; - fs::path driverPath = basePath / "libs" / platform / arch / "msodbcsql18.dll"; + fs::path driverPath = + basePath / "libs" / platform / arch / ("msodbcsql" MSODBCSQL_VERSION_MAJOR ".dll"); return driverPath.string(); #else diff --git a/mssql_python_odbc/__init__.py b/mssql_python_odbc/__init__.py index f51c2333..8355021a 100644 --- a/mssql_python_odbc/__init__.py +++ b/mssql_python_odbc/__init__.py @@ -10,30 +10,26 @@ wheel. It is not meant for direct consumption — install ``mssql-python`` instead, which depends on this package. -The public surface is :func:`get_driver_path`, which returns the absolute path -to the ODBC driver shared library for the current platform/architecture. The -native ``mssql_python.ddbc_bindings`` extension resolves the same location in -C++ (see ``GetOdbcLibsBaseDir`` / ``GetDriverPathCpp``); this Python API mirrors -that logic for tooling, tests, and diagnostics. +Driver-path resolution lives entirely in the native +``mssql_python.ddbc_bindings`` extension (``GetOdbcLibsBaseDir`` / +``GetDriverPathCpp``): it imports this package purely for its ``__file__`` and +appends ``libs///...`` itself. Keeping a single (C++) resolver +avoids a second copy of the arch/distro/filename logic that could silently +drift out of sync — see the drift guard in ``tests/test_000_dependencies.py``. """ import os -import platform -import sys -__all__ = ["get_driver_path", "get_libs_dir", "__version__"] +__all__ = ["get_libs_dir", "__version__"] -# Version tracks the bundled Microsoft ODBC Driver 18 for SQL Server release. +# Version tracks the bundled Microsoft ODBC Driver 18 for SQL Server release and +# is the single source of truth for the driver version. ``setup_odbc.py`` reads +# it for the wheel version, and the native build (``mssql_python/pybind/ +# CMakeLists.txt``) derives the driver filename version from it and injects it +# into ``GetDriverPathCpp`` -- so the C++ resolver and this package can never +# drift. Bump this one value to move to a new driver release. __version__ = "18.6.2" -# Driver shared-library file names per platform (must match the names produced -# by the ODBC driver packaging and expected by the native loader). -_DRIVER_FILENAME = { - "windows": "msodbcsql18.dll", - "linux": "libmsodbcsql-18.6.so.2.1", - "macos": "libmsodbcsql.18.dylib", -} - def get_libs_dir() -> str: """Return the absolute path to this package's ``libs/`` directory. @@ -44,81 +40,3 @@ def get_libs_dir() -> str: the driver. """ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs") - - -def _detect_arch() -> str: - """Return the architecture directory name for the current interpreter. - - Mirrors the compile-time detection in ``GetDriverPathCpp``: - * Windows uses ``x64`` / ``x86`` / ``arm64``. - * Linux and macOS use ``x86_64`` / ``arm64``. - """ - machine = platform.machine().lower() - - if sys.platform.startswith("win"): - if machine in ("amd64", "x86_64"): - return "x64" - if machine in ("arm64", "aarch64"): - return "arm64" - if machine in ("x86", "i386", "i686"): - return "x86" - raise OSError(f"Unsupported Windows architecture: {platform.machine()!r}") - - # Linux / macOS - if machine in ("x86_64", "amd64"): - return "x86_64" - if machine in ("arm64", "aarch64"): - return "arm64" - raise OSError(f"Unsupported architecture: {platform.machine()!r}") - - -def _detect_linux_distro_family() -> str: - """Return the Linux distro family directory name. - - Mirrors the ``/etc/*-release`` probing in ``GetDriverPathCpp`` so the Python - and C++ paths agree. - """ - if os.path.exists("/etc/alpine-release"): - return "alpine" - if os.path.exists("/etc/redhat-release") or os.path.exists("/etc/centos-release"): - return "rhel" - if os.path.exists("/etc/SuSE-release") or os.path.exists("/etc/SUSE-brand"): - return "suse" - return "debian_ubuntu" # default for Debian/Ubuntu and other glibc distros - - -def get_driver_path() -> str: - """Return the absolute path to the ODBC driver shared library. - - Resolves the platform/architecture (and, on Linux, the distro family) and - returns the full path to ``msodbcsql18`` inside this package. - - Raises: - OSError: if the platform/architecture is unsupported. - FileNotFoundError: if the resolved driver file is not present (e.g. the - package was built without this platform's binaries). - """ - libs_dir = get_libs_dir() - arch = _detect_arch() - - if sys.platform.startswith("win"): - driver_path = os.path.join(libs_dir, "windows", arch, _DRIVER_FILENAME["windows"]) - elif sys.platform.startswith("darwin"): - driver_path = os.path.join( - libs_dir, "macos", arch, "lib", _DRIVER_FILENAME["macos"] - ) - elif sys.platform.startswith("linux"): - distro = _detect_linux_distro_family() - driver_path = os.path.join( - libs_dir, "linux", distro, arch, "lib", _DRIVER_FILENAME["linux"] - ) - else: - raise OSError(f"Unsupported platform: {sys.platform!r}") - - if not os.path.isfile(driver_path): - raise FileNotFoundError( - f"ODBC driver not found at {driver_path!r}. The mssql-python-odbc " - f"package may not include binaries for this platform/architecture." - ) - - return driver_path diff --git a/setup_odbc.py b/setup_odbc.py index 0d83846d..7e87cc73 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -43,6 +43,22 @@ MIN_SETUPTOOLS = (62, 3, 0) +def _read_odbc_version() -> str: + """Return ``__version__`` from ``mssql_python_odbc/__init__.py``. + + Single source of truth for the package version: the value is defined once in + the package's ``__init__.py`` and read here (by regex, without importing the + package) so the wheel version can never drift from what the package reports + at runtime. + """ + init_file = PACKAGE_DIR / "__init__.py" + text = init_file.read_text(encoding="utf-8") + match = re.search(r'^__version__\s*=\s*["\']([^"\']+)["\']', text, re.MULTILINE) + if not match: + raise SystemExit(f"Could not find __version__ in {init_file}") + return match.group(1) + + def _require_min_setuptools() -> None: raw = setuptools.__version__ parts = tuple(int(m) for m in re.findall(r"\d+", raw)[:3]) @@ -197,7 +213,7 @@ def run(self): setup( name="mssql-python-odbc", - version="18.6.2", + version=_read_odbc_version(), description=( "Internal implementation package for mssql-python: Microsoft ODBC " "Driver 18 for SQL Server binaries. Not intended for direct use." diff --git a/tests/test_000_dependencies.py b/tests/test_000_dependencies.py index 5c50c10c..9398cb7a 100644 --- a/tests/test_000_dependencies.py +++ b/tests/test_000_dependencies.py @@ -6,6 +6,7 @@ import pytest import platform import os +import re import sys from pathlib import Path @@ -64,23 +65,12 @@ def _normalize_architecture(self): return arch_lower def _detect_linux_distro(self): - """Detect Linux distribution for driver path selection.""" - distro_name = "debian_ubuntu" # default - """ - #ifdef __linux__ - if (fs::exists("/etc/alpine-release")) { - platform = "alpine"; - } else if (fs::exists("/etc/redhat-release") || fs::exists("/etc/centos-release")) { - platform = "rhel"; - } else if (fs::exists("/etc/SuSE-release") || fs::exists("/etc/SUSE-brand")) { - platform = "suse"; - } else { - platform = "debian_ubuntu"; - } - - fs::path driverPath = basePath / "libs" / "linux" / platform / arch / "lib" / "libmsodbcsql-18.6.so.2.1"; - return driverPath.string(); + """Detect Linux distribution for driver path selection. + + Mirrors the ``/etc/*-release`` probing in the native ``GetDriverPathCpp`` + so the expected paths agree with what the C++ resolver looks for. """ + distro_name = "debian_ubuntu" # default try: if Path("/etc/alpine-release").exists(): distro_name = "alpine" @@ -95,6 +85,58 @@ def _detect_linux_distro(self): return distro_name + def _driver_version_parts(self): + """Return ``(major, minor)`` of the bundled msodbcsql driver. + + Derived from ``mssql_python_odbc.__version__`` -- the single source of + truth for the driver version. If that package is not installed, fall + back to the native resolver's own filename so the expected paths still + track whatever the compiled extension actually looks for (rather than a + hardcoded literal that could silently drift on a version bump). + + Only a missing package triggers the fallback: an installed-but-malformed + ``__version__`` is surfaced (not swallowed) so a corrupt package fails + loudly instead of silently masking behind the native resolver. + """ + try: + import mssql_python_odbc + except ImportError: + import mssql_python.ddbc_bindings as ddbc + + name = os.path.basename(ddbc.GetDriverPathCpp("base")) + match = re.search(r"(\d+)(?:\.(\d+))?", name) + if match is None: + raise AssertionError( + f"Could not derive driver version from resolved filename {name!r}" + ) + return match.group(1), (match.group(2) or "") + + parts = mssql_python_odbc.__version__.split(".") + if len(parts) < 2: + raise AssertionError( + "mssql_python_odbc.__version__ is malformed " + f"(expected 'major.minor[.patch]', got {mssql_python_odbc.__version__!r})" + ) + return parts[0], parts[1] + + def _driver_filename(self): + """Return the msodbcsql driver filename for the current platform. + + Built from the version parts (see :meth:`_driver_version_parts`) so it + stays in lockstep with the native resolver's ``GetDriverPathCpp``. + """ + major, minor = self._driver_version_parts() + if self.platform_name == "windows": + return f"msodbcsql{major}.dll" + if self.platform_name == "darwin": + return f"libmsodbcsql.{major}.dylib" + if not minor: + raise AssertionError( + f"Linux driver filename needs a minor version but none was derived (major={major!r})" + ) + # Linux embeds the major.minor plus the ELF soname suffix. + return f"libmsodbcsql-{major}.{minor}.so.2.1" + def get_expected_dependencies(self): """Get expected dependencies for the current platform and architecture.""" if self.platform_name == "windows": @@ -111,7 +153,7 @@ def _get_windows_dependencies(self): base_path = self.module_dir / "libs" / "windows" / self.normalized_arch dependencies = [ - base_path / "msodbcsql18.dll", + base_path / self._driver_filename(), base_path / "msodbcdiag18.dll", base_path / "mssql-auth.dll", base_path / "vcredist" / "msvcp140.dll", @@ -128,7 +170,7 @@ def _get_macos_dependencies(self): base_path = self.module_dir / "libs" / "macos" / arch / "lib" dependencies.extend( [ - base_path / "libmsodbcsql.18.dylib", + base_path / self._driver_filename(), base_path / "libodbcinst.2.dylib", ] ) @@ -149,7 +191,7 @@ def _get_linux_dependencies(self): base_path = self.module_dir / "libs" / "linux" / distro_name / runtime_arch / "lib" dependencies = [ - base_path / "libmsodbcsql-18.6.so.2.1", + base_path / self._driver_filename(), base_path / "libodbcinst.so.2", ] @@ -188,7 +230,11 @@ def get_expected_driver_path(self): if platform_name == "windows": driver_path = ( - Path(self.module_dir) / "libs" / "windows" / normalized_arch / "msodbcsql18.dll" + Path(self.module_dir) + / "libs" + / "windows" + / normalized_arch + / self._driver_filename() ) elif platform_name == "darwin": @@ -198,7 +244,7 @@ def get_expected_driver_path(self): / "macos" / normalized_arch / "lib" - / "libmsodbcsql.18.dylib" + / self._driver_filename() ) elif platform_name == "linux": @@ -210,7 +256,7 @@ def get_expected_driver_path(self): / distro_name / normalized_arch / "lib" - / "libmsodbcsql-18.6.so.2.1" + / self._driver_filename() ) else: @@ -339,7 +385,7 @@ def test_macos_universal_dependencies(self): for arch in ["arm64", "x86_64"]: base_path = dependency_tester.module_dir / "libs" / "macos" / arch / "lib" - msodbcsql_path = base_path / "libmsodbcsql.18.dylib" + msodbcsql_path = base_path / dependency_tester._driver_filename() libodbcinst_path = base_path / "libodbcinst.2.dylib" assert msodbcsql_path.exists(), f"macOS {arch} ODBC driver not found: {msodbcsql_path}" @@ -659,3 +705,75 @@ def test_ddbc_bindings_no_module_found_error(): assert python_version in expected_error assert architecture in expected_error assert extension in expected_error + + +class TestOdbcPackageSplit: + """Coverage for the standalone ``mssql-python-odbc`` driver package. + + The ODBC driver binaries are being split out of ``mssql-python`` into the + ``mssql-python-odbc`` package. Driver-path resolution has a single owner -- + the native C++ resolver (``GetDriverPathCpp``); the Python package only + ships the binaries and exposes ``__version__``. These tests therefore use + the native resolver rather than a duplicate Python path builder. + """ + + def test_odbc_package_ships_driver_for_platform(self): + """The odbc package (if installed) ships this platform's driver binary. + + Skips cleanly when ``mssql-python-odbc`` is not installed or its + ``libs/`` tree has not been synced (fresh source checkout / CI before + the binaries are placed). When ``libs/`` is present it MUST contain this + platform's driver, otherwise the package is broken. + """ + odbc = pytest.importorskip("mssql_python_odbc") + + libs_dir = odbc.get_libs_dir() + if not os.path.isdir(libs_dir): + pytest.skip( + "mssql_python_odbc/libs not present (fresh checkout or binaries " + "not built/synced for this platform)" + ) + + import mssql_python.ddbc_bindings as ddbc + + package_dir = os.path.dirname(os.path.abspath(odbc.__file__)) + driver_path = ddbc.GetDriverPathCpp(package_dir) + + assert os.path.isfile(driver_path), ( + "mssql-python-odbc ships a libs/ tree but no driver for this " + f"platform at: {driver_path}" + ) + + def test_cpp_driver_filename_matches_odbc_version(self): + """Guard against C++/Python version drift. + + The driver version in ``GetDriverPathCpp``'s resolved filename (e.g. + ``libmsodbcsql-18.6.so.2.1`` on Linux) is injected at build time from + ``mssql_python_odbc.__version__`` -- the single source of truth -- via + the ``MSODBCSQL_VERSION_*`` macros in ``CMakeLists.txt``, so a fresh + build cannot drift. This test additionally catches a *stale* compiled + extension being tested against updated sources (the filename baked into + the binary would then disagree with the current ``__version__``). Each + platform's CI validates its own compiled filename. + """ + odbc = pytest.importorskip("mssql_python_odbc") + + import mssql_python.ddbc_bindings as ddbc + + # GetDriverPathCpp only builds a path string; the base dir need not + # exist. It resolves for the host platform via compile-time #ifdefs. + filename = os.path.basename(ddbc.GetDriverPathCpp("base")) + major, minor = odbc.__version__.split(".")[:2] + + if platform.system().lower() == "linux": + # e.g. __version__ "18.6.2" -> "libmsodbcsql-18.6.so..." + assert ( + f"-{major}.{minor}.so" in filename + ), f"Linux driver filename {filename!r} disagrees with odbc __version__ {odbc.__version__!r}" + else: + # Windows (msodbcsql18.dll) / macOS (libmsodbcsql.18.dylib) key off + # the major version only. Match the exact digit run (word boundary) + # so e.g. major "18" does not spuriously match "msodbcsql182.dll". + assert re.search( + rf"(? bundled ``mssql_python`` libs when - the external package is not installed) is verified manually rather than here, - since exercising it requires uninstalling the package mid-run. -""" - -import os -import sys - -import pytest - -# Skip the whole module cleanly if the package is not importable. -mssql_python_odbc = pytest.importorskip("mssql_python_odbc") - - -EXPECTED_DRIVER_FILENAME = { - "win32": "msodbcsql18.dll", - "darwin": "libmsodbcsql.18.dylib", - "linux": "libmsodbcsql-18.6.so.2.1", -} - - -def _platform_key(): - if sys.platform.startswith("win"): - return "win32" - if sys.platform.startswith("darwin"): - return "darwin" - if sys.platform.startswith("linux"): - return "linux" - return sys.platform - - -def _package_dir(): - """Base directory the native loader appends ``libs`` to.""" - return os.path.dirname(os.path.abspath(mssql_python_odbc.__file__)) - - -def _libs_present(): - """True only when this platform's driver binary is actually present. - - Checks for the driver file rather than just the ``libs/`` directory, so a - stray or empty ``libs/`` in a dev checkout makes the driver tests skip - cleanly instead of failing. Mirrors the C++ resolver, which falls back to - the bundled libs unless the external package ships a real driver binary. - """ - try: - return os.path.isfile(mssql_python_odbc.get_driver_path()) - except OSError: - return False - - -_NO_LIBS_REASON = ( - "ODBC libs not present in package (fresh checkout, or not built/synced for " "this platform)" -) - - -class TestOdbcPackageMetadata: - def test_version_is_driver_version(self): - assert mssql_python_odbc.__version__ == "18.6.2" - - def test_public_api_present(self): - assert callable(mssql_python_odbc.get_driver_path) - assert callable(mssql_python_odbc.get_libs_dir) - - -class TestLibsDir: - def test_libs_dir_under_package(self): - libs_dir = mssql_python_odbc.get_libs_dir() - assert os.path.basename(libs_dir) == "libs" - assert os.path.normcase(libs_dir).startswith(os.path.normcase(_package_dir())) - - -class TestArchDetection: - def test_detect_arch_matches_platform(self): - arch = mssql_python_odbc._detect_arch() - if sys.platform.startswith("win"): - assert arch in ("x64", "x86", "arm64") - else: - assert arch in ("x86_64", "arm64") - - @pytest.mark.skipif( - not sys.platform.startswith("linux"), - reason="Linux-only distro-family detection", - ) - def test_detect_linux_distro_family(self): - distro = mssql_python_odbc._detect_linux_distro_family() - assert distro in ("alpine", "rhel", "suse", "debian_ubuntu") - - -class TestDriverPath: - @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) - def test_driver_path_layout_and_exists(self): - driver_path = mssql_python_odbc.get_driver_path() - # Correct driver filename for this platform. - assert os.path.basename(driver_path) == EXPECTED_DRIVER_FILENAME[_platform_key()] - # Resolved under the package's own libs directory. - assert mssql_python_odbc.get_libs_dir() in driver_path - # The resolved driver file actually exists on disk. - assert os.path.isfile(driver_path), f"driver not found at {driver_path}" - - @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) - def test_driver_path_contains_expected_platform_dir(self): - driver_path = mssql_python_odbc.get_driver_path().replace("\\", "/") - expected_segment = { - "win32": "/libs/windows/", - "darwin": "/libs/macos/", - "linux": "/libs/linux/", - }[_platform_key()] - assert expected_segment in driver_path - - -class TestPythonCppParity: - """The Python ``get_driver_path()`` must resolve to the exact same path the - native C++ loader (``GetDriverPathCpp``) computes for the same base dir, so - tooling/tests agree with the driver actually loaded at connect time.""" - - @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) - def test_get_driver_path_matches_cpp(self): - try: - from mssql_python import ddbc_bindings - except Exception as exc: # native extension not built / driver load failed - pytest.skip(f"ddbc_bindings unavailable: {exc}") - - get_cpp = getattr(ddbc_bindings, "GetDriverPathCpp", None) - if get_cpp is None: - pytest.skip("GetDriverPathCpp not exposed by this build") - - py_path = mssql_python_odbc.get_driver_path() - cpp_path = get_cpp(_package_dir()) - - def norm(p): - return os.path.normcase(os.path.normpath(p)) - - assert norm(py_path) == norm(cpp_path), ( - "Python/C++ driver path mismatch:\n" f" python={py_path}\n" f" cpp ={cpp_path}" - ) From 73733d47828f33c4e5bfe6e8f802588781210d22 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 13:19:17 +0530 Subject: [PATCH 11/15] Fix Linux pipeline uninstall parsing and ARM64 setup checks --- eng/pipelines/pr-validation-pipeline.yml | 34 ++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index d4ba85f5..4363f966 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -843,7 +843,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 debian_ubuntu driver library signatures:' - DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1072,26 +1072,32 @@ jobs: # Install dependencies in the ARM64 container if [ "$(distroName)" = "Ubuntu" ]; then docker exec test-container-$(distroName)-$(archName) bash -c " + set -euo pipefail export DEBIAN_FRONTEND=noninteractive export TZ=UTC ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone - apt-get update && + apt-get update apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev # Verify architecture uname -m dpkg --print-architecture + python3 --version + cmake --version " else # Debian ARM64 docker exec test-container-$(distroName)-$(archName) bash -c " + set -euo pipefail export DEBIAN_FRONTEND=noninteractive export TZ=UTC ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone - apt-get update && + apt-get update apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev # Verify architecture uname -m dpkg --print-architecture + python3 --version + cmake --version " fi displayName: 'Install basic dependencies in $(distroName) ARM64 container' @@ -1128,9 +1134,14 @@ jobs: - script: | # Install Python dependencies in the ARM64 container using virtual environment docker exec test-container-$(distroName)-$(archName) bash -c " + set -euo pipefail # Create a virtual environment python3 -m venv /opt/venv source /opt/venv/bin/activate + + # Ensure the expected runtime tools are available before build/test steps + command -v python >/dev/null 2>&1 + command -v cmake >/dev/null 2>&1 # Install dependencies in the virtual environment python -m pip install --upgrade pip @@ -1144,7 +1155,14 @@ jobs: - script: | # Build pybind bindings in the ARM64 container docker exec test-container-$(distroName)-$(archName) bash -c " + set -euo pipefail + if [ ! -f /opt/venv/bin/activate ]; then + echo '/opt/venv/bin/activate is missing. Python dependency setup did not complete.' + exit 1 + fi source /opt/venv/bin/activate + command -v python >/dev/null 2>&1 + command -v cmake >/dev/null 2>&1 cd mssql_python/pybind chmod +x build.sh ./build.sh @@ -1170,7 +1188,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 debian_ubuntu driver library signatures:' - DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/debian_ubuntu/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1385,7 +1403,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 rhel driver library signatures:' - DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/rhel/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1612,7 +1630,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 rhel driver library signatures:' - DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/rhel/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1848,7 +1866,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 alpine driver library signatures:' - DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/alpine/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " @@ -2101,7 +2119,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying arm64 alpine driver library signatures:' - DRIVER_MAJOR_MINOR=\$(python3 -c 'from pathlib import Path; import re; t=Path("mssql_python_odbc/__init__.py").read_text(encoding="utf-8"); m=re.search(r"^__version__\\s*=\\s*[\"\'](\\d+\\.\\d+)\\.\\d+[\"\']", t, re.M); print(m.group(1) if m else "")') + DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/alpine/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " From 087b39b432fba5fc1d33718cdbdefe85c3b24f9b Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 13:46:34 +0530 Subject: [PATCH 12/15] Fix Linux pipeline version parsing quoting --- eng/pipelines/pr-validation-pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 4363f966..fce61287 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -843,7 +843,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 debian_ubuntu driver library signatures:' - DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) + DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1188,7 +1188,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 debian_ubuntu driver library signatures:' - DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) + DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/debian_ubuntu/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1403,7 +1403,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 rhel driver library signatures:' - DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) + DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/rhel/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1630,7 +1630,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 rhel driver library signatures:' - DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) + DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/rhel/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 " @@ -1866,7 +1866,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 alpine driver library signatures:' - DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) + DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/alpine/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " @@ -2119,7 +2119,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying arm64 alpine driver library signatures:' - DRIVER_MAJOR_MINOR=\$(sed -nE "s/^__version__[[:space:]]*=[[:space:]]*['\"]([0-9]+\.[0-9]+)\.[0-9]+['\"].*/\1/p" mssql_python_odbc/__init__.py | head -1) + DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi ldd mssql_python/libs/linux/alpine/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " From 63a2117504b0c07129e9415811c151e826ee677e Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 13:59:27 +0530 Subject: [PATCH 13/15] Fix Linux pipeline host-side version parsing --- eng/pipelines/pr-validation-pipeline.yml | 42 ++++++++++++++---------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index fce61287..7562d71a 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -772,6 +772,9 @@ jobs: - script: | # Install ODBC driver in the container + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-$(distroName) bash -c " export DEBIAN_FRONTEND=noninteractive @@ -843,9 +846,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 debian_ubuntu driver library signatures:' - DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) - if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi - ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 + ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in $(distroName) container' @@ -1104,6 +1105,9 @@ jobs: - script: | # Install ODBC driver in the ARM64 container + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-$(distroName)-$(archName) bash -c " export DEBIAN_FRONTEND=noninteractive @@ -1188,9 +1192,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 debian_ubuntu driver library signatures:' - DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) - if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi - ldd mssql_python/libs/linux/debian_ubuntu/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 + ldd mssql_python/libs/linux/debian_ubuntu/arm64/lib/libmsodbcsql-${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in $(distroName) ARM64 container' @@ -1290,6 +1292,9 @@ jobs: - script: | # Install dependencies in the RHEL 9 container + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-rhel9 bash -c " # Enable CodeReady Builder repository for additional packages dnf update -y @@ -1403,9 +1408,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 rhel driver library signatures:' - DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) - if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi - ldd mssql_python/libs/linux/rhel/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 + ldd mssql_python/libs/linux/rhel/x86_64/lib/libmsodbcsql-${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in RHEL 9 container' @@ -1513,6 +1516,9 @@ jobs: - script: | # Install dependencies in the RHEL 9 ARM64 container + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-rhel9-arm64 bash -c " # Enable CodeReady Builder repository for additional packages dnf update -y @@ -1630,9 +1636,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled ODBC Driver and cleaned up libraries' echo 'Verifying arm64 rhel driver library signatures:' - DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) - if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi - ldd mssql_python/libs/linux/rhel/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 + ldd mssql_python/libs/linux/rhel/arm64/lib/libmsodbcsql-${DRIVER_MAJOR_MINOR}.so.2.1 " displayName: 'Uninstall ODBC Driver before running tests in RHEL 9 ARM64 container' @@ -1776,6 +1780,9 @@ jobs: - script: | # Install ODBC driver in the Alpine x86_64 container + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-alpine bash -c " # Detect architecture for ODBC driver download case \$(uname -m) in @@ -1866,9 +1873,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying x86_64 alpine driver library signatures:' - DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) - if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi - ldd mssql_python/libs/linux/alpine/x86_64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' + ldd mssql_python/libs/linux/alpine/x86_64/lib/libmsodbcsql-${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " displayName: 'Uninstall system ODBC Driver before running tests in Alpine x86_64 container' @@ -2028,6 +2033,9 @@ jobs: - script: | # Install ODBC driver in the Alpine ARM64 container + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-alpine-arm64 bash -c " # Detect architecture for ODBC driver download case \$(uname -m) in @@ -2119,9 +2127,7 @@ jobs: odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true echo 'Uninstalled system ODBC Driver and cleaned up libraries' echo 'Verifying arm64 alpine driver library signatures:' - DRIVER_MAJOR_MINOR=\$(perl -ne 'if(/^__version__\s*=\s*[\042\047]([0-9]+\.[0-9]+)\.[0-9]+[\042\047]/){print $1; exit}' mssql_python_odbc/__init__.py) - if [ -z "\$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi - ldd mssql_python/libs/linux/alpine/arm64/lib/libmsodbcsql-\${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' + ldd mssql_python/libs/linux/alpine/arm64/lib/libmsodbcsql-${DRIVER_MAJOR_MINOR}.so.2.1 || echo 'Driver library not found' " displayName: 'Uninstall system ODBC Driver before running tests in Alpine ARM64 container' From d9749ad17909080b42d380b09b972a9f316fdd42 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 14:06:24 +0530 Subject: [PATCH 14/15] Fix Ubuntu Linux pipeline version parsing --- eng/pipelines/pr-validation-pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 7562d71a..4721efc5 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -836,6 +836,9 @@ jobs: - script: | # Uninstall ODBC Driver before running tests + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-$(distroName) bash -c " export DEBIAN_FRONTEND=noninteractive apt-get remove --purge -y msodbcsql18 mssql-tools18 unixodbc-dev From 07888e92703bda45abc97c8dab0bb7e26043a7a6 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 14:12:54 +0530 Subject: [PATCH 15/15] Fix DRIVER_MAJOR_MINOR setup in all Linux uninstall steps --- eng/pipelines/pr-validation-pipeline.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 4721efc5..a97a47bc 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -1185,6 +1185,9 @@ jobs: - script: | # Uninstall ODBC Driver before running tests + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-$(distroName)-$(archName) bash -c " export DEBIAN_FRONTEND=noninteractive apt-get remove --purge -y msodbcsql18 mssql-tools18 unixodbc-dev @@ -1402,6 +1405,9 @@ jobs: - script: | # Uninstall ODBC Driver before running tests + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-rhel9 bash -c " dnf remove -y msodbcsql18 mssql-tools18 unixODBC-devel rm -f /usr/bin/sqlcmd @@ -1630,6 +1636,9 @@ jobs: - script: | # Uninstall ODBC Driver before running tests + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-rhel9-arm64 bash -c " dnf remove -y msodbcsql18 mssql-tools18 unixODBC-devel rm -f /usr/bin/sqlcmd @@ -1866,6 +1875,9 @@ jobs: - script: | # Uninstall ODBC Driver before running tests to use bundled libraries + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-alpine bash -c " # Remove system ODBC installation apk del msodbcsql18 mssql-tools18 unixodbc-dev || echo 'ODBC packages not installed via apk' @@ -2120,6 +2132,9 @@ jobs: - script: | # Uninstall ODBC Driver before running tests to use bundled libraries + DRIVER_VERSION=$(awk -F= '/^__version__/ {gsub(/[[:space:]"]/, "", $2); print $2; exit}' mssql_python_odbc/__init__.py) + DRIVER_MAJOR_MINOR=$(echo "$DRIVER_VERSION" | cut -d. -f1,2) + if [ -z "$DRIVER_MAJOR_MINOR" ]; then echo 'Error: failed to parse mssql_python_odbc.__version__'; exit 1; fi docker exec test-container-alpine-arm64 bash -c " # Remove system ODBC installation apk del msodbcsql18 mssql-tools18 unixodbc-dev || echo 'ODBC packages not installed via apk'