Merge pull request #71 from thatch/thatch/zstd-prereq

Build with zstd support

Author: thatch

Dockerfile

@@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y git htop build-essential \
     gdb lcov patchelf python3-pip python3-venv tcl \
     libexpat1-dev libffi-dev zlib1g-dev libgdbm-dev libgdbm-compat-dev \
     libssl-dev libsqlite3-dev uuid-dev \
-    liblzma-dev libbz2-dev
+    liblzma-dev libbz2-dev libzstd-dev
 
 RUN /usr/bin/python3 -mpip install -U pip setuptools
 

src/portable_python/cpython.py

@@ -8,7 +8,7 @@
 from runez.pyenv import Version
 
 from portable_python import LOG, patch_file, patch_folder, PPG, PythonBuilder
-from portable_python.external.xcpython import Bdb, Bzip2, Gdbm, LibFFI, Openssl, Readline, Sqlite, Uuid, Xz, Zlib
+from portable_python.external.xcpython import Bdb, Bzip2, Gdbm, LibFFI, Openssl, Readline, Sqlite, Uuid, Xz, Zlib, Zstd
 from portable_python.external.xtkinter import TkInter
 from portable_python.inspector import LibAutoCorrect, PythonInspector
 
@@ -96,7 +96,7 @@ def build_information(self):
 
     @classmethod
     def candidate_modules(cls):
-        return [LibFFI, Zlib, Xz, Bzip2, Readline, Openssl, Sqlite, Bdb, Gdbm, Uuid, TkInter]
+        return [LibFFI, Zlib, Zstd, Xz, Bzip2, Readline, Openssl, Sqlite, Bdb, Gdbm, Uuid, TkInter]
 
     @property
     def url(self):
@@ -172,6 +172,18 @@ def c_configure_args(self):
             yield f"-ltcl{version.mm}"
             yield f"-ltk{version.mm}"
 
+    def xenv_LIBZSTD_CFLAGS(self):
+        if self.version >= "3.14" and PPG.target.is_macos:
+            # Normally ./configure will autodetect using pkg-config, but
+            # this doesn't typically work on Mac (the pkg-config binary is
+            # in homebrew, which we omit from path) so we have to provide
+            # some hints about how to staticly include it.
+            yield f"-I{self.deps}/include"
+
+    def xenv_LIBZSTD_LIBS(self):
+        if self.version >= "3.14" and PPG.target.is_macos:
+            yield f"{self.deps_lib_dir}/libzstd.a"
+
     @runez.cached_property
     def prefix_lib_folder(self):
         """Path to <prefix>/lib/pythonM.m folder"""

src/portable_python/external/_inspect.py

@@ -10,6 +10,7 @@
     "_tkinter": "TCL_VERSION TK_VERSION",
     "_sqlite3": "sqlite_version version",
     "_ssl": "OPENSSL_VERSION",
+    "_zstd": "zstd_version",
     "dbm.gnu": "_GDBM_VERSION",
     "ensurepip": "_PIP_VERSION",
     "pyexpat": "version_info",

src/portable_python/external/xcpython.py

@@ -1,3 +1,4 @@
+import os.path
 from typing import ClassVar
 
 import runez
@@ -428,3 +429,39 @@ def _do_linux_compile(self):
         self.run_configure("./configure", self.c_configure_args())
         self.run_make()
         self.run_make("install")
+
+
+class Zstd(ModuleBuilder):
+    """
+    Newer compression format present in most 3.14+ builds
+    """
+
+    m_debian = "!libzstd-dev"
+    m_telltale = "{include}/zstd.h"
+
+    xenv_CFLAGS = "-fPIC"
+
+    def auto_select_reason(self):
+        if self.setup.python_spec.version >= "3.14":
+            if PPG.target.is_macos:
+                return "Required for 3.14 and up"  # Well, "expected" anyway
+            if not self.resolved_telltale:
+                return "Required for 3.14 and up"
+
+    @property
+    def url(self):
+        return (
+            self.cfg_url(self.version) or f"https://github.com/facebook/zstd/releases/download/v{self.version}/zstd-{self.version}.tar.gz"
+        )
+
+    @property
+    def version(self):
+        return self.cfg_version("1.5.7")
+
+    def _do_linux_compile(self):
+        # Notably, this does not build when given a relative path.
+        self.run_make(f"prefix={os.path.abspath(self.deps)}")
+        # the libdir on the resulting .dylib on Mac is wrong, but as long as we
+        # staticly compile this doesn't need a fixup.  I got as far as:
+        # "libdir=\\$(executable_path)/../lib")
+        self.run_make("install", f"prefix={os.path.abspath(self.deps)}")

src/portable_python/inspector.py

@@ -397,7 +397,7 @@ def get_lib_type(install_folder, path, basename):
 
 
 class PythonInspector:
-    default = "_bz2,_ctypes,_curses,_decimal,_dbm,_gdbm,_lzma,_tkinter,_sqlite3,_ssl,_uuid,pip,readline,pyexpat,setuptools,zlib"
+    default = "_bz2,_ctypes,_curses,_decimal,_dbm,_gdbm,_lzma,_tkinter,_sqlite3,_ssl,_uuid,_zstd,pip,readline,pyexpat,setuptools,zlib"
     additional = "_asyncio,_functools,_tracemalloc,dbm.gnu,ensurepip,ossaudiodev,spwd,sys,tkinter,venv,wheel"
 
     def __init__(self, spec, modules=None):