bump submodule and fix issues

benchmarks/requirements-bench.txt

@@ -18,6 +18,5 @@ pygments==2.20.0
 pytest==9.1.1
 pytest-codspeed==5.0.3
 python-dateutil==2.9.0.post0
-pytz==2026.2
 rich==15.0.0
 six==1.17.0

benchmarks/test_fetch_perf.py

@@ -67,7 +67,7 @@ def run() -> None:
 
 
 # small-N *_gate variants: at range(2048) the measured region is ~60% SQL compile + engine, ~40% fetch, so these
-# catch a fixed-cost regression (not a pure per-row one). Plus expensive scalar types (timestamptz pytz-per-row,
+# catch a fixed-cost regression (not a pure per-row one). Plus expensive scalar types (timestamptz zoneinfo per row,
 # blob, null-heavy), a heterogeneous per-cell-dispatch row, and the batched fetchmany loop.
 
 

duckdb/_relation.py

@@ -340,8 +340,8 @@ def _assemble(ctes: list[tuple[str, str]], body: str) -> str:
 def _stub(names: list[str], types: list[DuckDBPyType]) -> str | None:
     """An empty SELECT of typed NULLs standing in for these columns, or None where one has a type it cannot write.
 
-    A type's text leaves out its collation, which a column of its own carries as COLLATE; a collation on a type inside
-    a list or struct has no such spelling.
+    A 2.0 engine's type text leaves a collation out, so a column of its own carries it as COLLATE, and a collation on a
+    type inside a list or struct cannot be written from the type's text there.
     """
     columns: list[str] = []
     for name, type_ in zip(names, types, strict=True):

duckdb/_sql_keywords.py

@@ -33,6 +33,7 @@
     "attach": "unreserved",
     "attribute": "unreserved",
     "authorization": "type_function",
+    "auto": "unreserved",
     "backward": "unreserved",
     "before": "unreserved",
     "begin": "unreserved",
@@ -106,6 +107,7 @@
     "deferrable": "reserved",
     "deferred": "unreserved",
     "definer": "unreserved",
+    "defines": "unreserved",
     "delete": "unreserved",
     "delimiter": "unreserved",
     "delimiters": "unreserved",
@@ -240,6 +242,7 @@
     "map": "column_name/type_function",
     "mapping": "unreserved",
     "match": "unreserved",
+    "match_recognize": "reserved",
     "matched": "unreserved",
     "materialized": "unreserved",
     "maxvalue": "unreserved",
@@ -415,6 +418,7 @@
     "strip": "unreserved",
     "struct": "column_name/type_function",
     "subscription": "unreserved",
+    "subset": "unreserved",
     "substring": "column_name",
     "summarize": "reserved",
     "symmetric": "reserved",

duckdb/experimental/spark/sql/column.py

@@ -420,7 +420,7 @@ def _truncate(value: Expression, target: DuckDBPyType) -> Expression:
     Spark in ANSI mode does.
     """
     source_type = FunctionExpression("typeof", value)
-    is_string = source_type == ConstantExpression("VARCHAR")
+    is_string = _is_string(value)
     text = value.cast(VARCHAR)
     is_number = FunctionExpression(
         "regexp_full_match", text, ConstantExpression(r"\s*[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)\s*")
@@ -451,10 +451,18 @@ def _truncate(value: Expression, target: DuckDBPyType) -> Expression:
     )
 
 
+def _is_string(value: Expression) -> Expression:
+    """Whether ``value`` is a string at run time; a collated string's type reads VARCHAR('NOCASE') on 2.1 engines."""
+    return FunctionExpression(
+        "regexp_full_match", FunctionExpression("typeof", value), ConstantExpression(r"VARCHAR(\(.*\))?")
+    )
+
+
 def _to_binary(value: Expression) -> Expression:
     """Cast to binary like Spark, which takes a string's UTF-8 bytes where DuckDB reads escape sequences."""
-    is_string = FunctionExpression("typeof", value) == ConstantExpression("VARCHAR")
-    return CaseExpression(is_string, FunctionExpression("encode", value.cast(VARCHAR))).otherwise(value.cast(BLOB))
+    return CaseExpression(_is_string(value), FunctionExpression("encode", value.cast(VARCHAR))).otherwise(
+        value.cast(BLOB)
+    )
 
 
 def _children(dtype: DuckDBPyType) -> list[DuckDBPyType]:

duckdb/experimental/spark/sql/functions.py

@@ -3094,19 +3094,11 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
 
     Examples:
     --------
-    >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age"))
-    >>> df = df.orderBy(df.age)
-    >>> df.groupby("name").agg(first("age")).orderBy("name").show()
-    +-----+----------+
-    | name|first(age)|
-    +-----+----------+
-    |Alice|      NULL|
-    |  Bob|         5|
-    +-----+----------+
-
-    Now, to ignore any nulls we needs to set ``ignorenulls`` to `True`
+    The row a group's first value comes from is not set by the frame's order, so a group with a null and a value can
+    give either. ``ignorenulls`` is not supported here and raises.
 
-    >>> df.groupby("name").agg(first("age", ignorenulls=True)).orderBy("name").show()
+    >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age"))
+    >>> df.groupby("name").agg(first("age")).orderBy("name").show()
     +-----+----------+
     | name|first(age)|
     +-----+----------+
@@ -3149,19 +3141,11 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
 
     Examples:
     --------
-    >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age"))
-    >>> df = df.orderBy(df.age.desc())
-    >>> df.groupby("name").agg(last("age")).orderBy("name").show()
-    +-----+---------+
-    | name|last(age)|
-    +-----+---------+
-    |Alice|     NULL|
-    |  Bob|        5|
-    +-----+---------+
-
-    Now, to ignore any nulls we needs to set ``ignorenulls`` to `True`
+    The row a group's last value comes from is not set by the frame's order, so a group with a null and a value can
+    give either. ``ignorenulls`` is not supported here and raises.
 
-    >>> df.groupby("name").agg(last("age", ignorenulls=True)).orderBy("name").show()
+    >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age"))
+    >>> df.groupby("name").agg(last("age")).orderBy("name").show()
     +-----+---------+
     | name|last(age)|
     +-----+---------+

pyproject.toml

@@ -291,7 +291,6 @@ test = [ # dependencies used for running tests
     "py4j",
     "pyotp",
     "pyspark",
-    "pytz",
     "requests",
     "urllib3",
     "fsspec>=2022.11.0; sys_platform != 'win32' or platform_machine != 'ARM64'",
@@ -316,7 +315,6 @@ bench = [ # Pinned deps for the benchmark suite (see benchmarks/README.md). Mini
     "pytest",
     "pytest_codspeed",
     "polars>=1.33.0",
-    "pytz",
     "numpy<2; ( sys_platform != 'win32' or platform_machine != 'ARM64' ) and python_version < '3.12'",
     "numpy>=2; ( sys_platform != 'win32' or platform_machine != 'ARM64' ) and python_version >= '3.12'",
     "numpy>=2.3; sys_platform == 'win32' and platform_machine == 'ARM64'",
@@ -331,7 +329,6 @@ scripts = [ # dependencies used for running scripts
     "pandas",
     "polars>=1.33.0",
     "pyarrow; sys_platform != 'win32' or platform_machine != 'ARM64'",
-    "pytz"
 ]
 pypi = [ # dependencies used by the pypi cleanup script
     "pyotp>=2.9.0",

scripts/cache_data.json

@@ -258,7 +258,9 @@
         "type": "attribute",
         "full_path": "datetime.timezone",
         "name": "timezone",
-        "children": []
+        "children": [
+            "datetime.timezone.utc"
+        ]
     },
     "decimal": {
         "type": "module",
@@ -582,20 +584,6 @@
         "name": "Value",
         "children": []
     },
-    "pytz": {
-        "type": "module",
-        "full_path": "pytz",
-        "name": "pytz",
-        "children": [
-            "pytz.timezone"
-        ]
-    },
-    "pytz.timezone": {
-        "type": "attribute",
-        "full_path": "pytz.timezone",
-        "name": "timezone",
-        "children": []
-    },
     "types": {
         "type": "module",
         "full_path": "types",
@@ -858,5 +846,25 @@
         "full_path": "pandas.arrays.StringArray",
         "name": "StringArray",
         "children": []
+    },
+    "datetime.timezone.utc": {
+        "type": "attribute",
+        "full_path": "datetime.timezone.utc",
+        "name": "utc",
+        "children": []
+    },
+    "zoneinfo": {
+        "type": "module",
+        "full_path": "zoneinfo",
+        "name": "zoneinfo",
+        "children": [
+            "zoneinfo.ZoneInfo"
+        ]
+    },
+    "zoneinfo.ZoneInfo": {
+        "type": "attribute",
+        "full_path": "zoneinfo.ZoneInfo",
+        "name": "ZoneInfo",
+        "children": []
     }
 }

scripts/imports.py

@@ -60,6 +60,7 @@
 datetime.datetime.max
 datetime.datetime.combine
 datetime.timezone
+datetime.timezone.utc
 
 import decimal
 
@@ -122,9 +123,9 @@
 duckdb.filesystem.ModifiedMemoryFileSystem
 duckdb.Value
 
-import pytz
+import zoneinfo
 
-pytz.timezone
+zoneinfo.ZoneInfo
 
 import types
 

src/include/duckdb_python/import_cache/modules/datetime_module.hpp

@@ -20,6 +20,18 @@
 
 namespace duckdb {
 
+struct DatetimeTimezoneCacheItem : public PythonImportCacheItem {
+
+public:
+	DatetimeTimezoneCacheItem(optional_ptr<PythonImportCacheItem> parent)
+	    : PythonImportCacheItem("timezone", parent), utc("utc", this) {
+	}
+	~DatetimeTimezoneCacheItem() override {
+	}
+
+	PythonImportCacheItem utc;
+};
+
 struct DatetimeDatetimeCacheItem : public PythonImportCacheItem {
 
 public:
@@ -55,7 +67,7 @@ struct DatetimeCacheItem : public PythonImportCacheItem {
 public:
 	DatetimeCacheItem()
 	    : PythonImportCacheItem("datetime"), date(this), time("time", this), timedelta("timedelta", this),
-	      datetime(this), timezone("timezone", this) {
+	      datetime(this), timezone(this) {
 	}
 	~DatetimeCacheItem() override {
 	}
@@ -64,7 +76,7 @@ struct DatetimeCacheItem : public PythonImportCacheItem {
 	PythonImportCacheItem time;
 	PythonImportCacheItem timedelta;
 	DatetimeDatetimeCacheItem datetime;
-	PythonImportCacheItem timezone;
+	DatetimeTimezoneCacheItem timezone;
 };
 
 } // namespace duckdb

src/include/duckdb_python/import_cache/modules/pytz_module.hpp → src/include/duckdb_python/import_cache/modules/zoneinfo_module.hpp

@@ -2,7 +2,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb_python/import_cache/modules/pytz_module.hpp
+// duckdb_python/import_cache/modules/zoneinfo_module.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -20,18 +20,18 @@
 
 namespace duckdb {
 
-struct PytzCacheItem : public PythonImportCacheItem {
+struct ZoneinfoCacheItem : public PythonImportCacheItem {
 
 public:
-	static constexpr const char *Name = "pytz";
+	static constexpr const char *Name = "zoneinfo";
 
 public:
-	PytzCacheItem() : PythonImportCacheItem("pytz"), timezone("timezone", this) {
+	ZoneinfoCacheItem() : PythonImportCacheItem("zoneinfo"), ZoneInfo("ZoneInfo", this) {
 	}
-	~PytzCacheItem() override {
+	~ZoneinfoCacheItem() override {
 	}
 
-	PythonImportCacheItem timezone;
+	PythonImportCacheItem ZoneInfo;
 };
 
 } // namespace duckdb

src/include/duckdb_python/import_cache/python_import_cache.hpp

@@ -33,11 +33,11 @@ struct PythonImportCache {
 	PathlibCacheItem pathlib;
 	PolarsCacheItem polars;
 	DuckdbCacheItem duckdb;
-	PytzCacheItem pytz;
 	TypesCacheItem types;
 	TypingCacheItem typing;
 	UuidCacheItem uuid;
 	CollectionsCacheItem collections;
+	ZoneinfoCacheItem zoneinfo;
 
 public:
 	nb::handle AddCache(nb::object item);

src/include/duckdb_python/import_cache/python_import_cache_modules.hpp

@@ -8,8 +8,8 @@
 #include "duckdb_python/import_cache/modules/pathlib_module.hpp"
 #include "duckdb_python/import_cache/modules/polars_module.hpp"
 #include "duckdb_python/import_cache/modules/duckdb_module.hpp"
-#include "duckdb_python/import_cache/modules/pytz_module.hpp"
 #include "duckdb_python/import_cache/modules/types_module.hpp"
 #include "duckdb_python/import_cache/modules/typing_module.hpp"
 #include "duckdb_python/import_cache/modules/uuid_module.hpp"
-#include "duckdb_python/import_cache/modules/collections_module.hpp"
+#include "duckdb_python/import_cache/modules/collections_module.hpp"
+#include "duckdb_python/import_cache/modules/zoneinfo_module.hpp"

src/include/duckdb_python/pandas/column/pandas_numpy_column.hpp

@@ -13,11 +13,14 @@ class PandasNumpyColumn : public PandasColumn {
 		D_ASSERT(nb::hasattr(arr, "strides"));
 		auto first_stride = nb::cast<int64_t>(arr.attr("strides").attr("__getitem__")(0));
 		auto item_size = nb::cast<int64_t>(arr.attr("dtype").attr("itemsize"));
-		// The scan steps through whole items at aligned addresses, which a field of a packed structured array lacks
-		if (item_size <= 0 || first_stride % item_size != 0 || !nb::cast<bool>(arr.attr("flags").attr("aligned"))) {
+		// The scan steps forward through whole aligned items, which a reversed view or a packed structured field lacks
+		if (item_size <= 0 || first_stride < 0 || first_stride % item_size != 0 ||
+		    !nb::cast<bool>(arr.attr("flags").attr("aligned"))) {
 			array = NumpyArray(nb::module_::import_("numpy").attr("require")(arr, nb::arg("requirements") = "CA"));
 		}
-		stride = nb::cast<idx_t>(array.GetArray().attr("strides").attr("__getitem__")(0));
+		auto copied_stride = nb::cast<int64_t>(array.GetArray().attr("strides").attr("__getitem__")(0));
+		// numpy calls an array of one item contiguous whatever its stride, so the copy can leave it reversed
+		stride = UnsafeNumericCast<idx_t>(copied_stride < 0 ? item_size : copied_stride);
 	}
 
 public:

src/native/python_conversion.cpp

@@ -762,7 +762,9 @@ struct PythonVectorConversion {
 		if (result.GetType().id() != LogicalTypeId::BOOLEAN) {
 			throw TypeMismatchException(
 			    LogicalType::BOOLEAN, result.GetType(),
-			    "Python Conversion Failure: Expected a value of type %s, but got a value of type boolean");
+			    StringUtil::Format(
+			        "Python Conversion Failure: Expected a value of type %s, but got a value of type boolean",
+			        result.GetType().ToString()));
 		}
 		FlatVector::GetDataMutable<bool>(result)[result_offset] = val;
 	}