feat: support geospatial results on the kernel path

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Release History
 
 ## Unreleased
+- Add default-enabled kernel-backed GEOMETRY / GEOGRAPHY results, selected with `WithEnableGeoSpatialSupport` or the `enableGeoSpatialSupport` DSN parameter. Enabled mode returns EWKB with its per-row SRID and logical geo column metadata from `database/sql`; disabled mode returns WKT / EWKT reported as `STRING`. Arrow batch access retains the canonical `struct<srid,wkb>` value with separate OGC WKB in enabled mode.
 - **Behavior change:** `m2m.NewAuthenticatorWithScopes` now requests exactly the given scopes instead of always appending `all-apis`, so service principals with scoped OAuth secrets (e.g. `sql` only) can authenticate (databricks/databricks-sql-go#476). Empty scopes and `m2m.NewAuthenticator` still request `all-apis`; callers that relied on it being appended should pass no scopes, or add `all-apis` to their list if they need both. The SEA/kernel backend now forwards custom M2M scopes instead of rejecting them.
 - Let the kernel telemetry feature flag decide enablement when `enableTelemetry` is unset; explicit `true` and `false` continue to override it.
 

CONNECTION_PARAMETERS.md

@@ -155,12 +155,14 @@ ignored with a warning and still stripped from the SEA wire.
 |---|---|:---:|:---:|---|---|
 | `useArrowNativeDecimal` | `WithArrowNativeDecimal` | ✅ | ⚠️ | `false` | Thrift: return DECIMAL as native Arrow `decimal128` (lossless string when scanned via `database/sql`). The kernel path already renders DECIMAL as the exact string regardless, so the flag is inert there. |
 | | `WithKernelDecimalAsFloat(b)` | ❌ | ✅ | `false` | Scan top-level DECIMAL as lossy `float64` instead of the exact string. |
+| `enableGeoSpatialSupport` | `WithEnableGeoSpatialSupport(b)` | ❌ | ✅ | `true` | `true`: return top-level GEOMETRY / GEOGRAPHY as EWKB `[]byte` with the per-row SRID embedded and logical geo column metadata; `GetArrowBatches` retains the full `{srid,wkb}` struct with OGC WKB. `false`: return WKT / EWKT strings reported as `STRING`. |
 
-With the default client-result settings, results otherwise render **byte-for-byte
+Apart from the geospatial representation selected above, results render **byte-for-byte
 identically** on both backends (scalars,
 DECIMAL as exact string, TIMESTAMP / TIMESTAMP_NTZ shifted into the session time zone,
-INTERVAL, nested ARRAY / MAP / STRUCT and VARIANT as JSON, GEOMETRY / GEOGRAPHY as WKT,
-BINARY as `sql.RawBytes`).
+INTERVAL, nested ARRAY / MAP / STRUCT and VARIANT as JSON, and BINARY as
+`sql.RawBytes`). Set `enableGeoSpatialSupport=false` on the kernel path to match
+the Thrift path's WKT / EWKT geospatial values.
 
 ## Cloud Fetch
 

KERNEL_REV

@@ -1 +1 @@
-e5008635d35af1f83a88057c4cf4d97718e31ab4
+9c2e2e6182fc7285bcfff39b2177941bc804897c

README.md

@@ -270,6 +270,7 @@ session parameter on both backends.
 |---|---|---|---|---|
 | `useArrowNativeDecimal` | `WithArrowNativeDecimal` | Thrift only (inert on kernel) | `false` | Thrift: return DECIMAL as native Arrow `decimal128` (lossless string when scanned via `database/sql`). The kernel path already renders DECIMAL as the exact string regardless. |
 | | `WithKernelDecimalAsFloat(b)` | SEA only | `false` | Scan top-level DECIMAL as lossy `float64` instead of the exact string. |
+| `enableGeoSpatialSupport` | `WithEnableGeoSpatialSupport(b)` | SEA only | `true` | `true`: return top-level GEOMETRY / GEOGRAPHY as EWKB `[]byte` with the per-row SRID embedded and logical geo column metadata; `GetArrowBatches` retains the full `{srid,wkb}` struct with OGC WKB. `false`: return WKT / EWKT strings reported as `STRING`. |
 
 See [Cloud Fetch](#cloud-fetch), [TLS](#tls), and [Proxy](#proxy) for the remaining
 groups. Telemetry parameters are covered under [Telemetry](#telemetry).
@@ -358,10 +359,12 @@ The Thrift and kernel backends both honor the standard `HTTP_PROXY` / `HTTPS_PRO
 
 ## Data types
 
-Results render **byte-for-byte identically** on both backends. Scalars, DECIMAL (exact
-string), TIMESTAMP / TIMESTAMP_NTZ (shifted into the session time zone), INTERVAL, nested
-ARRAY / MAP / STRUCT and VARIANT (as JSON), and GEOMETRY / GEOGRAPHY (as WKT) are all
-supported. BINARY is returned as `sql.RawBytes`.
+Apart from the configurable geospatial representation, results render **byte-for-byte
+identically** on both backends. Scalars, DECIMAL (exact string), TIMESTAMP /
+TIMESTAMP_NTZ (shifted into the session time zone), INTERVAL, and nested ARRAY / MAP /
+STRUCT and VARIANT (as JSON) are supported. BINARY is returned as `sql.RawBytes`.
+Thrift returns GEOMETRY / GEOGRAPHY as WKT / EWKT; the kernel defaults to EWKB and
+matches Thrift when `enableGeoSpatialSupport=false`.
 
 Metadata is reached through SQL (`SHOW`, `DESCRIBE`, `information_schema`) on both
 backends — the driver exposes no `GetCatalogs`/`GetSchemas`/`GetTables`/`GetColumns` API

connector.go

@@ -316,6 +316,10 @@ func withUserConfig(ucfg config.UserConfig) ConnOption {
 		if ucfg.TokenCacheEnabledDSN {
 			kernelExperimental(c).TokenCacheEnabled = true
 		}
+		if ucfg.EnableGeoSpatialSupportDSN != nil {
+			enabled := *ucfg.EnableGeoSpatialSupportDSN
+			kernelExperimental(c).EnableGeoSpatialSupport = &enabled
+		}
 	}
 }
 
@@ -671,6 +675,28 @@ func WithKernelDecimalAsFloat(asFloat bool) ConnOption {
 	}
 }
 
+// WithEnableGeoSpatialSupport selects how the kernel path exposes top-level
+// GEOMETRY and GEOGRAPHY result values. True (the default) returns EWKB bytes
+// ([]byte), including the per-row SRID, and reports the logical GEOMETRY /
+// GEOGRAPHY type through database/sql metadata. False returns WKT / EWKT strings and reports STRING, matching the
+// existing string-mode driver contract. The per-row SRID remains present in the
+// Arrow struct returned by the driver's GetArrowBatches API in enabled mode;
+// that raw Arrow value retains Databricks' separate SRID and OGC WKB children.
+//
+// The conversion is entirely client-side and is never forwarded to SEA.
+// Nested geospatial values follow the driver's existing complex-value contract:
+// disabled mode renders WKT / EWKT leaves in JSON, while enabled mode preserves the
+// {"srid":...,"wkb":...} shape (with JSON base64 for WKB).
+//
+// EXPERIMENTAL, kernel-only: the default (Thrift) backend rejects this option at
+// connect time.
+func WithEnableGeoSpatialSupport(enabled bool) ConnOption {
+	return func(c *config.Config) {
+		value := enabled
+		kernelExperimental(c).EnableGeoSpatialSupport = &value
+	}
+}
+
 // WithKernelTrustedCerts adds a PEM CA-certificate bundle to the kernel's TLS
 // trust store on top of the system roots — for a corporate re-signing proxy or an
 // on-prem CA. Required (rather than relying on SSL_CERT_FILE) because the kernel's

connector_test.go

@@ -450,6 +450,19 @@ func TestNewConnector(t *testing.T) {
 		assert.Nil(t, coni.cfg.KernelExperimental)
 	})
 
+	t.Run("Connector test enableGeoSpatialSupport=false DSN param preserves explicit false", func(t *testing.T) {
+		ucfg, err := config.ParseDSN("token:supersecret@databricks-host:443/sql/1.0/endpoints/abc?enableGeoSpatialSupport=false")
+		require.NoError(t, err)
+		con, err := NewConnector(withUserConfig(ucfg))
+		require.NoError(t, err)
+
+		coni, ok := con.(*connector)
+		require.True(t, ok)
+		require.NotNil(t, coni.cfg.KernelExperimental)
+		require.NotNil(t, coni.cfg.KernelExperimental.EnableGeoSpatialSupport)
+		assert.False(t, *coni.cfg.KernelExperimental.EnableGeoSpatialSupport)
+	})
+
 	t.Run("Connector test WithTransport sets HTTPClient in CloudFetchConfig", func(t *testing.T) {
 		host := "databricks-host"
 		accessToken := "token"

internal/arrowscan/arrowscan.go

@@ -36,8 +36,10 @@ import (
 // date, timestamp, and top-level decimal (as an exact fixed-point string,
 // matching the Thrift path — a float64 would lose precision beyond ~17 digits).
 // Nested types (List/Map/Struct, and VARIANT which
-// arrives nested) render to a JSON string byte-identical to the Thrift path;
-// GEOMETRY arrives as a WKB/WKT string and is handled by the string arm. INTERVAL
+// arrives nested) render to a JSON string byte-identical to the Thrift path.
+// Top-level binary GEOMETRY / GEOGRAPHY is selected explicitly through
+// ScanOptions and combines its SRID and WKB children into EWKB []byte; string
+// mode is handled by the regular string arm. INTERVAL
 // day-time/year-month arrive as native arrow duration/month-interval and format to
 // the same string the Thrift path receives pre-formatted from the server. NULLs
 // map to nil. A genuinely unhandled type returns an error rather than a silently
@@ -106,20 +108,39 @@ func (c *StructKeyCache) keyPrefixes(st *arrow.StructType) []string {
 // StructKeyCache) so struct field-name keys are escaped once per result set
 // rather than once per row. Pass nil for the un-memoized one-shot behavior.
 func ScanCellCached(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache) (driver.Value, error) {
-	return scanCell(col, row, loc, keys, false)
+	return scanCell(col, row, loc, keys, ScanOptions{})
 }
 
 // ScanCellCachedDecimalFloat is ScanCellCached that, when decimalAsFloat is true,
 // scans a TOP-LEVEL Decimal128 to a lossy float64 instead of the exact string
 // (nested decimals still render exactly). Opt in via WithKernelDecimalAsFloat.
 func ScanCellCachedDecimalFloat(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache, decimalAsFloat bool) (driver.Value, error) {
-	return scanCell(col, row, loc, keys, decimalAsFloat)
+	return scanCell(col, row, loc, keys, ScanOptions{DecimalAsFloat: decimalAsFloat})
 }
 
-func scanCell(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache, decimalAsFloat bool) (driver.Value, error) {
+// ScanOptions controls top-level result adaptations that cannot be inferred from
+// an Arrow array alone. Logical GEOMETRY / GEOGRAPHY identity lives on the
+// enclosing schema Field, so kernelRows sets GeospatialBinary only for a field
+// carrying that metadata and a physical struct value.
+type ScanOptions struct {
+	DecimalAsFloat   bool
+	GeospatialBinary bool
+}
+
+// ScanCellCachedWithOptions is the full scanner entry point used by the kernel
+// rows implementation. Callers that do not need result-specific adaptations
+// should continue to use ScanCellCached.
+func ScanCellCachedWithOptions(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache, options ScanOptions) (driver.Value, error) {
+	return scanCell(col, row, loc, keys, options)
+}
+
+func scanCell(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache, options ScanOptions) (driver.Value, error) {
 	if col.IsNull(row) {
 		return nil, nil
 	}
+	if options.GeospatialBinary {
+		return scanGeospatialEWKB(col, row)
+	}
 	switch c := col.(type) {
 	case *array.Null:
 		return nil, nil
@@ -195,7 +216,7 @@ func scanCell(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache
 		return inLocation(timestampToTime(int64(c.Value(row)), dt.Unit), loc), nil
 	case *array.Decimal128:
 		dt := col.DataType().(*arrow.Decimal128Type)
-		if decimalAsFloat {
+		if options.DecimalAsFloat {
 			// Lossy fast path: float64, no per-cell string. Opt-in only.
 			return c.Value(row).ToFloat64(dt.Scale), nil
 		}
@@ -228,6 +249,41 @@ func scanCell(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache
 	}
 }
 
+// scanGeospatialEWKB validates Reyden's canonical top-level
+// struct<srid:int32,wkb:binary> value and embeds the sibling SRID into EWKB.
+// database/sql cannot return the two-child Arrow struct as a driver.Value, and
+// returning bare OGC WKB would lose the per-row SRID for GEOMETRY(ANY).
+func scanGeospatialEWKB(col arrow.Array, row int) (driver.Value, error) {
+	values, ok := col.(*array.Struct)
+	if !ok {
+		return nil, fmt.Errorf("binary geospatial column has arrow type %s, want struct<srid:int32,wkb:binary>", col.DataType())
+	}
+	fields := values.DataType().(*arrow.StructType).Fields()
+	if len(fields) != 2 || fields[0].Name != "srid" || fields[1].Name != "wkb" ||
+		fields[0].Type.ID() != arrow.INT32 || fields[1].Type.ID() != arrow.BINARY {
+		return nil, fmt.Errorf("binary geospatial column has arrow type %s, want struct<srid:int32,wkb:binary>", col.DataType())
+	}
+	srids, ok := values.Field(0).(*array.Int32)
+	if !ok {
+		return nil, fmt.Errorf("binary geospatial srid child has arrow type %s, want int32", values.Field(0).DataType())
+	}
+	wkbs, ok := values.Field(1).(*array.Binary)
+	if !ok {
+		return nil, fmt.Errorf("binary geospatial wkb child has arrow type %s, want binary", values.Field(1).DataType())
+	}
+	if srids.IsNull(row) {
+		return nil, fmt.Errorf("binary geospatial value at row %d has a null srid child", row)
+	}
+	if wkbs.IsNull(row) {
+		return nil, fmt.Errorf("binary geospatial value at row %d has a null wkb child", row)
+	}
+	ewkb, err := wkbToEWKB(wkbs.Value(row), srids.Value(row))
+	if err != nil {
+		return nil, fmt.Errorf("binary geospatial value at row %d: %w", row, err)
+	}
+	return ewkb, nil
+}
+
 // formatDayTimeInterval renders an arrow duration (in the given time unit) as the
 // Thrift path's "D HH:MM:SS.nnnnnnnnn" — days, then zero-padded hours:minutes:seconds
 // with 9 fractional digits, negated with a leading '-'.

internal/arrowscan/arrowscan_test.go

@@ -1,7 +1,9 @@
 package arrowscan
 
 import (
+	"bytes"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -684,3 +686,100 @@ func TestScanCellDecimalAsFloat(t *testing.T) {
 		t.Errorf("decimalAsFloat arm = %v, want ~99.98", fv)
 	}
 }
+
+func TestScanCellBinaryGeospatial(t *testing.T) {
+	pool := memory.NewGoAllocator()
+	wkb := []byte{
+		0x01, 0x01, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,
+	}
+	geoType := arrow.StructOf(
+		arrow.Field{Name: "srid", Type: arrow.PrimitiveTypes.Int32, Nullable: false},
+		arrow.Field{Name: "wkb", Type: arrow.BinaryTypes.Binary, Nullable: false},
+	)
+
+	build := func(outerValid, sridValid, wkbValid bool, srid int32) *array.Struct {
+		b := array.NewStructBuilder(pool, geoType)
+		defer b.Release()
+		b.Append(outerValid)
+		if sridValid {
+			b.FieldBuilder(0).(*array.Int32Builder).Append(srid)
+		} else {
+			b.FieldBuilder(0).(*array.Int32Builder).AppendNull()
+		}
+		if wkbValid {
+			b.FieldBuilder(1).(*array.BinaryBuilder).Append(wkb)
+		} else {
+			b.FieldBuilder(1).(*array.BinaryBuilder).AppendNull()
+		}
+		return b.NewStructArray()
+	}
+
+	t.Run("embeds each row SRID in EWKB", func(t *testing.T) {
+		for _, tc := range []struct {
+			srid       int32
+			sridLittle []byte
+		}{
+			{srid: 4326, sridLittle: []byte{0xe6, 0x10, 0x00, 0x00}},
+			{srid: 3857, sridLittle: []byte{0x11, 0x0f, 0x00, 0x00}},
+		} {
+			arr := build(true, true, true, tc.srid)
+			got, err := ScanCellCachedWithOptions(arr, 0, nil, nil, ScanOptions{GeospatialBinary: true})
+			arr.Release()
+			if err != nil {
+				t.Fatal(err)
+			}
+			want := append([]byte{0x01, 0x01, 0x00, 0x00, 0x20}, tc.sridLittle...)
+			want = append(want, wkb[5:]...)
+			if !bytes.Equal(got.([]byte), want) {
+				t.Errorf("SRID %d EWKB = %x, want %x", tc.srid, got, want)
+			}
+		}
+	})
+
+	t.Run("outer null wins", func(t *testing.T) {
+		arr := build(false, false, false, 0)
+		defer arr.Release()
+		got, err := ScanCellCachedWithOptions(arr, 0, nil, nil, ScanOptions{GeospatialBinary: true})
+		if err != nil || got != nil {
+			t.Errorf("outer-null geo = (%v, %v), want (nil, nil)", got, err)
+		}
+	})
+
+	t.Run("rejects null srid child", func(t *testing.T) {
+		arr := build(true, false, true, 0)
+		defer arr.Release()
+		_, err := ScanCellCachedWithOptions(arr, 0, nil, nil, ScanOptions{GeospatialBinary: true})
+		if err == nil || !strings.Contains(err.Error(), "null srid") {
+			t.Errorf("error = %v, want null-srid validation error", err)
+		}
+	})
+
+	t.Run("rejects null wkb child", func(t *testing.T) {
+		arr := build(true, true, false, 4326)
+		defer arr.Release()
+		_, err := ScanCellCachedWithOptions(arr, 0, nil, nil, ScanOptions{GeospatialBinary: true})
+		if err == nil || !strings.Contains(err.Error(), "null wkb") {
+			t.Errorf("error = %v, want null-wkb validation error", err)
+		}
+	})
+
+	t.Run("rejects a non-canonical struct", func(t *testing.T) {
+		badType := arrow.StructOf(
+			arrow.Field{Name: "srid", Type: arrow.PrimitiveTypes.Int64},
+			arrow.Field{Name: "bytes", Type: arrow.BinaryTypes.Binary},
+		)
+		b := array.NewStructBuilder(pool, badType)
+		defer b.Release()
+		b.Append(true)
+		b.FieldBuilder(0).(*array.Int64Builder).Append(4326)
+		b.FieldBuilder(1).(*array.BinaryBuilder).Append(wkb)
+		arr := b.NewStructArray()
+		defer arr.Release()
+		_, err := ScanCellCachedWithOptions(arr, 0, nil, nil, ScanOptions{GeospatialBinary: true})
+		if err == nil || !strings.Contains(err.Error(), "want struct<srid:int32,wkb:binary>") {
+			t.Errorf("error = %v, want canonical-shape validation error", err)
+		}
+	})
+}