diff --git a/bindings/go/predicate.go b/bindings/go/predicate.go index e8d36de0..1bf53870 100644 --- a/bindings/go/predicate.go +++ b/bindings/go/predicate.go @@ -253,8 +253,14 @@ var errConsumedPredicate = fmt.Errorf("paimon: predicate already consumed or nil // PredicateBuilder creates filter predicates for a table. // It holds a Go-level reference to the Table and does not own any C resources, // so there is no Close() method. +// +// caseSensitive is baked into every predicate this builder produces: the core +// resolves a column when the predicate is constructed, so it cannot be changed +// afterwards. Use Table.PredicateBuilder for exact matching or +// Table.PredicateBuilderWithCaseSensitive to opt out. type PredicateBuilder struct { - table *Table + table *Table + caseSensitive bool } // Eq creates an equality predicate: column = value. @@ -333,16 +339,21 @@ func (pb *PredicateBuilder) NotIn(column string, values ...any) (*Predicate, err // buildLeafPredicate is a helper for comparison predicates that take (table, column, datum). func (pb *PredicateBuilder) buildLeafPredicate( - ffiVar *FFI[func(*paimonTable, *byte, paimonDatumC) (*paimonPredicate, error)], + ffiVar *leafPredicateFFI, column string, datum Datum, ) (*Predicate, error) { t := pb.table if t.inner == nil { return nil, ErrClosed } - createFn := ffiVar.symbol(t.ctx) cCol := append([]byte(column), 0) - inner, err := createFn(t.inner, &cCol[0], datum.inner) + var inner *paimonPredicate + var err error + if pb.caseSensitive { + inner, err = ffiVar.exact.symbol(t.ctx)(t.inner, &cCol[0], datum.inner) + } else { + inner, err = ffiVar.folded.symbol(t.ctx)(t.inner, &cCol[0], datum.inner, false) + } runtime.KeepAlive(cCol) runtime.KeepAlive(datum) if err != nil { @@ -354,16 +365,21 @@ func (pb *PredicateBuilder) buildLeafPredicate( // buildNullPredicate is a helper for IS NULL / IS NOT NULL predicates. func (pb *PredicateBuilder) buildNullPredicate( - ffiVar *FFI[func(*paimonTable, *byte) (*paimonPredicate, error)], + ffiVar *nullPredicateFFI, column string, ) (*Predicate, error) { t := pb.table if t.inner == nil { return nil, ErrClosed } - createFn := ffiVar.symbol(t.ctx) cCol := append([]byte(column), 0) - inner, err := createFn(t.inner, &cCol[0]) + var inner *paimonPredicate + var err error + if pb.caseSensitive { + inner, err = ffiVar.exact.symbol(t.ctx)(t.inner, &cCol[0]) + } else { + inner, err = ffiVar.folded.symbol(t.ctx)(t.inner, &cCol[0], false) + } runtime.KeepAlive(cCol) if err != nil { return nil, err @@ -374,7 +390,7 @@ func (pb *PredicateBuilder) buildNullPredicate( // buildInPredicate is a helper for IS IN / IS NOT IN predicates. func (pb *PredicateBuilder) buildInPredicate( - ffiVar *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr) (*paimonPredicate, error)], + ffiVar *inPredicateFFI, column string, values []any, ) (*Predicate, error) { t := pb.table @@ -389,13 +405,20 @@ func (pb *PredicateBuilder) buildInPredicate( } datums[i] = d.inner } - createFn := ffiVar.symbol(t.ctx) cCol := append([]byte(column), 0) var datumsPtr unsafe.Pointer if len(datums) > 0 { datumsPtr = unsafe.Pointer(&datums[0]) } - inner, err := createFn(t.inner, &cCol[0], datumsPtr, uintptr(len(datums))) + var inner *paimonPredicate + var err error + if pb.caseSensitive { + inner, err = ffiVar.exact.symbol(t.ctx)(t.inner, &cCol[0], datumsPtr, uintptr(len(datums))) + } else { + inner, err = ffiVar.folded.symbol(t.ctx)( + t.inner, &cCol[0], datumsPtr, uintptr(len(datums)), false, + ) + } runtime.KeepAlive(cCol) runtime.KeepAlive(datums) runtime.KeepAlive(values) @@ -462,12 +485,72 @@ var ffiPredicateFree = newFFI(ffiOpts{ } }) -var ffiPredicateEqual = newPredicateLeafFFI("paimon_predicate_equal") -var ffiPredicateNotEqual = newPredicateLeafFFI("paimon_predicate_not_equal") -var ffiPredicateLessThan = newPredicateLeafFFI("paimon_predicate_less_than") -var ffiPredicateLessOrEqual = newPredicateLeafFFI("paimon_predicate_less_or_equal") -var ffiPredicateGreaterThan = newPredicateLeafFFI("paimon_predicate_greater_than") -var ffiPredicateGreaterOrEqual = newPredicateLeafFFI("paimon_predicate_greater_or_equal") +var ffiPredicateEqual = newPredicateLeafFFIPair("paimon_predicate_equal") +var ffiPredicateNotEqual = newPredicateLeafFFIPair("paimon_predicate_not_equal") +var ffiPredicateLessThan = newPredicateLeafFFIPair("paimon_predicate_less_than") +var ffiPredicateLessOrEqual = newPredicateLeafFFIPair("paimon_predicate_less_or_equal") +var ffiPredicateGreaterThan = newPredicateLeafFFIPair("paimon_predicate_greater_than") +var ffiPredicateGreaterOrEqual = newPredicateLeafFFIPair("paimon_predicate_greater_or_equal") + +// Every predicate constructor in the C ABI comes in two flavours: the plain +// symbol, which always matches column names exactly, and an additive +// `_with_case_sensitive` variant taking a trailing `bool`. The two share one +// implementation, the plain one passing `true`, so pairing them buys nothing at +// runtime; it keeps the default path off the new symbols, so existing callers run +// exactly the code they ran before, and lets the default follow the C side if it +// ever redefines what the plain symbol means. The flag is written as an explicit +// 0/1 byte for the same reason as in read_builder.go: Rust `bool` occupies one +// byte and admits no value but 0 or 1. +type leafPredicateFFI struct { + exact *FFI[func(*paimonTable, *byte, paimonDatumC) (*paimonPredicate, error)] + folded *FFI[func(*paimonTable, *byte, paimonDatumC, bool) (*paimonPredicate, error)] +} + +func newPredicateLeafFFIPair(sym string) *leafPredicateFFI { + return &leafPredicateFFI{ + exact: newPredicateLeafFFI(sym), + folded: newPredicateLeafCaseFFI(sym + "_with_case_sensitive"), + } +} + +// newPredicateLeafCaseFFI wraps the (table, column, datum, case_sensitive) form. +func newPredicateLeafCaseFFI( + sym string, +) *FFI[func(*paimonTable, *byte, paimonDatumC, bool) (*paimonPredicate, error)] { + return newFFI(ffiOpts{ + sym: contextKey(sym), + rType: &typeResultPredicate, + aTypes: []*ffi.Type{ + &ffi.TypePointer, &ffi.TypePointer, &typePaimonDatum, &ffi.TypeUint8, + }, + }, func(ctx context.Context, ffiCall ffiCall) func(*paimonTable, *byte, paimonDatumC, bool) (*paimonPredicate, error) { + return func( + table *paimonTable, column *byte, datum paimonDatumC, caseSensitive bool, + ) (*paimonPredicate, error) { + flag := boolByte(caseSensitive) + var result resultPredicate + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&table), + unsafe.Pointer(&column), + unsafe.Pointer(&datum), + unsafe.Pointer(&flag), + ) + if result.error != nil { + return nil, parseError(ctx, result.error) + } + return result.predicate, nil + } + }) +} + +// boolByte renders a Go bool as the single byte a Rust `bool` argument expects. +func boolByte(value bool) uint8 { + if value { + return 1 + } + return 0 +} // newPredicateLeafFFI creates an FFI wrapper for comparison predicate functions // with signature: (table, column, datum) -> result_predicate. @@ -493,8 +576,46 @@ func newPredicateLeafFFI(sym string) *FFI[func(*paimonTable, *byte, paimonDatumC }) } -var ffiPredicateIsNull = newPredicateNullFFI("paimon_predicate_is_null") -var ffiPredicateIsNotNull = newPredicateNullFFI("paimon_predicate_is_not_null") +var ffiPredicateIsNull = newPredicateNullFFIPair("paimon_predicate_is_null") +var ffiPredicateIsNotNull = newPredicateNullFFIPair("paimon_predicate_is_not_null") + +type nullPredicateFFI struct { + exact *FFI[func(*paimonTable, *byte) (*paimonPredicate, error)] + folded *FFI[func(*paimonTable, *byte, bool) (*paimonPredicate, error)] +} + +func newPredicateNullFFIPair(sym string) *nullPredicateFFI { + return &nullPredicateFFI{ + exact: newPredicateNullFFI(sym), + folded: newPredicateNullCaseFFI(sym + "_with_case_sensitive"), + } +} + +// newPredicateNullCaseFFI wraps the (table, column, case_sensitive) form. +func newPredicateNullCaseFFI( + sym string, +) *FFI[func(*paimonTable, *byte, bool) (*paimonPredicate, error)] { + return newFFI(ffiOpts{ + sym: contextKey(sym), + rType: &typeResultPredicate, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypeUint8}, + }, func(ctx context.Context, ffiCall ffiCall) func(*paimonTable, *byte, bool) (*paimonPredicate, error) { + return func(table *paimonTable, column *byte, caseSensitive bool) (*paimonPredicate, error) { + flag := boolByte(caseSensitive) + var result resultPredicate + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&table), + unsafe.Pointer(&column), + unsafe.Pointer(&flag), + ) + if result.error != nil { + return nil, parseError(ctx, result.error) + } + return result.predicate, nil + } + }) +} // newPredicateNullFFI creates an FFI wrapper for null-check predicate functions // with signature: (table, column) -> result_predicate. @@ -519,8 +640,53 @@ func newPredicateNullFFI(sym string) *FFI[func(*paimonTable, *byte) (*paimonPred }) } -var ffiPredicateIsIn = newPredicateInFFI("paimon_predicate_is_in") -var ffiPredicateIsNotIn = newPredicateInFFI("paimon_predicate_is_not_in") +var ffiPredicateIsIn = newPredicateInFFIPair("paimon_predicate_is_in") +var ffiPredicateIsNotIn = newPredicateInFFIPair("paimon_predicate_is_not_in") + +type inPredicateFFI struct { + exact *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr) (*paimonPredicate, error)] + folded *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr, bool) (*paimonPredicate, error)] +} + +func newPredicateInFFIPair(sym string) *inPredicateFFI { + return &inPredicateFFI{ + exact: newPredicateInFFI(sym), + folded: newPredicateInCaseFFI(sym + "_with_case_sensitive"), + } +} + +// newPredicateInCaseFFI wraps the (table, column, datums, len, case_sensitive) form. +func newPredicateInCaseFFI( + sym string, +) *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr, bool) (*paimonPredicate, error)] { + return newFFI(ffiOpts{ + sym: contextKey(sym), + rType: &typeResultPredicate, + aTypes: []*ffi.Type{ + &ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypeUint8, + }, + }, func(ctx context.Context, ffiCall ffiCall) func(*paimonTable, *byte, unsafe.Pointer, uintptr, bool) (*paimonPredicate, error) { + return func( + table *paimonTable, column *byte, datums unsafe.Pointer, datumsLen uintptr, + caseSensitive bool, + ) (*paimonPredicate, error) { + flag := boolByte(caseSensitive) + var result resultPredicate + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&table), + unsafe.Pointer(&column), + unsafe.Pointer(&datums), + unsafe.Pointer(&datumsLen), + unsafe.Pointer(&flag), + ) + if result.error != nil { + return nil, parseError(ctx, result.error) + } + return result.predicate, nil + } + }) +} // newPredicateInFFI creates an FFI wrapper for IN/NOT IN predicate functions // with signature: (table, column, datums, datums_len) -> result_predicate. diff --git a/bindings/go/read_builder.go b/bindings/go/read_builder.go index 177c5953..a6d335c6 100644 --- a/bindings/go/read_builder.go +++ b/bindings/go/read_builder.go @@ -56,6 +56,24 @@ func (rb *ReadBuilder) WithProjection(columns []string) error { return projFn(rb.inner, columns) } +// WithCaseSensitive sets whether the names given to WithProjection must match +// the schema exactly. The default is true. With false, names are matched by +// ASCII case folding, and a name that folds onto two different schema columns is +// rejected as ambiguous. Either way the returned records carry the schema's own +// spelling, not the requested one. +// +// This does not affect predicates. A predicate resolves its column when it is +// built, so its case sensitivity comes from which builder produced it — +// Table.PredicateBuilder or Table.PredicateBuilderWithCaseSensitive — and is +// unaffected by this setting. Call order relative to WithProjection does not +// matter: projection names are resolved in NewRead. +func (rb *ReadBuilder) WithCaseSensitive(caseSensitive bool) error { + if rb.inner == nil { + return ErrClosed + } + return ffiReadBuilderWithCaseSensitive.symbol(rb.ctx)(rb.inner, caseSensitive) +} + // WithFilter sets a filter predicate for scan planning and read-side pruning. // // The predicate is used in two phases: @@ -177,6 +195,31 @@ var ffiReadBuilderWithProjection = newFFI(ffiOpts{ } }) +// Rust `bool` is a single byte whose only valid values are 0 and 1, so the +// argument is declared as a 1-byte integer (the ffi package documents TypeUint8 +// as the way to pass a bool) and an explicit 0/1 is written into it via +// boolByte. This is the binding's first non-pointer scalar smaller than 4 bytes, +// so do not copy a wider type here. +var ffiReadBuilderWithCaseSensitive = newFFI(ffiOpts{ + sym: "paimon_read_builder_with_case_sensitive", + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypeUint8}, +}, func(ctx context.Context, ffiCall ffiCall) func(rb *paimonReadBuilder, caseSensitive bool) error { + return func(rb *paimonReadBuilder, caseSensitive bool) error { + flag := boolByte(caseSensitive) + var errPtr *paimonError + ffiCall( + unsafe.Pointer(&errPtr), + unsafe.Pointer(&rb), + unsafe.Pointer(&flag), + ) + if errPtr != nil { + return parseError(ctx, errPtr) + } + return nil + } +}) + var ffiReadBuilderWithFilter = newFFI(ffiOpts{ sym: "paimon_read_builder_with_filter", rType: &ffi.TypePointer, diff --git a/bindings/go/table.go b/bindings/go/table.go index d41a84bd..0ced03c0 100644 --- a/bindings/go/table.go +++ b/bindings/go/table.go @@ -45,9 +45,21 @@ func (t *Table) Close() { }) } -// PredicateBuilder returns a builder for creating filter predicates on this table. +// PredicateBuilder returns a builder for creating filter predicates on this +// table, matching column names exactly. func (t *Table) PredicateBuilder() *PredicateBuilder { - return &PredicateBuilder{table: t} + return &PredicateBuilder{table: t, caseSensitive: true} +} + +// PredicateBuilderWithCaseSensitive returns a predicate builder that resolves +// column names by ASCII case folding when caseSensitive is false. A name that +// folds onto two different schema columns is rejected as ambiguous. +// +// Case sensitivity is fixed when the predicate is built, which is why it belongs +// to the builder rather than to ReadBuilder.WithCaseSensitive — that setting +// covers projection only. +func (t *Table) PredicateBuilderWithCaseSensitive(caseSensitive bool) *PredicateBuilder { + return &PredicateBuilder{table: t, caseSensitive: caseSensitive} } // NewReadBuilder creates a ReadBuilder for this table. diff --git a/bindings/go/tests/paimon_test.go b/bindings/go/tests/paimon_test.go index 621292a6..2cadddc5 100644 --- a/bindings/go/tests/paimon_test.go +++ b/bindings/go/tests/paimon_test.go @@ -879,3 +879,154 @@ func TestReadWithProjection(t *testing.T) { } } } + +// TestReadWithCaseInsensitiveProjection covers ReadBuilder.WithCaseSensitive. +// The fixture columns are lowercase, so an uppercase projection is the reverse +// of the Hive/Spark case it exists for, and exercises the same code path. +func TestReadWithCaseInsensitiveProjection(t *testing.T) { + table := openTestTable(t) + + // Default: exact matching, so the uppercase names do not resolve. The + // failure surfaces in NewRead, not in WithProjection. + rbDefault, err := table.NewReadBuilder() + if err != nil { + t.Fatalf("Failed to create read builder: %v", err) + } + defer rbDefault.Close() + if err := rbDefault.WithProjection([]string{"ID", "NAME"}); err != nil { + t.Fatalf("WithProjection should defer resolution: %v", err) + } + if _, err := rbDefault.NewRead(); err == nil { + t.Fatal("expected NewRead to reject an uppercase projection by default") + } + + // Explicitly asking for case sensitivity must behave like the default. This + // pins the `true` value of the flag, not just the `false` one. + rbExact, err := table.NewReadBuilder() + if err != nil { + t.Fatalf("Failed to create read builder: %v", err) + } + defer rbExact.Close() + if err := rbExact.WithCaseSensitive(true); err != nil { + t.Fatalf("WithCaseSensitive(true) failed: %v", err) + } + if err := rbExact.WithProjection([]string{"ID", "NAME"}); err != nil { + t.Fatalf("WithProjection failed: %v", err) + } + if _, err := rbExact.NewRead(); err == nil { + t.Fatal("expected WithCaseSensitive(true) to keep rejecting uppercase names") + } + + // Opting out resolves the names, and readRows asserts the records carry the + // schema's own lowercase spelling. Both call orders must agree, because the + // projection is resolved in NewRead rather than when it is set. + for _, order := range []string{"flag-first", "projection-first"} { + rb, err := table.NewReadBuilder() + if err != nil { + t.Fatalf("%s: failed to create read builder: %v", order, err) + } + if order == "flag-first" { + if err := rb.WithCaseSensitive(false); err != nil { + rb.Close() + t.Fatalf("%s: WithCaseSensitive(false) failed: %v", order, err) + } + if err := rb.WithProjection([]string{"ID", "NAME"}); err != nil { + rb.Close() + t.Fatalf("%s: WithProjection failed: %v", order, err) + } + } else { + if err := rb.WithProjection([]string{"ID", "NAME"}); err != nil { + rb.Close() + t.Fatalf("%s: WithProjection failed: %v", order, err) + } + if err := rb.WithCaseSensitive(false); err != nil { + rb.Close() + t.Fatalf("%s: WithCaseSensitive(false) failed: %v", order, err) + } + } + rows := readRows(t, rb) + rb.Close() + if len(rows) != 3 { + t.Fatalf("%s: expected 3 rows, got %d: %v", order, len(rows), rows) + } + } +} + +// TestPredicateBuilderCaseSensitivity covers the predicate half. A predicate +// resolves its column when it is built, so the choice belongs to the builder and +// ReadBuilder.WithCaseSensitive has no bearing on it. +func TestPredicateBuilderCaseSensitivity(t *testing.T) { + table := openTestTable(t) + + // Exact is the default, and asking for it explicitly must agree. + for _, exact := range []struct { + name string + pb *paimon.PredicateBuilder + }{ + {"default", table.PredicateBuilder()}, + {"explicitly-exact", table.PredicateBuilderWithCaseSensitive(true)}, + } { + if _, err := exact.pb.Eq("ID", int32(1)); err == nil { + t.Fatalf("%s: expected Eq on an uppercase column to fail", exact.name) + } + if _, err := exact.pb.IsNotNull("NAME"); err == nil { + t.Fatalf("%s: expected IsNotNull on an uppercase column to fail", exact.name) + } + if _, err := exact.pb.In("ID", int32(1)); err == nil { + t.Fatalf("%s: expected In on an uppercase column to fail", exact.name) + } + lower, err := exact.pb.Eq("id", int32(1)) + if err != nil { + t.Fatalf("%s: exact lowercase name should still work: %v", exact.name, err) + } + lower.Close() + } + + // All three constructor shapes have their own C entry point, so each needs + // its own case: (column, datum), (column) and (column, datums, len). + folded := table.PredicateBuilderWithCaseSensitive(false) + notNull, err := folded.IsNotNull("NAME") + if err != nil { + t.Fatalf("folded IsNotNull failed: %v", err) + } + notNull.Close() + + eq, err := folded.Eq("ID", int32(2)) + if err != nil { + t.Fatalf("folded Eq failed: %v", err) + } + rbEq, err := table.NewReadBuilder() + if err != nil { + t.Fatalf("Failed to create read builder: %v", err) + } + defer rbEq.Close() + if err := rbEq.WithFilter(eq); err != nil { + t.Fatalf("WithFilter failed: %v", err) + } + rows := readRows(t, rbEq) + if len(rows) != 1 || rows[0].id != 2 { + t.Fatalf("folded Eq should select id=2, got %v", rows) + } + + in, err := folded.In("ID", int32(1), int32(3)) + if err != nil { + t.Fatalf("folded In failed: %v", err) + } + rbIn, err := table.NewReadBuilder() + if err != nil { + t.Fatalf("Failed to create read builder: %v", err) + } + defer rbIn.Close() + if err := rbIn.WithFilter(in); err != nil { + t.Fatalf("WithFilter failed: %v", err) + } + rows = readRows(t, rbIn) + if len(rows) != 2 { + t.Fatalf("folded In should select two rows, got %v", rows) + } + for _, r := range rows { + if r.id != 1 && r.id != 3 { + t.Fatalf("folded In returned an unexpected row: %v", rows) + } + } +} diff --git a/docs/src/go-binding.md b/docs/src/go-binding.md index 40acda2d..92139c8d 100644 --- a/docs/src/go-binding.md +++ b/docs/src/go-binding.md @@ -485,6 +485,34 @@ if err := rb.WithProjection([]string{"id", "name"}); err != nil { // Continue with scan-then-read as above... ``` +### Case-Insensitive Column Names + +Projected names must match the schema exactly by default. Pass `false` to +`WithCaseSensitive` to match by ASCII case folding instead — useful for tables +whose columns were declared in upper case by Hive or Spark. A name that folds onto +two schema columns differing only in case is rejected as ambiguous. + +```go +if err := rb.WithCaseSensitive(false); err != nil { + log.Fatal(err) +} +if err := rb.WithProjection([]string{"ID", "NAME"}); err != nil { + log.Fatal(err) +} +``` + +Call order does not matter: names that differ from the schema only in case are +resolved in `NewRead`, which is also where such a name surfaces as an error. A +name that matches no column under any casing is still rejected by +`WithProjection` itself. + +The returned records always carry the schema's own spelling, not the spelling you +asked for, so downstream column lookups stay stable. + +This setting covers projection only. A predicate resolves its column when it is +built, so its case sensitivity comes from which builder created it — see +[Case-Insensitive Predicates](#case-insensitive-predicates). + ## Filter Push-Down Filter push-down prunes data at two levels: @@ -544,6 +572,28 @@ if err := rb.WithFilter(pred); err != nil { // Continue with scan-then-read... ``` +### Case-Insensitive Predicates + +`Table.PredicateBuilder` matches column names exactly. Use +`PredicateBuilderWithCaseSensitive(false)` for a builder that folds ASCII case +instead; every predicate it produces resolves its column that way. + +```go +pb := table.PredicateBuilderWithCaseSensitive(false) + +// Resolves to the schema's "id" column. +pred, err := pb.Eq("ID", int32(1)) +``` + +The choice is fixed when the predicate is built, which is why it belongs to the +builder and not to the read builder. `ReadBuilder.WithCaseSensitive` does not +affect predicates, and a predicate from a case-folding builder keeps that +behaviour whatever the read builder is set to. + +The predicate stores the schema's own spelling of the column, not the one you +passed, so file-level pruning and row-level filtering apply to it exactly as they +would to an exactly-spelled predicate. + ### Compound Predicates Combine predicates with `And`, `Or`, and `Not`. The `predicate` sub-package provides variadic helpers: