Skip to content

feat(go): support case-insensitive column names on reads #796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jackylee-ch wants to merge 1 commit into
base: main
Choose a base branch
from
+444 −22
Open

feat(go): support case-insensitive column names on reads #796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 186 additions & 20 deletions bindings/go/predicate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,8 +253,14 @@ var errConsumedPredicate = fmt.Errorf("paimon: predicate already consumed or nil
// PredicateBuilder creates filter predicates for a table.
// It holds a Go-level reference to the Table and does not own any C resources,
// so there is no Close() method.
//
// caseSensitive is baked into every predicate this builder produces: the core
// resolves a column when the predicate is constructed, so it cannot be changed
// afterwards. Use Table.PredicateBuilder for exact matching or
// Table.PredicateBuilderWithCaseSensitive to opt out.
type PredicateBuilder struct {
table *Table
table *Table
caseSensitive bool
}

// Eq creates an equality predicate: column = value.
Expand Down Expand Up @@ -333,16 +339,21 @@ func (pb *PredicateBuilder) NotIn(column string, values ...any) (*Predicate, err

// buildLeafPredicate is a helper for comparison predicates that take (table, column, datum).
func (pb *PredicateBuilder) buildLeafPredicate(
ffiVar *FFI[func(*paimonTable, *byte, paimonDatumC) (*paimonPredicate, error)],
ffiVar *leafPredicateFFI,
column string, datum Datum,
) (*Predicate, error) {
t := pb.table
if t.inner == nil {
return nil, ErrClosed
}
createFn := ffiVar.symbol(t.ctx)
cCol := append([]byte(column), 0)
inner, err := createFn(t.inner, &cCol[0], datum.inner)
var inner *paimonPredicate
var err error
if pb.caseSensitive {
inner, err = ffiVar.exact.symbol(t.ctx)(t.inner, &cCol[0], datum.inner)
} else {
inner, err = ffiVar.folded.symbol(t.ctx)(t.inner, &cCol[0], datum.inner, false)
}
runtime.KeepAlive(cCol)
runtime.KeepAlive(datum)
if err != nil {
Expand All @@ -354,16 +365,21 @@ func (pb *PredicateBuilder) buildLeafPredicate(

// buildNullPredicate is a helper for IS NULL / IS NOT NULL predicates.
func (pb *PredicateBuilder) buildNullPredicate(
ffiVar *FFI[func(*paimonTable, *byte) (*paimonPredicate, error)],
ffiVar *nullPredicateFFI,
column string,
) (*Predicate, error) {
t := pb.table
if t.inner == nil {
return nil, ErrClosed
}
createFn := ffiVar.symbol(t.ctx)
cCol := append([]byte(column), 0)
inner, err := createFn(t.inner, &cCol[0])
var inner *paimonPredicate
var err error
if pb.caseSensitive {
inner, err = ffiVar.exact.symbol(t.ctx)(t.inner, &cCol[0])
} else {
inner, err = ffiVar.folded.symbol(t.ctx)(t.inner, &cCol[0], false)
}
runtime.KeepAlive(cCol)
if err != nil {
return nil, err
Expand All @@ -374,7 +390,7 @@ func (pb *PredicateBuilder) buildNullPredicate(

// buildInPredicate is a helper for IS IN / IS NOT IN predicates.
func (pb *PredicateBuilder) buildInPredicate(
ffiVar *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr) (*paimonPredicate, error)],
ffiVar *inPredicateFFI,
column string, values []any,
) (*Predicate, error) {
t := pb.table
Expand All @@ -389,13 +405,20 @@ func (pb *PredicateBuilder) buildInPredicate(
}
datums[i] = d.inner
}
createFn := ffiVar.symbol(t.ctx)
cCol := append([]byte(column), 0)
var datumsPtr unsafe.Pointer
if len(datums) > 0 {
datumsPtr = unsafe.Pointer(&datums[0])
}
inner, err := createFn(t.inner, &cCol[0], datumsPtr, uintptr(len(datums)))
var inner *paimonPredicate
var err error
if pb.caseSensitive {
inner, err = ffiVar.exact.symbol(t.ctx)(t.inner, &cCol[0], datumsPtr, uintptr(len(datums)))
} else {
inner, err = ffiVar.folded.symbol(t.ctx)(
t.inner, &cCol[0], datumsPtr, uintptr(len(datums)), false,
)
}
runtime.KeepAlive(cCol)
runtime.KeepAlive(datums)
runtime.KeepAlive(values)
Expand Down Expand Up @@ -462,12 +485,72 @@ var ffiPredicateFree = newFFI(ffiOpts{
}
})

var ffiPredicateEqual = newPredicateLeafFFI("paimon_predicate_equal")
var ffiPredicateNotEqual = newPredicateLeafFFI("paimon_predicate_not_equal")
var ffiPredicateLessThan = newPredicateLeafFFI("paimon_predicate_less_than")
var ffiPredicateLessOrEqual = newPredicateLeafFFI("paimon_predicate_less_or_equal")
var ffiPredicateGreaterThan = newPredicateLeafFFI("paimon_predicate_greater_than")
var ffiPredicateGreaterOrEqual = newPredicateLeafFFI("paimon_predicate_greater_or_equal")
var ffiPredicateEqual = newPredicateLeafFFIPair("paimon_predicate_equal")
var ffiPredicateNotEqual = newPredicateLeafFFIPair("paimon_predicate_not_equal")
var ffiPredicateLessThan = newPredicateLeafFFIPair("paimon_predicate_less_than")
var ffiPredicateLessOrEqual = newPredicateLeafFFIPair("paimon_predicate_less_or_equal")
var ffiPredicateGreaterThan = newPredicateLeafFFIPair("paimon_predicate_greater_than")
var ffiPredicateGreaterOrEqual = newPredicateLeafFFIPair("paimon_predicate_greater_or_equal")

// Every predicate constructor in the C ABI comes in two flavours: the plain
// symbol, which always matches column names exactly, and an additive
// `_with_case_sensitive` variant taking a trailing `bool`. The two share one
// implementation, the plain one passing `true`, so pairing them buys nothing at
// runtime; it keeps the default path off the new symbols, so existing callers run
// exactly the code they ran before, and lets the default follow the C side if it
// ever redefines what the plain symbol means. The flag is written as an explicit
// 0/1 byte for the same reason as in read_builder.go: Rust `bool` occupies one
// byte and admits no value but 0 or 1.
type leafPredicateFFI struct {
exact *FFI[func(*paimonTable, *byte, paimonDatumC) (*paimonPredicate, error)]
folded *FFI[func(*paimonTable, *byte, paimonDatumC, bool) (*paimonPredicate, error)]
}

func newPredicateLeafFFIPair(sym string) *leafPredicateFFI {
return &leafPredicateFFI{
exact: newPredicateLeafFFI(sym),
folded: newPredicateLeafCaseFFI(sym + "_with_case_sensitive"),
}
}

// newPredicateLeafCaseFFI wraps the (table, column, datum, case_sensitive) form.
func newPredicateLeafCaseFFI(
sym string,
) *FFI[func(*paimonTable, *byte, paimonDatumC, bool) (*paimonPredicate, error)] {
return newFFI(ffiOpts{
sym: contextKey(sym),
rType: &typeResultPredicate,
aTypes: []*ffi.Type{
&ffi.TypePointer, &ffi.TypePointer, &typePaimonDatum, &ffi.TypeUint8,
},
}, func(ctx context.Context, ffiCall ffiCall) func(*paimonTable, *byte, paimonDatumC, bool) (*paimonPredicate, error) {
return func(
table *paimonTable, column *byte, datum paimonDatumC, caseSensitive bool,
) (*paimonPredicate, error) {
flag := boolByte(caseSensitive)
var result resultPredicate
ffiCall(
unsafe.Pointer(&result),
unsafe.Pointer(&table),
unsafe.Pointer(&column),
unsafe.Pointer(&datum),
unsafe.Pointer(&flag),
)
if result.error != nil {
return nil, parseError(ctx, result.error)
}
return result.predicate, nil
}
})
}

// boolByte renders a Go bool as the single byte a Rust `bool` argument expects.
func boolByte(value bool) uint8 {
if value {
return 1
}
return 0
}

// newPredicateLeafFFI creates an FFI wrapper for comparison predicate functions
// with signature: (table, column, datum) -> result_predicate.
Expand All @@ -493,8 +576,46 @@ func newPredicateLeafFFI(sym string) *FFI[func(*paimonTable, *byte, paimonDatumC
})
}

var ffiPredicateIsNull = newPredicateNullFFI("paimon_predicate_is_null")
var ffiPredicateIsNotNull = newPredicateNullFFI("paimon_predicate_is_not_null")
var ffiPredicateIsNull = newPredicateNullFFIPair("paimon_predicate_is_null")
var ffiPredicateIsNotNull = newPredicateNullFFIPair("paimon_predicate_is_not_null")

type nullPredicateFFI struct {
exact *FFI[func(*paimonTable, *byte) (*paimonPredicate, error)]
folded *FFI[func(*paimonTable, *byte, bool) (*paimonPredicate, error)]
}

func newPredicateNullFFIPair(sym string) *nullPredicateFFI {
return &nullPredicateFFI{
exact: newPredicateNullFFI(sym),
folded: newPredicateNullCaseFFI(sym + "_with_case_sensitive"),
}
}

// newPredicateNullCaseFFI wraps the (table, column, case_sensitive) form.
func newPredicateNullCaseFFI(
sym string,
) *FFI[func(*paimonTable, *byte, bool) (*paimonPredicate, error)] {
return newFFI(ffiOpts{
sym: contextKey(sym),
rType: &typeResultPredicate,
aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypeUint8},
}, func(ctx context.Context, ffiCall ffiCall) func(*paimonTable, *byte, bool) (*paimonPredicate, error) {
return func(table *paimonTable, column *byte, caseSensitive bool) (*paimonPredicate, error) {
flag := boolByte(caseSensitive)
var result resultPredicate
ffiCall(
unsafe.Pointer(&result),
unsafe.Pointer(&table),
unsafe.Pointer(&column),
unsafe.Pointer(&flag),
)
if result.error != nil {
return nil, parseError(ctx, result.error)
}
return result.predicate, nil
}
})
}

// newPredicateNullFFI creates an FFI wrapper for null-check predicate functions
// with signature: (table, column) -> result_predicate.
Expand All @@ -519,8 +640,53 @@ func newPredicateNullFFI(sym string) *FFI[func(*paimonTable, *byte) (*paimonPred
})
}

var ffiPredicateIsIn = newPredicateInFFI("paimon_predicate_is_in")
var ffiPredicateIsNotIn = newPredicateInFFI("paimon_predicate_is_not_in")
var ffiPredicateIsIn = newPredicateInFFIPair("paimon_predicate_is_in")
var ffiPredicateIsNotIn = newPredicateInFFIPair("paimon_predicate_is_not_in")

type inPredicateFFI struct {
exact *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr) (*paimonPredicate, error)]
folded *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr, bool) (*paimonPredicate, error)]
}

func newPredicateInFFIPair(sym string) *inPredicateFFI {
return &inPredicateFFI{
exact: newPredicateInFFI(sym),
folded: newPredicateInCaseFFI(sym + "_with_case_sensitive"),
}
}

// newPredicateInCaseFFI wraps the (table, column, datums, len, case_sensitive) form.
func newPredicateInCaseFFI(
sym string,
) *FFI[func(*paimonTable, *byte, unsafe.Pointer, uintptr, bool) (*paimonPredicate, error)] {
return newFFI(ffiOpts{
sym: contextKey(sym),
rType: &typeResultPredicate,
aTypes: []*ffi.Type{
&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypeUint8,
},
}, func(ctx context.Context, ffiCall ffiCall) func(*paimonTable, *byte, unsafe.Pointer, uintptr, bool) (*paimonPredicate, error) {
return func(
table *paimonTable, column *byte, datums unsafe.Pointer, datumsLen uintptr,
caseSensitive bool,
) (*paimonPredicate, error) {
flag := boolByte(caseSensitive)
var result resultPredicate
ffiCall(
unsafe.Pointer(&result),
unsafe.Pointer(&table),
unsafe.Pointer(&column),
unsafe.Pointer(&datums),
unsafe.Pointer(&datumsLen),
unsafe.Pointer(&flag),
)
if result.error != nil {
return nil, parseError(ctx, result.error)
}
return result.predicate, nil
}
})
}

// newPredicateInFFI creates an FFI wrapper for IN/NOT IN predicate functions
// with signature: (table, column, datums, datums_len) -> result_predicate.
Expand Down
43 changes: 43 additions & 0 deletions bindings/go/read_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,24 @@ func (rb *ReadBuilder) WithProjection(columns []string) error {
return projFn(rb.inner, columns)
}

// WithCaseSensitive sets whether the names given to WithProjection must match
// the schema exactly. The default is true. With false, names are matched by
// ASCII case folding, and a name that folds onto two different schema columns is
// rejected as ambiguous. Either way the returned records carry the schema's own
// spelling, not the requested one.
//
// This does not affect predicates. A predicate resolves its column when it is
// built, so its case sensitivity comes from which builder produced it —
// Table.PredicateBuilder or Table.PredicateBuilderWithCaseSensitive — and is
// unaffected by this setting. Call order relative to WithProjection does not
// matter: projection names are resolved in NewRead.
func (rb *ReadBuilder) WithCaseSensitive(caseSensitive bool) error {
if rb.inner == nil {
return ErrClosed
}
return ffiReadBuilderWithCaseSensitive.symbol(rb.ctx)(rb.inner, caseSensitive)
}

// WithFilter sets a filter predicate for scan planning and read-side pruning.
//
// The predicate is used in two phases:
Expand Down Expand Up @@ -177,6 +195,31 @@ var ffiReadBuilderWithProjection = newFFI(ffiOpts{
}
})

// Rust `bool` is a single byte whose only valid values are 0 and 1, so the
// argument is declared as a 1-byte integer (the ffi package documents TypeUint8
// as the way to pass a bool) and an explicit 0/1 is written into it via
// boolByte. This is the binding's first non-pointer scalar smaller than 4 bytes,
// so do not copy a wider type here.
var ffiReadBuilderWithCaseSensitive = newFFI(ffiOpts{
sym: "paimon_read_builder_with_case_sensitive",
rType: &ffi.TypePointer,
aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypeUint8},
}, func(ctx context.Context, ffiCall ffiCall) func(rb *paimonReadBuilder, caseSensitive bool) error {
return func(rb *paimonReadBuilder, caseSensitive bool) error {
flag := boolByte(caseSensitive)
var errPtr *paimonError
ffiCall(
unsafe.Pointer(&errPtr),
unsafe.Pointer(&rb),
unsafe.Pointer(&flag),
)
if errPtr != nil {
return parseError(ctx, errPtr)
}
return nil
}
})

var ffiReadBuilderWithFilter = newFFI(ffiOpts{
sym: "paimon_read_builder_with_filter",
rType: &ffi.TypePointer,
Expand Down
16 changes: 14 additions & 2 deletions bindings/go/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,21 @@ func (t *Table) Close() {
})
}

// PredicateBuilder returns a builder for creating filter predicates on this table.
// PredicateBuilder returns a builder for creating filter predicates on this
// table, matching column names exactly.
func (t *Table) PredicateBuilder() *PredicateBuilder {
return &PredicateBuilder{table: t}
return &PredicateBuilder{table: t, caseSensitive: true}
}

// PredicateBuilderWithCaseSensitive returns a predicate builder that resolves
// column names by ASCII case folding when caseSensitive is false. A name that
// folds onto two different schema columns is rejected as ambiguous.
//
// Case sensitivity is fixed when the predicate is built, which is why it belongs
// to the builder rather than to ReadBuilder.WithCaseSensitive — that setting
// covers projection only.
func (t *Table) PredicateBuilderWithCaseSensitive(caseSensitive bool) *PredicateBuilder {
return &PredicateBuilder{table: t, caseSensitive: caseSensitive}
}

// NewReadBuilder creates a ReadBuilder for this table.
Expand Down
Loading
Loading