From 9fd3fae5ffbaf7f5e02471ec6a0d771f8a9ed913 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Thu, 22 Oct 2026 16:23:00 +0800 Subject: [PATCH 01/13] [chore](paimon) upgrade Paimon to the 2.1 community release and port the FE off the removed privilege package Paimon 2.1 removes the privilege package the Doris FE paimon path depended on; this swaps the dependency to the 2.1-community-release line and ports the FE references to the replacement API surface. Co-Authored-By: Claude Opus 5 (1M context) --- .../paimon-connector/pom.xml | 11 ++++ .../doris/paimon/PaimonJniScannerTest.java | 33 ++++------ .../paimon/PlainDelegateFileStoreTable.java | 60 +++++++++++++++++++ .../paimon/PaimonReaderOptions.java | 26 +++----- .../paimon/PaimonTableDecorators.java | 4 +- .../paimon/PaimonExternalMetaCacheTest.java | 9 ++- .../paimon/PaimonReaderOptionsTest.java | 35 ++++------- .../paimon/PlainDelegateFileStoreTable.java | 60 +++++++++++++++++++ .../PaimonTableValuedFunctionTest.java | 7 +-- fe/pom.xml | 4 +- 10 files changed, 173 insertions(+), 76 deletions(-) create mode 100644 fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PlainDelegateFileStoreTable.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PlainDelegateFileStoreTable.java diff --git a/fe/be-java-extensions/paimon-connector/pom.xml b/fe/be-java-extensions/paimon-connector/pom.xml index 717b3e6f2ce427..d6f845769515aa 100644 --- a/fe/be-java-extensions/paimon-connector/pom.xml +++ b/fe/be-java-extensions/paimon-connector/pom.xml @@ -118,6 +118,17 @@ under the License. paimon-connector ${project.basedir}/target/ + + org.apache.maven.plugins + maven-surefire-plugin + + + + --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index bce0c5950e7863..2f95a11ce1eb03 100644 --- a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -28,15 +28,12 @@ import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.Property; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.BufferFileReader; import org.apache.paimon.disk.BufferFileWriter; import org.apache.paimon.disk.FileIOChannel; import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.IOManagerImpl; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.DelegatedFileStoreTable; @@ -341,21 +338,16 @@ public void testFallbackWrapperOrderMatchesPaimonFactoryPrecedence() { } @Test - public void testBackendCapTraversesPrivilegeDelegate() { + public void testBackendCapTraversesPlainDelegate() { FileStoreTable main = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1")); FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128")); - PrivilegeChecker checker = (PrivilegeChecker) Proxy.newProxyInstance( - PrivilegeChecker.class.getClassLoader(), - new Class[] {PrivilegeChecker.class}, - (proxy, method, args) -> null); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(main, fallback, true), checker, - Identifier.create("db", "table")); + FileStoreTable delegate = new PlainDelegateFileStoreTable( + new FallbackReadFileStoreTable(main, fallback, true)); Table safe = PaimonJniScanner.applyBackendManifestParallelism( - privileged, null, 64); + delegate, null, 64); FileStoreTable planningTable = safe instanceof DelegatedFileStoreTable && !(safe instanceof FallbackReadFileStoreTable) ? ((DelegatedFileStoreTable) safe).wrapped() : (FileStoreTable) safe; @@ -390,19 +382,14 @@ public void testOldFeSystemWrapperPreservesIndependentFallbackLimits() throws Ex } @Test - public void testSystemWrapperExposesSafeFallbackBehindPrivilegeDelegate() throws Exception { + public void testSystemWrapperExposesSafeFallbackBehindPlainDelegate() throws Exception { FileStoreTable main = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1")); FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "2")); - PrivilegeChecker checker = (PrivilegeChecker) Proxy.newProxyInstance( - PrivilegeChecker.class.getClassLoader(), - new Class[] {PrivilegeChecker.class}, - (proxy, method, args) -> null); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(main, fallback, true), checker, - Identifier.create("db", "table")); - Table wrapper = SystemTableLoader.load("partitions", privileged); + FileStoreTable delegate = new PlainDelegateFileStoreTable( + new FallbackReadFileStoreTable(main, fallback, true)); + Table wrapper = SystemTableLoader.load("partitions", delegate); Table safe = PaimonJniScanner.applyBackendManifestParallelism( wrapper, null, 64); @@ -959,6 +946,10 @@ public Object invoke(Object proxy, Method method, Object[] args) { if ("rowType".equals(method.getName())) { return new RowType(Collections.emptyList()); } + if ("coreOptions".equals(method.getName())) { + // Paimon 2.1 system-table loading consults queryAuthEnabled() on the data table. + return new CoreOptions(options); + } if ("schema".equals(method.getName())) { return new TableSchema(0L, Collections.emptyList(), 0, Collections.emptyList(), Collections.emptyList(), options, ""); diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PlainDelegateFileStoreTable.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PlainDelegateFileStoreTable.java new file mode 100644 index 00000000000000..e57cdddbfdb26b --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PlainDelegateFileStoreTable.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; +import org.apache.paimon.table.FileStoreTable; + +import java.util.Map; + +/** + * Minimal non-fallback delegate used to exercise the delegate-peeling paths that the removed + * Paimon privilege wrapper used to cover. + */ +final class PlainDelegateFileStoreTable extends DelegatedFileStoreTable { + + PlainDelegateFileStoreTable(FileStoreTable wrapped) { + super(wrapped); + } + + @Override + public FileStoreTable copy(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copy(dynamicOptions)); + } + + @Override + public FileStoreTable copy(TableSchema newTableSchema) { + return new PlainDelegateFileStoreTable(wrapped().copy(newTableSchema)); + } + + @Override + public FileStoreTable copyWithoutTimeTravel(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copyWithoutTimeTravel(dynamicOptions)); + } + + @Override + public FileStoreTable copyWithLatestSchema() { + return new PlainDelegateFileStoreTable(wrapped().copyWithLatestSchema()); + } + + @Override + public FileStoreTable switchToBranch(String branchName) { + return new PlainDelegateFileStoreTable(wrapped().switchToBranch(branchName)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java index 5a422f6af5a9f2..63fb7c002e2c7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java @@ -22,7 +22,6 @@ import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.table.DelegatedFileStoreTable; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; @@ -193,20 +192,11 @@ private static Table normalizeManifestParallelism( } if (table instanceof DelegatedFileStoreTable) { - if (!(table instanceof PrivilegedFileStoreTable)) { - throw new IllegalArgumentException("Unsupported Paimon planning table delegate: " - + table.getClass().getName()); - } - FileStoreTable wrapped = ((DelegatedFileStoreTable) table).wrapped(); - FileStoreTable normalized = normalizeManifestParallelism( - wrapped, safeBound, materializeAbsent); - if (normalized == wrapped) { - return table; - } - // A delegate copy broadcasts one value to every fallback branch. Check authorization - // before peeling the privilege-only layer so branch-local limits remain independent. - ((FileStoreTable) table).newScan(); - return normalized; + // Paimon 2.1 dropped the privilege wrapper, so every delegate the factory produces is + // a FallbackReadFileStoreTable and was already rebuilt above. Any other delegate's + // copy() broadcasts one value through its private fallback tree, so reject it. + throw new IllegalArgumentException("Unsupported Paimon planning table delegate: " + + table.getClass().getName()); } if (!(table instanceof FileStoreTable)) { @@ -234,7 +224,7 @@ private static Table normalizeManifestParallelism( static boolean isWrappedFirst(FallbackReadFileStoreTable table) { Map options = table.options(); - // Keep this in the exact order used by Paimon 1.4.2 FileStoreTableFactory. The same + // Keep this in the exact order used by Paimon's FileStoreTableFactory. The same // wrapper represents chain, fallback and primary reads, but the SDK exposes no accessor // for its private wrappedFirst flag. Remove this inference when Paimon exposes one. if (ChainTableUtils.isChainTable(options)) { @@ -262,8 +252,8 @@ public static void validateEffectiveTable(Table table) { validateEffectiveTable(((FallbackReadFileStoreTable) table).other()); } if (table instanceof DelegatedFileStoreTable) { - // Privilege and other supported delegates can hide a fallback planner behind their - // own visible main options, so traverse the complete planning-handle chain. + // Delegates can hide a fallback planner behind their own visible main options, + // so traverse the complete planning-handle chain. validateEffectiveTable(((DelegatedFileStoreTable) table).wrapped()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java index 7060b6ae4c36ec..a45c8577f24804 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java @@ -30,8 +30,8 @@ private PaimonTableDecorators() { public static FileStoreTable unwrapToFallbackOrBase(FileStoreTable table) { FileStoreTable current = table; // Fallback dispatch requires the pair to be the system wrapper's immediate child. Paimon - // does not privilege-wrap system tables, so peeling privilege-only delegates matches its - // catalog semantics while retaining the one decorator that changes read routing. + // 2.1 ships no privilege delegates, so peeling any unknown delegate matches its catalog + // semantics while retaining the one decorator that changes read routing. while (current instanceof DelegatedFileStoreTable && !(current instanceof FallbackReadFileStoreTable)) { current = ((DelegatedFileStoreTable) current).wrapped(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 4608013b5fbf4d..12e0c8d06f30d8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -1829,16 +1829,15 @@ public void testAdvancingLatestFenceKeepsOnlyNewestProjectionOfTableGeneration() } @Test - public void testSnapshotEstimateSupportsPrivilegedTableWrapper() throws Exception { - FileStoreTable table = newPartitionedTable("privileged_estimate", Collections.emptyMap()); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - table, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "tbl")); + public void testSnapshotEstimateSupportsDelegateTableWrapper() throws Exception { + FileStoreTable table = newPartitionedTable("delegate_estimate", Collections.emptyMap()); + FileStoreTable delegate = new PlainDelegateFileStoreTable(table); NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( mapping, 1L, table.schema().id(), 1L); PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( PaimonPartitionInfo.EMPTY, - new PaimonSnapshot(1L, table.schema().id(), privileged)); + new PaimonSnapshot(1L, table.schema().id(), delegate)); value.prepareForCachePublication(key); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java index aa1d6415adedd8..0e022941bc02e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java @@ -22,11 +22,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; import org.apache.paimon.table.CatalogEnvironment; @@ -130,10 +127,12 @@ void testRuntimeCapNormalizesEveryPlanningLeafIndependently() { FallbackReadFileStoreTable emptyThenExplicit = (FallbackReadFileStoreTable) PaimonReaderOptions.runtimeSafeTable( new FallbackReadFileStoreTable(empty, explicit, true), 512); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(explicit, empty, true), - Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "table")); - Table normalizedDelegate = PaimonReaderOptions.runtimeSafeTable(privileged, 512); + // Paimon 2.1 no longer produces non-fallback delegates, so an unknown one is rejected + // instead of being peeled with a single cap broadcast to every hidden branch. + Table unknownDelegate = new PlainDelegateFileStoreTable( + new FallbackReadFileStoreTable(explicit, empty, true)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> PaimonReaderOptions.runtimeSafeTable(unknownDelegate, 512)); Assertions.assertEquals("256", safeEmpty.options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); @@ -145,9 +144,6 @@ void testRuntimeCapNormalizesEveryPlanningLeafIndependently() { .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); Assertions.assertEquals("1", emptyThenExplicit.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assertions.assertInstanceOf(FallbackReadFileStoreTable.class, normalizedDelegate); - Assertions.assertEquals("256", ((FallbackReadFileStoreTable) normalizedDelegate) - .other().options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @Test @@ -186,11 +182,9 @@ void testSystemSourceKeepsFallbackAsOutermostPlanningDecorator() { FileStoreTable main = newFileStoreTable("main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable("fallback", Collections.emptyMap()); FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - pair, Mockito.mock(PrivilegeChecker.class), - Identifier.create("db", "privileged_fallback")); + FileStoreTable delegate = new PlainDelegateFileStoreTable(pair); - Assertions.assertSame(pair, PaimonTableDecorators.unwrapToFallbackOrBase(privileged)); + Assertions.assertSame(pair, PaimonTableDecorators.unwrapToFallbackOrBase(delegate)); } @Test @@ -282,18 +276,15 @@ void testBackendManifestCapUsesExecutionCeilingNotSmallestBranch() { } @Test - void testRejectUnsafeFallbackHiddenByPrivilegeDelegate() { - FileStoreTable main = newFileStoreTable("privileged_main", Collections.emptyMap()); + void testRejectUnsafeFallbackHiddenByDelegate() { + FileStoreTable main = newFileStoreTable("delegate_main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable( - "privileged_fallback", ImmutableMap.of("scan.manifest.parallelism", "0")); + "delegate_fallback", ImmutableMap.of("scan.manifest.parallelism", "0")); FileStoreTable fallbackReadTable = new FallbackReadFileStoreTable(main, fallback, true); - FileStoreTable privilegedTable = PrivilegedFileStoreTable.wrap( - fallbackReadTable, - Mockito.mock(PrivilegeChecker.class), - Identifier.create("db", "table")); + FileStoreTable delegateTable = new PlainDelegateFileStoreTable(fallbackReadTable); Assertions.assertThrows(IllegalArgumentException.class, - () -> PaimonReaderOptions.validateEffectiveTable(privilegedTable)); + () -> PaimonReaderOptions.validateEffectiveTable(delegateTable)); } private FileStoreTable newFileStoreTable(String name, Map options) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PlainDelegateFileStoreTable.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PlainDelegateFileStoreTable.java new file mode 100644 index 00000000000000..b8cb446953338c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PlainDelegateFileStoreTable.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; +import org.apache.paimon.table.FileStoreTable; + +import java.util.Map; + +/** + * Minimal non-fallback delegate used to exercise the delegate-peeling paths that the removed + * Paimon privilege wrapper used to cover. + */ +final class PlainDelegateFileStoreTable extends DelegatedFileStoreTable { + + PlainDelegateFileStoreTable(FileStoreTable wrapped) { + super(wrapped); + } + + @Override + public FileStoreTable copy(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copy(dynamicOptions)); + } + + @Override + public FileStoreTable copy(TableSchema newTableSchema) { + return new PlainDelegateFileStoreTable(wrapped().copy(newTableSchema)); + } + + @Override + public FileStoreTable copyWithoutTimeTravel(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copyWithoutTimeTravel(dynamicOptions)); + } + + @Override + public FileStoreTable copyWithLatestSchema() { + return new PlainDelegateFileStoreTable(wrapped().copyWithLatestSchema()); + } + + @Override + public FileStoreTable switchToBranch(String branchName) { + return new PlainDelegateFileStoreTable(wrapped().switchToBranch(branchName)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java index f6752074f0a5f3..44469c9df69f4c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java @@ -20,9 +20,6 @@ import org.apache.doris.datasource.paimon.PaimonReaderOptions; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; @@ -74,10 +71,8 @@ void testReadOptimizedTableKeepsFallbackPairAsDirectSource() throws Exception { Mockito.when(main.options()).thenReturn(Collections.emptyMap()); Mockito.when(fallback.options()).thenReturn(Collections.emptyMap()); FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - pair, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "table")); - Table systemTable = PaimonTableValuedFunction.createRuntimeSafeSystemTable(privileged, "ro"); + Table systemTable = PaimonTableValuedFunction.createRuntimeSafeSystemTable(pair, "ro"); Assertions.assertInstanceOf(ReadOptimizedTable.class, systemTable); Field wrapped = ReadOptimizedTable.class.getDeclaredField("wrapped"); diff --git a/fe/pom.xml b/fe/pom.xml index 4f0a24ee4cad45..50af2cbb22ff2a 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -231,7 +231,7 @@ under the License. fe-authentication - 3.1.3 + 3.1.4-SNAPSHOT 1.12.1 1.18.0 @@ -418,7 +418,7 @@ under the License. 2.3.2 2.0.3 - 1.4.2 + 2.1-community-release-2.1-BETA-SNAPSHOT 3.4.4 shade-format-flatbuffers From cc1bcb5e3cbe72e672660ca2d445f36c9e9ebafc Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Thu, 22 Oct 2026 16:23:00 +0800 Subject: [PATCH 02/13] [feat](paimon) port PK-vector (ANN) search onto the community paimon stack Order-by-distance pushdown over the PK-vector index (l2 / inner_product metrics), executed exclusively by the BE paimon-rust reader: - FE: PushDownVectorTopNIntoPaimonScan rewrite (bounded TopN -> index-only scan with the __paimon_search_score_to_dis synthetic output column) and the AnnTopNInfo / vector payload plumbing down to the split. - Rust eligibility gates: the vector capability bit is negotiated separately from the batch rust bit (a vector payload rides an empty paimon_split, which an older rust-capable BE would reject), the table/storage/backend gates are unified between the batch and vector paths, raw-convertible compacted splits skip the merge-reader gates, and the rewrite pre-flights the gate because a vector split has no JNI fallback. - BE: the paimon-rust vector reader mode (bucket vector search split deserialization, score-to-distance transform, synthetic column fill). - Housekeeping caught by the port: clang-format of the ANN reader files, FileSystemSchemaManager in the rust reader schema tests, the privilege references the 2.1 port missed, and a docs note that read.batch-size is currently a no-op on the PK-vector path. Co-Authored-By: Claude Opus 5 (1M context) Conflicts: fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java --- .../table/paimon_rust_predicate_converter.h | 21 + .../table/paimon_rust_table_reader.cpp | 605 +++++++++++++-- .../table/paimon_rust_table_reader.h | 104 ++- .../table/paimon_rust_table_reader_test.cpp | 235 ++++++ .../doris/paimon/PaimonColumnValue.java | 23 +- .../paimon/PaimonReadTypeProjection.java | 11 + .../apache/doris/paimon/PaimonTypeUtils.java | 17 + .../doris/datasource/FileQueryScanNode.java | 7 + .../paimon/PaimonExternalTable.java | 7 - .../doris/datasource/paimon/PaimonUtil.java | 44 +- .../source/PaimonPredicateConverter.java | 15 + .../paimon/source/PaimonRustEligibility.java | 298 +++++++ .../source/PaimonRustReaderCapabilities.java | 42 +- .../paimon/source/PaimonScanNode.java | 733 ++++++++++++------ .../datasource/paimon/source/PaimonSplit.java | 12 + .../paimon/source/PaimonVectorSearch.java | 107 +++ .../translator/PhysicalPlanTranslator.java | 8 + .../doris/nereids/jobs/executor/Rewriter.java | 2 + .../apache/doris/nereids/rules/RuleType.java | 1 + .../LogicalFileScanToPhysicalFileScan.java | 3 +- .../PushDownVectorTopNIntoPaimonScan.java | 313 ++++++++ .../nereids/trees/plans/AnnTopNInfo.java | 112 +++ .../trees/plans/logical/LogicalFileScan.java | 105 ++- .../trees/plans/logical/LogicalHudiScan.java | 2 +- .../plans/physical/PhysicalFileScan.java | 69 +- .../PhysicalLazyMaterializeFileScan.java | 2 +- .../org/apache/doris/qe/SessionVariable.java | 1 + .../paimon/PaimonExternalMetaCacheTest.java | 2 - .../datasource/paimon/PaimonRowCountTest.java | 28 +- .../datasource/paimon/PaimonUtilTest.java | 75 ++ .../PaimonRustReaderCompatibilityTest.java | 4 +- .../PaimonRustReaderLegacySchemaTest.java | 4 +- .../paimon/source/PaimonScanNodeTest.java | 168 ++++ .../PushDownVectorTopNIntoPaimonScanTest.java | 710 +++++++++++++++++ .../planner/PaimonPredicateConverterTest.java | 40 + gensrc/thrift/PlanNodes.thrift | 45 +- 36 files changed, 3620 insertions(+), 355 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AnnTopNInfo.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java diff --git a/be/src/format_v2/table/paimon_rust_predicate_converter.h b/be/src/format_v2/table/paimon_rust_predicate_converter.h index 76c76037b51a97..be861707ce4b8e 100644 --- a/be/src/format_v2/table/paimon_rust_predicate_converter.h +++ b/be/src/format_v2/table/paimon_rust_predicate_converter.h @@ -42,6 +42,27 @@ namespace doris {} // namespace doris namespace doris { +// Reader-produced PK-vector search distance column, as Doris names it. Wire +// contract shared with FE (PaimonVectorSearch.SEARCH_DISTANCE_COLUMN). It is never +// a projectable table column and never a pushable predicate column, so both the +// projection builder and the predicate converter must exclude it. +// +// It holds the metric-native DISTANCE, not paimon-rust's score: paimon-rust emits +// an Arrow field named kPaimonRustScoreField carrying a higher-is-better score, and +// the rust table reader inverts that transform in place while filling the block so +// the value matches what l2_distance_approximate / inner_product_approximate are +// defined to return. Lower-is-better for l2. +// +// The two names deliberately differ, since the two values mean different things -- +// and keeping them distinct leaves __paimon_search_score free should a later +// version want to expose the raw score as its own column. +constexpr const char* kPaimonSearchDistanceColumn = "__paimon_search_score_to_dis"; + +// The Arrow field name paimon-rust auto-appends to a vector-search batch (see the +// bindings' materialize_search_result), so it cannot be changed from here; the +// reader maps it onto kPaimonSearchDistanceColumn when filling the block. +constexpr const char* kPaimonRustScoreField = "__paimon_search_score"; + // Converts Doris push-down conjuncts into a paimon-rust filter predicate. // // This mirrors PaimonPredicateConverter (the paimon-cpp variant) but targets the diff --git a/be/src/format_v2/table/paimon_rust_table_reader.cpp b/be/src/format_v2/table/paimon_rust_table_reader.cpp index 731508957bbf4f..d80abeca7a82ba 100644 --- a/be/src/format_v2/table/paimon_rust_table_reader.cpp +++ b/be/src/format_v2/table/paimon_rust_table_reader.cpp @@ -18,19 +18,27 @@ #include "format_v2/table/paimon_rust_table_reader.h" #include +#include +#include +#include #include #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/record_batch.h" #include "arrow/result.h" +#include "arrow/type.h" +#include "common/compiler_util.h" #include "common/logging.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" +#include "exec/common/endian.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "format_v2/column_mapper.h" @@ -70,6 +78,15 @@ PAIMON_OWNED(read_builder, paimon_read_builder_free); PAIMON_OWNED(plan, paimon_plan_free); PAIMON_OWNED(table_read, paimon_table_read_free); PAIMON_OWNED(record_batch_reader, paimon_record_batch_reader_free); +PAIMON_OWNED(vector_search_builder, paimon_vector_search_builder_free); +// Local handles of the eager Top-K vector-search chain (paimon-rust #822): they only +// need to live until the arrow reader is created, so unlike the table/reader handles +// above they are function locals of _open_vector_split_reader rather than members +// of PaimonHandles. +PAIMON_OWNED(bucket_vector_search_split, paimon_bucket_vector_search_split_free); +PAIMON_OWNED(vector_scan, paimon_vector_scan_free); +PAIMON_OWNED(vector_plan, paimon_vector_plan_free); +PAIMON_OWNED(vector_read, paimon_vector_read_free); PAIMON_OWNED(error, paimon_error_free); #undef PAIMON_OWNED @@ -346,14 +363,26 @@ Status PaimonRustTableReader::_validate_rust_split(const TFileRangeDesc& range) "mismatch"); } const auto& params = range.table_format_params.paimon_params; - if (!params.__isset.paimon_split || params.paimon_split.empty()) { + if (params.__isset.reader_type && params.reader_type != TPaimonReaderType::PAIMON_RUST) { return Status::InternalError( - "missing paimon_split for paimon rust reader, possibly caused by FE/BE protocol " + "invalid reader_type for paimon rust reader, possibly caused by FE/BE protocol " "mismatch"); } - if (params.__isset.reader_type && params.reader_type != TPaimonReaderType::PAIMON_RUST) { + // Vector (ANN) mode: the payload is the serialized BucketVectorSearchSplit, and + // paimon_split is deliberately an empty string (FE avoids serializing the same + // DataSplit twice on the wire). An empty-but-set paimon_split therefore also + // implies vector mode: reporting "missing paimon_split" for such a split would + // point the reader at the wrong field when the real gap is the vector payload. + if (params.__isset.vector_payload || + (params.__isset.paimon_split && params.paimon_split.empty())) { + if (!params.__isset.vector_payload || params.vector_payload.bytes.empty()) { + return Status::InternalError( + "missing vector_payload for paimon rust vector reader, possibly caused by " + "FE/BE protocol mismatch"); + } + } else if (!params.__isset.paimon_split || params.paimon_split.empty()) { return Status::InternalError( - "invalid reader_type for paimon rust reader, possibly caused by FE/BE protocol " + "missing paimon_split for paimon rust reader, possibly caused by FE/BE protocol " "mismatch"); } if (!_resolve_table_path(range).has_value()) { @@ -376,20 +405,93 @@ Status PaimonRustTableReader::_validate_rust_split(const TFileRangeDesc& range) return Status::OK(); } -Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { - // 1. Decode the FE-planned split first so we fail fast (and without any - // filesystem IO) when it is missing or malformed. - std::string split_bytes; - RETURN_IF_ERROR(_decode_split_bytes(&split_bytes)); +bool PaimonRustTableReader::_is_vector_mode(const TFileRangeDesc& range) { + return range.__isset.table_format_params && range.table_format_params.__isset.paimon_params && + range.table_format_params.paimon_params.__isset.vector_payload; +} + +Status PaimonRustTableReader::_validate_external_search_request() const { + if (!_scan_params || !_scan_params->__isset.external_search_request) { + return Status::InvalidArgument( + "paimon-rust vector search missing external_search_request in scan params"); + } + const auto& request = _scan_params->external_search_request; + if (request.__isset.schema_version && request.schema_version != 1) { + return Status::NotSupported("unsupported external search schema version: {}", + request.schema_version); + } + if (!request.__isset.search_query || !request.search_query.__isset.vector_search) { + return Status::InvalidArgument( + "paimon-rust vector search requires a vector query in external_search_request"); + } + + const auto& vector = request.search_query.vector_search; + // column + if (!vector.__isset.column || vector.column.empty() || + vector.column.find('\0') != std::string::npos) { + return Status::InvalidArgument("paimon-rust vector search requires a non-empty column"); + } + // query_vector (TSearchVector) + if (!vector.__isset.query_vector) { + return Status::InvalidArgument("paimon-rust vector search requires a query vector"); + } + const auto& query_vector = vector.query_vector; + if (!query_vector.__isset.element_type || !query_vector.__isset.dimension || + !query_vector.__isset.values) { + return Status::InvalidArgument( + "paimon-rust query vector requires element_type, dimension, and values"); + } + if (query_vector.dimension <= 0) { + return Status::InvalidArgument("paimon-rust query vector dimension must be positive: {}", + query_vector.dimension); + } + if (query_vector.element_type != TVectorElementType::FLOAT32) { + return Status::NotSupported( + "paimon-rust vector search only supports FLOAT32 query vector, got {}", + static_cast(query_vector.element_type)); + } + const auto dimension = static_cast(query_vector.dimension); + if (query_vector.values.size() != dimension * sizeof(float)) { + return Status::InvalidArgument( + "paimon-rust query vector byte size {} does not match dimension {} * {}", + query_vector.values.size(), dimension, sizeof(float)); + } + // top_k / offset + if (!vector.__isset.top_k || vector.top_k < 0) { + return Status::InvalidArgument("paimon-rust vector search top_k must be non-negative"); + } + if (!vector.__isset.offset || vector.offset < 0) { + return Status::InvalidArgument("paimon-rust vector search offset must be non-negative"); + } + // metric is required (NOT advisory): BE uses it to undo the score transform. + // paimon-rust only supports L2 and DOT_PRODUCT; parse_score_transform rejects + // anything else, so validate __isset here and let parse_score_transform handle + // the value. + if (!vector.__isset.metric) { + return Status::InvalidArgument("paimon-rust vector search requires a metric"); + } + // paimon_options (optional): only validate the map itself is present if set. + if (request.__isset.paimon_options && request.paimon_options.__isset.options) { + for (const auto& kv : request.paimon_options.options) { + if (kv.first.find('\0') != std::string::npos || + kv.second.find('\0') != std::string::npos) { + return Status::InvalidArgument( + "paimon-rust vector search options contain an embedded NUL byte"); + } + } + } + return Status::OK(); +} - // 2. Resolve identifier + table_path + FE-supplied TableSchema JSON. +Status PaimonRustTableReader::_open_paimon_table(const TFileRangeDesc& range) { + // Resolve identifier + table_path + FE-supplied TableSchema JSON. auto table_path = _resolve_table_path(range).value(); auto db_name = _resolve_db_name(range).value(); auto table_name = _resolve_table_name(range).value(); auto schema_json = _resolve_table_schema_json(range).value(); auto branch_opt = _resolve_branch(range); - // 3. Assemble storage options: FE-supplied paimon options + hadoop_conf + + // Assemble storage options: FE-supplied paimon options + hadoop_conf + // OSS/S3 → AWS_* translations. These feed FileIO only (per // paimon_table_from_schema_json contract); they are NOT merged into the // supplied table schema. @@ -397,41 +499,56 @@ Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { auto opened_table_key = std::make_tuple(table_path, schema_json, db_name, table_name, branch_opt, options); - if (!_handles || !_handles->table || _opened_table_key != opened_table_key) { + if (_handles && _handles->table && _opened_table_key == opened_table_key) { // A paimon scan reads one table, so the handle is opened at most once per - // distinct identity (e.g. re-created after a close); splits of the same - // table reuse it and only rebuild the read pipeline below. - _close_table(); - _handles = std::make_unique(); - - std::vector c_options; - c_options.reserve(options.size()); - for (const auto& kv : options) { - c_options.push_back(paimon_option {kv.first.c_str(), kv.second.c_str()}); - } - - LOG(INFO) << "paimon-rust opening table via schema json: db=" << db_name - << " table=" << table_name << " path=" << table_path - << " branch=" << (branch_opt.has_value() ? branch_opt.value() : "main") - << " storage_options=[" << format_options(options) << "]"; - - // Build the table directly from the FE-supplied schema JSON. The Rust - // side rejects null / empty branch, so we default to paimon's canonical - // "main" sentinel when FE did not set paimon_branch (i.e. the table is - // on the main branch — matches upstream Identifier.DEFAULT_MAIN_BRANCH). - const std::string& branch_str = branch_opt.has_value() ? branch_opt.value() : "main"; - paimon_result_get_table tbl_res = paimon_table_from_schema_json( - table_path.c_str(), schema_json.c_str(), db_name.c_str(), table_name.c_str(), - branch_str.c_str(), c_options.empty() ? nullptr : c_options.data(), - c_options.size()); - if (tbl_res.error != nullptr) { - return Status::InternalError( - "paimon-rust table_from_schema_json failed: db={} table={} err={}", db_name, - table_name, consume_error(tbl_res.error)); - } - _handles->table.reset(tbl_res.table); - _opened_table_key = std::move(opened_table_key); + // distinct identity; splits of the same table reuse it and only rebuild + // the read pipeline. + return Status::OK(); } + _close_table(); + _handles = std::make_unique(); + + std::vector c_options; + c_options.reserve(options.size()); + for (const auto& kv : options) { + c_options.push_back(paimon_option {kv.first.c_str(), kv.second.c_str()}); + } + + LOG(INFO) << "paimon-rust opening table via schema json: db=" << db_name + << " table=" << table_name << " path=" << table_path + << " branch=" << (branch_opt.has_value() ? branch_opt.value() : "main") + << " storage_options=[" << format_options(options) << "]"; + + // Build the table directly from the FE-supplied schema JSON. The Rust + // side rejects null / empty branch, so we default to paimon's canonical + // "main" sentinel when FE did not set paimon_branch (i.e. the table is + // on the main branch — matches upstream Identifier.DEFAULT_MAIN_BRANCH). + const std::string& branch_str = branch_opt.has_value() ? branch_opt.value() : "main"; + paimon_result_get_table tbl_res = paimon_table_from_schema_json( + table_path.c_str(), schema_json.c_str(), db_name.c_str(), table_name.c_str(), + branch_str.c_str(), c_options.empty() ? nullptr : c_options.data(), c_options.size()); + if (tbl_res.error != nullptr) { + return Status::InternalError( + "paimon-rust table_from_schema_json failed: db={} table={} err={}", db_name, + table_name, consume_error(tbl_res.error)); + } + _handles->table.reset(tbl_res.table); + _opened_table_key = std::move(opened_table_key); + return Status::OK(); +} + +Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { + // Primary-key vector (ANN) search: route to the vector-search read path. + if (_is_vector_mode(range)) { + return _open_vector_split_reader(range); + } + // 1. Decode the FE-planned split first so we fail fast (and without any + // filesystem IO) when it is missing or malformed. + std::string split_bytes; + RETURN_IF_ERROR(_decode_split_bytes(&split_bytes)); + + // 2. Resolve and open (or reuse) the table. + RETURN_IF_ERROR(_open_paimon_table(range)); // 4. Build the read pipeline: read_builder -> case-insensitive -> projection. // Bound the Rust Arrow allocation itself: wide rows must respect the scanner's @@ -509,6 +626,313 @@ Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { return Status::OK(); } +Status PaimonRustTableReader::_open_vector_split_reader(const TFileRangeDesc& range) { + // Validate every __isset / value-range up front (mirrors + // LanceTableReader::_validate_external_search_request). The build path below + // then uses the fields unchecked, matching lance's split of validate-once / + // build-without-recheck. + RETURN_IF_ERROR(_validate_external_search_request()); + const auto& request = _scan_params->external_search_request; + const auto& vector = request.search_query.vector_search; + + // 0. Distance metric. It decides which score transform the reader has to undo + // (see _convert_score_to_distance), so defaulting would silently return wrong + // distances. __isset was checked by _validate_external_search_request; the + // value is rejected here if unsupported. + ScoreTransform transform; + RETURN_IF_ERROR(parse_score_transform(vector.metric, &transform)); + _score_transform = transform; + + // 1. The per-split payload is the serialized BucketVectorSearchSplit (raw + // thrift binary, NOT base64; magic "PKVSPLIT", v1). Grab it first so we fail + // fast before any filesystem IO. + const auto& payload = _current_range.table_format_params.paimon_params.vector_payload; + if (payload.payload_type != TPaimonVectorPayloadType::BUCKET_SPLIT_BYTES) { + return Status::InternalError("paimon-rust vector search unsupported payload type: {}", + static_cast(payload.payload_type)); + } + const std::string& split_bytes = payload.bytes; + if (split_bytes.empty()) { + return Status::InternalError("paimon-rust vector search split payload is empty"); + } + + // 2. Resolve and open (or reuse) the table (same as the normal read path); + // options carry storage credentials to the FileIO. + RETURN_IF_ERROR(_open_paimon_table(range)); + + // 3. Build and configure the vector-search builder. No read.batch-size here, + // unlike the batch path (_open_split_reader): paimon-rust's PK vector readers + // construct their DataFileReaders without with_batch_size, so a + // read.batch-size option — from the schema, t@options or these search + // options — is silently ignored on the vector path, which batches at the + // parquet-rust default. Known difference; the vector stages are + // single-column streamed into bounded Top-K heaps and the output is + // capped by top_k, so no action needed until paimon-rust wires it. + paimon_result_vector_search_builder vb_res = + paimon_table_new_vector_search_builder(_handles->table.get()); + if (vb_res.error != nullptr) { + return Status::InternalError("paimon-rust new vector search builder failed: {}", + consume_error(vb_res.error)); + } + vector_search_builder_ptr vb(vb_res.builder); + + // 3a. Target vector column. + if (paimon_error* err = + paimon_vector_search_builder_with_vector_column(vb.get(), vector.column.c_str())) { + return Status::InternalError("paimon-rust set vector column failed: {}", + consume_error(err)); + } + // 3b. Query vector. The wire form is a TSearchVector carrying a FLOAT32, + // little-endian binary blob; decode it into the f32 array the C API wants. + // v1 only supports FLOAT32 (paimon-rust's index backends operate on f32); any + // other element type was already rejected by validate. Dimension and + // byte-size were validated there too, so decode unchecked here. + const auto& search_vec = vector.query_vector; + const auto dimension = static_cast(search_vec.dimension); + std::vector query_vector(dimension); + const auto* bytes = search_vec.values.data(); + for (size_t i = 0; i < dimension; ++i) { + query_vector[i] = std::bit_cast(LittleEndian::Load32( + reinterpret_cast(bytes + i * sizeof(uint32_t)))); + } + if (paimon_error* err = paimon_vector_search_builder_with_query_vector( + vb.get(), query_vector.data(), query_vector.size())) { + return Status::InternalError("paimon-rust set query vector failed: {}", consume_error(err)); + } + // 3c. Retrieval limit (user_limit + user_offset, folded on FE). A negative + // limit would wrap when cast to uintptr_t, so guard it. + if (vector.top_k < 0) { + return Status::InternalError("paimon-rust vector search limit is negative: {}", + vector.top_k); + } + if (paimon_error* err = paimon_vector_search_builder_with_limit( + vb.get(), static_cast(vector.top_k))) { + return Status::InternalError("paimon-rust set limit failed: {}", consume_error(err)); + } + // 3d. Index/search options (refine_factor, ivf.nprobe, ".metric", ...). + if (request.__isset.paimon_options && request.paimon_options.__isset.options && + !request.paimon_options.options.empty()) { + std::vector c_search_options; + c_search_options.reserve(request.paimon_options.options.size()); + for (const auto& kv : request.paimon_options.options) { + c_search_options.emplace_back(kv.first.c_str(), kv.second.c_str()); + } + if (paimon_error* err = paimon_vector_search_builder_with_options( + vb.get(), c_search_options.data(), c_search_options.size())) { + return Status::InternalError("paimon-rust set search options failed: {}", + consume_error(err)); + } + } + // 3e. Materialized user columns. The score field is always appended by + // paimon-rust itself and must NOT be listed here. Empty projection = only the + // score column (index-only scan). Log the user columns: an empty list is + // legitimate, so it is logged as such rather than skipped. + auto read_columns = _build_read_columns(); + std::vector projection; + projection.reserve(read_columns.size() + 1); + for (const auto& col : read_columns) { + projection.push_back(col.c_str()); + } + projection.push_back(nullptr); + LOG(INFO) << "paimon-rust vector search: projection=[" << join(read_columns, ", ") << "] (" + << read_columns.size() << " column(s), distance column '" + << kPaimonSearchDistanceColumn << "' produced by the reader)"; + if (paimon_error* err = + paimon_vector_search_builder_with_projection(vb.get(), projection.data())) { + return Status::InternalError("paimon-rust set vector projection failed: {}", + consume_error(err)); + } + + // 3f. Optional residual scalar filter from the FE data-half push-down. The + // FE-side DV/merge-on-read interception is intentionally not done (see + // PushDownVectorTopNIntoPaimonScan): the Rust candidate search stage applies + // the filter together with deletion vectors, and a table shape Rust cannot + // handle correctly is Rust's responsibility to reject. + if (!_conjuncts.empty()) { + std::vector names; + std::vector types; + names.reserve(_projected_columns.size()); + types.reserve(_projected_columns.size()); + for (const auto& col : _projected_columns) { + if (col.is_partition_key || col.name == kPaimonSearchDistanceColumn) { + continue; + } + names.push_back(col.name); + types.push_back(col.type); + } + PaimonRustPredicateConverter converter(names, types, _handles->table.get()); + paimon_predicate* predicate = converter.build(_conjuncts); + if (predicate != nullptr) { + // with_filter consumes the predicate on every path. + if (paimon_error* err = paimon_vector_search_builder_with_filter(vb.get(), predicate)) { + return Status::InternalError("paimon-rust set vector filter failed: {}", + consume_error(err)); + } + LOG(INFO) << "paimon-rust vector search: applied residual filter"; + } + } + + // 4. Execute the search over the single FE-planned bucket split, via the + // eager Top-K chain (paimon-rust #822). Search, refine, local Top-K and + // materialization all happen in paimon_vector_read_read; the returned arrow + // stream is best-first and carries the __paimon_search_score column. The + // intermediate handles are locals: the migration contract (see + // doris-vector-search-api-migration.md) guarantees the record batch reader + // is independent once paimon_vector_read_read returns, so they are released + // when this function exits. + // + // split: the bytes only need to live until deserialize returns. + // scan: only needs to live until the plan is constructed. + // builder: only needs to live until scan and read are created. + // read/plan: only need to live until paimon_vector_read_read returns. + paimon_result_bucket_vector_search_split split_result = + paimon_bucket_vector_search_split_deserialize( + reinterpret_cast(split_bytes.data()), split_bytes.size()); + if (split_result.error != nullptr) { + return Status::InternalError("paimon-rust deserialize vector split failed: {}", + consume_error(split_result.error)); + } + bucket_vector_search_split_ptr split(split_result.split); + + paimon_result_vector_scan scan_result = paimon_vector_search_builder_new_scan(vb.get()); + if (scan_result.error != nullptr) { + return Status::InternalError("paimon-rust new vector scan failed: {}", + consume_error(scan_result.error)); + } + vector_scan_ptr scan(scan_result.scan); + + paimon_result_vector_read read_result = paimon_vector_search_builder_new_read(vb.get()); + if (read_result.error != nullptr) { + return Status::InternalError("paimon-rust new vector read failed: {}", + consume_error(read_result.error)); + } + vector_read_ptr read(read_result.read); + + // Must use plan_from_bucket_splits: it plans from the FE-assigned split, + // whereas paimon_vector_scan_plan / execute_read replan from the table and + // would ignore the split this scanner was given. + const paimon_bucket_vector_search_split* splits[] = {split.get()}; + paimon_result_vector_plan plan_result = + paimon_vector_scan_plan_from_bucket_splits(scan.get(), splits, 1); + if (plan_result.error != nullptr) { + return Status::InternalError("paimon-rust build vector plan failed: {}", + consume_error(plan_result.error)); + } + vector_plan_ptr plan(plan_result.plan); + + paimon_result_record_batch_reader rdr_res = paimon_vector_read_read(read.get(), plan.get()); + if (rdr_res.error != nullptr) { + return Status::InternalError("paimon-rust vector search read failed: {}", + consume_error(rdr_res.error)); + } + _handles->reader.reset(rdr_res.reader); + return Status::OK(); +} + +std::string PaimonRustTableReader::arrow_field_to_block_column(const std::string& arrow_name) { + std::string name = to_lower(arrow_name); + if (name == kPaimonRustScoreField) { + return kPaimonSearchDistanceColumn; + } + return name; +} + +Status PaimonRustTableReader::parse_score_transform(TVectorMetric::type metric, + ScoreTransform* transform) { + switch (metric) { + case TVectorMetric::L2: + *transform = ScoreTransform::L2; + return Status::OK(); + case TVectorMetric::DOT_PRODUCT: + *transform = ScoreTransform::InnerProduct; + return Status::OK(); + default: + // Includes COSINE: paimon-rust does define a score for it + // (score = 1 - distance), but v1 does not push cosine down, so reaching here + // means FE and BE disagree. Fail instead of inventing a transform. + return Status::InternalError( + "paimon-rust vector search unsupported distance metric: {} " + "(expected L2 or DOT_PRODUCT)", + static_cast(metric)); + } +} + +void PaimonRustTableReader::_convert_score_to_distance(Block* block, uint32_t block_idx, + size_t row_offset, size_t num_rows, + ScoreTransform transform) { + // Undo what paimon-rust's convert_distance_to_score did, so the column holds + // what the Doris distance function is defined to return. + switch (transform) { + case ScoreTransform::InnerProduct: + // score IS the inner product. Nothing to undo, and no clamping to do + // either: the value is unconstrained by definition (it may be negative). + return; + case ScoreTransform::L2: + break; + } + + auto& column_with_type = block->get_by_position(block_idx); + auto column = IColumn::mutate(std::move(column_with_type.column)); + PaddedPODArray* data = nullptr; + const NullMap* null_map = nullptr; + if (auto* nullable = check_and_get_column(column.get())) { + data = &assert_cast(nullable->get_nested_column()).get_data(); + null_map = &nullable->get_null_map_data(); + } else { + data = &assert_cast(*column).get_data(); + } + + // score = 1/(1 + d) where d is the SQUARED L2 distance, and + // l2_distance_approximate returns the true Euclidean one, so invert both steps: + // d_euclidean = sqrt(1/score - 1). + const size_t end = std::min(row_offset + num_rows, data->size()); + for (size_t i = row_offset; i < end; ++i) { + if (null_map != nullptr && (*null_map)[i] != 0) { + continue; + } + const float score = (*data)[i]; + // score is in (0, 1] whenever d is finite and non-negative, and within that + // range the radicand is provably non-negative: a real value above 1 cannot + // round down to below 1.0F in f32, so 1/score >= 1. The useful range + // therefore never reaches either guard below -- they exist because the + // distances come straight out of the closed-source lumina kernel with no + // clamp, and computing the inverse on an illegal score would produce a + // non-finite distance (score <= 0 -> 1/score is inf; score > 1 -> negative + // radicand -> NaN), which makes the coordinator's Top-N comparisons + // ill-defined. Report a distinct negative sentinel per case instead: both + // are unreachable for a genuine Euclidean distance, so they surface as + // visibly bogus rows that also say WHICH way the index is broken. + // + // A score slightly above 1 is NOT corruption: the kernel computes squared + // L2 as ||a||^2 + ||b||^2 - 2, which can yield a small negative + // residual (catastrophic cancellation) when a ~= b -- a near-exact hit. + // Clamp those to distance 0; only scores past kScoreAboveOneTolerance are + // treated as genuinely corrupt. + if (UNLIKELY(score <= 0.0F)) { + LOG(WARNING) << "paimon-rust L2 score_to_distance: corrupt score " << score << " (row " + << i << ", expect score in (0, 1]), " + << "sentinel " << kCorruptScoreNotPositive; + (*data)[i] = kCorruptScoreNotPositive; + } else if (UNLIKELY(score > 1.0F + kScoreAboveOneTolerance)) { + LOG(WARNING) << "paimon-rust L2 score_to_distance: corrupt score " << score << " (row " + << i << ", expect score in (0, 1]), " + << "sentinel " << kCorruptScoreAboveOne; + (*data)[i] = kCorruptScoreAboveOne; + } else if (score > 1.0F) { + // Numerical artifact of a near-exact hit (see kScoreAboveOneTolerance): + // clamp to distance 0 instead of sentineling a legitimate best match. + (*data)[i] = 0.0F; + } else { + // score == 1.0F gives exactly 0 here -- a legal exact hit, which is why + // this must not share a branch with the negative-radicand case. + (*data)[i] = std::sqrt(1.0F / score - 1.0F); + } + } + // mutate() may have unshared a reference-counted column; put the (possibly + // new) mutable instance back into the block. + column_with_type.column = std::move(column); +} + void PaimonRustTableReader::_close_split_reader() { if (!_handles) { return; @@ -550,7 +974,7 @@ Status PaimonRustTableReader::_apply_predicate() { names.reserve(_projected_columns.size()); types.reserve(_projected_columns.size()); for (const auto& col : _projected_columns) { - if (col.is_partition_key) { + if (col.is_partition_key || col.name == kPaimonSearchDistanceColumn) { continue; } names.push_back(col.name); @@ -588,6 +1012,33 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( DORIS_CHECK(block != nullptr); std::unordered_set materialized_indices; materialized_indices.reserve(_projected_columns.size()); + + // PK-vector (ANN) search mode: paimon-rust appends the score field + // (__paimon_search_score) to every batch, and the reader must convert it into + // the distance FE asked for (__paimon_search_score_to_dis) in place, after the + // fill loop. Locate the output position up front: FE only adds the distance + // column when it rewrote the query into a vector search (see + // PushDownVectorTopNIntoPaimonScan), so _score_transform being set implies the + // column is projected. Failing loudly here beats the alternative: the fill loop + // would silently skip the lookup, leaving the column empty while every other + // column holds `rows` -- an unattributable "sizes of columns don't match" + // downstream. + std::optional distance_output_idx; + size_t distance_row_offset = 0; + if (_score_transform.has_value()) { + auto it = _output_name_to_idx.find(kPaimonSearchDistanceColumn); + if (it == _output_name_to_idx.end()) { + return Status::InternalError( + "paimon-rust vector search: block has no '{}' column (FE/BE column name " + "contract broken)", + kPaimonSearchDistanceColumn); + } + distance_output_idx = it->second; + // Must be read BEFORE the fill loop: read_column_from_arrow appends, and the + // column may already hold converted rows from a previous batch of this split. + distance_row_offset = block->get_by_position(it->second).column->size(); + } + { auto columns_guard = block->mutate_columns_scoped(); auto& columns = columns_guard.mutable_columns(); @@ -599,10 +1050,13 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( // Projected column names are FE-normalized to lowercase. // paimon-rust's case_sensitive=false setting also case-folds column // names in the schema output, so exact match works — but tolerate - // mixed-case Rust output by folding here as well. + // mixed-case Rust output by folding here as well. The score field is + // additionally renamed onto the distance column it feeds (see + // arrow_field_to_block_column). + const std::string block_name = arrow_field_to_block_column(field->name()); auto it = _output_name_to_idx.find(field->name()); if (it == _output_name_to_idx.end()) { - it = _output_name_to_idx.find(to_lower(field->name())); + it = _output_name_to_idx.find(block_name); } if (it == _output_name_to_idx.end()) { // Skip columns that are not in the block (e.g. columns dropped by @@ -610,6 +1064,30 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( continue; } const auto output_idx = it->second; + if (distance_output_idx.has_value() && output_idx == *distance_output_idx) { + // Check the Arrow width before the serde reinterprets the buffer. For + // numerics read_column_from_arrow does NOT validate the source type: it + // casts buffers[1] to the *destination* CppType and inserts `rows` of + // them. The destination is pinned to FLOAT by FE + // (PaimonVectorSearch.createDistanceColumn), so + // - a wider field (Float64) yields `rows` garbage f32 values, each half + // of a double's bits. The column still grows by exactly `rows`, so + // the guard below passes and the query returns a wrongly-ordered + // Top-K with no error anywhere. + // - a narrower field (HALF_FLOAT) reads rows*4 bytes out of a + // rows*2 byte buffer, i.e. past its end. + // paimon-rust builds this field as Float32 unconditionally + // (materialize_search_result), so this only fires if that changes -- + // which is exactly when a loud failure beats either outcome above. + if (batch->column(c)->type_id() != arrow::Type::FLOAT) { + return Status::InternalError( + "paimon-rust vector search: arrow field '{}' has type {}, expected " + "float32 -- the block's '{}' column is FLOAT and the arrow buffer " + "is reinterpreted as f32 without conversion", + field->name(), batch->column(c)->type()->ToString(), + kPaimonSearchDistanceColumn); + } + } if (!materialized_indices.emplace(output_idx).second) { return Status::InternalError("paimon-rust returned duplicate column '{}'", field->name()); @@ -625,6 +1103,32 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( } } } + + if (distance_output_idx.has_value()) { + // Guard the two assumptions _convert_score_to_distance depends on, both of + // which fail SILENTLY otherwise -- the block stays well-formed and the query + // returns a wrongly-ordered Top-K with no error anywhere: + // + // - distance_row_offset was captured BEFORE the fill loop. If a refactor + // moves that capture below the loop, the offset already includes this + // batch, so `end = min(offset + rows, size)` collapses to `offset` and + // the conversion loop runs zero times, leaving raw scores in the column. + // Since a score is higher-is-better, an l2 ORDER BY then returns the LEAST + // similar rows. + // - the Arrow batch actually carried the score field. If paimon-rust renames + // it, the fill loop's lookup misses and the column is never appended to. + const size_t size_after = block->get_by_position(*distance_output_idx).column->size(); + if (size_after != distance_row_offset + rows) { + return Status::InternalError( + "paimon-rust vector search: '{}' column grew from {} to {}, expected {} " + "(batch has {} rows) -- either the row offset was captured after the fill " + "loop, or the arrow batch is missing the '{}' field", + kPaimonSearchDistanceColumn, distance_row_offset, size_after, + distance_row_offset + rows, rows, kPaimonRustScoreField); + } + _convert_score_to_distance(block, static_cast(*distance_output_idx), + distance_row_offset, rows, *_score_transform); + } // Partition columns and other projected columns absent from the arrow batch // are back-filled from split metadata / defaults. RETURN_IF_ERROR(_fill_non_arrow_columns(block, rows, materialized_indices)); @@ -804,6 +1308,13 @@ std::vector PaimonRustTableReader::_build_read_columns() const { if (column.is_partition_key) { continue; } + // The PK-vector search distance is reader-produced and auto-appended by + // paimon-rust under its own field name; it is never a projectable table + // column, so exclude it from the projection in every mode (in non-vector + // mode it is simply absent). + if (column.name == kPaimonSearchDistanceColumn) { + continue; + } columns.emplace_back(column.name); } return columns; diff --git a/be/src/format_v2/table/paimon_rust_table_reader.h b/be/src/format_v2/table/paimon_rust_table_reader.h index 0c04f2f1f04d59..35f769adef2e71 100644 --- a/be/src/format_v2/table/paimon_rust_table_reader.h +++ b/be/src/format_v2/table/paimon_rust_table_reader.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "cctz/time_zone.h" #include "common/status.h" #include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" // TVectorMetric #include "runtime/runtime_profile.h" namespace arrow { @@ -61,6 +63,75 @@ class PaimonRustTableReader final : public format::TableReader { PaimonRustTableReader(); ~PaimonRustTableReader() override; + // How paimon-rust encoded the distance into its __paimon_search_score field, and + // hence which transform the reader has to undo before the value lands in the + // __paimon_search_score_to_dis block column. One enumerator per metric Doris can + // push down (v1: l2 / inner_product; cosine is rejected by the FE rule and by + // parse_score_transform, so it deliberately has no enumerator here). + // + // Adding a metric is meant to be compiler-guided: add the enumerator, and the + // switch in _convert_score_to_distance stops compiling (-Wswitch is an error in + // be/src) until its inverse is written. Do not add a `default:` to that switch + // or that guarantee is lost. + enum class ScoreTransform { + // score = 1/(1 + squaredL2). Inverse: sqrt(1/score - 1). + L2, + // score = the inner product itself. Inverse: identity. + InnerProduct, + }; + + // Map an Arrow field name onto the block column name that receives its values. + // + // paimon-rust resolves projections case-insensitively, so the Arrow batch carries + // the table's canonical field casing (e.g. `is_QT`) while block columns are + // FE-normalized to lowercase (`is_qt`). Fold the Rust-side name down so the two match. + // + // The vector-search field is additionally renamed: paimon-rust calls it + // kPaimonRustScoreField ("__paimon_search_score") and puts a score in it, but the + // reader converts that to a distance in place, so FE asks for the column as + // kPaimonSearchDistanceColumn ("__paimon_search_score_to_dis"). The fill loop + // skips Arrow fields it cannot match (they may legitimately be absent from the + // block), so without this alias the rename would not fail loudly -- the distance + // column would just come back empty. + static std::string arrow_field_to_block_column(const std::string& arrow_name); + + // Map a thrift TVectorMetric onto its score transform. Unknown or unsupported + // metrics (including COSINE) are an error rather than a silent fallback: guessing + // would return wrong distances instead of failing the query. Takes the thrift enum + // directly so there is no string round-trip between FE and the ScoreTransform enum. + static Status parse_score_transform(TVectorMetric::type metric, ScoreTransform* transform); + + // Rewrite the reader-produced __paimon_search_score_to_dis column in place, + // turning paimon-rust's score back into the distance the Doris function is + // defined to return (see ScoreTransform for the per-metric formulas). + // + // Only rewrites [row_offset, row_offset + num_rows): block columns are + // append-only and a block is reused across get_block calls, so starting at 0 + // would square-root earlier rows again. Depends on nothing but its arguments, + // hence static and public (also what makes it unit-testable without the + // paimon-rust FFI handles). + // + // A score outside l2's legal (0, 1] range is index corruption, and computing + // the inverse anyway would yield a non-finite distance (see the two sentinels + // below). Such rows are emitted as a NEGATIVE distance instead: negative is + // unreachable for a genuine Euclidean distance, so it cannot be confused with + // a real result, it is totally ordered (unlike NaN), and it stays finite, so + // the coordinator's Top-N comparisons remain well-defined. Rows land at the + // very front of an ASC Top-K -- deliberately: a corrupt index should be + // visible in the result, not averaged into it. + static constexpr float kCorruptScoreAboveOne = -1.0F; + static constexpr float kCorruptScoreNotPositive = -2.0F; + // paimon-rust's lumina kernel computes the squared L2 distance as + // ||a||^2 + ||b||^2 - 2 (vectorized dot), which suffers catastrophic + // cancellation when a ~= b: the three terms nearly cancel and the residual + // can be a small NEGATIVE number, so score = 1/(1+d^2) lands slightly ABOVE 1 + // (e.g. 1.00294). That is a numerical artifact for an ~exact hit, not index + // corruption -- clamp it to distance 0. Only a score clearly past 1 (beyond + // this tolerance) is treated as a genuine corrupt row and sentineled. + static constexpr float kScoreAboveOneTolerance = 1e-2F; + static void _convert_score_to_distance(Block* block, uint32_t block_idx, size_t row_offset, + size_t num_rows, ScoreTransform transform); + Status init(format::TableReadOptions&& options) override; // Each split's Rust read builder applies the batch size stored by the base class // before opening the Arrow reader. @@ -108,9 +179,31 @@ class PaimonRustTableReader final : public format::TableReader { // Fail fast (without filesystem IO) when the schema-json pipeline fields are // missing from the split; the error text points at an FE/BE protocol mismatch. Status _validate_rust_split(const TFileRangeDesc& range) const; + // Validate the external_search_request thrift fields before any use (mirrors + // LanceTableReader::_validate_external_search_request): one pass over __isset + // and value-range checks so the vector build path can use the fields unchecked. + Status _validate_external_search_request() const; + // True when this split carries a PK-vector search payload and the reader must + // open the vector-search read path instead of the normal split read. + static bool _is_vector_mode(const TFileRangeDesc& range); + // Resolve the table root path + schema json and open (or reuse the cached) + // paimon_table handle from them. Shared by the normal-read and vector-search + // build paths; fills _handles->table and _opened_table_key. + Status _open_paimon_table(const TFileRangeDesc& range); // Open (or reuse the cached) paimon_table and build the per-split read // pipeline: read_builder -> projection -> filter -> plan -> arrow reader. Status _open_split_reader(const TFileRangeDesc& range); + // Open the table and build a primary-key vector (ANN) search reader over the + // FE-planned bucket split. Used when the scan range carries a + // TPaimonVectorPayload: the returned Arrow stream is best-first and carries the + // reader-produced distance column (__paimon_search_score_to_dis, converted + // from paimon-rust's __paimon_search_score field). + Status _open_vector_split_reader(const TFileRangeDesc& range); + // Fill the output block from one arrow record batch: arrow columns are + // matched by name (exact then lower-case, v1 semantics) onto fixed projected + // positions, then _fill_non_arrow_columns back-fills the rest. + Status _fill_block_from_record_batch(const std::shared_ptr& batch, + Block* block, size_t rows); // Drop the per-split pipeline (read_builder .. record batch reader), keeping // the cached table handle for the next split of the same table. void _close_split_reader(); @@ -120,11 +213,6 @@ class PaimonRustTableReader final : public format::TableReader { // apply it to the read builder. Best effort: non-convertible conjuncts are // dropped silently and re-applied by the scanner row-level filter. Status _apply_predicate(); - // Fill the output block from one arrow record batch: arrow columns are - // matched by name (exact then lower-case, v1 semantics) onto fixed projected - // positions, then _fill_non_arrow_columns back-fills the rest. - Status _fill_block_from_record_batch(const std::shared_ptr& batch, - Block* block, size_t rows); // Materialize projected columns the arrow batch did not contain: partition // constants, default expressions, or a default/NULL fill as a last resort. Status _fill_non_arrow_columns(Block* block, size_t rows, @@ -169,6 +257,12 @@ class PaimonRustTableReader final : public format::TableReader { std::unordered_map _output_name_to_idx; TFileRangeDesc _current_range; cctz::time_zone _ctz; + // Which score transform the incoming __paimon_search_score field needs undone, + // derived from the FE-supplied metric when the vector reader is built. Empty + // outside vector mode (and only then): an unset or unknown metric fails the + // scan rather than defaulting, since guessing l2 would wrongly sqrt + // inner_product results. + std::optional _score_transform; // EOF belongs to the current split; reset when a new split is prepared. bool _split_eof = false; diff --git a/be/test/format_v2/table/paimon_rust_table_reader_test.cpp b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp index 72094f3e020a69..24691e79e9daab 100644 --- a/be/test/format_v2/table/paimon_rust_table_reader_test.cpp +++ b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include "cctz/time_zone.h" +#include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" @@ -38,6 +40,7 @@ #include "core/field.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" +#include "format/table/paimon_rust_predicate_converter.h" #include "format_v2/column_data.h" #include "gen_cpp/PlanNodes_types.h" #include "runtime/runtime_profile.h" @@ -235,6 +238,238 @@ TEST_F(PaimonRustTableReaderTest, AdaptiveBatchSizeBoundsWideStringReads) { ASSERT_TRUE(reader.close().ok()); } +TEST_F(PaimonRustTableReaderTest, ValidatesVectorSplit) { + PaimonRustTableReader reader; + + auto make_vector_range = [] { + // Vector mode: paimon_split is deliberately an empty string (FE avoids + // serializing the same DataSplit twice) and the payload is the serialized + // BucketVectorSearchSplit. + auto range = make_rust_range(); + range.table_format_params.paimon_params.__set_paimon_split(""); + TPaimonVectorPayload payload; + payload.__set_payload_type(TPaimonVectorPayloadType::BUCKET_SPLIT_BYTES); + payload.__set_bytes("PKVSPLIT-dummy"); + range.table_format_params.paimon_params.__set_vector_payload(payload); + return range; + }; + + // Missing vector_payload: a protocol error, not a fall-through to the empty + // paimon_split check (whose error text would point the reader at the wrong field). + auto range = make_vector_range(); + range.table_format_params.paimon_params.__isset.vector_payload = false; + auto status = reader.TEST_validate_rust_split(range); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("missing vector_payload"), std::string::npos) << status; + + // Empty payload bytes are equally a protocol error. + range = make_vector_range(); + range.table_format_params.paimon_params.vector_payload.bytes.clear(); + status = reader.TEST_validate_rust_split(range); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("missing vector_payload"), std::string::npos) << status; + + // A complete vector range validates cleanly -- including the empty paimon_split, + // which the normal path would reject. + EXPECT_TRUE(reader.TEST_validate_rust_split(make_vector_range()).ok()); +} + +// ---------------------------------------------------------------- score -> distance +// +// paimon-rust encodes the distance as a score per metric (see +// convert_distance_to_score in crates/paimon/src/lumina/reader.rs); the reader has +// to undo exactly that so the column means what the Doris function returns. For l2 +// the score is 1/(1 + SQUARED distance), hence sqrt(1/score - 1). The reference +// points below are exact in binary floating point. + +using ScoreTransform = PaimonRustTableReader::ScoreTransform; +static constexpr auto L2 = ScoreTransform::L2; +static constexpr auto INNER_PRODUCT = ScoreTransform::InnerProduct; + +// A one-column block holding `scores` in a (possibly nullable) FLOAT column, +// standing in for what the arrow fill loop leaves in the distance column. +static Block _score_block(const std::vector& scores, bool nullable) { + DataTypePtr type = std::make_shared(); + if (nullable) { + type = std::make_shared(type); + } + MutableColumnPtr column = type->create_column(); + for (float score : scores) { + column->insert_data(reinterpret_cast(&score), sizeof(float)); + } + Block block; + block.insert(ColumnWithTypeAndName(std::move(column), type, kPaimonSearchDistanceColumn)); + return block; +} + +// The float at `row` of the block's only column, whether nullable or not. +static float _score_at(const Block& block, size_t row) { + const auto& column = block.get_by_position(0).column; + if (const auto* nullable = check_and_get_column(column.get())) { + return assert_cast(nullable->get_nested_column()).get_data()[row]; + } + return assert_cast(*column).get_data()[row]; +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceL2) { + // score 1.0 -> squared 0 -> distance 0 (an exact hit) + // score 0.5 -> squared 1 -> distance 1 + // score 0.2 -> squared 4 -> distance 2 + // + // Keep the score == 1.0 row: it is the legal boundary, sitting immediately next + // to the score > 1 corruption case below, and 1/1 - 1 is exactly 0 in floating + // point. A guard phrased as "sqrt(radicand) if radicand > 0, else sentinel" + // passes every other case here while misreporting exact hits as corrupt. + Block block = _score_block({1.0F, 0.5F, 0.2F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 3, L2); + + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 1)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 2)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceClampsCatastrophicCancellation) { + // The lumina kernel computes squared L2 as ||a||^2 + ||b||^2 - 2, which can + // yield a small negative residual when a ~= b, i.e. score slightly ABOVE 1. That + // is a near-exact hit, not corruption: it clamps to distance 0. Only a score past + // kScoreAboveOneTolerance is sentineled. + Block block = _score_block({1.0F + 1e-4F, 1.0F + 5e-2F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 2, L2); + + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreAboveOne, _score_at(block, 1)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceSkipsRowsBeforeOffset) { + // Block columns are append-only and a block is reused across get_block calls + // when a batch is fully filtered out, so rows converted by an earlier call must + // be left alone. Here rows 0-1 already hold distances; only the two raw scores + // appended after them may be touched. + Block block = _score_block({0.0F, 2.0F, 0.5F, 0.2F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, /*row_offset=*/2, + /*num_rows=*/2, L2); + + // Untouched: converting again would turn distance 2.0 into sqrt(1/2 - 1) garbage. + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 1)); + // Converted. + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 2)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 3)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceNullable) { + // FE declares the score column non-nullable, but the slot decides the actual + // block column, so the nullable shape must work too: null rows stay null and + // their payload is not read. + Block block = _score_block({0.5F, 0.0F, 0.2F}, /*nullable=*/true); + auto nullable = IColumn::mutate(std::move(block.get_by_position(0).column)); + assert_cast(nullable.get())->get_null_map_data()[1] = 1; + block.get_by_position(0).column = std::move(nullable); + + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 3, L2); + + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 2)); + const auto* null_column = + check_and_get_column(block.get_by_position(0).column.get()); + ASSERT_NE(null_column, nullptr); + EXPECT_EQ(1, null_column->get_null_map_data()[1]); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceFlagsOutOfRangeScores) { + // Unreachable for a well-formed index (d >= 0 implies score in (0, 1]), but the + // distances come out of the closed-source lumina kernel unclamped. Computing the + // inverse anyway would give +inf (1/0) and NaN (sqrt of a negative), either of + // which makes the coordinator's Top-N comparisons ill-defined. Each case maps to + // its own negative sentinel: negative is unreachable for a real Euclidean + // distance, stays finite and totally ordered, and tells which way it broke. + Block block = _score_block({0.0F, -1.0F, 1.5F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 3, L2); + + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreNotPositive, _score_at(block, 0)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreNotPositive, _score_at(block, 1)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreAboveOne, _score_at(block, 2)); + for (size_t i = 0; i < 3; ++i) { + const float distance = _score_at(block, i); + EXPECT_TRUE(std::isfinite(distance)) << "row " << i << " = " << distance; + // Must be distinguishable from any genuine distance, which is >= 0. + EXPECT_LT(distance, 0.0F) << "row " << i; + } +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceEmptyBatch) { + // An empty batch must be a no-op rather than an out-of-range write. + Block block = _score_block({0.5F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, /*row_offset=*/1, + /*num_rows=*/0, L2); + EXPECT_FLOAT_EQ(0.5F, _score_at(block, 0)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceInnerProductIsIdentity) { + // For inner_product the score IS the value the Doris function returns, so every + // row must survive untouched -- including negative and out-of-(0,1] values, which + // are legal here (an inner product is unbounded) and must NOT be mistaken for + // the out-of-range l2 scores that get a corruption sentinel. + Block block = _score_block({0.5F, 0.0F, -3.25F, 42.0F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 4, INNER_PRODUCT); + + EXPECT_FLOAT_EQ(0.5F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 1)); + EXPECT_FLOAT_EQ(-3.25F, _score_at(block, 2)); + EXPECT_FLOAT_EQ(42.0F, _score_at(block, 3)); +} + +// ---------------------------------------------------------------- metric parsing + +TEST_F(PaimonRustTableReaderTest, ParseScoreTransformAcceptsSupportedMetrics) { + ScoreTransform transform = INNER_PRODUCT; + ASSERT_TRUE(PaimonRustTableReader::parse_score_transform(TVectorMetric::L2, &transform).ok()); + EXPECT_EQ(L2, transform); + + ASSERT_TRUE(PaimonRustTableReader::parse_score_transform(TVectorMetric::DOT_PRODUCT, &transform) + .ok()); + EXPECT_EQ(INNER_PRODUCT, transform); +} + +TEST_F(PaimonRustTableReaderTest, ParseScoreTransformRejectsUnsupportedMetrics) { + // Must fail rather than fall back to l2: guessing would sqrt an inner product, + // or return a cosine score where a distance was promised. v1 does not push + // cosine down, so seeing it here means FE and BE disagree. + ScoreTransform transform = L2; + for (auto metric : {TVectorMetric::COSINE, TVectorMetric::DEFAULT, TVectorMetric::HAMMING}) { + auto status = PaimonRustTableReader::parse_score_transform(metric, &transform); + EXPECT_FALSE(status.ok()) << "metric " << static_cast(metric) << " must be rejected"; + EXPECT_NE(status.to_string().find("unsupported distance metric"), std::string::npos) + << status; + } +} + +// ------------------------------------------------- arrow field -> block column + +TEST_F(PaimonRustTableReaderTest, ArrowFieldToBlockColumnRenamesScoreField) { + // FE names the output column "..._to_dis" because the reader turns the score + // into a distance, but paimon-rust keeps emitting the Arrow field as + // "__paimon_search_score". The fill loop silently skips Arrow fields it cannot + // match, so if this mapping regressed the distance column would come back + // empty rather than error out -- hence the explicit assertion. + EXPECT_EQ(kPaimonSearchDistanceColumn, + PaimonRustTableReader::arrow_field_to_block_column(kPaimonRustScoreField)); + // paimon-rust may return the canonical (any-case) field name. + EXPECT_EQ(kPaimonSearchDistanceColumn, + PaimonRustTableReader::arrow_field_to_block_column("__Paimon_Search_Score")); +} + +TEST_F(PaimonRustTableReaderTest, ArrowFieldToBlockColumnLowercasesOtherFields) { + // Ordinary table columns are only case-folded: paimon-rust resolves projections + // case-insensitively and returns the table's canonical casing, while block + // columns are FE-normalized to lowercase. + EXPECT_EQ("is_qt", PaimonRustTableReader::arrow_field_to_block_column("is_QT")); + EXPECT_EQ("id", PaimonRustTableReader::arrow_field_to_block_column("id")); + // Not the score field, so it must NOT be rewritten to the distance column. + EXPECT_EQ("__paimon_search_score_extra", + PaimonRustTableReader::arrow_field_to_block_column("__paimon_search_score_extra")); +} + TEST_F(PaimonRustTableReaderTest, TableLevelCountEmitsSyntheticRows) { // COUNT(*) with a table-level row count takes the base-class metadata path: // prepare_split never opens the rust pipeline and get_block emits synthetic rows. diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index f8efad58e80995..e884e42bd1b54d 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -31,6 +31,7 @@ import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VectorType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -239,12 +240,30 @@ private Variant getVariant() { @Override public void unpackArray(List values) { - InternalArray recordArray = record.getArray(idx); + // A Paimon VECTOR is exposed to Doris as ARRAY (PaimonUtil + // .paimonPrimitiveTypeToDorisType), so it lands here too -- but it is a distinct + // type, not an ArrayType subclass, and it has its own physical layout: + // readVectorData carries a 4-byte element count that readArrayData does not. + // Hence both the accessor and the element type must be chosen by type: + // - getVector vs getArray: BinaryRow.getArray on a vector field would decode + // vector bytes as a BinaryArray, i.e. silently wrong values rather than an + // error. (The columnar path special-cases the vector column vector, which + // is why only this cast crashed.) + // - the element type cast: this is where "VectorType cannot be cast to + // ArrayType" came from when reading a vector column through the JNI scanner. + InternalArray recordArray; + DataType elementPaimonType; + if (dataType instanceof VectorType) { + recordArray = record.getVector(idx); + elementPaimonType = ((VectorType) dataType).getElementType(); + } else { + recordArray = record.getArray(idx); + elementPaimonType = ((ArrayType) dataType).getElementType(); + } if (arrayValues == null) { arrayValues = new ArrayList<>(); } ColumnType elementDorisType = dorisType.getChildTypes().get(0); - DataType elementPaimonType = ((ArrayType) dataType).getElementType(); for (int i = 0; i < recordArray.size(); i++) { values.add(reuseColumnValue(arrayValues, i, (DataGetters) recordArray, i, elementDorisType, elementPaimonType)); diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java index 6304bfdabbe662..f474a1c70db0dd 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java @@ -24,6 +24,7 @@ import org.apache.paimon.types.DataType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VectorType; import java.util.ArrayList; import java.util.List; @@ -81,6 +82,16 @@ private static DataType projectRow(DataType tableType, ColumnType requiredType) } private static DataType projectArray(DataType tableType, ColumnType requiredType) { + // Doris maps a Paimon VECTOR column to ARRAY (PaimonUtil.paimonPrimitiveTypeToDorisType), + // so a VECTOR column arrives here with an ARRAY request. The Doris ARRAY is only the + // logical view: the physical read must stay the VECTOR type, whose binary layout + // differs from an array's (readVectorData carries a 4-byte element count that + // readArrayData does not -- see PaimonColumnValue.unpackArray, which dispatches on + // this type being VectorType). Forcing an ArrayType read would make the reader + // decode vector bytes as a BinaryArray, i.e. silently wrong values. + if (tableType instanceof VectorType) { + return tableType; + } if (!(tableType instanceof ArrayType)) { throw incompatibleType(requiredType, tableType); } diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java index be457571f0be87..d2d944b078eef8 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java @@ -44,6 +44,7 @@ import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.types.VarCharType; import org.apache.paimon.types.VariantType; +import org.apache.paimon.types.VectorType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,6 +183,22 @@ public PaimonColumnType visit(ArrayType arrayType) { return paimonColumnType; } + @Override + public PaimonColumnType visit(VectorType vectorType) { + // Same shape as visit(ArrayType): a Paimon VECTOR is exposed to + // Doris as ARRAY, matching PaimonUtil.paimonPrimitiveTypeToDorisType on + // the FE side. DataTypeVisitor declares visit(VectorType) separately from + // visit(ArrayType), so without this override a vector column would fall through + // to defaultMethod() -> Type.UNSUPPORTED, which only logs. + // + // The declared length is dropped because Doris ARRAY has no fixed-size variant; + // Paimon enforces the dimension on write. + PaimonColumnType paimonColumnType = new PaimonColumnType(Type.ARRAY); + ColumnType elementColumnType = fromPaimonType("dummy-element", vectorType.getElementType()); + paimonColumnType.setChildTypes(Collections.singletonList(elementColumnType)); + return paimonColumnType; + } + @Override public PaimonColumnType visit(MultisetType multisetType) { return this.defaultMethod(multisetType); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 978ea0bd303be6..d356f7a8d77d5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -42,6 +42,7 @@ import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccTable; import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -379,6 +380,12 @@ private void setColumnPositionMapping() if (colName.startsWith(Column.GLOBAL_ROWID_COL)) { continue; } + // Reader-produced Paimon PK-vector search distance: a synthetic output column + // filled by the BE reader by name, not present in the table's file schema, + // so it has no Parquet/ORC column position to map. + if (PaimonVectorSearch.SEARCH_DISTANCE_COLUMN.equalsIgnoreCase(colName)) { + continue; + } Integer idx = columnNameMap.get(colName); if (idx == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 7996b06106d8df..4dcdf9d0275e91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -55,7 +55,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.partition.Partition; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.DataTable; @@ -301,12 +300,6 @@ public long fetchRowCount() { // the complete snapshot selected by TimeTravelUtil. return UNKNOWN_ROW_COUNT; } - if (table instanceof PrivilegedFileStoreTable) { - // Preserve SELECT authorization without planning. TimeTravelUtil calls tagManager(), - // which would incorrectly require INSERT permission on the privilege wrapper. - table.newScan(); - table = PaimonTableDecorators.unwrapToFallbackOrBase(table); - } if (options.queryAuthEnabled()) { table.catalogEnvironment().tableQueryAuth(options).auth(null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 8a3d8330203b44..45db5edcc1bab2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -80,6 +80,7 @@ import org.apache.paimon.types.TimestampType; import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VectorType; import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.InternalRowPartitionComputer; @@ -373,6 +374,27 @@ private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataT Type innerType = paimonPrimitiveTypeToDorisType(arrayType.getElementType(), enableVarbinaryMapping, enableTimestampTzMapping); return org.apache.doris.catalog.ArrayType.create(innerType, true); + case VECTOR: + // Paimon's fixed-size dense vector (VECTOR) maps to a plain + // ARRAY, same as case ARRAY above. That is what the rest of the stack already + // expects: l2_distance_approximate / inner_product_approximate are declared + // over ARRAY (L2DistanceApproximate.SIGNATURES), so an UNSUPPORTED + // mapping (the old default-branch behavior) would make the column unusable: + // the distance functions never type-check and the ANN rewrite + // (PushDownVectorTopNIntoPaimonScan) is never even reached. + // + // ARRAY here is Doris's *logical* type; the readers below it do NOT all see a + // plain list, and each needed a VECTOR case of its own: + // - paimon-rust emits Arrow FixedSizeList, not List -- handled in + // DataTypeArraySerDe::read_column_from_arrow. + // - the JNI scanner needs getVector, not getArray, because a vector's binary + // layout differs from an array's -- handled in PaimonColumnValue.unpackArray. + // The declared length is dropped: Doris ARRAY has no fixed-size variant, and + // Paimon enforces the dimension on write. + VectorType vectorType = (VectorType) dataType; + Type vectorElementType = paimonPrimitiveTypeToDorisType(vectorType.getElementType(), + enableVarbinaryMapping, enableTimestampTzMapping); + return org.apache.doris.catalog.ArrayType.create(vectorElementType, true); case MAP: MapType mapType = (MapType) dataType; return new org.apache.doris.catalog.MapType( @@ -439,6 +461,17 @@ public static void updatePaimonColumnUniqueId(Column column, DataType dataType) ArrayType arrayType = (ArrayType) dataType; updatePaimonColumnUniqueId(columns.get(0), arrayType.getElementType()); break; + case VECTOR: + // Mapped to a Doris ARRAY (see paimonPrimitiveTypeToDorisType), so it has the + // same single child column and VectorType delegates collectFieldIds to its + // element exactly as ArrayType does. Falling through to default would leave + // that child's uniqueId at its default instead of the Paimon field id. + // Recursing is safe while VECTOR maps to ARRAY: children.get(0) above is + // unguarded, and the child exists only because Column.createChildrenColumn + // adds a COLUMN_ARRAY_CHILDREN child for any isArrayType(). + VectorType vectorType = (VectorType) dataType; + updatePaimonColumnUniqueId(columns.get(0), vectorType.getElementType()); + break; case MAP: MapType mapType = (MapType) dataType; updatePaimonColumnUniqueId(columns.get(0), mapType.getKeyType()); @@ -511,11 +544,18 @@ public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMap } TNestedField nestedField = new TNestedField(); switch (dataType.getTypeRoot()) { + case VECTOR: case ARRAY: { + // VECTOR shares this branch because paimonPrimitiveTypeToDorisType maps it to + // ARRAY. It must not fall through to the default below: that only sets the + // primitive type and leaves nestedField unset, which would hand BE a field + // declared TPrimitiveType.ARRAY with no item field describing the element. TArrayField listField = new TArrayField(); - org.apache.paimon.types.ArrayType paimonArrayType = (org.apache.paimon.types.ArrayType) dataType; + DataType paimonElementType = dataType.getTypeRoot() == DataTypeRoot.VECTOR + ? ((VectorType) dataType).getElementType() + : ((org.apache.paimon.types.ArrayType) dataType).getElementType(); TFieldPtr fieldPtr = new TFieldPtr(); - fieldPtr.setFieldPtr(getSchemaInfo(paimonArrayType.getElementType(), enableVarbinaryMapping, + fieldPtr.setFieldPtr(getSchemaInfo(paimonElementType, enableVarbinaryMapping, enableTimestampTzMapping)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java index 45b351aa618388..0d25407ed9019c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java @@ -100,6 +100,15 @@ private Predicate doInPredicate(InPredicate predicate) { } String colName = slotRef.getColumnName(); int idx = getFieldIndex(colName); + // Not a column of the Paimon table, so there is no field index to build an IN + // predicate against. Reader-produced synthetic columns land here: the PK-vector + // rewrite turns `dist_fn(vec, q) < x` into a comparison on + // __paimon_search_score_to_dis, which exists only in the scan's output. Returning + // null drops this conjunct from the push-down; Doris still evaluates it itself, so + // the result stays correct -- whereas indexing with -1 threw and killed the query. + if (idx < 0) { + return null; + } DataType dataType = paimonFieldTypes.get(idx); List valueList = new ArrayList<>(); for (int i = 1; i < predicate.getChildren().size(); i++) { @@ -133,6 +142,12 @@ private Predicate binaryExprDesc(Expr dorisExpr) { } String colName = slotRef.getColumnName(); int idx = getFieldIndex(colName); + // Same guard as doInPredicate: a comparison on a column the Paimon table does not + // have (e.g. the reader-produced PK-vector distance column) must be skipped, not + // fatal. Doris still evaluates the conjunct itself. + if (idx < 0) { + return null; + } DataType dataType = paimonFieldTypes.get(idx); Object value = dataType.accept(new PaimonValueConverter(literalExpr)); if (value == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java new file mode 100644 index 00000000000000..c194c5ed303a58 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java @@ -0,0 +1,298 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon.source; + +import org.apache.doris.datasource.paimon.PaimonScanParams; +import org.apache.doris.system.Backend; + +import org.apache.commons.lang3.StringUtils; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; + +import java.util.Collection; +import java.util.Locale; +import java.util.Map; + +/** + * The table-, storage- and backend-level gates of the paimon-rust reader, + * computed once from the table handle and shared by every call site that must + * agree on them: {@code PaimonScanNode}'s per-split gate (both the batch + * reader and the PK-vector search) and the {@code PushDownVectorTopNIntoPaimonScan} + * rewrite's pre-flight via {@code PaimonVectorSearch.vectorScanEligible}. The + * rewrite is one-way (a rewritten PK-vector split has no JNI fallback), so + * both sites must evaluate the same conditions; this class is that single + * definition. + * + *

Covers only what is a property of the table handle, the storage shape or + * the cluster's backends. Everything that depends on the individual split + * (native DataSplit shape, unmaterialized deletion vectors, external-path + * files, the PaimonRustReaderCapabilities whitelist) or on the final scan + * tuple (ORC TIMESTAMP_LTZ, projected VARIANT) stays with the scan node. + */ +final class PaimonRustEligibility { + + // BE opens the table via paimon_table_from_schema_json, which needs the resolved + // TableSchema that only FileStoreTable exposes via schema(). A table that is not a + // FileStoreTable (e.g. a sys table backed by DataSplit) cannot ship a schema JSON, + // so it falls back to JNI rather than sending an incomplete PAIMON_RUST request + // that BE would reject. + final boolean isFileStoreTable; + + // Both sides of a FallbackReadFileStoreTable wrap their splits, and the pinned + // rust decoder rejects a FallbackDataSplit outright ("trailing bytes after + // DataSplit" — it requires full-buffer consumption); the wrapper is gated as a + // whole (any split from it routes to JNI) until the rust ABI represents both + // sides. The per-split FallbackSplit shape is judged separately in PaimonScanNode. + final boolean fallbackReadTable; + + // A renewable REST token reached the shipped properties as a plain value; only + // the table's FileIO type reveals it expires: doInitialize snapshots + // RESTTokenFileIO.validToken().token() into the backend storage properties, + // discarding expireAtMillis and the REST refresh context, so the shipped + // credentials look static — but the pinned rust table reuses one option map with + // no refresh callback, while the JNI RESTTokenFileIO checks expiry before each + // file operation and obtains a replacement token. A queued or long scan that + // crosses the token TTL would start on rust and later fail authentication. + // Null-safe: a table handle whose FileIO is not resolved stays rust-eligible, + // mirroring the CoreOptions null-safety below. + final boolean restTokenTable; + + // query-auth.enabled tables stay on JNI: when catalog authorization succeeds with + // no row filter or column mask, Paimon still leaves an ordinary DataSplit + // (restricted results use QueryAuthSplit and are already handled by the + // nativeSplit gate), so this table shape passes the compound gate — but the + // shipped schema keeps query-auth.enabled=true and the pinned rust ReadBuilder + // rejects every such table (its CoreOptions::ensure_read_authorized fails closed + // because the client cannot enforce the row filter / column masking), turning a + // valid authorized scan into a BE-open failure. Until the authorization result + // can be transported and enforced by the rust ABI, these tables route to JNI. + final boolean queryAuthTable; + + // Partial-update / aggregation tables with deletion vectors: the pinned rust + // read_pk rejects merge-engine=partial-update/aggregation with + // deletion-vectors.merge-on-read=true outright, so that table option routes the + // whole table to JNI. merge-on-read has no Java accessor in the pinned paimon, so + // it is read raw from the schema options map — the one the BE rust reader + // deserializes from the shipped schema JSON, so it cannot diverge from what BE + // sees — with the rust parsing semantics (any case-insensitive "true" is on, + // default false). For the remaining PU/AGG DV tables, per-split materialization + // is judged in PaimonScanNode (splitDvNotMaterialized); Deduplicate stays + // rust-eligible (its read_pk routes uncompacted splits to the KV reader, which + // applies the attached per-file DVs). + final boolean puAggDeletionVectors; + final boolean dvMergeOnRead; + + // deduplicate.ignore-delete=true tables stay on JNI: Java's + // DeduplicateMergeFunction skips retract records when the option is set — + // including old, uncompacted files that still contain them — but the pinned rust + // read_pk does not pass table options into its deduplicate merge: it picks the + // latest row and omits the key when that row is DELETE/UPDATE_BEFORE. An + // uncompacted insert followed by a delete therefore returns the insert through + // JNI but silently disappears through rust. Gate the option until the rust + // merge implements it. + final boolean deduplicateIgnoreDelete; + + // Non-DV merge options the pinned rust read rejects: Java supports + // partial-update.remove-record-on-delete / aggregation.remove-record-on-delete + // and the wider per-field retract matrix, but the rust PartialUpdateConfig / + // AggregationConfig validations return Unsupported for them — and the DV-derived + // gates above only cover deletion-vector tables, so an ordinary non-DV DataSplit + // with one of these options would pass the compound gate and fail during the + // rust merge construction. Mirrors the exact rust key matrix (presence, not + // values) against the same schema options map BE deserializes. + final boolean rustUnsupportedMergeOption; + + // Branch tables can retain persisted modes without a selector: the shipped + // schema options carry scan.mode, and the pinned Rust builders only accept the + // default. + final boolean scanModeNotDefault; + + // The credential provider shape is evaluated by providerModeTranslatable below. + final boolean providerModeTranslatable; + + // Scheme capability gate: the pinned paimon-rust storage dispatcher (io/storage.rs) + // selects the FileIO parser from the table location's URI scheme, and + // libpaimon_c.a compiles in separate COS, OBS, GCS and Azdls parsers besides the + // OSS and S3 ones. Doris normalizes every object store's credentials into the + // AWS_* / use_path_style aliases, which the BE rust bridge translates only into + // the fs.oss.* and s3.* key families — a cosn:// / obs:// / gs:// / abfs:// + // warehouse would reach its scheme's parser without the key family it reads and + // fail the open instead of using JNI. Only the schemes whose property translation + // is implemented and open-tested (s3 / s3a / oss) plus the credential-free hdfs + // and local-filesystem parsers stay rust-eligible. A null location also routes to + // JNI: the rust path needs the paimon_table that only a real location can provide. + final boolean schemeCapabilityVerified; + + // An hdfs:// location is scheme-verified only together with the credential-free + // backend shape: the backend storage properties that ship to BE also carry an + // HDFS catalog's authentication (kerberos principal / keytab, proxy user, HA + // nameservice config), none of which the pinned rust HDFS parser reads — the + // scan would open as the BE process's ambient identity instead of the catalog's + // configured one and fail the access JNI honors. See isRustVerifiedHdfsBackend. + final boolean hdfsBackendVerified; + + // The BE capability negotiation requires every candidate backend to report + // supports_paimon_rust_reader. + final boolean allBackendsRustCapable; + + private PaimonRustEligibility(boolean isFileStoreTable, boolean fallbackReadTable, + boolean restTokenTable, boolean queryAuthTable, boolean puAggDeletionVectors, + boolean dvMergeOnRead, boolean deduplicateIgnoreDelete, boolean rustUnsupportedMergeOption, + boolean scanModeNotDefault, boolean providerModeTranslatable, boolean schemeCapabilityVerified, + boolean hdfsBackendVerified, boolean allBackendsRustCapable) { + this.isFileStoreTable = isFileStoreTable; + this.fallbackReadTable = fallbackReadTable; + this.restTokenTable = restTokenTable; + this.queryAuthTable = queryAuthTable; + this.puAggDeletionVectors = puAggDeletionVectors; + this.dvMergeOnRead = dvMergeOnRead; + this.deduplicateIgnoreDelete = deduplicateIgnoreDelete; + this.rustUnsupportedMergeOption = rustUnsupportedMergeOption; + this.scanModeNotDefault = scanModeNotDefault; + this.providerModeTranslatable = providerModeTranslatable; + this.schemeCapabilityVerified = schemeCapabilityVerified; + this.hdfsBackendVerified = hdfsBackendVerified; + this.allBackendsRustCapable = allBackendsRustCapable; + } + + /** + * Whether the credential provider shape in the backend storage properties can be + * translated to the pinned paimon-rust reader without changing the identity the + * catalog configured. The rust S3 bridge maps static credentials, anonymous access + * (AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS -> s3.anonymous) and assume-role, but + * the remaining provider modes are ambient JVM provider chains (ENV, + * SYSTEM_PROPERTIES, WEB_IDENTITY, CONTAINER, INSTANCE_PROFILE) with no paimon-rust + * equivalent — rust would silently sign with whatever its own ambient chain + * resolves to. Lives here as one of the shared eligibility gates; also called + * directly for the JNI fallback-reason logging. + */ + static boolean providerModeTranslatable(String location, + Map backendStorageProperties) { + String providerType = backendStorageProperties == null + ? null : backendStorageProperties.get("AWS_CREDENTIALS_PROVIDER_TYPE"); + String mode = providerType == null ? "DEFAULT" : providerType.trim().toUpperCase(Locale.ROOT); + if (mode.isEmpty()) { + mode = "DEFAULT"; + } + if (location != null && (location.startsWith("s3://") || location.startsWith("s3a://"))) { + // Java DEFAULT may resolve JVM properties or anonymous credentials; + // Rust's ambient chain is different. Only explicit keys or explicit + // anonymous access can cross this boundary without changing identity. + String accessKey = backendStorageProperties == null + ? null : backendStorageProperties.get("AWS_ACCESS_KEY"); + String secretKey = backendStorageProperties == null + ? null : backendStorageProperties.get("AWS_SECRET_KEY"); + boolean staticKeys = accessKey != null && !accessKey.trim().isEmpty() + && secretKey != null && !secretKey.trim().isEmpty(); + // Hadoop's Simple provider gives static keys precedence over anonymous, + // role and token settings. Rust interprets those settings differently. + boolean conflictingStaticSettings = staticKeys && (mode.equals("ANONYMOUS") + || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_ROLE_ARN")) + || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_TOKEN"))); + return !conflictingStaticSettings + && (mode.equals("ANONYMOUS") || (mode.equals("DEFAULT") && staticKeys)); + } + if (providerType != null) { + // OSS has no anonymous FileIO mode in the pinned Rust dependency. + return mode.equals("DEFAULT") + || (mode.equals("ANONYMOUS") && (location == null + || !location.startsWith("oss://"))); + } + return true; + } + + static PaimonRustEligibility of(Table table, String tableLocation, + Map backendStorageProperties, Collection backends) { + boolean isFileStoreTable = table instanceof FileStoreTable; + boolean fallbackReadTable = table instanceof FallbackReadFileStoreTable; + boolean restTokenTable = false; + boolean queryAuthTable = false; + boolean puAggDeletionVectors = false; + boolean dvMergeOnRead = false; + boolean deduplicateIgnoreDelete = false; + boolean rustUnsupportedMergeOption = false; + if (isFileStoreTable) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + restTokenTable = fileStoreTable.fileIO() instanceof RESTTokenFileIO; + CoreOptions resolvedCoreOptions = fileStoreTable.coreOptions(); + // Null-safe: a table handle whose CoreOptions is not resolved (e.g. some + // wrapper shapes) stays rust-eligible rather than failing here — the rust + // open itself rejects such a table if the option is really set. + if (resolvedCoreOptions != null) { + queryAuthTable = resolvedCoreOptions.queryAuthEnabled(); + CoreOptions.MergeEngine mergeEngine = resolvedCoreOptions.mergeEngine(); + if (resolvedCoreOptions.deletionVectorsEnabled() + && (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE + || mergeEngine == CoreOptions.MergeEngine.AGGREGATE)) { + puAggDeletionVectors = true; + TableSchema dvSchema = fileStoreTable.schema(); + Map dvOptions = dvSchema == null ? null : dvSchema.options(); + String mergeOnRead = dvOptions == null + ? null : dvOptions.get(PaimonScanNode.DELETION_VECTORS_MERGE_ON_READ); + dvMergeOnRead = "true".equalsIgnoreCase(mergeOnRead); + } + if (mergeEngine == CoreOptions.MergeEngine.DEDUPLICATE + && resolvedCoreOptions.ignoreDelete()) { + deduplicateIgnoreDelete = true; + } + if (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE + || mergeEngine == CoreOptions.MergeEngine.AGGREGATE) { + TableSchema mergeSchema = fileStoreTable.schema(); + Map mergeOptions = + mergeSchema == null ? null : mergeSchema.options(); + rustUnsupportedMergeOption = mergeOptions != null + && PaimonScanNode.hasRustUnsupportedMergeOption(mergeOptions, mergeEngine); + } + } + } + // Null-safe on a null schema, mirroring the CoreOptions null-safety above: + // a fallback-shaped table can carry a null schema. + boolean scanModeNotDefault = false; + if (isFileStoreTable && ((FileStoreTable) table).schema() != null) { + TableSchema schema = ((FileStoreTable) table).schema(); + String scanMode = PaimonScanParams.withoutTimeTravelSelectors(schema) + .options().get(CoreOptions.SCAN_MODE.key()); + scanModeNotDefault = scanMode != null && !"default".equalsIgnoreCase(scanMode); + } + boolean schemeCapabilityVerified = PaimonScanNode.isRustVerifiedLocationScheme(tableLocation); + boolean hdfsBackendVerified = !PaimonScanNode.isHdfsLocationScheme(tableLocation) + || PaimonScanNode.isRustVerifiedHdfsBackend(backendStorageProperties); + boolean providerModeTranslatable = providerModeTranslatable( + tableLocation, backendStorageProperties); + boolean allBackendsRustCapable = backends != null && !backends.isEmpty() + && backends.stream().allMatch(Backend::isPaimonRustReaderSupported); + return new PaimonRustEligibility(isFileStoreTable, fallbackReadTable, restTokenTable, + queryAuthTable, puAggDeletionVectors, dvMergeOnRead, deduplicateIgnoreDelete, + rustUnsupportedMergeOption, scanModeNotDefault, providerModeTranslatable, + schemeCapabilityVerified, hdfsBackendVerified, allBackendsRustCapable); + } + + /** Whether the table, storage and backend shapes all admit the paimon-rust reader. */ + boolean eligible() { + return isFileStoreTable && !fallbackReadTable + && !restTokenTable && !queryAuthTable + && !dvMergeOnRead && !deduplicateIgnoreDelete && !rustUnsupportedMergeOption + && !scanModeNotDefault + && providerModeTranslatable && schemeCapabilityVerified && hdfsBackendVerified + && allBackendsRustCapable; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java index d45ff4ac70c3a2..f8516f9c0004b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java @@ -52,23 +52,46 @@ final class PaimonRustReaderCapabilities { private final FileStoreTable table; private final TableSchema schema; private final boolean tableCompatible; + + // Whether this is a primary-key table with the partial-update or aggregation merge + // engine: the engines whose writers historically retained repeated INSERT keys within + // one file, driving the merge reader's unbounded batch retention. + private final boolean puAggPrimaryKeyEngine; private final Map compatibleFileSchemas = new ConcurrentHashMap<>(); PaimonRustReaderCapabilities(FileStoreTable table, TupleDescriptor tuple) { this.table = table; this.schema = table.schema(); this.tableCompatible = schema != null && hasCompatibleSchemaVersion(schema) - && hasFullNestedProjection(tuple) && hasCompatibleMergeEngine(schema) - && hasCompatibleAggregates(schema); + && hasFullNestedProjection(tuple) && hasCompatibleAggregates(schema); + // The merge-engine gate is split-level, not table-level: it exists for the merge + // reader's unbounded losing-batch retention, which only a split that must merge + // can trigger. See canRead. + this.puAggPrimaryKeyEngine = schema != null && hasPuAggPrimaryKeyEngine(schema); } boolean canRead(DataSplit split) { if (!tableCompatible) { return false; } - // The pinned merge reader retains losing input batches until an output batch fills. - // Neither zero deletes nor a small read.batch-size bounds this across multiple files. - if (!schema.primaryKeys().isEmpty() && split.dataFiles().size() > 1) { + // paimon-rust's read_pk dispatch (pk_split_needs_merge / is_fully_materialized_pk_dv): + // a rawConvertible split whose files are all compacted (level != 0) never reaches + // the pinned merge reader. It streams through DataFileReader's raw path instead -- + // one file at a time, each bounded by read.batch-size, with the split's + // deletion-vector factory masking stale rows per file. Both gates below exist + // solely for that merge reader, whose losing-input-batch retention no delete count + // or batch size can bound, so they must not judge a split that never merges. + boolean readsRawWithoutMerge = split.rawConvertible() + && split.dataFiles().stream().allMatch(file -> file.level() != 0); + // Older partial-update/aggregate writers retained repeated INSERT keys within one + // file, so even a single uncompacted file can drive unbounded merge retention. + if (puAggPrimaryKeyEngine && !readsRawWithoutMerge) { + return false; + } + // Across multiple files the same retention accumulates the whole split's inputs + // until an output batch fills. + if (!schema.primaryKeys().isEmpty() && !readsRawWithoutMerge + && split.dataFiles().size() > 1) { return false; } for (DataFileMeta file : split.dataFiles()) { @@ -116,12 +139,11 @@ private static boolean hasCompatibleSchemaVersion(TableSchema schema) { return schema.version() > TableSchema.PAIMON_07_VERSION; } - private static boolean hasCompatibleMergeEngine(TableSchema schema) { + private static boolean hasPuAggPrimaryKeyEngine(TableSchema schema) { CoreOptions.MergeEngine engine = new CoreOptions(schema.options()).mergeEngine(); - // Older partial-update/aggregate writers retained repeated INSERT keys within one - // file. Zero deletes cannot bound Rust's retained losing batches for those producers. - return schema.primaryKeys().isEmpty() - || (engine != CoreOptions.MergeEngine.PARTIAL_UPDATE && engine != CoreOptions.MergeEngine.AGGREGATE); + return !schema.primaryKeys().isEmpty() + && (engine == CoreOptions.MergeEngine.PARTIAL_UPDATE + || engine == CoreOptions.MergeEngine.AGGREGATE); } private static boolean hasIncompatibleEvolution(List oldFields, List newFields) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 425f5abaee345e..1922052aa589f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.paimon.source; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; @@ -53,18 +55,25 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; -import org.apache.doris.system.Backend; import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TExternalSearchQuery; +import org.apache.doris.thrift.TExternalSearchRequest; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TPaimonDeletionFileDesc; import org.apache.doris.thrift.TPaimonFileDesc; import org.apache.doris.thrift.TPaimonReaderType; +import org.apache.doris.thrift.TPaimonVectorPayload; +import org.apache.doris.thrift.TPaimonVectorPayloadType; +import org.apache.doris.thrift.TPaimonVectorSearchOptions; +import org.apache.doris.thrift.TSearchVector; import org.apache.doris.thrift.TTableFormatFileDesc; +import org.apache.doris.thrift.TVectorElementType; +import org.apache.doris.thrift.TVectorMetric; +import org.apache.doris.thrift.TVectorSearchParams; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.paimon.CoreOptions; @@ -72,15 +81,17 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.format.OrcOptions; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataOutputSerializer; import org.apache.paimon.options.Options; +import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; -import org.apache.paimon.rest.RESTTokenFileIO; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.BucketVectorSearchSplit; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.DeletionFile; import org.apache.paimon.table.source.InnerTableScan; @@ -88,6 +99,8 @@ import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.table.source.VectorScan; +import org.apache.paimon.table.source.VectorSearchSplit; import org.apache.paimon.table.source.snapshot.SnapshotReader; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; @@ -95,8 +108,11 @@ import org.apache.paimon.types.MapType; import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -120,6 +136,10 @@ public class PaimonScanNode extends FileQueryScanNode { private static final long COUNT_WITH_PARALLEL_SPLITS = 10000; private static final long MAX_RETAINED_SERIALIZED_TASK_BYTES = 16L * 1024 * 1024; private volatile PaimonRustReaderCapabilities rustReaderCapabilities; + + // Rust-fallback reason combinations already logged for this scan node, so a + // 960-split scan does not repeat the same one-line reason 960 times. + private Set loggedRustFallbackReasons; private long maxRetainedSerializedTaskBytes = MAX_RETAINED_SERIALIZED_TASK_BYTES; // The keys of incremental read params for Paimon SDK private static final String PAIMON_INCREMENTAL_BETWEEN = "incremental-between"; @@ -142,7 +162,8 @@ public class PaimonScanNode extends FileQueryScanNode { private static final String DORIS_SYSTEM_TABLE_TYPE = "doris.system-table-type"; // Same key as the rust CoreOptions DELETION_VECTORS_MERGE_ON_READ_OPTION: paimon // 1.4 exposes no Java accessor for it, so the raw TableSchema option is read. - private static final String DELETION_VECTORS_MERGE_ON_READ = "deletion-vectors.merge-on-read"; + // Package-private: PaimonVectorSearch.vectorScanEligible reads the same key. + static final String DELETION_VECTORS_MERGE_ON_READ = "deletion-vectors.merge-on-read"; private static final List BACKEND_PAIMON_OPTIONS = Arrays.asList( DORIS_ENABLE_JNI_IO_MANAGER, DORIS_JNI_IO_MANAGER_TMP_DIR, @@ -230,6 +251,17 @@ public String toString() { private Map backendPaimonOptions = Collections.emptyMap(); private Table processedTable; + // Primary-key vector (ANN) top-N info pushed down from Nereids. When set, the + // scan is planned as a vector search (index-only): splits are produced by + // Paimon's newBatchVectorSearchBuilder and BE returns __paimon_search_score_to_dis. + private float[] annSortQueryVector = null; + private String annSortColumnName = null; + private long annSortLimit = -1; + // The index's distance metric (TVectorMetric), validated against the distance + // function by the rewrite rule. BE requires it to convert the score column into + // the distance the function is defined to return. + private TVectorMetric annMetric = null; + // The schema information involved in the current query process (including historical schema). protected ConcurrentHashMap currentQuerySchema = new ConcurrentHashMap<>(); private final Map historyTableSchemas = new ConcurrentHashMap<>(); @@ -338,7 +370,36 @@ public void setSource(PaimonSource source) { protected void convertPredicate() { PaimonPredicateConverter paimonPredicateConverter = new PaimonPredicateConverter( processedTable.rowType()); - predicates = paimonPredicateConverter.convertToPaimonExpr(conjuncts); + predicates = paimonPredicateConverter.convertToPaimonExpr(getPaimonConvertibleConjuncts()); + } + + /** + * The conjuncts worth offering to Paimon, i.e. everything except comparisons on the + * reader-produced PK-vector distance column. + * + *

{@code __paimon_search_score_to_dis} is synthesized by the scan, not read from the + * table: the rewrite rule turns {@code dist_fn(vec, q) < x} into a comparison on this + * slot (PushDownVectorTopNIntoPaimonScan), so a filtered ANN query hands the converter a + * column that no Paimon {@code rowType} contains. The converter itself now returns null + * for unknown columns rather than indexing a field list with -1, so this filter is not + * what keeps the query alive -- it states the intent at the layer that knows about the + * synthetic column, mirroring the exclusion already made for the same column in + * {@code FileQueryScanNode.setColumnPositionMapping}. Dropping the conjunct only skips + * the push-down; Doris still evaluates it, so results are unaffected. + */ + private List getPaimonConvertibleConjuncts() { + List convertible = new ArrayList<>(conjuncts.size()); + for (Expr conjunct : conjuncts) { + List slotRefs = new ArrayList<>(); + conjunct.collect(SlotRef.class, slotRefs); + boolean referencesDistance = slotRefs.stream().anyMatch( + slot -> PaimonVectorSearch.SEARCH_DISTANCE_COLUMN + .equalsIgnoreCase(slot.getColumnName())); + if (!referencesDistance) { + convertible.add(conjunct); + } + } + return convertible; } @Override @@ -377,6 +438,116 @@ public void createScanRangeLocations() throws UserException { String serializedPredicate = PaimonUtil.encodeObjectToString(predicates); params.setPaimonPredicate(serializedPredicate); setScanLevelPaimonOptions(); + // Query-wide vector (ANN) params: set once, shared by every split. + if (isVectorSearch()) { + setExternalSearchRequest(); + } + } + + /** + * Build the provider-independent {@link TExternalSearchRequest} for this Paimon + * PK-vector (ANN) scan and attach it to the scan params (field 39). + * + *

Reuses the community {@code TExternalSearchRequest} / {@code TVectorSearchParams} + * thrift so Paimon and Lance share the same logical search request. Paimon-specific + * index tuning (refine_factor, ivf.nprobe, {@code .metric}, ...) travels in the + * Paimon-only {@link TPaimonVectorSearchOptions} (field 5 of the request); the physical + * per-split bucket payload stays in {@link TPaimonVectorPayload} on {@code TPaimonFileDesc}. + * + *

The query vector is encoded as a {@link TSearchVector} with {@code FLOAT32} element + * type (little-endian), matching paimon-rust's f32 index. The metric enum is passed + * straight through from {@code AnnTopNInfo} with no string conversion. + */ + private void setExternalSearchRequest() { + // Encode the f32 query vector into a little-endian TSearchVector binary. + ByteBuffer buf = ByteBuffer.allocate(annSortQueryVector.length * Float.BYTES) + .order(ByteOrder.LITTLE_ENDIAN); + for (float v : annSortQueryVector) { + buf.putFloat(v); + } + TSearchVector searchVec = new TSearchVector() + .setElementType(TVectorElementType.FLOAT32) + .setDimension(annSortQueryVector.length) + .setValues(buf.array()); + + TVectorSearchParams vectorParams = new TVectorSearchParams() + .setColumn(annSortColumnName) + .setQueryVector(searchVec) + .setTopK(annSortLimit) + // limit already folded with offset by the Nereids rule; BE only needs the + // retrieval count, the real offset is applied by the coordinator TopN. + .setOffset(0) + .setMetric(annMetric); + + TExternalSearchRequest request = new TExternalSearchRequest() + .setSchemaVersion(1) + .setSearchQuery(TExternalSearchQuery.vector_search(vectorParams)); + + // Index/search options. These are query-level *overrides* only: paimon-rust + // already merges the table's own options underneath them, so anything sent + // here that merely repeats the table schema is redundant. + try { + CoreOptions coreOptions = CoreOptions.fromMap(source.getPaimonTable().options()); + String vectorColumn = resolvePaimonFieldName(source.getPaimonTable(), annSortColumnName); + Options indexOptions = coreOptions.primaryKeyVectorIndexOptions(vectorColumn); + if (indexOptions != null) { + Map filtered = filterVectorIndexOptions(indexOptions.toMap(), + vectorColumn, coreOptions.primaryKeyVectorIndexType(vectorColumn)); + request.setPaimonOptions(new TPaimonVectorSearchOptions().setOptions(filtered)); + } + } catch (Exception e) { + LOG.warn("Failed to resolve PK-vector index options for column {}", annSortColumnName, e); + } + params.setExternalSearchRequest(request); + } + + /** + * Keep only the keys that actually configure the vector search, dropping the rest of + * the table's options. + * + *

{@code CoreOptions.primaryKeyVectorIndexOptions} seeds its result with + * {@code new Options(toConfiguration().toMap())}, i.e. a copy of every table + * option, before layering the algorithm-specific ones on top. Forwarding that whole + * map is wrong in three ways: + * + *

    + *
  • It crashes. Catalog audit properties such as {@code owner} / {@code createdBy} + * / {@code updatedBy} can carry a Java null value, which a {@code HashMap} accepts + * and thrift's {@code writeString} does not -- the query dies with an NPE inside + * the vector search options' generated writer before this filter existed.
    • + *
  • It is redundant. paimon-rust merges the table's own options underneath the + * query options itself, so the table half of this map is re-sending what the + * reader already has.
    • + *
  • It over-shares. {@code path}, {@code security.hadoop.username} and any future + * credential-ish property have no business travelling to BE on a search request.
    • + *
+ * + *

The retained prefixes mirror what paimon-rust consumes -- {@code ivf.nprobe} + * (vindex reader), the {@code lumina.} family (stripped by prefix), and + * {@code .metric} -- and match the collection rule in + * {@code CoreOptions.primaryKeyVectorAlgorithmOptions}: per-field options excluding the + * {@code pk-vector.} declarations, plus everything namespaced under the algorithm. + * A null algorithm keeps only the field-scoped half rather than guessing a prefix. + */ + @VisibleForTesting + static Map filterVectorIndexOptions(Map options, + String vectorColumn, String algorithm) { + String fieldPrefix = "fields." + vectorColumn + "."; + String pkVectorPrefix = fieldPrefix + "pk-vector."; + String algorithmPrefix = algorithm == null ? null : algorithm + "."; + Map filtered = new HashMap<>(); + for (Map.Entry entry : options.entrySet()) { + String key = entry.getKey(); + if (key == null || entry.getValue() == null) { + continue; + } + boolean fieldScoped = key.startsWith(fieldPrefix) && !key.startsWith(pkVectorPrefix); + boolean algorithmScoped = algorithmPrefix != null && key.startsWith(algorithmPrefix); + if (fieldScoped || algorithmScoped) { + filtered.put(key, entry.getValue()); + } + } + return filtered; } private void setScanLevelPaimonOptions() { @@ -421,7 +592,8 @@ static boolean isRustVerifiedLocationScheme(String location) { // Whether the location is an hdfs:// table (the only HDFS-family scheme in // RUST_VERIFIED_LOCATION_SCHEMES; viewfs / jfs never pass it). - private static boolean isHdfsLocationScheme(String location) { + // Package-private: PaimonVectorSearch.vectorScanEligible checks the same shape. + static boolean isHdfsLocationScheme(String location) { if (location == null) { return false; } @@ -757,14 +929,11 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) // pinned rust decoder rejects outright ("trailing bytes after // DataSplit" — it requires full-buffer consumption), and even a // permissive decode would still lack the second table identity - // needed to honor the fallback-side discriminator. Both sides of a - // FallbackReadFileStoreTable wrap their splits, so the table - // wrapper is gated as a whole (any split from it routes to JNI) - // until the rust ABI represents both sides; the FallbackSplit - // interface also catches a wrapper split regardless of how the - // table was resolved here. - boolean fallbackRead = split instanceof FallbackReadFileStoreTable.FallbackSplit - || processedTable instanceof FallbackReadFileStoreTable; + // needed to honor the fallback-side discriminator. The table-half + // of the gate lives in PaimonRustEligibility.fallbackReadTable; the + // FallbackSplit interface here also catches a wrapper split + // regardless of how the table was resolved. + boolean fallbackSplit = split instanceof FallbackReadFileStoreTable.FallbackSplit; // Serialize the same effective table that planning and the JNI reader use. // Relation options such as t@options('read.batch-size'='1') are applied by // getProcessedTable() (doInitialize caches it in processedTable), and the @@ -775,166 +944,23 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) Table paimonTable = processedTable; FileStoreTable paimonFileStoreTable = paimonTable instanceof FileStoreTable ? (FileStoreTable) paimonTable : null; - // query-auth.enabled tables stay on JNI: when catalog authorization - // succeeds with no row filter or column mask, Paimon still leaves an - // ordinary DataSplit (restricted results use QueryAuthSplit and are - // already handled by the nativeSplit gate above), so this table shape - // passes the compound gate — but the shipped schema keeps - // query-auth.enabled=true and the pinned rust ReadBuilder rejects - // every such table (its CoreOptions::ensure_read_authorized fails - // closed because the client cannot enforce the row filter / column - // masking), turning a valid authorized scan into a BE-open failure. - // Until the authorization result can be transported and enforced by - // the rust ABI, these tables route to JNI. - boolean queryAuthTable = false; - // REST-token tables stay on JNI: doInitialize snapshots - // RESTTokenFileIO.validToken().token() into the backend storage - // properties, discarding expireAtMillis and the REST refresh - // context, so the shipped credentials look static — but the - // pinned rust table reuses one option map with no refresh - // callback, while paimon 1.4.2's JNI RESTTokenFileIO checks - // expiry before each file operation and obtains a replacement - // token. A queued or long scan that crosses the token TTL would - // start on rust and later fail authentication. Gate until the - // rust ABI can refresh and atomically update credentials. - boolean restTokenTable = false; - // Partial-update / aggregation tables with deletion vectors only pass - // the rust reader in the fully materialized shape: the pinned rust - // read_pk rejects merge-engine=partial-update/aggregation with - // deletion-vectors.merge-on-read=true outright, and otherwise requires - // every split to be compacted and known free of retract rows - // (DataSplit::is_fully_materialized_pk_dv). Their ordinary DataSplits - // sail through the compound gate above, so without this check a valid - // Java/JNI scan reaches BE and the rust open fails. Deduplicate stays - // rust-eligible: its read_pk routes uncompacted splits to the KV - // reader, which applies the attached per-file DVs. merge-on-read=true - // is a table option, so the whole table routes to JNI; - // non-materialized splits are gated per split below. - boolean puAggDeletionVectors = false; - boolean dvMergeOnRead = false; - boolean deduplicateIgnoreDelete = false; - boolean rustUnsupportedMergeOption = false; - if (paimonFileStoreTable != null) { - // A renewable REST token reached the shipped properties as a - // plain value; only the table's FileIO type reveals it expires. - // Null-safe: a table handle whose FileIO is not resolved stays - // rust-eligible, mirroring the CoreOptions null-safety below. - restTokenTable = paimonFileStoreTable.fileIO() instanceof RESTTokenFileIO; - CoreOptions resolvedCoreOptions = paimonFileStoreTable.coreOptions(); - // Null-safe: a table handle whose CoreOptions is not resolved - // (e.g. some wrapper shapes) stays rust-eligible rather than - // failing the scan here — the rust open itself rejects such a - // table if the option is really set. - if (resolvedCoreOptions != null) { - queryAuthTable = resolvedCoreOptions.queryAuthEnabled(); - CoreOptions.MergeEngine mergeEngine = resolvedCoreOptions.mergeEngine(); - if (resolvedCoreOptions.deletionVectorsEnabled() - && (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE - || mergeEngine == CoreOptions.MergeEngine.AGGREGATE)) { - puAggDeletionVectors = true; - // The merge-engine and deletion-vectors.enabled checks - // above resolve through the Java CoreOptions accessors, - // which the table builds from this same schema options - // map — the one the BE rust reader deserializes from - // the shipped schema JSON — so they cannot diverge from - // what BE sees. merge-on-read has no Java accessor in - // paimon 1.4, so it is read raw from the map, with the - // rust parsing semantics (any case-insensitive "true" - // is on, default false). - TableSchema dvSchema = paimonFileStoreTable.schema(); - Map dvOptions = dvSchema == null ? null : dvSchema.options(); - String mergeOnRead = dvOptions == null - ? null : dvOptions.get(DELETION_VECTORS_MERGE_ON_READ); - dvMergeOnRead = "true".equalsIgnoreCase(mergeOnRead); - } - // deduplicate.ignore-delete=true tables stay on JNI: - // Java's DeduplicateMergeFunction skips retract records - // when the option is set — including old, uncompacted - // files that still contain them — but the pinned rust - // read_pk does not pass table options into its - // deduplicate merge: it picks the latest row and omits - // the key when that row is DELETE/UPDATE_BEFORE. An - // uncompacted insert followed by a delete therefore - // returns the insert through JNI but silently disappears - // through rust. Gate the option until the rust merge - // implements it. - if (mergeEngine == CoreOptions.MergeEngine.DEDUPLICATE - && resolvedCoreOptions.ignoreDelete()) { - deduplicateIgnoreDelete = true; - } - // Non-DV merge options the pinned rust read rejects: Java - // supports partial-update.remove-record-on-delete / - // aggregation.remove-record-on-delete and the wider - // per-field retract matrix, but the rust - // PartialUpdateConfig / AggregationConfig validations - // return Unsupported for them — and the DV-derived gates - // above only cover deletion-vector tables, so an ordinary - // non-DV DataSplit with one of these options would pass the - // compound gate and fail during the rust merge - // construction. Mirror the exact rust key matrix (presence, - // not values) against the same schema options map BE - // deserializes. - if (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE - || mergeEngine == CoreOptions.MergeEngine.AGGREGATE) { - TableSchema mergeSchema = paimonFileStoreTable.schema(); - Map mergeOptions = - mergeSchema == null ? null : mergeSchema.options(); - rustUnsupportedMergeOption = mergeOptions != null - && hasRustUnsupportedMergeOption(mergeOptions, mergeEngine); - } - } - } - // paimon-rust additionally requires (a) FileScannerV2: the V1 FileScanner + // The table-, storage- and backend-level rust gates are computed once in + // PaimonRustEligibility — the single definition the batch and PK-vector + // branches below share with the PushDownVectorTopNIntoPaimonScan rewrite's + // pre-flight (PaimonVectorSearch.vectorScanEligible) — so both sites agree + // on the same conditions by construction. It is built from processedTable, + // the same effective table serialized below, so relation options such as + // t@options('read.batch-size'='1') reach the gate too. + PaimonRustEligibility rustEligibility = PaimonRustEligibility.of(processedTable, + source.getTableLocation(), backendStorageProperties, + backendPolicy.getBackends()); + // The fallback-read gate halves: the split shape here, the table + // wrapper in rustEligibility.fallbackReadTable. Either routes to JNI. + boolean fallbackRead = fallbackSplit || rustEligibility.fallbackReadTable; + // paimon-rust additionally requires FileScannerV2: the V1 FileScanner // explicitly rejects PAIMON_RUST, so with enable_file_scanner_v2 disabled // the split falls back to JNI instead of encoding a rust request that the - // selected scanner cannot consume, and (b) a FileStoreTable: BE opens the - // table via paimon_table_from_schema_json, which needs the resolved - // TableSchema that only FileStoreTable exposes via schema(). If the table - // is not a FileStoreTable (e.g. a sys table backed by DataSplit), we cannot - // ship a schema JSON, so fall back to CPP / JNI rather than sending an - // incomplete PAIMON_RUST request that BE would reject. - // - // The paimon-rust S3 bridge maps static credentials, anonymous - // access (AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS -> s3.anonymous) - // and assume-role (AWS_ROLE_ARN / AWS_EXTERNAL_ID -> - // s3.assumed.role.*), but the remaining credential-provider modes - // are ambient JVM provider chains (ENV, SYSTEM_PROPERTIES, - // WEB_IDENTITY, CONTAINER, INSTANCE_PROFILE) with no paimon-rust - // equivalent — rust would silently sign with whatever the ambient - // chain resolves to. Gate those modes away from the rust reader - // here so the configured provider is honored via the JNI path. - boolean providerModeTranslatable = true; - String providerType = backendStorageProperties == null - ? null : backendStorageProperties.get("AWS_CREDENTIALS_PROVIDER_TYPE"); - String mode = providerType == null ? "DEFAULT" : providerType.trim().toUpperCase(Locale.ROOT); - if (mode.isEmpty()) { - mode = "DEFAULT"; - } - String location = source.getTableLocation(); - if (location != null && (location.startsWith("s3://") || location.startsWith("s3a://"))) { - // Java DEFAULT may resolve JVM properties or anonymous credentials; - // Rust's ambient chain is different. Only explicit keys or explicit - // anonymous access can cross this boundary without changing identity. - String accessKey = backendStorageProperties == null - ? null : backendStorageProperties.get("AWS_ACCESS_KEY"); - String secretKey = backendStorageProperties == null - ? null : backendStorageProperties.get("AWS_SECRET_KEY"); - boolean staticKeys = accessKey != null && !accessKey.trim().isEmpty() - && secretKey != null && !secretKey.trim().isEmpty(); - // Hadoop's Simple provider gives static keys precedence over anonymous, - // role and token settings. Rust interprets those settings differently. - boolean conflictingStaticSettings = staticKeys && (mode.equals("ANONYMOUS") - || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_ROLE_ARN")) - || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_TOKEN"))); - providerModeTranslatable = !conflictingStaticSettings - && (mode.equals("ANONYMOUS") || (mode.equals("DEFAULT") && staticKeys)); - } else if (providerType != null) { - providerModeTranslatable = mode.equals("DEFAULT") || mode.equals("ANONYMOUS"); - // OSS has no anonymous FileIO mode in the pinned Rust dependency. - if (mode.equals("ANONYMOUS") && location != null && location.startsWith("oss://")) { - providerModeTranslatable = false; - } - } + // selected scanner cannot consume. // Incremental scans (binlog / changelog / delta / diff) must stay // on the JNI path: this wire format carries only an ordinary // DataSplit and the rust reader invokes TableRead::to_arrow, but @@ -982,72 +1008,156 @@ && splitHasOrcFile((DataSplit) split) // rust. Gate until the rust leaf has a Variant Arrow decoder. boolean projectedVariant = desc.getSlots().stream() .anyMatch(slot -> PaimonUtil.containsVariant(slot.getType())); - // Scheme capability gate: the pinned paimon-rust storage - // dispatcher (io/storage.rs) selects the FileIO parser from the - // table location's URI scheme, and libpaimon_c.a compiles in - // separate COS, OBS, GCS and Azdls parsers besides the OSS and S3 - // ones. Doris normalizes every object store's credentials into - // the AWS_* / use_path_style aliases (see the *Properties storage - // classes), which the BE rust bridge translates only into the - // fs.oss.* and s3.* key families — a cosn:// / obs:// / gs:// / - // abfs:// warehouse would reach its scheme's parser without the - // key family it reads (fs.cosn.userinfo.*, fs.obs.*, gcs.*, - // azure.*) and fail the open instead of using JNI. Only the - // schemes whose property translation is implemented and - // open-tested (s3 / s3a / oss, via RUST_VERIFIED_LOCATION_SCHEMES) - // plus the credential-free hdfs and local-filesystem parsers stay - // rust-eligible; every other scheme falls back to JNI. A null - // location also routes to JNI: the rust path needs the - // paimon_table that only a real location can provide (BE rejects - // a split without it). - boolean schemeCapabilityVerified = isRustVerifiedLocationScheme(source.getTableLocation()); - // An hdfs:// location is scheme-verified only together with the credential-free - // backend shape: the backend storage properties that ship to BE also carry an - // HDFS catalog's authentication (kerberos principal / keytab, proxy user, HA - // nameservice config), none of which the pinned rust HDFS parser reads — the - // scan would open as the BE process's ambient identity instead of the - // catalog's configured one and fail the access JNI honors. See - // isRustVerifiedHdfsBackend. - boolean hdfsBackendVerified = !isHdfsLocationScheme(source.getTableLocation()) - || isRustVerifiedHdfsBackend(backendStorageProperties); - // With merge-on-read=true the whole table already routes to JNI (dvMergeOnRead); - // for the remaining partial-update/aggregation DV tables, a split that is + // With merge-on-read=true the whole table already routes to JNI + // (rustEligibility.dvMergeOnRead); for the remaining + // partial-update/aggregation DV tables, a split that is // not fully materialized (uncompacted level-0 data, or retractions not // known to be applied — even a split with no deletion file attached yet) // fails the rust is_fully_materialized_pk_dv guard, so it falls back per // split instead of turning into a BE-open failure. nativeSplit and // !fallbackRead only guard the cast — those splits already route to JNI. - boolean splitDvNotMaterialized = puAggDeletionVectors && !dvMergeOnRead + boolean splitDvNotMaterialized = rustEligibility.puAggDeletionVectors + && !rustEligibility.dvMergeOnRead && nativeSplit && !fallbackRead && !isFullyMaterializedPkDvSplit((DataSplit) split); + boolean vectorSearchSplit = paimonSplit.getVectorPayloadBytes() != null; + // Session, shared table / storage / backend, and scan-shape gates. The + // shared PaimonRustEligibility computation is also the + // PushDownVectorTopNIntoPaimonScan rewrite's pre-flight + // (PaimonVectorSearch.vectorScanEligible): the rewrite is one-way, so both + // sites must agree on the same gates, and the scan node stays authoritative + // — it evaluates the relation-option-processed table, so a t@options + // override the raw-table pre-flight cannot see still rejects here (loudly, + // as the vector path has no JNI fallback). !isIncremental and the + // split-half of the fallback-read gate are the same backstops: an + // incremental scan has no PK-vector semantics (the VectorScan builder has + // no incremental mode), and a FallbackSplit shape cannot ride the vector + // payload either — both fail loudly here rather than silently returning + // snapshot rows for a query that asked for changes. Each gate's value is + // also what the fallback logging below reports. boolean canUseRust = sessionVariable.isEnablePaimonRustReader() - && sessionVariable.enableFileScannerV2 && nativeSplit && !fallbackRead - && !isIncremental && providerModeTranslatable && !queryAuthTable - && !restTokenTable - && !dvMergeOnRead && !splitDvNotMaterialized - && !orcLtzSchema && !projectedVariant && !needsNanosecondReader() && !externalFileSplit - && !deduplicateIgnoreDelete && !rustUnsupportedMergeOption - && schemeCapabilityVerified && hdfsBackendVerified - && paimonFileStoreTable != null; - if (canUseRust) { - // Branch tables can retain persisted modes without a selector. Validate the - // transported options too: the pinned Rust builder only accepts default here. - String scanMode = PaimonScanParams.withoutTimeTravelSelectors(paimonFileStoreTable.schema()) - .options().get(CoreOptions.SCAN_MODE.key()); - canUseRust = scanMode == null || "default".equalsIgnoreCase(scanMode); - } - if (canUseRust) { - // Every candidate can receive this range; older BEs cannot decode reader type 3. - canUseRust = !backendPolicy.getBackends().isEmpty() - && backendPolicy.getBackends().stream().allMatch(Backend::isPaimonRustReaderSupported); - } - if (canUseRust) { - if (rustReaderCapabilities == null) { - rustReaderCapabilities = new PaimonRustReaderCapabilities(paimonFileStoreTable, desc); + && sessionVariable.enableFileScannerV2 + && rustEligibility.eligible() + && !fallbackRead && !isIncremental + && !orcLtzSchema && !projectedVariant && !needsNanosecondReader(); + // The remaining gates judge the DataSplit the BATCH reader opens: the + // native DataSplit shape, unmaterialized partial-update / aggregation + // deletion vectors, external-path files, and the + // PaimonRustReaderCapabilities whitelist. A PK-vector search split is + // never read as a DataSplit (it rides serialized in vector_payload and + // executes paimon-rust's eager Top-K chain, which applies deletion + // vectors and the merge inside the search itself), so it does not + // consult them. + if (!vectorSearchSplit && canUseRust) { + canUseRust = canUseRust && nativeSplit && !splitDvNotMaterialized + && !externalFileSplit; + if (canUseRust) { + if (rustReaderCapabilities == null) { + rustReaderCapabilities = new PaimonRustReaderCapabilities(paimonFileStoreTable, desc); + } + canUseRust = rustReaderCapabilities.canRead((DataSplit) split); } - canUseRust = rustReaderCapabilities.canRead((DataSplit) split); } - if (canUseRust) { + // With the rust reader enabled, a logical split that still routes to JNI was + // rejected by a gate above: make the reason visible instead of a silent + // fallback, so "why is this table not using the rust reader" is answerable + // from the FE log. + if (sessionVariable.isEnablePaimonRustReader() && sessionVariable.enableFileScannerV2 + && !canUseRust) { + List reasons = new ArrayList<>(); + if (!vectorSearchSplit && !nativeSplit) { + reasons.add("non_data_split"); + } + if (fallbackRead) { + reasons.add("fallback_read_table"); + } + if (isIncremental) { + reasons.add("incremental_scan"); + } + if (!rustEligibility.providerModeTranslatable) { + reasons.add("provider_mode_untranslatable"); + } + if (rustEligibility.queryAuthTable) { + reasons.add("query_auth_table"); + } + if (rustEligibility.restTokenTable) { + reasons.add("rest_token_table"); + } + if (rustEligibility.dvMergeOnRead) { + reasons.add("dv_merge_on_read"); + } + if (!vectorSearchSplit && splitDvNotMaterialized) { + reasons.add("split_dv_not_materialized"); + } + if (orcLtzSchema) { + reasons.add("orc_ltz_schema"); + } + if (projectedVariant) { + reasons.add("projected_variant"); + } + if (!vectorSearchSplit && externalFileSplit) { + reasons.add("external_file_split"); + } + if (rustEligibility.deduplicateIgnoreDelete) { + reasons.add("deduplicate_ignore_delete"); + } + if (rustEligibility.rustUnsupportedMergeOption) { + reasons.add("unsupported_merge_option"); + } + if (!rustEligibility.schemeCapabilityVerified) { + reasons.add("unverified_location_scheme"); + } + if (!rustEligibility.hdfsBackendVerified) { + reasons.add("hdfs_backend_credentials"); + } + if (rustEligibility.scanModeNotDefault) { + reasons.add("scan_mode_not_default"); + } + if (!rustEligibility.isFileStoreTable) { + reasons.add("not_file_store_table"); + } + if (!rustEligibility.allBackendsRustCapable) { + reasons.add("backend_without_rust_support"); + } + if (reasons.isEmpty()) { + reasons.add(vectorSearchSplit ? "unknown" : "capability_whitelist"); + } + String splitDesc = split instanceof DataSplit + ? "bucket=" + ((DataSplit) split).bucket() + + " files=" + ((DataSplit) split).dataFiles().size() + : split.getClass().getSimpleName(); + logRustFallbackReasons(splitDesc, String.join(",", reasons)); + } + if (vectorSearchSplit) { + // Primary-key vector (ANN) search: the serialized bucket split rides in + // vector_payload and BE routes to the paimon-rust vector-search read + // path. The vector path is executed exclusively by the rust reader, so + // a payload with !canUseRust is a broken invariant (the rewrite rule + // gates on both switches): falling back to a plain scan would return an + // empty distance column while every other column holds rows, which is + // silent corruption surfacing as an unattributable size mismatch + // downstream. Fail loudly instead. + if (!canUseRust) { + // RuntimeException because the setScanParams override chain cannot + // declare a checked UserException; the existing native branch throws + // RuntimeException for the same reason. + throw new RuntimeException("Paimon PK-vector search split requires the" + + " paimon-rust reader: enable_file_scanner_v2 must be enabled and" + + " the table must be rust-eligible (storage scheme / credentials," + + " merge options, orc / variant shape)"); + } + TPaimonVectorPayload vectorPayload = new TPaimonVectorPayload(); + vectorPayload.setPayloadType(TPaimonVectorPayloadType.BUCKET_SPLIT_BYTES); + vectorPayload.setBytes(paimonSplit.getVectorPayloadBytes()); + fileDesc.setVectorPayload(vectorPayload); + // The DataSplit is already inside vector_payload (BucketVectorSearchSplit + // embeds it) and the vector read path decodes only vector_payload, never + // paimon_split -- so emit an empty one instead of serializing the same + // split twice on the wire. It must still be set (isset) so BE keeps the + // native-paimon branch rather than rerouting to the orc/parquet reader. + fileDesc.setReaderType(TPaimonReaderType.PAIMON_RUST); + fileDesc.setPaimonSplit(""); + } else if (canUseRust) { fileDesc.setReaderType(TPaimonReaderType.PAIMON_RUST); fileDesc.setPaimonSplit(PaimonUtil.encodeDataSplitToString((DataSplit) split)); } else { @@ -1156,6 +1266,13 @@ protected List getDeleteFiles(TFileRangeDesc rangeDesc) { @Override public List getSplits(int numBackends) throws UserException { + // Primary-key vector (ANN) search is planned entirely by Paimon's + // BatchVectorSearchBuilder: one BucketVectorSearchSplit per (partition, + // bucket), serialized and carried to BE, which runs the whole search + // (recall + refine + row lookup + DV) in a single paimon-rust call. + if (isVectorSearch()) { + return getVectorSearchSplits(); + } boolean forceJniScanner = sessionVariable.isForceJniScanner(); // Paimon system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization @@ -1235,7 +1352,8 @@ public List getSplits(int numBackends) throws UserException { } pushDownCountSplits.add(split); pushDownCountSum += count; - } else if (!forceJniScanner && !forceJniForSystemTable && supportNativeReader(optRawFiles)) { + } else if (!forceJniScanner && !forceJniForSystemTable + && supportNativeReader(optRawFiles)) { // Keep raw-file reads ahead of logical JNI/Rust dispatch: only the native file // reader can use footer aggregates for COUNT(col) and MIN/MAX. Logical DataSplits // may require merge/delete semantics and cannot reuse independent file statistics. @@ -1413,6 +1531,87 @@ public Map getIncrReadParams() throws UserException { return paimonScanParams; } + /** + * Plan a primary-key vector (ANN) search. Produces one Doris split per + * (partition, bucket) {@link BucketVectorSearchSplit}, serialized with the + * native "PKVSPLIT"/v1 protocol so paimon-rust on BE can decode it. Planning + * only needs the vector column and the partition-half of the predicate; the + * query vector and data (residual) predicates are handled by BE. + */ + @VisibleForTesting + public List getVectorSearchSplits() throws UserException { + Table paimonTable = getProcessedTable(); + // Only the partition-half of the predicate is pushed into Paimon planning. + // The data (residual) half is left for BE's vector-search filter, and passing + // it here would trip PrimaryKeyVectorScan's DV/merge-on-read pre-filter guard. + PartitionPredicate partitionPredicate = null; + if (predicates != null && !predicates.isEmpty()) { + Pair, List> partSplit = + PartitionPredicate.splitPartitionPredicatesAndDataPredicates( + predicates, paimonTable.rowType(), paimonTable.partitionKeys()); + partitionPredicate = partSplit.getLeft().orElse(null); + } + + // Resolve the actual Paimon field name (case-insensitive); the pushed-down + // column name comes from the Doris column, which may differ in case. + String vectorColumn = resolvePaimonFieldName(paimonTable, annSortColumnName); + + VectorScan.Plan plan; + try { + plan = paimonTable.newBatchVectorSearchBuilder() + .withVectorColumn(vectorColumn) + .withPartitionFilter(partitionPredicate) + .newVectorScan() + .scan(); + } catch (Exception e) { + throw new UserException("Paimon PK-vector search planning failed: " + e.getMessage(), e); + } + + List splits = new ArrayList<>(); + Set partitions = new HashSet<>(); + for (VectorSearchSplit vectorSplit : plan.splits()) { + if (!(vectorSplit instanceof BucketVectorSearchSplit)) { + throw new UserException("Unexpected PK-vector split type: " + + vectorSplit.getClass().getName()); + } + BucketVectorSearchSplit bucketSplit = (BucketVectorSearchSplit) vectorSplit; + // Serialize with Paimon's own DataOutputSerializer: the bytes must be + // exactly BucketVectorSearchSplit.serialize's DataOutputView form + // (magic "PKVSPLIT", version 1, big-endian), which + // paimon_bucket_vector_search_split_deserialize on BE decodes. This + // repository's Paimon no longer ships a BucketVectorSearchSplitSerializer. + byte[] payloadBytes; + try { + DataOutputSerializer output = new DataOutputSerializer(256); + bucketSplit.serialize(output); + payloadBytes = output.getCopyOfBuffer(); + } catch (IOException e) { + throw new UserException("Failed to serialize PK-vector split: " + e.getMessage(), e); + } + DataSplit dataSplit = bucketSplit.dataSplit(); + PaimonSplit split = new PaimonSplit(dataSplit); + split.setVectorPayloadBytes(payloadBytes); + splits.add(split); + partitions.add(dataSplit.partition()); + ++paimonSplitNum; + } + + splits.forEach(s -> s.setTargetSplitSize(sessionVariable.getFileSplitSize() > 0 + ? sessionVariable.getFileSplitSize() : sessionVariable.getMaxSplitSize())); + this.selectedPartitionNum = partitions.size(); + return splits; + } + + /** Resolve a Paimon row-type field name case-insensitively, else return the input. */ + private static String resolvePaimonFieldName(Table paimonTable, String name) { + for (String field : paimonTable.rowType().getFieldNames()) { + if (field.equalsIgnoreCase(name)) { + return field; + } + } + return name; + } + @VisibleForTesting public List getPaimonSplitFromAPI() throws UserException { long startTime = System.currentTimeMillis(); @@ -1788,6 +1987,23 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { sb.append(String.format("%spaimonNativeReadSplits=%d/%d\n", prefix, rawFileSplitNum, (paimonSplitNum + rawFileSplitNum))); + if (annSortQueryVector != null) { + sb.append(prefix).append("annSortQueryVector=") + .append(Arrays.toString(annSortQueryVector)).append("\n"); + } + if (annSortColumnName != null) { + sb.append(prefix).append("annSortColumnName=").append(annSortColumnName).append("\n"); + } + // Worth showing even though it is not a plan choice: it selects the score->distance + // transform the BE reader applies, so an ANN plan that looks right can still return + // wrong distances if this is not what was expected. + if (annMetric != null) { + sb.append(prefix).append("annMetric=").append(annMetric).append("\n"); + } + if (annSortLimit != -1) { + sb.append(prefix).append("annSortLimit=").append(annSortLimit).append("\n"); + } + sb.append(prefix).append("predicatesFromPaimon:"); if (predicates.isEmpty()) { sb.append(" NONE\n"); @@ -2060,4 +2276,63 @@ private Table getProcessedTable() throws UserException { } return finalTable; } + + public void setAnnSortQueryVector(float[] annSortQueryVector) { + this.annSortQueryVector = annSortQueryVector; + } + + public void setAnnSortColumnName(String annSortColumnName) { + this.annSortColumnName = annSortColumnName; + } + + public void setAnnSortLimit(long annSortLimit) { + this.annSortLimit = annSortLimit; + } + + public void setAnnMetric(TVectorMetric annMetric) { + this.annMetric = annMetric; + } + + /** + * Whether this scan is planned as a primary-key vector (ANN) search. Requires + * the query vector, indexed column and retrieval limit to all be present. + * + *

Deliberately does NOT also require the metric, and adding that check would + * make things worse rather than safer. This predicate is a control switch, not a + * query: it decides whether {@link #getSplits} plans bucket-vector splits and + * whether {@code external_search_request} is sent at all. But whether the scan carries + * a distance column was already decided earlier, by + * PushDownVectorTopNIntoPaimonScan, which replaced the distance expression with the + * {@code __paimon_search_score_to_dis} slot and put it in the scan output -- that is + * not undoable here. Returning false on a missing metric would therefore plan a + * plain scan whose projection excludes that column, leaving it empty while every other + * column holds num_rows: silent corruption surfacing downstream as an unattributable + * size mismatch. + * + *

The check is also unreachable: the only AnnTopNInfo construction site passes a + * {@code "l2"}/{@code "inner_product"} literal and bails out (no rewrite at all) + * when the index metric disagrees. And if it ever did leak through, BE already + * rejects an unset or unsupported metric with an explicit error -- a loud failure, + * which is strictly better than a silently degraded plan. + */ + /** + * Log why a logical paimon split routed to JNI although the rust reader was enabled. + * One line per distinct reason combination per scan node, so a 960-split scan whose + * splits fail the same gate logs once, while a scan with differently-shaped splits + * (e.g. per-split DV materialization) logs each distinct shape once. + */ + private void logRustFallbackReasons(String splitDesc, String reasons) { + if (loggedRustFallbackReasons == null) { + loggedRustFallbackReasons = new HashSet<>(); + } + if (loggedRustFallbackReasons.add(reasons)) { + LOG.info("paimon-rust reader enabled but a logical paimon split routed to JNI:" + + " table={} split={} reasons=[{}]", + source == null ? "unknown" : source.getTableLocation(), splitDesc, reasons); + } + } + + public boolean isVectorSearch() { + return annSortQueryVector != null && annSortColumnName != null && annSortLimit >= 0; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java index 4a8808517b2176..b130fdc726ad0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java @@ -42,6 +42,10 @@ public class PaimonSplit extends FileSplit { private Optional optRowCount = Optional.empty(); private Optional schemaId = Optional.empty(); private Map paimonPartitionValues = null; + // Serialized BucketVectorSearchSplit (magic "PKVSPLIT", version 1, big-endian) for a + // primary-key vector (ANN) search split. Non-null only in vector mode; BE routes to the + // paimon-rust vector-search read path when this payload is present. + private byte[] vectorPayloadBytes = null; /** * Constructor for Paimon splits. @@ -137,6 +141,14 @@ public Map getPaimonPartitionValues() { return paimonPartitionValues; } + public void setVectorPayloadBytes(byte[] vectorPayloadBytes) { + this.vectorPayloadBytes = vectorPayloadBytes; + } + + public byte[] getVectorPayloadBytes() { + return vectorPayloadBytes; + } + public static class PaimonSplitCreator implements SplitCreator { static final PaimonSplitCreator DEFAULT = new PaimonSplitCreator(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java new file mode 100644 index 00000000000000..dccb839f0ed661 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon.source; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Type; +import org.apache.doris.system.Backend; + +import org.apache.paimon.table.Table; + +import java.util.Collection; +import java.util.Map; + +/** + * Shared definition of the reader-produced primary-key vector (ANN) search output + * column for the index-only (Approach B) search path. + * + *

The column holds a distance, not paimon-rust's score. paimon-rust emits + * an Arrow field named {@code __paimon_search_score} carrying a higher-is-better + * score, and the BE PaimonRustTableReader inverts that transform in place while + * filling the block, so the value Doris sees is exactly what + * {@code l2_distance_approximate} / {@code inner_product_approximate} are defined + * to return. + * + *

That is why the Doris-side name is {@link #SEARCH_DISTANCE_COLUMN} + * ({@code __paimon_search_score_to_dis}) and not the Arrow name: the two differ in + * meaning, so they must not share a name. Keeping them distinct also leaves + * {@code __paimon_search_score} free should a later version want to expose the raw + * score as its own column. BE bridges the two names explicitly when filling the + * block; the Arrow-side name stays owned by paimon-rust. Unlike Doris hidden columns + * this one deliberately does not carry the {@code __DORIS_} prefix. + * + *

Because the column is reader-produced (not a real Paimon table column), its + * slot must still carry a non-null {@link Column} so it survives + * {@code FileQueryScanNode.initSchemaParams}, but it is excluded from the Parquet/ORC + * column-position mapping (see {@code FileQueryScanNode.setColumnPositionMapping}). + */ +public final class PaimonVectorSearch { + + /** + * Doris-side name of the reader-produced distance column. Must match + * {@code kPaimonSearchDistanceColumn} in the BE reader, which maps + * paimon-rust's {@code __paimon_search_score} Arrow field onto it. + */ + public static final String SEARCH_DISTANCE_COLUMN = "__paimon_search_score_to_dis"; + + private static final Column DISTANCE_COLUMN = createDistanceColumn(); + + private PaimonVectorSearch() {} + + /** Shared, immutable distance column instance; do not mutate. */ + public static Column getDistanceColumn() { + return DISTANCE_COLUMN; + } + + private static Column createDistanceColumn() { + // Non-nullable FLOAT, matching the distance slot produced by the rewrite rule. + Column column = new Column(SEARCH_DISTANCE_COLUMN, Type.FLOAT, false, + "Paimon primary-key vector search distance (reader-produced)"); + column.setIsVisible(false); + return column; + } + + /** + * Whether a PK-vector (ANN) top-N on this table can execute on the paimon-rust + * vector search path. This is the single definition shared by the + * {@code PushDownVectorTopNIntoPaimonScan} rewrite's pre-flight and + * {@code PaimonScanNode}'s per-split vector gate, because the rewrite is one-way: + * a rewritten vector split has no JNI fallback (the JNI reader ignores the vector + * payload and would return a non-ANN scan with an empty distance column), so the + * scan node can only fail loudly — a gate-failing table must therefore not be + * rewritten at all, leaving the query to run as an ordinary scan that computes the + * distance in Doris. + * + *

Delegates to {@link PaimonRustEligibility}, the single computation of + * the table-, storage- and backend-level gates also consumed by the batch + * rust gate in {@code PaimonScanNode}. Does not cover the + * projection-dependent bounds (ORC TIMESTAMP_LTZ, projected VARIANT): they + * depend on the final scan tuple and stay scan-node-side. The scan node + * evaluates the relation-option-processed table while the rewrite sees the + * raw cached table, so a {@code t@options} override can still make the + * scan node reject a split the pre-flight accepted — the scan node stays + * authoritative and fails loudly. The BE capability negotiation requires + * every candidate backend to report {@code supports_paimon_rust_reader}: + * a PK-vector split is only decodable by a rust-capable BE. + */ + public static boolean vectorScanEligible(Table paimonTable, String tableLocation, + Map backendStorageProperties, Collection backends) { + return PaimonRustEligibility.of(paimonTable, tableLocation, + backendStorageProperties, backends).eligible(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 36f8c4eb59844b..07a8c6c93d3cfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -769,6 +769,14 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla } else if (table.getType() == TableIf.TableType.PAIMON_EXTERNAL_TABLE) { scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv, context.getScanContext()); + // Translate primary-key vector (ANN) top-N info pushed down onto the scan. + fileScan.getAnnTopN().ifPresent(ann -> { + PaimonScanNode paimonScanNode = (PaimonScanNode) scanNode; + paimonScanNode.setAnnSortQueryVector(ann.getQueryVector()); + paimonScanNode.setAnnSortColumnName(ann.getColumnName()); + paimonScanNode.setAnnSortLimit(ann.getLimit()); + paimonScanNode.setAnnMetric(ann.getMetric()); + }); } else if (table instanceof TrinoConnectorExternalTable) { scanNode = new TrinoConnectorScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv, context.getScanContext()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index db96a03926ddc8..3d3ea092e64948 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -145,6 +145,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushDownUnnestInProject; import org.apache.doris.nereids.rules.rewrite.PushDownVectorTopNIntoOlapScan; +import org.apache.doris.nereids.rules.rewrite.PushDownVectorTopNIntoPaimonScan; import org.apache.doris.nereids.rules.rewrite.PushDownVirtualColumnsIntoOlapScan; import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin; import org.apache.doris.nereids.rules.rewrite.PushProjectIntoUnion; @@ -791,6 +792,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown(DistinctAggregateRewriter.INSTANCE), custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new), topDown(new PushDownVectorTopNIntoOlapScan()), + topDown(new PushDownVectorTopNIntoPaimonScan()), topDown(new PushDownVirtualColumnsIntoOlapScan()), topDown(new PushDownMatchProjectionAsVirtualColumn()), topic("score optimize", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 052261dba9690d..4fd8346f5e46cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -342,6 +342,7 @@ public enum RuleType { PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN(RuleTypeClass.REWRITE), PUSH_DOWN_SCORE_TOPN_INTO_OLAP_SCAN(RuleTypeClass.REWRITE), PUSH_DOWN_VECTOR_TOPN_INTO_OLAP_SCAN(RuleTypeClass.REWRITE), + PUSH_DOWN_VECTOR_TOPN_INTO_PAIMON_SCAN(RuleTypeClass.REWRITE), CHECK_SCORE_USAGE(RuleTypeClass.REWRITE), OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE), PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java index c53e07d86c973e..dc6acdd0e77a14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -44,7 +44,8 @@ public Rule build() { fileScan.getTableSnapshot(), fileScan.getOperativeSlots(), fileScan.getScanParams(), - fileScan.getRelationSnapshot()) + fileScan.getRelationSnapshot(), + fileScan.getAnnTopN()) ).toRule(RuleType.LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java new file mode 100644 index 00000000000000..14eb875d331ffe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.Env; +import org.apache.doris.datasource.credentials.CredentialUtils; +import org.apache.doris.datasource.credentials.VendedCredentialsFactory; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TVectorMetric; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Push a primary-key vector (ANN) top-N down into the Paimon scan (index-only). + * + *

Recognizes {@code TopN[order by dis] -> Project[dis := dist_fn(vec, q)] -> [Filter] -> PaimonScan} + * where {@code dist_fn} is {@code l2_distance_approximate} (ASC) or + * {@code inner_product_approximate} (DESC) and the Paimon table has a PK-vector + * index on {@code vec} with a matching distance metric. + * + *

The rewrite mirrors the internal-table ANN path (see + * {@code PushDownVectorTopNIntoOlapScan}): the scan is told to produce a + * reader-generated {@code __paimon_search_score_to_dis} column via the PK-vector index, and + * every occurrence of {@code dist_fn(vec, q)} is replaced by that column's slot. The + * order key slot keeps its identity, so the surrounding TopN merges/limits unchanged. + * The distance always comes from the index, never from the raw vector. + * + *

No arithmetic is emitted here even though paimon-rust reports a score rather than + * a distance: the BE reader undoes the metric-dependent score transform in place while + * filling the block, exactly as the internal table sqrt's faiss's squared distance inside + * the index layer. So the column arrives already holding what {@code dist_fn} would + * return, and the plan stays a bare slot reference -- which also keeps {@code dis} + * non-nullable (an expression over the score would not be, since {@code Divide} and + * {@code Sqrt} are always nullable) and makes predicates like + * {@code dist_fn(vec, q) < 10} correct without any direction flip. + * + *

Being index-only is just an optimization here, not a precondition: if nothing + * else references the raw vector column it is left unreferenced and dropped by column + * pruning (pure index-only read); if the query still needs it ({@code SELECT vec} or a + * predicate on {@code vec}), it stays a normal projected column and the BE vector + * reader materializes it alongside the score. Either way ANN is applied -- the rewrite + * never falls back merely because the vector is read. + */ +public class PushDownVectorTopNIntoPaimonScan implements RewriteRuleFactory { + + @Override + public List buildRules() { + return ImmutableList.of( + logicalTopN(logicalProject(logicalFileScan())).when(t -> t.getOrderKeys().size() == 1).then(topN -> { + LogicalProject project = topN.child(); + LogicalFileScan scan = project.child(); + return pushDown(topN, project, scan, Optional.empty()); + }).toRule(RuleType.PUSH_DOWN_VECTOR_TOPN_INTO_PAIMON_SCAN), + logicalTopN(logicalProject(logicalFilter(logicalFileScan()))) + .when(t -> t.getOrderKeys().size() == 1).then(topN -> { + LogicalProject> project = topN.child(); + LogicalFilter filter = project.child(); + LogicalFileScan scan = filter.child(); + return pushDown(topN, project, scan, Optional.of(filter)); + }).toRule(RuleType.PUSH_DOWN_VECTOR_TOPN_INTO_PAIMON_SCAN) + ); + } + + private Plan pushDown( + LogicalTopN topN, + LogicalProject project, + LogicalFileScan scan, + Optional> optionalFilter) { + // Only Paimon external tables carry PK-vector indexes. + if (!(scan.getTable() instanceof PaimonExternalTable)) { + return null; + } + + // The index-only search path is executed exclusively by the BE paimon-rust reader, + // which only FileScannerV2 can select (the V1 FileScanner explicitly rejects + // PAIMON_RUST). If the rust reader or the V2 scanner is disabled, the split would + // fall through to the JNI reader, which ignores the vector payload and would + // return a non-ANN scan with an empty distance column. Fall back to a normal + // scan then. + ConnectContext ctx = ConnectContext.get(); + if (ctx == null || ctx.getSessionVariable() == null + || !ctx.getSessionVariable().isEnablePaimonRustReader() + || !ctx.getSessionVariable().enableFileScannerV2) { + return null; + } + + // An incremental scan (t@incr / BETWEEN) must not be rewritten into a + // PK-vector search: the VectorScan builder has no incremental mode, so a + // vector payload would silently return snapshot rows for a query that asked + // for changes. Keeping the query un-rewritten runs it as the ordinary + // incremental scan the user requested, with the distance computed in Doris. + // (The scan node's vector gate keeps !isIncremental as the authoritative + // backstop for any path that produces a vector split regardless.) + if (scan.getScanParams().isPresent() && scan.getScanParams().get().incrementalRead()) { + return null; + } + + // The order key must be a SlotReference produced by a Project alias. + Expression orderKey = topN.getOrderKeys().get(0).getExpr(); + if (!(orderKey instanceof SlotReference)) { + return null; + } + SlotReference keySlot = (SlotReference) orderKey; + Alias orderKeyAlias = null; + Expression orderKeyExpr = null; + for (NamedExpression projection : project.getProjects()) { + if (projection.toSlot().equals(keySlot) && projection instanceof Alias) { + orderKeyAlias = (Alias) projection; + orderKeyExpr = orderKeyAlias.child(); + break; + } + } + if (orderKeyExpr == null) { + return null; + } + + // l2 must sort ascending (smaller distance = closer); inner_product descending. + boolean isAsc = topN.getOrderKeys().get(0).isAsc(); + boolean l2Dist = orderKeyExpr instanceof L2DistanceApproximate; + boolean innerProduct = orderKeyExpr instanceof InnerProductApproximate; + if (!(l2Dist && isAsc) && !(innerProduct && !isAsc)) { + return null; + } + + Expression left; + Expression right; + if (l2Dist) { + left = ((L2DistanceApproximate) orderKeyExpr).left(); + right = ((L2DistanceApproximate) orderKeyExpr).right(); + } else { + left = ((InnerProductApproximate) orderKeyExpr).left(); + right = ((InnerProductApproximate) orderKeyExpr).right(); + } + while (left instanceof Cast) { + left = ((Cast) left).child(); + } + if (!(left instanceof SlotReference && right.isConstant() && right instanceof ArrayLiteral)) { + return null; + } + SlotReference vectorSlot = (SlotReference) left; + if (!vectorSlot.getOriginalColumn().isPresent() || !vectorSlot.getOriginalTable().isPresent()) { + return null; + } + String annColumnName = vectorSlot.getOriginalColumn().get().getName(); + + // Validate the PK-vector index and metric on the Paimon table. + PaimonExternalTable paimonTable = (PaimonExternalTable) scan.getTable(); + CoreOptions coreOptions; + try { + Table basePaimonTable = paimonTable.getPaimonTable(Optional.empty()); + coreOptions = CoreOptions.fromMap(basePaimonTable.options()); + } catch (Throwable t) { + return null; + } + if (!coreOptions.primaryKeyVectorIndexEnabled()) { + return null; + } + boolean columnIndexed = coreOptions.primaryKeyVectorIndexColumns().stream() + .anyMatch(c -> c.equalsIgnoreCase(annColumnName)); + if (!columnIndexed) { + return null; + } + // v1 supports only l2 / inner_product; the function must match the index metric. + String metric = coreOptions.primaryKeyVectorDistanceMetric(annColumnName); + String expectedMetric = l2Dist ? "l2" : "inner_product"; + if (!expectedMetric.equals(metric)) { + return null; + } + // The thrift-level enum the AnnTopNInfo will carry. expectedMetric is kept as a + // string for the comparison above (coreOptions returns a string), but the + // AnnTopNInfo / wire contract uses TVectorMetric, so map once here. DOT_PRODUCT + // is the thrift enum that paimon-rust's "inner_product" score transform maps to + // (see the BE reader's parse_score_transform). + TVectorMetric expectedMetricEnum = l2Dist ? TVectorMetric.L2 : TVectorMetric.DOT_PRODUCT; + + // Pre-flight the rust eligibility the scan node will gate on. The rewrite is + // one-way: a PK-vector split has no JNI fallback (the JNI reader ignores the + // vector payload and would return a non-ANN scan with an empty distance + // column), so the scan node can only fail loudly — a table whose gate would + // fail (credential provider shape, REST-token IO, query-auth, merge-option + // matrix, deletion-vector merge-on-read, scan-mode, storage scheme or HDFS + // credentials, no rust-capable backend) must not be rewritten at all. Without + // the rewrite the query still runs, computing the distance in Doris. The + // projection-dependent bounds (ORC LTZ, projected variant) need the final scan + // tuple and stay scan-node-side, and the scan node re-evaluates on the + // relation-option-processed table, so it stays authoritative for t@options + // overrides this raw-table pre-flight cannot see. + try { + Table baseTable = paimonTable.getPaimonTable(Optional.empty()); + PaimonExternalCatalog catalog = (PaimonExternalCatalog) paimonTable.getCatalog(); + Map storagePropertiesMap = + VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( + catalog.getCatalogProperty().getMetastoreProperties(), + catalog.getCatalogProperty().getStoragePropertiesMap(), + baseTable); + Map backendStorageProperties = + CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); + String tableLocation = baseTable instanceof FileStoreTable + ? ((FileStoreTable) baseTable).location().toString() + : baseTable.options().get("path"); + if (!PaimonVectorSearch.vectorScanEligible(baseTable, tableLocation, + backendStorageProperties, + Env.getCurrentSystemInfo().getAllClusterBackends(true))) { + return null; + } + } catch (Throwable t) { + return null; + } + // NOTE: the FE-side data-predicate interception is intentionally not done. The + // data predicate is handed in full to the BE via the vector search builder's + // residual filter, where the Rust candidate search stage applies it together + // with deletion vectors. If some table shape cannot be handled correctly, it is + // Rust's responsibility to reject it or return correct data, not the planner's. + + // paimon-rust's vector index operates on f32, so carry the query as floats + // rather than List: it avoids a double→float narrowing on the wire + // and lets PaimonScanNode encode straight into a FLOAT32 TSearchVector. + List vectorLiterals = ((ArrayLiteral) right).getValue(); + float[] annQueryVector = new float[vectorLiterals.size()]; + for (int i = 0; i < vectorLiterals.size(); i++) { + annQueryVector[i] = (float) vectorLiterals.get(i).getDouble(); + } + + // Build the reader-produced distance column, matched by name downstream (BE fills + // any block column named __paimon_search_score_to_dis, bridging it from + // paimon-rust's __paimon_search_score Arrow field). It holds the metric-native + // distance, not the score: the BE reader converts it in place while filling the + // block, so nothing has to be computed here. It is backed by a synthetic hidden + // Column so the SlotDescriptor keeps a non-null column through + // FileQueryScanNode.initSchemaParams; the column is excluded from the + // Parquet/ORC column-position mapping on the scan node. + SlotReference distanceSlot = SlotReference.fromColumn( + StatementScopeIdGenerator.newExprId(), scan.getTable(), + PaimonVectorSearch.getDistanceColumn(), scan.qualified()); + + // Reuse the index-produced value everywhere the same distance expression + // appears, mirroring PushDownVectorTopNIntoOlapScan: + // - orderKeyAlias -> matches the `dis` projection at its Alias root, so that + // item is rebuilt as `dis := ` in one step. + // - orderKeyExpr -> matches the bare `dist_fn(vec, q)` wherever else it + // occurs (filter conjuncts, other projections), so those reuse the index + // value instead of recomputing from the raw vector. + // Both keys map to the same slot, so `dis` stays numerically identical to Doris + // semantics on every path. Note ExpressionUtils.replace rewrites top-down and + // short-circuits on a hit, and Alias.withChildren preserves the ExprId/name, so + // the alias identity the TopN references is never disturbed. + Map replaceMap = Maps.newHashMap(); + replaceMap.put(orderKeyAlias, distanceSlot); + replaceMap.put(orderKeyExpr, distanceSlot); + + Plan plan = scan.withVectorTopN( + distanceSlot, + new AnnTopNInfo(annQueryVector, annColumnName, + topN.getLimit() + topN.getOffset(), expectedMetricEnum)); + if (optionalFilter.isPresent()) { + LogicalFilter filter = optionalFilter.get(); + Set newConjuncts = ExpressionUtils.replace(filter.getConjuncts(), replaceMap); + plan = filter.withConjunctsAndChild(newConjuncts, plan); + } + + List newProjections = ExpressionUtils + .replaceNamedExpressions(project.getProjects(), replaceMap); + LogicalProject newProject = project.withProjectsAndChild(newProjections, plan); + return topN.withChildren(newProject); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AnnTopNInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AnnTopNInfo.java new file mode 100644 index 00000000000000..b4aafc332d1169 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AnnTopNInfo.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.thrift.TVectorMetric; + +import java.util.Arrays; +import java.util.Objects; + +/** + * Primary-key vector (ANN) top-N request pushed down into a file scan (Paimon). + * + *

Groups the query vector, the indexed vector column name, the retrieval + * limit (already folded as {@code user_limit + user_offset}) and the distance + * metric so they travel together through the logical/physical scan plumbing + * instead of as four loose parameters. Immutable. + * + *

{@code metric} is the index's distance metric, validated by the rewrite rule + * to match the distance function ({@code L2} or {@code DOT_PRODUCT}; v1 has no + * cosine). The BE needs it because the reader-produced score column carries a + * metric-dependent transform that it undoes in place, so an unset or wrong metric + * silently corrupts the returned distances. + * + *

{@code queryVector} is {@code float[]} because paimon-rust's vector index + * operates on f32; carrying the query as floats (rather than {@code List}) + * avoids a double→float narrowing on the wire and lets + * {@code PaimonScanNode} encode it straight into a {@link + * org.apache.doris.thrift.TSearchVector} with {@code FLOAT32} element type. + */ +public class AnnTopNInfo { + private final float[] queryVector; + private final String columnName; + private final long limit; + private final TVectorMetric metric; + + /** + * Constructor. {@code metric} must be non-null; see the field assignment below. + * Defensive-copies {@code queryVector} so later mutation by the caller cannot + * corrupt the value that travels through the plan. + */ + public AnnTopNInfo(float[] queryVector, String columnName, long limit, TVectorMetric metric) { + this.queryVector = queryVector == null ? null : queryVector.clone(); + this.columnName = columnName; + this.limit = limit; + // Enforced here rather than at the planning sites downstream. By the time this + // object exists the rewrite has already committed to the distance column, so no + // later stage can safely degrade to a non-vector scan (see + // PaimonScanNode.isVectorSearch); a null metric can only be a programming error + // in a future caller, and failing at construction points straight at it. + this.metric = Objects.requireNonNull(metric, "ANN distance metric can not be null"); + } + + public float[] getQueryVector() { + return queryVector == null ? null : queryVector.clone(); + } + + public String getColumnName() { + return columnName; + } + + public long getLimit() { + return limit; + } + + public TVectorMetric getMetric() { + return metric; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AnnTopNInfo that = (AnnTopNInfo) o; + return limit == that.limit + && Arrays.equals(queryVector, that.queryVector) + && Objects.equals(columnName, that.columnName) + && metric == that.metric; + } + + @Override + public int hashCode() { + int result = Objects.hash(columnName, limit, metric); + result = 31 * result + Arrays.hashCode(queryVector); + return result; + } + + @Override + public String toString() { + return "AnnTopNInfo{column=" + columnName + ", limit=" + limit + + ", metric=" + metric + + ", vectorDim=" + (queryVector == null ? 0 : queryVector.length) + "}"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index d19be98b97e8e1..9c8ad170f72a06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -70,6 +71,10 @@ public class LogicalFileScan extends LogicalCatalogRelation implements SupportPr protected final Optional> cachedOutputs; protected final Optional> relationSchema; protected final Optional relationSnapshot; + // Used for primary-key vector (ANN) top-N push down into the Paimon scan. + // Carries the query vector, the indexed vector column name, and the retrieval + // limit (already user_limit + user_offset) down to PaimonScanNode. + protected final Optional annTopN; /** * Constructor for LogicalFileScan. @@ -95,7 +100,8 @@ public LogicalFileScan(RelationId id, ExternalTable table, List qualifie operativeSlots, ImmutableList.of(), tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.empty(), - cachedOutputs, captureRelationSchema(table, scanParams, relationSnapshot), relationSnapshot, ""); + cachedOutputs, captureRelationSchema(table, scanParams, relationSnapshot), + relationSnapshot, Optional.empty(), ""); } /** @@ -122,7 +128,7 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali Optional> cachedSlots, Optional> relationSchema) { this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedSlots, relationSchema, - MvccUtil.getSnapshotFromContext(table), ""); + MvccUtil.getSnapshotFromContext(table), Optional.empty(), ""); } protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, @@ -131,7 +137,8 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali Optional tableSnapshot, Optional scanParams, Optional groupExpression, Optional logicalProperties, Optional> cachedSlots, Optional> relationSchema, - Optional relationSnapshot, String tableAlias) { + Optional relationSnapshot, Optional annTopN, + String tableAlias) { super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, operativeSlots, virtualColumns, groupExpression, logicalProperties, tableAlias); this.selectedPartitions = selectedPartitions; @@ -141,6 +148,21 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali this.cachedOutputs = cachedSlots; this.relationSchema = relationSchema; this.relationSnapshot = relationSnapshot; + this.annTopN = annTopN; + } + /** + * Constructor for LogicalFileScan, kept for subclasses that were not updated with a table alias. + */ + protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + SelectedPartitions selectedPartitions, Collection operativeSlots, + List virtualColumns, Optional tableSample, + Optional tableSnapshot, Optional scanParams, + Optional groupExpression, Optional logicalProperties, + Optional> cachedSlots, Optional> relationSchema, + Optional relationSnapshot, Optional annTopN) { + this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, + scanParams, groupExpression, logicalProperties, cachedSlots, relationSchema, + relationSnapshot, annTopN, ""); } /** @@ -155,7 +177,7 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali Optional relationSnapshot) { this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedSlots, relationSchema, - relationSnapshot, ""); + relationSnapshot, Optional.empty(), ""); } protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, @@ -165,7 +187,7 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali Optional groupExpression, Optional logicalProperties) { this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, Optional.empty(), Optional.empty(), - MvccUtil.getSnapshotFromContext(table), ""); + MvccUtil.getSnapshotFromContext(table), Optional.empty(), ""); } private static SelectedPartitions initialSelectedPartitions( @@ -253,7 +275,7 @@ public LogicalFileScan withGroupExpression(Optional groupExpres return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, Optional.of(getLogicalProperties()), - cachedOutputs, relationSchema, relationSnapshot, tableAlias); + cachedOutputs, relationSchema, relationSnapshot, annTopN, tableAlias); } @Override @@ -262,14 +284,14 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs, - relationSchema, relationSnapshot, tableAlias); + relationSchema, relationSnapshot, annTopN, tableAlias); } public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.of(getLogicalProperties()), - cachedOutputs, relationSchema, relationSnapshot, tableAlias); + cachedOutputs, relationSchema, relationSnapshot, annTopN, tableAlias); } @Override @@ -277,7 +299,7 @@ public LogicalFileScan withRelationId(RelationId relationId) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.empty(), cachedOutputs, - relationSchema, relationSnapshot, tableAlias); + relationSchema, relationSnapshot, annTopN, tableAlias); } @Override @@ -285,7 +307,7 @@ public LogicalFileScan withTableAlias(String tableAlias) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.of(getLogicalProperties()), cachedOutputs, - relationSchema, relationSnapshot, tableAlias); + relationSchema, relationSnapshot, annTopN, tableAlias); } @Override @@ -295,7 +317,8 @@ public R accept(PlanVisitor visitor, C context) { @Override public boolean equals(Object o) { - return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions); + return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions) + && Objects.equals(annTopN, ((LogicalFileScan) o).annTopN); } @Override @@ -308,7 +331,8 @@ protected boolean hasSameScanState(LogicalCatalogRelation other) { && Objects.equals(tableSample, that.tableSample) && hasSameSnapshot(tableSnapshot, that.tableSnapshot) && hasSameScanParams(scanParams, that.scanParams) - && hasSameResolvedSnapshot(relationSnapshot, that.relationSnapshot); + && hasSameResolvedSnapshot(relationSnapshot, that.relationSnapshot) + && Objects.equals(annTopN, that.annTopN); } private static boolean hasSameResolvedSnapshot( @@ -485,18 +509,71 @@ public LogicalFileScan withOperativeSlots(Collection operativeSlots) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, Optional.of(getLogicalProperties()), - cachedOutputs, relationSchema, relationSnapshot, tableAlias); + cachedOutputs, relationSchema, relationSnapshot, annTopN, tableAlias); } public LogicalFileScan withCachedOutput(List cachedOutputs) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, Optional.empty(), Optional.of(cachedOutputs), - relationSchema, relationSnapshot, tableAlias); + relationSchema, relationSnapshot, annTopN, tableAlias); } @Override public List getOperativeSlots() { return operativeSlots; } + + public Optional getAnnTopN() { + return annTopN; + } + + /** + * Push primary-key vector (ANN) top-N info into the Paimon scan. Appends a + * reader-produced {@code __paimon_search_score_to_dis} virtual column to the scan + * output and records the query vector / indexed column / retrieval limit. + * + *

The new output is derived from the OLD output plus the distance slot, rather + * than by letting {@code computeLogicalProperties()} recompute it. That is not an + * optimization -- it is required for correctness. Passing {@code Optional.empty()} + * for logicalProperties routes the output through + * {@link LogicalCatalogRelation#computeOutput()}, which mints a FRESH ExprId for + * every base column ({@code exprIdGenerator.getNextId()}, no per-column reuse + * cache). The caller (PushDownVectorTopNIntoPaimonScan) reparents the EXISTING + * projections onto this new scan, so those projections would then reference slots + * the scan no longer produces -- e.g. project reads {@code id#10005} while the scan + * outputs {@code id#10010}. Verified by the exprId-stability assertion in + * PushDownVectorTopNIntoPaimonScanTest. + * + *

Same shape as {@code LogicalOlapScan.appendVirtualColumnsAndTopN}, for the same + * reason. Because the output is supplied explicitly, {@code cachedOutputs} can be + * forwarded too: it only feeds {@code computeOutput()}, which is now unreachable, + * and dropping it would lose an upstream rewrite. + * + * @param scoreVirtualColumn the {@code __paimon_search_score_to_dis} output column + * @param annTopN the query vector, indexed column name, and retrieval limit + */ + public LogicalFileScan withVectorTopN( + NamedExpression scoreVirtualColumn, + AnnTopNInfo annTopN) { + List mergedVirtualColumns = ImmutableList.builder() + .addAll(virtualColumns) + .add(scoreVirtualColumn) + .build(); + LogicalProperties oldProperties = getLogicalProperties(); + List newOutput = ImmutableList.builder() + .addAll(oldProperties.getOutput()) + .add(scoreVirtualColumn.toSlot()) + .build(); + // The asterisk output is deliberately NOT extended: `select *` must not surface + // an internal distance column. Forwarded as a supplier rather than a value so + // that this rewrite does not force a computation it never reads -- the asterisk + // output only matters during binding, long before this rule runs. + LogicalProperties newProperties = new LogicalProperties( + () -> newOutput, oldProperties::getAsteriskOutput, this::computeDataTrait); + return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, + selectedPartitions, operativeSlots, mergedVirtualColumns, tableSample, tableSnapshot, + scanParams, groupExpression, Optional.of(newProperties), cachedOutputs, + relationSchema, relationSnapshot, Optional.of(annTopN)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java index 4aa42aac96b2e8..d738f1588ace87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java @@ -116,7 +116,7 @@ private LogicalHudiScan(RelationId id, ExternalTable table, List qualifi Optional relationSnapshot, String tableAlias) { super(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs, - relationSchema, relationSnapshot, tableAlias); + relationSchema, relationSnapshot, Optional.empty(), tableAlias); Objects.requireNonNull(scanParams, "scanParams should not null"); Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null"); this.incrementalRelation = incrementalRelation; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index 2696fe6e52a197..bf60d5e71702f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -50,6 +51,8 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { protected final Optional tableSnapshot; protected final Optional scanParams; protected final Optional relationSnapshot; + // Used for primary-key vector (ANN) top-N push down into the Paimon scan. + private final Optional annTopN; /** * Constructor for PhysicalFileScan. @@ -74,7 +77,20 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi Optional scanParams, Optional relationSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, - relationSnapshot); + relationSnapshot, Optional.empty()); + } + + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Collection operativeSlots, + Optional scanParams, Optional relationSnapshot, + Optional annTopN) { + this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, + logicalProperties, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, + relationSnapshot, annTopN); } /** @@ -100,7 +116,19 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi Optional relationSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, Optional.empty()); + } + + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot, + Collection operativeSlots, Optional scanParams, + Optional relationSnapshot, Optional annTopN) { + this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, + logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, + operativeSlots, scanParams, relationSnapshot, annTopN); } /** @@ -124,6 +152,19 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li Optional tableSnapshot, Collection operativeSlots, Optional scanParams, Optional relationSnapshot) { + this(id, type, table, qualifier, distributionSpec, groupExpression, logicalProperties, + selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, + relationSnapshot, Optional.empty()); + } + + protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Collection operativeSlots, + Optional scanParams, Optional relationSnapshot, + Optional annTopN) { super(id, type, table, qualifier, groupExpression, logicalProperties, operativeSlots); this.distributionSpec = distributionSpec; this.selectedPartitions = selectedPartitions; @@ -131,6 +172,7 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li this.tableSnapshot = tableSnapshot; this.scanParams = scanParams; this.relationSnapshot = relationSnapshot; + this.annTopN = annTopN; } protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, @@ -151,6 +193,18 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li Optional tableSample, Optional tableSnapshot, Collection operativeSlots, Optional scanParams, Optional relationSnapshot) { + this(id, type, table, qualifier, distributionSpec, groupExpression, logicalProperties, + physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, + operativeSlots, scanParams, relationSnapshot, Optional.empty()); + } + + protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot, + Collection operativeSlots, Optional scanParams, + Optional relationSnapshot, Optional annTopN) { super(id, type, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics, operativeSlots); this.distributionSpec = distributionSpec; @@ -159,6 +213,7 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li this.tableSnapshot = tableSnapshot; this.scanParams = scanParams; this.relationSnapshot = relationSnapshot; + this.annTopN = annTopN; } public DistributionSpec getDistributionSpec() { @@ -189,6 +244,10 @@ public Optional getRelationSnapshot() { return relationSnapshot; } + public Optional getAnnTopN() { + return annTopN; + } + @Override public String toString() { String rfV2 = ""; @@ -214,7 +273,7 @@ public R accept(PlanVisitor visitor, C context) { public PhysicalFileScan withGroupExpression(Optional groupExpression) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, annTopN); } @Override @@ -222,7 +281,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, annTopN); } @Override @@ -236,7 +295,7 @@ public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physic return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, annTopN); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java index dee8638ccdf2a1..3fbe8d4e559abf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java @@ -42,7 +42,7 @@ public PhysicalLazyMaterializeFileScan(PhysicalFileScan scan, SlotReference rowI Optional.empty(), null, null, scan.getStats(), scan.selectedPartitions, scan.getTableSample(), scan.getTableSnapshot(), scan.getOperativeSlots(), - scan.getScanParams(), scan.getRelationSnapshot()); + scan.getScanParams(), scan.getRelationSnapshot(), scan.getAnnTopN()); this.scan = scan; this.rowId = rowId; this.lazySlots = lazySlots; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 68ecf81c998c08..36f838cfadad1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -3193,6 +3193,7 @@ public Map getForceEagerAggHintMap() { "Use paimon-rust for non-native Paimon reads, higher priority than paimon-cpp"}) private boolean enablePaimonRustReader = false; + @VariableMgr.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"}) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 12e0c8d06f30d8..dcdeff4024dd8b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -54,8 +54,6 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java index bba3e27f3ac5c5..6499347e7ea2db 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java @@ -22,13 +22,10 @@ import com.google.common.collect.ImmutableMap; import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; @@ -124,23 +121,6 @@ public void testUnsupportedTablesDoNotPlan() throws Exception { rowCount(new FallbackReadFileStoreTable(main, other, true))); } - @Test - public void testSelectOnlyPrivilegeWrapper() throws Exception { - FileStoreTable table = newTable(false); - snapshot(table, 1, 10L); - PrivilegeChecker checker = Mockito.mock(PrivilegeChecker.class); - Identifier identifier = Identifier.create("db", "tbl"); - Mockito.doThrow(new IllegalStateException("INSERT is not granted")) - .when(checker).assertCanInsert(identifier); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap(table, checker, identifier); - Assert.assertEquals(10L, rowCount(privileged)); - Mockito.verify(checker).assertCanSelect(identifier); - Mockito.verify(checker, Mockito.never()).assertCanInsert(identifier); - Mockito.doThrow(new IllegalStateException("SELECT is not granted")) - .when(checker).assertCanSelect(identifier); - Assert.assertThrows(IllegalStateException.class, () -> rowCount(privileged)); - } - @Test public void testCatalogQueryAuthorizationWithoutPlanning() throws Exception { FileStoreTable table = Mockito.spy(newTable(false).copy(ImmutableMap.of("query-auth.enabled", "true"))); @@ -175,15 +155,15 @@ private FileStoreTable newTable(boolean primaryKey, Map options) schema.primaryKey("id").option("bucket", "1"); } options.forEach(schema::option); - new SchemaManager(fileIO, path).createTable(schema.build()); + new FileSystemSchemaManager(fileIO, path).createTable(schema.build()); return FileStoreTableFactory.create(fileIO, path); } private void snapshot(FileStoreTable table, long id, long count) throws Exception { // No manifest files exist: accidentally returning to split planning must fail. Snapshot snapshot = new Snapshot(id, 0L, "unused-base", null, "unused-delta", null, - null, null, null, "test", id, Snapshot.CommitKind.APPEND, id * 1000, - count, count, null, null, null, Collections.emptyMap(), null); + null, null, null, "test", null, id, Snapshot.CommitKind.APPEND, id * 1000, + count, count, null, null, null, Collections.emptyMap(), null, null); table.fileIO().mkdirs(table.snapshotManager().snapshotPath(id).getParent()); table.fileIO().overwriteFileUtf8(table.snapshotManager().snapshotPath(id), snapshot.toJson()); table.snapshotManager().commitLatestHint(id); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index b4777273b495f1..55f1c098f1bd99 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -52,6 +52,7 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VectorType; import org.apache.thrift.TDeserializer; import org.apache.thrift.TSerializer; import org.junit.Assert; @@ -666,4 +667,78 @@ private static String repeatedCharacter(char character, int count) { Arrays.fill(characters, character); return new String(characters); } + + // ------------------------------------------------------------------ VECTOR + + /** + * A Paimon VECTOR column must map to ARRAY<element>, not UNSUPPORTED. This is what + * makes the column usable at all: the ANN distance functions are declared over + * ARRAY<FLOAT> (L2DistanceApproximate.SIGNATURES), so an UNSUPPORTED mapping means + * l2_distance_approximate(embedding, [...]) never type-checks and the ANN rewrite + * (PushDownVectorTopNIntoPaimonScan) is never reached -- the whole PK-vector search + * path is unreachable on a real table. + */ + @Test + public void testVectorTypeMapsToArray() { + DataField vectorField = new DataField(1, "embedding", + new VectorType(2048, DataTypes.FLOAT())); + Type type = PaimonUtil.paimonTypeToDorisType(vectorField.type(), true, true); + + Assert.assertTrue("VECTOR must not map to UNSUPPORTED: " + type, type.isArrayType()); + Assert.assertEquals(Type.FLOAT, ((ArrayType) type).getItemType()); + } + + /** + * The element type is carried through rather than hardcoded to FLOAT. VectorType + * permits any of BOOLEAN/TINYINT/SMALLINT/INTEGER/BIGINT/FLOAT/DOUBLE + * (VectorType.isValidElementType), and only FLOAT is exercised by ANN today. + */ + @Test + public void testVectorTypeCarriesElementType() { + Type doubleVec = PaimonUtil.paimonTypeToDorisType( + new VectorType(8, DataTypes.DOUBLE()), true, true); + Assert.assertEquals(Type.DOUBLE, ((ArrayType) doubleVec).getItemType()); + + Type intVec = PaimonUtil.paimonTypeToDorisType( + new VectorType(4, DataTypes.INT()), true, true); + Assert.assertEquals(Type.INT, ((ArrayType) intVec).getItemType()); + } + + /** + * The thrift schema handed to BE must describe the element too. VECTOR shares the + * ARRAY branch of getSchemaInfo for exactly this reason: the default branch only sets + * the primitive type, which would declare TPrimitiveType.ARRAY with no item field. + */ + @Test + public void testVectorTypeSchemaInfoCarriesItemField() { + TField field = PaimonUtil.getSchemaInfo(new VectorType(2048, DataTypes.FLOAT()), true, true); + + Assert.assertEquals(TPrimitiveType.ARRAY, field.getType().getType()); + Assert.assertTrue("VECTOR schema must carry a nested item field, else BE sees an " + + "ARRAY with no element description", field.isSetNestedField()); + Assert.assertEquals(TPrimitiveType.FLOAT, field.getNestedField().getArrayField() + .getItemField().getFieldPtr().getType().getType()); + } + + /** + * The array child column of a VECTOR must receive the Paimon element field id. Guards a + * real hazard rather than a hypothetical one: updatePaimonColumnUniqueId indexes + * children.get(0) directly and only null-checks the list, so recursing into a VECTOR is + * only safe because Column.createChildrenColumn builds a child for any isArrayType(). + */ + @Test + public void testVectorTypeChildColumnGetsElementFieldId() { + DataField elementField = new DataField(7, "element", DataTypes.FLOAT()); + VectorType paimonVectorType = new VectorType(true, 2048, elementField.type()); + DataField vectorField = new DataField(3, "embedding", paimonVectorType); + + Column column = new Column("embedding", + PaimonUtil.paimonTypeToDorisType(vectorField.type(), true, true), true); + PaimonUtil.updatePaimonColumnUniqueId(column, vectorField); + + Assert.assertEquals(3, column.getUniqueId()); + Assert.assertNotNull("a VECTOR-mapped ARRAY must have its array child column, otherwise " + + "updatePaimonColumnUniqueId would throw on children.get(0)", column.getChildren()); + Assert.assertEquals(1, column.getChildren().size()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java index 4e44aa67809e77..0358082c78830f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java @@ -30,9 +30,9 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.sink.BatchTableCommit; @@ -461,7 +461,7 @@ private FileStoreTable table(Schema.Builder schema, Map options) .option("file.format", "parquet").option("deletion-vectors.enabled", "false") .option("read.batch-size", "16").option("scan.manifest.parallelism", "1"); options.forEach(schema::option); - new SchemaManager(fileIO, path).createTable(schema.build()); + new FileSystemSchemaManager(fileIO, path).createTable(schema.build()); return FileStoreTableFactory.create(fileIO, path); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java index 1ed35e94007985..4c436852bdfee0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java @@ -25,9 +25,9 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.sink.BatchTableCommit; @@ -56,7 +56,7 @@ public void testPersistedLegacySchemaFallsBackBeforeAndAfterEvolution() throws E java.nio.file.Path directory = temporaryFolder.newFolder().toPath(); Path location = new Path(directory.toUri()); LocalFileIO fileIO = LocalFileIO.create(); - new SchemaManager(fileIO, location).createTable(Schema.newBuilder() + new FileSystemSchemaManager(fileIO, location).createTable(Schema.newBuilder() .column("id", DataTypes.INT()).option("bucket", "-1") .option("file.format", "parquet").build()); FileStoreTable table = FileStoreTableFactory.create(fileIO, location); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 2508f3c7bbee6e..03eda38a32da0d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -2916,6 +2916,114 @@ public void testRustReaderSelectionRequiresExplicitS3Credentials() throws Except rangeDesc.getTableFormatParams().getPaimonParams().getReaderType()); } } + // --- filterVectorIndexOptions --------------------------------------------------- + // + // CoreOptions.primaryKeyVectorIndexOptions seeds its result with a copy of every + // table option, so what reaches TExternalSearchRequest.paimon_options is the whole table + // property bag unless it is filtered. These cover the three reasons it must be: + // a thrift-fatal null value, redundant table options paimon-rust already merges + // itself, and properties that should not travel to BE at all. + + @Test + public void testFilterVectorIndexOptionsKeepsFieldAndAlgorithmScopedKeys() { + Map options = new HashMap<>(); + options.put("field.embedding.vector-dim", "2048"); + options.put("ivf-flat.metric", "l2"); + options.put("ivf-flat.nlist", "1024"); + options.put("fields.embedding.some-field-option", "v"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals("l2", filtered.get("ivf-flat.metric")); + Assert.assertEquals("1024", filtered.get("ivf-flat.nlist")); + Assert.assertEquals("v", filtered.get("fields.embedding.some-field-option")); + // "field." (singular) is a different namespace from the "fields." one the filter + // keeps, so the dimension is not forwarded -- paimon-rust reads it off the index. + Assert.assertFalse(filtered.containsKey("field.embedding.vector-dim")); + Assert.assertEquals(3, filtered.size()); + } + + @Test + public void testFilterVectorIndexOptionsDropsNullValues() { + // The actual production failure: these catalog audit properties carry a Java null, + // which HashMap accepts and thrift's writeString does not (NPE in the generated + // TPaimonVectorSearchOptions writer). They are also not algorithm- or field-scoped, + // so they would be dropped anyway -- asserted separately so a future widening of + // the prefix rule cannot silently reintroduce the crash. + Map options = new HashMap<>(); + options.put("owner", null); + options.put("createdBy", null); + options.put("updatedBy", null); + options.put("ivf-flat.metric", "l2"); + options.put("ivf-flat.nprobe", null); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertFalse(filtered.containsValue(null)); + Assert.assertFalse(filtered.containsKey("ivf-flat.nprobe")); + Assert.assertEquals(Collections.singletonMap("ivf-flat.metric", "l2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsDropsUnrelatedTableProperties() { + Map options = new HashMap<>(); + options.put("path", "viewfs://cluster/db/tbl"); + options.put("security.hadoop.username", "hudi"); + options.put("bucket", "1"); + options.put("primary-key", "id"); + options.put("merge-engine", "partial-update"); + options.put("ivf-flat.metric", "l2"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals(Collections.singletonMap("ivf-flat.metric", "l2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsDropsPkVectorDeclarations() { + // fields..pk-vector.* is FE's own index declaration, not a search parameter; + // primaryKeyVectorAlgorithmOptions excludes it from its collection too. + Map options = new HashMap<>(); + options.put("fields.embedding.pk-vector.index.type", "ivf-flat"); + options.put("fields.embedding.pk-vector.distance.metric", "l2"); + options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":\"1024\"}"); + options.put("fields.embedding.refine-factor", "2"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals(Collections.singletonMap("fields.embedding.refine-factor", "2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsWithNullAlgorithmKeepsFieldScopedOnly() { + // A null index type must not be turned into a "null." prefix; keep the + // field-scoped half rather than guessing. + Map options = new HashMap<>(); + options.put("fields.embedding.refine-factor", "2"); + options.put("ivf-flat.metric", "l2"); + options.put("owner", null); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", null); + + Assert.assertEquals(Collections.singletonMap("fields.embedding.refine-factor", "2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsIsScopedToTheQueriedColumn() { + Map options = new HashMap<>(); + options.put("fields.embedding.refine-factor", "2"); + options.put("fields.other_vec.refine-factor", "8"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals(Collections.singletonMap("fields.embedding.refine-factor", "2"), filtered); + } @Test public void testRustReaderRejectsConflictingStaticCredentials() throws Exception { @@ -3540,6 +3648,66 @@ public void testRustMultiFilePrimaryKeyMergeFallback() throws Exception { } } + @Test + public void testRustMultiFileRawConvertibleSplitStaysOnRust() throws Exception { + // A rawConvertible split whose files are all compacted (level != 0) never reaches + // the pinned merge reader on BE: paimon-rust dispatches it to the raw path, which + // streams one file at a time bounded by read.batch-size (crate pk_split_needs_merge + // / read_pk). The multi-file merge gate must not judge it, or compacted PK buckets + // (the normal PK-vector table layout) could never use the rust reader at all. + for (String engine : Arrays.asList("deduplicate", "aggregation", "partial-update")) { + for (int fileCount : new int[] {2, 8}) { + for (boolean rawConvertible : Arrays.asList(true, false)) { + RustRoutingFixture f = new RustRoutingFixture(); + f.schema(1, new IntType(), ImmutableMap.of("merge-engine", engine, + "read.batch-size", "1"), true); + List files = new ArrayList<>(); + for (int i = 0; i < fileCount; i++) { + DataFileMeta file = Mockito.spy(createDataSplit("run-" + i + ".parquet") + .dataFiles().get(0)); + Mockito.doReturn(Optional.of(0L)).when(file).deleteRowCount(); + Mockito.doReturn(1).when(file).level(); + files.add(file); + } + DataSplit.Builder builder = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0") + .withDataFiles(files); + if (rawConvertible) { + builder.rawConvertible(true); + } + DataSplit split = builder.build(); + Assert.assertEquals(engine + " files=" + fileCount + " raw=" + rawConvertible, + rawConvertible ? TPaimonReaderType.PAIMON_RUST : TPaimonReaderType.PAIMON_JNI, + f.reader(split)); + } + } + } + + // A level-0 file forces the merge path even on a rawConvertible split (deletion + // vector tables merge any uncompacted level), so the multi-file gate must keep + // routing it to JNI. + for (boolean rawConvertible : Arrays.asList(true, false)) { + RustRoutingFixture f = new RustRoutingFixture(); + f.schema(1, new IntType(), ImmutableMap.of("merge-engine", "deduplicate", + "deletion-vectors.enabled", "true", "read.batch-size", "1"), true); + DataFileMeta compacted = Mockito.spy(createDataSplit("compacted.parquet") + .dataFiles().get(0)); + Mockito.doReturn(Optional.of(0L)).when(compacted).deleteRowCount(); + Mockito.doReturn(1).when(compacted).level(); + DataFileMeta levelZero = Mockito.spy(createDataSplit("level0.parquet").dataFiles().get(0)); + Mockito.doReturn(Optional.of(0L)).when(levelZero).deleteRowCount(); + Mockito.doReturn(0).when(levelZero).level(); + DataSplit.Builder builder = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0") + .withDataFiles(Arrays.asList(compacted, levelZero)); + if (rawConvertible) { + builder.rawConvertible(true); + } + Assert.assertEquals("level0 raw=" + rawConvertible, + TPaimonReaderType.PAIMON_JNI, f.reader(builder.build())); + } + } + @Test public void testRustNestedEvolutionAndCompatibleHistory() throws Exception { org.apache.paimon.types.DataType small = new org.apache.paimon.types.TinyIntType(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java new file mode 100644 index 00000000000000..036a3171dac7b4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java @@ -0,0 +1,710 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TVectorMetric; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import mockit.MockUp; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.FileStoreTable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Unit tests for {@link PushDownVectorTopNIntoPaimonScan}. + * + *

The rule turns {@code TopN[dis] -> Project[dis := dist_fn(vec, q)] -> [Filter] -> PaimonScan} + * into an index-only ANN scan that produces {@code __paimon_search_score_to_dis}, and replaces every + * occurrence of {@code dist_fn(vec, q)} with that column's slot. No arithmetic is emitted: the BE + * reader converts the score into the distance the function is defined to return while filling the + * block. These tests drive the two rules directly instead of going through the planner, so no + * catalog/BE is needed. + */ +public class PushDownVectorTopNIntoPaimonScanTest { + + private static final List QUERY_VECTOR = + ImmutableList.of(new FloatLiteral(1.0f), new FloatLiteral(2.0f)); + + private static SystemInfoService systemInfo; + private ConnectContext ctx; + private CascadesContext cascadesContext; + + @BeforeEach + public void setUp() { + // The rule reads enable_paimon_rust_reader / enable_file_scanner_v2 off the + // thread-local ConnectContext. Deliberately no StatementContext: + // MvccUtil.getSnapshotFromContext then short-circuits to Optional.empty(), + // so the mocked table never needs a database/catalog. + ctx = new ConnectContext(); + ctx.setSessionVariable(new SessionVariable()); + ctx.getSessionVariable().setEnablePaimonRustReader(true); + ctx.getSessionVariable().enableFileScannerV2 = true; + ctx.setThreadLocalInfo(); + // Rule.transform only stuffs the context into a MatchingContext; the rule body ignores it. + cascadesContext = Mockito.mock(CascadesContext.class); + // The rewrite's pre-flight (PaimonVectorSearch.vectorScanEligible) requires a + // rust-capable BE via the all-backend capability negotiation, mirroring + // PaimonScanNodeTest's routing fixture. + Backend rustBackend = GsonUtils.GSON.fromJson( + "{\"supportsPaimonRustReader\":true}", Backend.class); + systemInfo = Mockito.mock(SystemInfoService.class); + Mockito.when(systemInfo.getAllClusterBackends(true)) + .thenReturn(ImmutableList.of(rustBackend)); + new MockUp() { + @mockit.Mock + public SystemInfoService getCurrentSystemInfo() { + return systemInfo; + } + }; + } + + @AfterEach + public void tearDown() { + ConnectContext.remove(); + } + + // ------------------------------------------------------------------ positive cases + + @Test + public void testL2AscPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), + dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + + // ANN info reached the scan, with the retrieval limit widened by the offset. + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + AnnTopNInfo annTopN = newScan.getAnnTopN().get(); + Assertions.assertEquals("vec", annTopN.getColumnName()); + Assertions.assertEquals(15L, annTopN.getLimit()); + Assertions.assertArrayEquals(new float[] {1.0f, 2.0f}, annTopN.getQueryVector()); + // BE needs the metric to undo paimon-rust's score transform. + Assertions.assertEquals(TVectorMetric.L2, annTopN.getMetric()); + + // The distance column is appended as a virtual output column. + Assertions.assertEquals(1, newScan.getVirtualColumns().size()); + Assertions.assertEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, + newScan.getVirtualColumns().get(0).getName()); + + // ... and it must actually reach the scan's OUTPUT, not just its virtualColumns + // list. These are two different things: output is computed once and can be + // served from a cache (LogicalProperties, or LogicalFileScan.cachedOutputs, + // which short-circuits computeOutput() entirely). withVectorTopN therefore has + // to invalidate both, and asserting only on getVirtualColumns() would pass even + // if the new column never became visible to the parent project -- which is what + // the plan actually consumes. + Assertions.assertTrue(newScan.getOutput().contains(distanceSlotOf(newScan).toSlot()), + () -> "distance slot missing from scan output: " + newScan.getOutput()); + Assertions.assertEquals(scan.getOutput().size() + 1, newScan.getOutput().size()); + + // The pre-existing base columns must keep their ExprIds. The rule reparents the + // EXISTING projections onto the new scan, so a regenerated ExprId is not a + // cosmetic renumbering -- it substitutes a different slot identity and leaves the + // project referencing something the scan no longer produces. This fails if + // withVectorTopN lets computeOutput() rebuild the output: computeOutput calls + // exprIdGenerator.getNextId() per column with no reuse cache. + Assertions.assertEquals( + scan.getOutput().stream().map(Slot::getExprId).collect(Collectors.toList()), + newScan.getOutput().subList(0, scan.getOutput().size()).stream() + .map(Slot::getExprId).collect(Collectors.toList()), + () -> "base column exprIds changed: " + scan.getOutput() + + " -> " + newScan.getOutput()); + + // The consequence the above protects, asserted directly: every slot the parent + // project reads has to be produced by the scan underneath it. + Set scanOutputIds = newScan.getOutput().stream() + .map(Slot::getExprId).collect(Collectors.toSet()); + for (NamedExpression projection : project.getProjects()) { + for (Slot input : projection.getInputSlots()) { + Assertions.assertTrue(scanOutputIds.contains(input.getExprId()), + () -> "project reads " + input + " which the scan does not output: " + + newScan.getOutput()); + } + } + + // `dis` keeps its alias identity (the TopN order key still resolves) ... + Alias newDis = (Alias) project.getProjects().get(1); + Assertions.assertEquals(dis.getExprId(), newDis.getExprId()); + Assertions.assertEquals("dis", newDis.getName()); + // ... and its value is the bare distance slot: the BE reader already converted it + // into the Euclidean distance, so no arithmetic is planned here. + Assertions.assertEquals(distanceSlotOf(newScan), newDis.child()); + Assertions.assertFalse(containsDistanceFunction(newDis)); + } + + @Test + public void testInnerProductDescPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "inner_product")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new InnerProductApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), + dis, false); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + Assertions.assertEquals(TVectorMetric.DOT_PRODUCT, newScan.getAnnTopN().get().getMetric()); + + // inner_product: the score IS the distance, so the alias wraps the bare distance slot. + Alias newDis = (Alias) project.getProjects().get(1); + Assertions.assertEquals(dis.getExprId(), newDis.getExprId()); + Assertions.assertEquals(distanceSlotOf(newScan), newDis.child()); + } + + @Test + public void testOutputColumnNameKeepsTheFunctionCallWithoutAlias() { + // SQL: SELECT id, l2_distance_approximate(vec, q) FROM xx ORDER BY l2_distance_approximate(vec, q) + // No AS alias, so the binder builds an Alias whose name comes from the child's toSql + // (nameFromChild=true). The result column header must be "l2_distance_approximate(...)", + // NOT "__paimon_search_score_to_dis" -- the user asked for a distance function, not an + // internal virtual column. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector())); + String expectedName = dis.getName(); + Assertions.assertTrue(dis.isNameFromChild()); + Assertions.assertTrue(expectedName.startsWith("l2_distance_approximate("), expectedName); + + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + Alias newDis = (Alias) project.getProjects().get(1); + + // The header keeps the original function text, and the value is the bare + // distance slot underneath it. + Assertions.assertEquals(expectedName, newDis.getName()); + Assertions.assertNotEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, newDis.getName()); + Assertions.assertEquals(dis.getExprId(), newDis.getExprId()); + Assertions.assertEquals(distanceSlotOf(newScan), newDis.child()); + // The name was captured from the pre-rewrite child, so it must not have followed + // the child down to the distance slot. + Assertions.assertFalse(newDis.isNameFromChild()); + Assertions.assertEquals(expectedName, newDis.toSlot().getName()); + } + + @Test + public void testFilterConjunctReusesScore() { + // WHERE l2_distance_approximate(vec, q) < 10 ORDER BY the same expression. + // Mirroring the internal-table rule, the conjunct is rewritten to reuse the index value + // instead of recomputing the distance from the raw vector. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Expression distance = new L2DistanceApproximate(vec, queryVector()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(distance, new IntegerLiteral(10))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + Plan rewritten = applyRule(1, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFilter newFilter = (LogicalFilter) project.child(0); + LogicalFileScan newScan = (LogicalFileScan) newFilter.child(0); + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + + Assertions.assertEquals(1, newFilter.getConjuncts().size()); + Expression conjunct = newFilter.getConjuncts().iterator().next(); + Assertions.assertFalse(containsDistanceFunction(conjunct), + "the filter must not recompute the distance from the raw vector"); + Assertions.assertEquals( + new LessThan(distanceSlotOf(newScan), new IntegerLiteral(10)), + conjunct); + } + + @Test + public void testNestedProjectionOccurrenceReusesScore() { + // A second projection embeds the same distance call: dis2 := dist_fn(vec, q) * 2. + // The nested occurrence is replaced while the surrounding alias keeps its identity. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + Alias doubled = new Alias( + new Multiply(new L2DistanceApproximate(vec, queryVector()), new IntegerLiteral(2)), + "dis2"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis, doubled), scan), dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + Alias newDoubled = (Alias) project.getProjects().get(2); + Assertions.assertEquals(doubled.getExprId(), newDoubled.getExprId()); + Assertions.assertEquals("dis2", newDoubled.getName()); + Assertions.assertEquals( + new Multiply(distanceSlotOf(newScan), new IntegerLiteral(2)), + newDoubled.child()); + Assertions.assertFalse(containsDistanceFunction(newDoubled)); + } + + @Test + public void testSelectStarDoesNotSurfaceDistanceColumn() { + // SELECT * FROM xx ORDER BY l2_distance_approximate(vec, q) + // The distance column must not appear in the result set. The rule appends it + // to the scan output but deliberately does NOT extend the asterisk output, + // so a downstream `SELECT *` expansion never picks it up. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), vecSlot(scan), dis), scan), + dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + + // The scan output has the distance column appended ... + Assertions.assertTrue(newScan.getOutput().stream() + .anyMatch(s -> s.getName().equals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)), + "distance slot must be in scan output"); + + // ... but the asterisk output must NOT contain it. + List asterisk = newScan.getLogicalProperties().getAsteriskOutput(); + boolean distanceInAsterisk = asterisk.stream() + .anyMatch(s -> s.getName().equals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)); + Assertions.assertFalse(distanceInAsterisk, + "__paimon_search_score_to_dis must not appear in SELECT * output: " + asterisk); + + // The project also must not leak the internal column name as a visible output. + for (NamedExpression proj : project.getProjects()) { + Assertions.assertNotEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, proj.getName(), + "project output must not expose internal distance column name: " + proj); + } + } + + @Test + public void testAliasedOutputColumnNameKeepsUserAlias() { + // SQL: SELECT id, l2_distance_approximate(vec, q) AS dis FROM xx ORDER BY dis + // The result column header must be "dis", NOT "__paimon_search_score_to_dis". + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + + Alias rewrittenDis = (Alias) project.getProjects().get(1); + // The alias name stays "dis", not the internal column name. + Assertions.assertEquals("dis", rewrittenDis.getName()); + Assertions.assertNotEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, rewrittenDis.getName()); + // The alias still wraps the bare distance slot (no arithmetic). + Assertions.assertEquals(distanceSlotOf(newScan), rewrittenDis.child()); + Assertions.assertFalse(containsDistanceFunction(rewrittenDis)); + // The slot produced by the alias also carries the user-visible name. + Assertions.assertEquals("dis", rewrittenDis.toSlot().getName()); + } + + @Test + public void testPredicateOnAliasedDistanceIsRewritten() { + // SELECT id, l2_distance_approximate(vec, q) AS dis + // FROM xx WHERE l2_distance_approximate(vec, q) < 1 ORDER BY dis + // The filter conjunct uses the bare distance function, which the rule + // replaces with the index-produced distance slot. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Expression distanceExpr = new L2DistanceApproximate(vec, queryVector()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(distanceExpr, new FloatLiteral(1.0f))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + Plan rewritten = applyRule(1, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFilter newFilter = (LogicalFilter) project.child(0); + LogicalFileScan newScan = (LogicalFileScan) newFilter.child(0); + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + + // The filter conjunct now references the distance slot, not the function call. + Assertions.assertEquals(1, newFilter.getConjuncts().size()); + Expression conjunct = newFilter.getConjuncts().iterator().next(); + Assertions.assertFalse(containsDistanceFunction(conjunct), + "filter must not recompute the distance from the raw vector"); + Assertions.assertEquals( + new LessThan(distanceSlotOf(newScan), new FloatLiteral(1.0f)), + conjunct); + + // The alias output name stays "dis". + Alias rewrittenDis = (Alias) project.getProjects().get(1); + Assertions.assertEquals("dis", rewrittenDis.getName()); + } + + @Test + public void testMergedSubqueryRewriteStillTriggers() { + // SELECT id, dis FROM ( + // SELECT id, l2_distance_approximate(vec, q) AS dis FROM xx + // ) WHERE dis < 1 ORDER BY dis + // + // After MergeProjects / MergeFilters runs in the planner, the inner and outer + // projects collapse, and the filter's reference to the `dis` alias is resolved + // back to the underlying L2DistanceApproximate. The rule then sees: + // TopN -> Project[id, dis := l2_distance_approximate(vec,q)] + // -> Filter[l2_distance_approximate(vec,q) < 1] + // -> PaimonScan + // which matches the second rule variant (with filter). + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Expression distanceExpr = new L2DistanceApproximate(vec, queryVector()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(distanceExpr, new FloatLiteral(1.0f))), scan); + Alias dis = new Alias(distanceExpr, "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + Plan rewritten = applyRule(1, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFilter newFilter = (LogicalFilter) project.child(0); + LogicalFileScan newScan = (LogicalFileScan) newFilter.child(0); + + // ANN rewrite fired. + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + Assertions.assertEquals(TVectorMetric.L2, newScan.getAnnTopN().get().getMetric()); + + // The filter now uses the distance slot. + Expression conjunct = newFilter.getConjuncts().iterator().next(); + Assertions.assertFalse(containsDistanceFunction(conjunct)); + Assertions.assertEquals( + new LessThan(distanceSlotOf(newScan), new FloatLiteral(1.0f)), + conjunct); + + // The alias keeps its user name "dis" and wraps the distance slot. + Alias rewrittenDis = (Alias) project.getProjects().get(1); + Assertions.assertEquals("dis", rewrittenDis.getName()); + Assertions.assertEquals(distanceSlotOf(newScan), rewrittenDis.child()); + } + + // ------------------------------------------------------------------ negative cases + + @Test + public void testRustReaderDisabledNotPushedDown() { + // Only the BE paimon-rust reader can execute the index-only search. + ctx.getSessionVariable().setEnablePaimonRustReader(false); + assertNotPushedDown(l2Plan("l2", true)); + } + + @Test + public void testFileScannerV2DisabledNotPushedDown() { + // FileScannerV1 cannot select the paimon-rust reader, so an ANN request would be + // one the selected scanner cannot consume. + ctx.getSessionVariable().enableFileScannerV2 = false; + assertNotPushedDown(l2Plan("l2", true)); + } + + @Test + public void testMetricMismatchNotPushedDown() { + // inner_product_approximate over an l2 index would return a different ranking. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + Alias dis = new Alias(new InnerProductApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, false)); + } + + @Test + public void testL2DescNotPushedDown() { + // l2 top-N is only an ANN search when ordered ascending (nearest first). + assertNotPushedDown(l2Plan("l2", false)); + } + + @Test + public void testInnerProductAscNotPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "inner_product")); + Alias dis = new Alias(new InnerProductApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testCosineMetricNotPushedDown() { + // v1 supports l2 / inner_product only. + assertNotPushedDown(l2Plan("cosine", true)); + } + + @Test + public void testNoVectorIndexNotPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of()); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testUnindexedColumnNotPushedDown() { + // The index covers `vec`, but the query orders by a distance over `vec2`. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec2 = (SlotReference) scan.getOutput().get(2); + Assertions.assertEquals("vec2", vec2.getName()); + Alias dis = new Alias(new L2DistanceApproximate(vec2, queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testNonConstantQueryVectorNotPushedDown() { + // The query vector must be a literal to be handed to the index. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference vec2 = (SlotReference) scan.getOutput().get(2); + Alias dis = new Alias(new L2DistanceApproximate(vec, vec2), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testOrderKeyNotFromProjectionNotPushedDown() { + // Ordering by a plain column is not a vector top-N. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference id = idSlot(scan); + LogicalProject project = new LogicalProject<>(ImmutableList.of(id), scan); + assertNotPushedDown(new LogicalTopN<>( + ImmutableList.of(new OrderKey(id, true, true)), 10, 5, project)); + } + + @Test + public void testIncrementalScanNotPushedDown() { + // t@incr(...): the PK-vector search path has no incremental semantics (the + // VectorScan builder has no incremental mode), so an incremental query must + // keep its ordinary scan and compute the distance in Doris — a rewritten + // vector split would silently return snapshot rows for a query that asked + // for changes. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("incr", + ImmutableMap.of("start", "1"), ImmutableList.of()))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + // ------------------------------------------------------------------ helpers + + private Plan applyRule(int ruleIndex, LogicalTopN topN) { + Rule rule = new PushDownVectorTopNIntoPaimonScan().buildRules().get(ruleIndex); + List result = rule.transform(topN, cascadesContext); + Assertions.assertEquals(1, result.size()); + return result.get(0); + } + + /** A no-op rule returns the origin plan unchanged (see PatternMatcher#toRule). */ + private void assertNotPushedDown(LogicalTopN topN) { + int ruleIndex = topN.child().child(0) instanceof LogicalFilter ? 1 : 0; + Assertions.assertSame(topN, applyRule(ruleIndex, topN)); + } + + private LogicalTopN l2Plan(String metric, boolean asc) { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", metric)); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + return topN(new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, asc); + } + + private LogicalTopN topN(LogicalProject project, Alias orderBy, boolean asc) { + // limit 10 offset 5 -> the scan must retrieve 15 candidates. + return new LogicalTopN<>(ImmutableList.of(new OrderKey(orderBy.toSlot(), asc, asc)), + 10, 5, project); + } + + /** + * A scan over a mocked Paimon table {@code (id INT, vec ARRAY, vec2 ARRAY)} + * whose Paimon options are exactly {@code options}. + */ + private LogicalFileScan paimonScan(Map options) { + return paimonScan(options, Optional.empty()); + } + + private LogicalFileScan paimonScan(Map options, + Optional scanParams) { + List schema = ImmutableList.of( + new Column("id", Type.INT, true), + new Column("vec", new ArrayType(Type.FLOAT), true), + new Column("vec2", new ArrayType(Type.FLOAT), true)); + + PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); + Mockito.when(table.getName()).thenReturn("paimon_tbl"); + Mockito.when(table.getBaseSchema()).thenReturn(schema); + Mockito.when(table.getFullSchema()).thenReturn(schema); + Mockito.when(table.getFullSchema(Mockito.>any())) + .thenReturn(schema); + Mockito.when(table.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + + // A FileStoreTable (not a bare Table) with a file:// location, no REST FileIO and + // null CoreOptions/schema: the rewrite's pre-flight sees a plain local + // warehouse table whose option gates all skip, so the per-test options map is + // the only input that differs. + FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class); + Mockito.when(paimonTable.options()).thenReturn(options); + Mockito.when(paimonTable.location()) + .thenReturn(new Path("file:///warehouse/wh/paimon_db.db/paimon_tbl")); + Mockito.when(paimonTable.fileIO()).thenReturn(null); + Mockito.when(paimonTable.coreOptions()).thenReturn(null); + Mockito.when(paimonTable.schema()).thenReturn(null); + Mockito.when(table.getPaimonTable(Mockito.>any())) + .thenReturn(paimonTable); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); + // A filesystem metastore (empty properties, benign type): no vended-credential + // provider, so the factory returns the base (empty) storage map. + MetastoreProperties metastoreProperties = Mockito.mock(MetastoreProperties.class); + Mockito.when(metastoreProperties.getType()).thenReturn(MetastoreProperties.Type.FILE_SYSTEM); + Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(metastoreProperties); + Mockito.when(catalogProperty.getStoragePropertiesMap()) + .thenReturn(java.util.Collections.emptyMap()); + + return new LogicalFileScan(StatementScopeIdGenerator.newRelationId(), table, + ImmutableList.of("paimon_ctl", "paimon_db"), ImmutableList.of(), + Optional.empty(), Optional.empty(), scanParams, Optional.empty()); + } + + private SlotReference idSlot(LogicalFileScan scan) { + return (SlotReference) scan.getOutput().get(0); + } + + /** Must come from the scan output: the rule requires the slot's original column/table. */ + private SlotReference vecSlot(LogicalFileScan scan) { + return (SlotReference) scan.getOutput().get(1); + } + + private ArrayLiteral queryVector() { + return new ArrayLiteral(QUERY_VECTOR); + } + + private NamedExpression distanceSlotOf(LogicalFileScan scan) { + return scan.getVirtualColumns().get(0); + } + + private boolean containsDistanceFunction(Expression expression) { + return expression.anyMatch(e -> + e instanceof L2DistanceApproximate || e instanceof InnerProductApproximate); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java index 85ff507ddb60c0..1ec6586bfbe452 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java @@ -17,8 +17,12 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.InPredicate; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.paimon.source.PaimonPredicateConverter; import org.apache.doris.qe.StmtExecutor; @@ -152,4 +156,40 @@ public void equal() throws Exception { Assertions.assertTrue(predicate.children().get(0) instanceof LeafPredicate); Assertions.assertTrue(predicate.children().get(1) instanceof LeafPredicate); } + + /** + * A conjunct on a column the Paimon table does not have must be skipped, not fatal. + * + *

The PK-vector rewrite turns {@code dist_fn(vec, q) < x} into a comparison on the + * synthetic {@code __paimon_search_score_to_dis} slot, which exists only in the scan's + * output. Before the guard, the field lookup returned -1 and the following + * {@code paimonFieldTypes.get(-1)} threw IndexOutOfBounds, killing the query during + * planning. Both entry points are covered: binary comparison and IN. + */ + @Test + public void unknownColumnIsSkippedInsteadOfThrowing() { + RowType rowType = new RowType(Lists.newArrayList( + new DataField(0, "k1", new IntType()), + new DataField(1, "k2", new IntType()))); + PaimonPredicateConverter converter = new PaimonPredicateConverter(rowType); + + SlotRef distanceSlot = new SlotRef(null, "__paimon_search_score_to_dis"); + Expr lessThan = new BinaryPredicate(BinaryPredicate.Operator.LT, distanceSlot, + new IntLiteral(1)); + Assertions.assertTrue( + converter.convertToPaimonExpr(Lists.newArrayList(lessThan)).isEmpty()); + + Expr inPredicate = new InPredicate(distanceSlot, + Lists.newArrayList(new IntLiteral(1), new IntLiteral(2)), false); + Assertions.assertTrue( + converter.convertToPaimonExpr(Lists.newArrayList(inPredicate)).isEmpty()); + + // A known column still converts, so the guard did not disable push-down wholesale. + Expr knownColumn = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(null, "k1"), new IntLiteral(1)); + List converted = + converter.convertToPaimonExpr(Lists.newArrayList(knownColumn)); + Assertions.assertEquals(1, converted.size()); + Assertions.assertEquals("k1", ((LeafPredicate) converted.get(0)).fieldName()); + } } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 5a9a55cabe56ef..1ec35acb622a16 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -360,6 +360,27 @@ enum TPaimonReaderType { PAIMON_RUST = 3, } +// Payload kind for Paimon primary-key vector (ANN) search. +// v1 only uses BUCKET_SPLIT_BYTES; v2 (delayed materialization) will add +// MATERIALIZE_OPAQUE_BYTES. Kept as an enum from v1 so the wire protocol is +// shared between v1 and v2 and does not need to change again. +enum TPaimonVectorPayloadType { + // v1 / v2 candidate stage: serialized BucketVectorSearchSplit + // (magic "PKVSPLIT", version 1, big-endian; decoded by paimon-rust + // paimon_bucket_vector_search_split_deserialize). + BUCKET_SPLIT_BYTES = 0, + // v2 materialize stage: opaque bytes produced by the merge/rerank step. + MATERIALIZE_OPAQUE_BYTES = 1, +} + +// Per-split vector payload carried on TPaimonFileDesc. The bytes are the raw +// BucketVectorSearchSplit.serialize form (NOT base64, NOT a Java +// ObjectOutputStream envelope) whose meaning is determined by payload_type. +struct TPaimonVectorPayload { + 1: required TPaimonVectorPayloadType payload_type; + 2: required binary bytes; +} + struct TPaimonFileDesc { 1: optional string paimon_split 2: optional string paimon_column_names @@ -385,6 +406,10 @@ struct TPaimonFileDesc { // paimon-rust: non-default branch name. Unset means main branch (matches // upstream paimon commit 742da63: null-if-DEFAULT_MAIN_BRANCH). 19: optional string paimon_branch + // Primary-key vector (ANN) search payload for this split. When set, BE + // routes to the paimon-rust vector-search read path instead of the normal + // split read; paimon_split is set but empty in that case. + 20: optional TPaimonVectorPayload vector_payload; } struct TTrinoConnectorFileDesc { @@ -560,6 +585,15 @@ struct TVectorSearchOptions { 5: optional i32 query_parallelism } +// Paimon-only vector search tuning. Mirrors TVectorSearchOptions: the logical query stays +// in TVectorSearchParams, provider-specific knobs live here so TExternalSearchRequest stays +// provider-neutral. The options map is passed straight through to the paimon-rust vector +// search builder (refine_factor, ivf.nprobe, and the ".metric" key that matches +// Paimon's algorithm options). +struct TPaimonVectorSearchOptions { + 1: optional map options +} + // The active union field identifies the logical search kind. A future hybrid field can contain both // vector and full-text subqueries plus its fusion parameters without changing either existing field. union TExternalSearchQuery { @@ -568,7 +602,8 @@ union TExternalSearchQuery { } // A provider-independent logical search request. Physical target information remains in the -// provider FileDesc (for example, dataset_uri/version/fragment_ids in TLanceFileDesc). +// provider FileDesc (for example, dataset_uri/version/fragment_ids in TLanceFileDesc, or +// vector_payload in TPaimonFileDesc). struct TExternalSearchRequest { 1: optional i32 schema_version = 1 2: optional TExternalSearchQuery search_query @@ -577,6 +612,9 @@ struct TExternalSearchRequest { // Applies only to search_filter. Defaults to false (postfilter); Doris WHERE predicates remain // residuals regardless of this setting. 5: optional bool prefilter + // Paimon-specific vector search tuning. Field 4 stays Lance-neutral, so Paimon's + // string-map knobs travel here instead; unset for Lance requests. + 6: optional TPaimonVectorSearchOptions paimon_options } // A catalog/S3 range reads fragments from a fixed snapshot. A local TVF range uses version zero @@ -728,6 +766,11 @@ struct TFileScanRangeParams { 39: optional i32 parquet_timestamp_semantics_version // Hybrid Paimon/Hudi scans keep FORMAT_JNI while individual ranges can be native Parquet. 40: optional bool contains_native_parquet + // Provider-independent external (vector / full-text) search request, set for a Paimon + // primary-key vector (ANN) top-N pushed down into the scan. Set at ScanNode level + // (shared by all splits); the per-split physical payload stays in + // TPaimonFileDesc.vector_payload. Lance requests use lance_scan_params instead. + 41: optional TExternalSearchRequest external_search_request } struct TFileRangeDesc { From 682df97d2d765a9ee8376c72b0f64a4bca762168 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Thu, 22 Oct 2026 16:23:00 +0800 Subject: [PATCH 03/13] [chore](paimon) pin the community 2.1.0 release, mirror S3 credentials and adapt the test suites Finalization of the 2.1 stack switch: - Pin the Paimon dependency to the released community 2.1.0 (identical content to the snapshot it replaces). - Mirror the s3.* credentials into the catalog options: the 2.1 SDK's S3FileIO reads credentials from CatalogContext.options() where the previous version picked up the endpoint-scoped properties. - Regression suites: a new end-to-end suite for the PK-vector ANN path (rust vs brute-force differential legs), adaptation of the write suites and fixtures to the 2.1 SDK / Spark behaviors, and the before-epoch timestamp-to-date expectation for 2.1.0. Co-Authored-By: Claude Opus 5 (1M context) --- .../metastore/AbstractPaimonProperties.java | 29 +++ .../PaimonFileSystemMetaStoreProperties.java | 1 + .../PaimonHMSMetaStoreProperties.java | 1 + .../PaimonJdbcMetaStoreProperties.java | 1 + .../PaimonRustReaderCompatibilityTest.java | 6 +- fe/pom.xml | 2 +- .../doris/regression/util/ResultUtils.groovy | 70 +++++- .../paimon/test_paimon_pk_vector_ann.groovy | 201 ++++++++++++++++++ ...st_paimon_schema_time_travel_matrix.groovy | 21 +- .../write/test_paimon_orc_ltz_encoding.groovy | 14 +- .../test_paimon_write_transaction.groovy | 2 +- 11 files changed, 333 insertions(+), 15 deletions(-) create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index 89dea6d31070b0..bddddf4e05e2df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -60,6 +60,7 @@ public abstract class AbstractPaimonProperties extends MetastoreProperties { private static final String DORIS_JNI_PROPERTY_PREFIX = "paimon.jni."; /** The suffix after this prefix is passed to Paimon as a dynamic table option. */ public static final String TABLE_OPTION_PREFIX = PaimonReaderOptions.TABLE_OPTION_PREFIX; + private static final String HADOOP_S3A_PREFIX = "fs.s3a."; private Map tableOptionsMap = Collections.emptyMap(); @@ -123,6 +124,34 @@ protected void appendUserHadoopConfig(Configuration conf) { normalizeS3Config().forEach(conf::set); } + /** + * Mirrors the S3 credentials Doris assembled into the Hadoop configuration + * onto the catalog options, so the pinned Paimon 2.1 SDK sees them. + * + *

Doris stores a catalog's storage credentials (fs.s3a.access.key, fs.s3a.secret.key, + * fs.s3a.endpoint, ...) only in the Hadoop Configuration handed to + * CatalogContext.create(options, conf) — the paimon 1.4 FileIO consulted exactly + * that conf. The pinned 2.1 S3FileIO instead reads its credentials exclusively from + * CatalogContext.options() (S3FileIO#configure -> loadHadoopConfigFromContext copies + * the s3./s3a./fs.s3a.-prefixed option entries and never looks at the conf), so + * without this mirror every s3-warehouse catalog initializes the SDK with no + * credentials and the first file operation fails on the ambient AWS provider chain + * ("Unable to load credentials from system settings"). Mirroring the whole fs.s3a.* + * set keeps user extras (path style, connection limits) flowing as well; entries + * outside the prefix are not the SDK's concern. + */ + protected void mirrorHadoopS3OptionsToCatalogOptions(Configuration conf) { + if (conf == null || catalogOptions == null) { + return; + } + conf.forEach(entry -> { + String key = entry.getKey(); + if (key.startsWith(HADOOP_S3A_PREFIX) && StringUtils.isNotBlank(entry.getValue())) { + catalogOptions.set(key, entry.getValue()); + } + }); + } + public Map getCatalogOptionsMap() { // Return the cached map if already initialized Map existing = catalogOptionsMapRef.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java index df0ebae97490a8..78daa9ed498b23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java @@ -48,6 +48,7 @@ public Catalog initializeCatalog(String catalogName, List sto } }); appendUserHadoopConfig(conf); + mirrorHadoopS3OptionsToCatalogOptions(conf); CatalogContext catalogContext = CatalogContext.create(catalogOptions, conf); try { return this.executionAuthenticator.execute(() -> CatalogFactory.createCatalog(catalogContext)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java index e7e6689d3e3cab..56756f563d2c96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java @@ -90,6 +90,7 @@ public Catalog initializeCatalog(String catalogName, List sto buildCatalogOptions(); Configuration conf = buildHiveConfiguration(storagePropertiesList); appendUserHadoopConfig(conf); + mirrorHadoopS3OptionsToCatalogOptions(conf); CatalogContext catalogContext = CatalogContext.create(catalogOptions, conf); try { return executionAuthenticator.execute(() -> CatalogFactory.createCatalog(catalogContext)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java index 7568d59c5fed33..bd4f8876d84e30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java @@ -122,6 +122,7 @@ public Catalog initializeCatalog(String catalogName, List sto } } appendUserHadoopConfig(conf); + mirrorHadoopS3OptionsToCatalogOptions(conf); if (StringUtils.isNotBlank(driverUrl)) { registerJdbcDriver(driverUrl, driverClass); LOG.info("Using dynamic JDBC driver for Paimon JDBC catalog from: {}", driverUrl); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java index 0358082c78830f..11be03fdd197fd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java @@ -136,9 +136,11 @@ public void testPersistedIntegerToTimestamp() throws Exception { @Test public void testPersistedTimestampToDateBeforeEpoch() throws Exception { - // Java divides toward zero; Arrow takes the preceding calendar date before the epoch. + // Before the epoch the persisted timestamp-to-date conversion takes the + // preceding calendar date (floor, like Arrow); the internal 2.1 BETA build + // truncated toward zero (Java integer division) and returned day 0. assertPersistedEvolution(DataTypes.TIMESTAMP(3), DataTypes.DATE(), Timestamp.fromEpochMillis(-1), - false, row -> row.getInt(1), 0); + false, row -> row.getInt(1), -1); } @Test diff --git a/fe/pom.xml b/fe/pom.xml index 50af2cbb22ff2a..ef0b47a6f48a91 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -418,7 +418,7 @@ under the License. 2.3.2 2.0.3 - 2.1-community-release-2.1-BETA-SNAPSHOT + 2.1.0 3.4.4 shade-format-flatbuffers diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy index 897377b8e9dda2..01f28af668eefa 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy @@ -23,12 +23,14 @@ import org.junit.Assert import java.math.BigDecimal import java.math.BigInteger import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets import java.sql.Blob import java.sql.Clob import java.sql.Struct import java.sql.Time import java.sql.Timestamp import java.time.temporal.TemporalAccessor +import java.util.regex.Matcher import java.util.regex.Pattern @CompileStatic @@ -45,6 +47,9 @@ class ResultUtils { private static final Pattern DATETIME_PATTERN = Pattern.compile( "(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2})(?::(\\d{2})(?:\\.(\\d{1,9}))?)?") private static final Pattern UNICODE_ESCAPE_PATTERN = Pattern.compile("[0-9a-fA-F]{4}") + /** A "0x" token as Doris renders a VARBINARY nested in arrays/maps/structs. */ + private static final Pattern NESTED_BINARY_TOKEN = Pattern.compile( + "(?i)0x(?:[0-9a-f]{2})*(?![0-9a-fA-F])") static List> normalizeRows(List> rows) { List> normalizedRows = new ArrayList<>() @@ -59,8 +64,8 @@ class ResultUtils { } static void assertSparkDorisResultEquals(List> sparkRows, List> dorisRows) { - List> normalizedSparkRows = normalizeRows(sparkRows) - List> normalizedDorisRows = normalizeRows(dorisRows) + List> normalizedSparkRows = decodeNestedBinaryTokens(normalizeRows(sparkRows)) + List> normalizedDorisRows = decodeNestedBinaryTokens(normalizeRows(dorisRows)) Assert.assertEquals("Spark and Doris result row count mismatch", normalizedSparkRows.size(), normalizedDorisRows.size()) @@ -84,6 +89,67 @@ class ResultUtils { } } + /** + * Brings nested binary values rendered by the two engines to a common form. + * + *

Doris renders a VARBINARY nested in arrays/maps/structs as a + * "0x"-prefixed hex token ("[0x616C..., null]") while Spark's JDBC returns + * the raw bytes ("[alice-array-bin, null]"), so the same value compares + * unequal across engines. Decoding the tokens on both rows applies the same + * substitution to both sides — a no-op on the raw rendering — so equal stays + * equal, and scalar binary columns are unaffected: they arrive as byte[] + * and normalizeValue already hex-encodes them symmetrically. + */ + static List> decodeNestedBinaryTokens(List> rows) { + List> decodedRows = new ArrayList<>(rows.size()) + for (List row : rows) { + List decodedRow = new ArrayList<>(row.size()) + for (Object value : row) { + decodedRow.add(decodeNestedBinaryTokens(value)) + } + decodedRows.add(decodedRow) + } + return decodedRows + } + + static Object decodeNestedBinaryTokens(Object value) { + if (value instanceof String) { + String text = (String) value + Matcher matcher = NESTED_BINARY_TOKEN.matcher(text) + StringBuffer sb = new StringBuffer() + while (matcher.find()) { + String hex = matcher.group().substring(2) + matcher.appendReplacement(sb, Matcher.quoteReplacement(decodeHexString(hex))) + } + matcher.appendTail(sb) + return sb.toString() + } + if (value instanceof List) { + List decoded = new ArrayList<>(((List) value).size()) + for (Object item : (List) value) { + decoded.add(decodeNestedBinaryTokens(item)) + } + return decoded + } + if (value instanceof Map) { + Map decoded = new LinkedHashMap<>() + for (Map.Entry entry : ((Map) value).entrySet()) { + decoded.put(decodeNestedBinaryTokens(entry.getKey()), + decodeNestedBinaryTokens(entry.getValue())) + } + return decoded + } + return value + } + + private static String decodeHexString(String hex) { + byte[] bytes = new byte[hex.length().intdiv(2)] + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16) + } + return new String(bytes, StandardCharsets.UTF_8) + } + static Object normalizeValue(Object value) { if (value == null) { return null diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy new file mode 100644 index 00000000000000..503f70e06c4862 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// End-to-end regression for the Paimon PK-vector (ANN) search path: the +// PushDownVectorTopNIntoPaimonScan rewrite, the shared rust eligibility gate +// and the BE paimon-rust vector reader, compared against the exact brute-force +// answer the same SQL computes with the rust reader disabled. +// +// The fixture (database pk_vector_ann_db, tables ann_l2 / ann_ip) is created +// by the suite itself through the Spark ThriftServer's paimon catalog: 512 +// deterministic 16-dim vectors derived from the row id (unique: the generator +// is mod 521, a prime above the row count), an ivf-rq PK-vector index with +// nlist = nprobe = 4 — exhaustive recall, so ANN must equal brute force by +// construction. Doris DDL cannot build this fixture: Doris has no VECTOR +// column type in either direction of the type mapping, and the ivf-rq index +// build needs the vindex factory jars that only the Spark server carries +// (paimon-vector + paimon-vector-index-java; see regression.md). +// +// Float precision note: the ANN distances are computed in f32 by paimon-rust +// while the brute-force leg computes in Doris doubles, so near-tied neighbors +// can swap positions between the legs. The assertions therefore compare the +// top-1 exactly (it is the probe row itself, distance 0), the first three in +// order, and the whole top-N as a set. +import org.apache.doris.regression.action.ProfileAction + +suite("test_paimon_pk_vector_ann", "p0,external") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disabled paimon test") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_paimon_pk_vector_ann" + String database = "pk_vector_ann_db" + int rows = 512 + String dimension = "16" + + // Session overrides are captured for the finally block the way the sibling + // suites do. + def originalRustReader = sql("select @@enable_paimon_rust_reader")[0][0] + def originalV2 = sql("select @@enable_file_scanner_v2")[0][0] + def originalEnableProfile = sql("select @@enable_profile")[0][0] + + try { + sql "set enable_profile=true" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + // The s3.region property is required: paimon-rust's S3 client rejects a + // missing region, while the JNI reader falls back to the SDK default. + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.region' = 'us-east-1', + 'use_path_style' = 'true' + )""" + + // ------------------------------------------------------------------ fixture + // Recreated on every run, so the suite is self-contained. The vector + // column is declared ARRAY + 'field.embedding.vector-dim': the 2.1 + // connector turns that into a paimon VECTOR column, which no + // Doris DDL can express. The write commit builds the ivf-rq index inline + // through the vindex factory jars loaded on the Spark server. + String vectorOptions = '{"ivf-rq.nlist":"4","ivf-rq.dimension":"' + dimension + '",' + + '"ivf-rq.rq.bits":"4","ivf-rq.train.sample-ratio":"1"}' + String generatedRows = """ + SELECT id, CAST(id % 7 AS INT), + transform(sequence(0, 15), + i -> CAST(((id * 17 + i * 13) % 521 - 260) AS FLOAT) / 260.0) + FROM (SELECT explode(sequence(1, ${rows})) AS id)""" + [false, true].each { innerProduct -> + def table = innerProduct ? "ann_ip" : "ann_l2" + def metric = innerProduct ? "inner_product" : "l2" + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${database}; + DROP TABLE IF EXISTS paimon.${database}.${table}; + CREATE TABLE paimon.${database}.${table} ( + id BIGINT, tag INT, embedding ARRAY + ) USING paimon TBLPROPERTIES ( + 'primary-key' = 'id', 'bucket' = '2', + 'merge-engine' = 'partial-update', + 'deletion-vectors.enabled' = 'true', + 'deletion-vectors.merge-on-read' = 'false', + 'changelog-producer' = 'none', + 'vector-field' = 'embedding', + 'field.embedding.vector-dim' = '${dimension}', + 'pk-vector.index.columns' = 'embedding', + 'fields.embedding.pk-vector.index.type' = 'ivf-rq', + 'fields.embedding.pk-vector.distance.metric' = '${metric}', + 'fields.embedding.pk-vector.index.options' = '${vectorOptions}', + 'ivf.nprobe' = '4' + ); + INSERT INTO paimon.${database}.${table} ${generatedRows}; + """ + } + sql """switch ${catalogName}""" + sql "use ${database}" + def fixtureCount = sql("select count(*) from ann_l2") + assertEquals(rows.toLong(), fixtureCount[0][0], "fixture must have ${rows} rows") + + // The probe vector is the embedding of id=7 itself, taken from the data: + // its top-1 neighbor must be id=7 with distance 0, and every leg below + // uses the same in-data vector so ANN and brute force are comparable + // without any golden values. + def probeRow = sql("select embedding from ann_l2 where id = 7") + String probe = probeRow[0][0].toString() + + def idsOf = { rowsResult -> rowsResult.collect { it[0] } } + def compareTopN = { String table, boolean innerProduct, int n -> + def fn = innerProduct ? "inner_product_approximate" : "l2_distance_approximate" + def orderClause = innerProduct ? "desc" : "" + sql "set enable_paimon_rust_reader=0" + def brute = idsOf(sql("""select id from ${table} + order by ${fn}(embedding, ${probe}) ${orderClause} limit ${n}""")) + sql "set enable_paimon_rust_reader=1" + def ann = idsOf(sql("""select id from ${table} + order by ${fn}(embedding, ${probe}) ${orderClause} limit ${n}""")) + assertEquals(brute.first(), ann.first(), "top-1 must match exactly") + assertEquals(brute.take(3), ann.take(3), + "top-3 order must match: brute=${brute} ann=${ann}") + assertEquals(brute.sort(), ann.sort(), + "top-${n} sets must match: brute=${brute} ann=${ann}") + } + + // ------------------------------------------------------------------ l2 + def profileAction = new ProfileAction(context) + compareTopN("ann_l2", false, 5) + assertEquals(7L, idsOf(sql("""select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 1"""))[0], + "the probe row itself must be top-1") + + // The ANN leg must actually have executed on the rust vector reader — + // otherwise the equality above could pass on a silent brute-force scan. + def annQuery = """select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""" + sql annQuery + def queryId = sql("select last_query_id()")[0][0] + def profile = profileAction.getProfile(queryId.toString(), ["PaimonRustReader"]) + assertTrue(profile.contains("PaimonRustReader"), + "ANN query must run through the paimon-rust vector reader") + + // Non-index-only projection: the raw vector column is still readable + // alongside the distance; tag = (id - 1) % 7 by the fixture generator. + def annTagged = sql("""select id, tag from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""") + annTagged.each { row -> + assertEquals(row[0] % 7, row[1], "tag must be id % 7") + } + + // Residual filter: the filter rides to BE's vector-search stage and + // must agree with the filtered brute force (as a set; see the header). + sql "set enable_paimon_rust_reader=0" + def bruteFiltered = idsOf(sql("""select id from ann_l2 where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annFiltered = idsOf(sql("""select id from ann_l2 where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteFiltered, annFiltered, "filtered ANN must match filtered brute force") + + // Rewrite disabled by the scanner switch: still correct, just slower. + sql "set enable_file_scanner_v2=0" + def noRewrite = idsOf(sql("""select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_file_scanner_v2=1" + sql "set enable_paimon_rust_reader=0" + def bruteTop5 = idsOf(sql("""select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteTop5, noRewrite, "without FileScannerV2 the query must brute-force") + + // -------------------------------------------------------- inner product + compareTopN("ann_ip", true, 5) + } finally { + sql "set enable_paimon_rust_reader=${originalRustReader}" + sql "set enable_file_scanner_v2=${originalV2}" + sql "set enable_profile=${originalEnableProfile}" + try { + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } catch (Throwable t) { + logger.info("drop catalog failed: " + t) + } + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy index 274bc1c67ab069..0f85825e9d783c 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -57,14 +57,19 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { } def snapshotCommitTimeMillis = { String tableName, String snapshotId -> - List> rows = spark_paimon """ - -- Paimon's TIMESTAMP_NTZ commit time is a UTC wall clock; convert it to the - -- Spark session zone before casting so unix_millis keeps the actual instant. - SELECT unix_millis(CAST( - convert_timezone('UTC', current_timezone(), commit_time) AS TIMESTAMP - )) - FROM paimon.${dbName}.`${tableName}\$snapshots` - WHERE snapshot_id = ${snapshotId} + // Read the commit time through Doris so the epoch reference matches the + // scan.creation-time-millis resolution (the earliest snapshot committed at + // or after the value) on the same clock: Doris' unix_timestamp keeps the + // fractional seconds, so * 1000 is the exact commit instant. The previous + // Spark-side convert_timezone + unix_millis conversion assumed the NTZ + // commit time is a UTC wall clock, which the 2.1 ThriftServer no longer + // holds — the value came out shifted by the session UTC offset (8h into + // the future) and every creation-time scan silently resolved to the latest + // snapshot. + List> rows = sql """ + select cast(unix_timestamp(commit_time) * 1000 as bigint) + from ${catalogName}.${dbName}.${tableName}\$snapshots + where snapshot_id = ${snapshotId} """ assertEquals(1, rows.size()) return rows[0][0].toString() diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy index fc3f30477c50ac..c1b10059f7b25d 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy @@ -55,7 +55,19 @@ suite("test_paimon_orc_ltz_encoding", "p0,external,paimon") { SHOW TBLPROPERTIES paimon.${dbName}.${tableName} ('${legacyKey}') """)) } - assertEncoding("t_modern", "false") + // The paimon 1.4 connector materialized unset property defaults into the stored + // table options, so t_modern (created without the legacy flag) read back as + // "false". The 2.1 connector no longer persists unset defaults, so the same + // query fails with "does not have property" — which still means the modern + // default. Anything else (a persisted "true", or any other error) must fail. + boolean modernDefault = false + try { + assertEncoding("t_modern", "false") + modernDefault = true + } catch (Throwable t) { + modernDefault = t.toString().contains("does not have property") + } + assertTrue(modernDefault, "t_modern must keep the modern (non-legacy) encoding default") sql """CREATE TABLE t_legacy (event_time TIMESTAMPTZ(6)) ENGINE=paimon PROPERTIES ('file.format' = 'orc', '${legacyKey}' = 'true')""" sql """CREATE TABLE IF NOT EXISTS t_legacy (event_time TIMESTAMPTZ(6)) ENGINE=paimon""" diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy index bfdeffa376bbe6..715bcd178c2c4f 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy @@ -447,7 +447,7 @@ suite("test_paimon_write_transaction", "p0,external,paimon") { sql """INSERT INTO t_failed_write VALUES (2, 'accepted_before_error'), (3, repeat('x', 1048576))""" - exception "The record exceeds the maximum size of a sort buffer" + exception "The record exceeds the maximum size of a" } qt_txn_failed_write_after """SELECT id, payload FROM t_failed_write ORDER BY id""" qt_txn_failed_snapshot_after """SELECT COUNT(*) FROM t_failed_write\$snapshots""" From 492063c5a3c4a6ed722ddf257c02353a56e1f317 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Tue, 13 Oct 2026 19:13:37 +0800 Subject: [PATCH 04/13] [fix](paimon) make the PK-vector rewrite sound for bounded top-N (PR review) Addresses the P1/P2 review findings on PushDownVectorTopNIntoPaimonScan: P1 - Keep residual filtering before the bounded vector top-N. The BE paimon-rust predicate converter skips conjuncts it cannot represent (and the synthetic distance column is outside its field registry), and the engine re-applies those residuals on the returned block -- after the top-K rows were already selected. With candidate distances 0, 2 and 3, a `WHERE dis > 1 ORDER BY dis LIMIT 1` fetched only the distance-0 row and returned nothing. The rewrite now fires with a Filter only when every conjunct provably rides into the vector search: a comparison / IN / null-check on a table column whose Doris type is in the converter's supported set (mirrors PaimonRustPredicateConverter._is_supported_slot_type). Anything else declines the rewrite and the query computes the distance in Doris, which is always correct. Verified end to end: the differential leg caught the empty result on the unfixed build and passes on the fixed one. P1 - Do not strip value-changing casts from the indexed vector. A legal cast chain can change vector values (an ARRAY -> ARRAY -> ARRAY round trip loses precision), so searching the original vectors while the query means the casted ones changes the ranking. The rewrite now accepts a direct indexed column slot only. P1 - Preserve the relation's file-creation-time restriction on the vector path. The ordinary path drops manifest entries older than the bound, but the PK-vector search plans its own splits and never consults the bound, so old files' nearest neighbors could leak into the result. A scan carrying scan.file-creation-time-millis now declines the rewrite (snapshot / tag / version selectors are resolved into the table before the vector search and stay eligible). P2 - Check the projection capabilities before committing the rewrite. A PK-vector split has no JNI fallback, so a query whose projected columns would set the projectedVariant bound (nested Variant) or an ORC table with TIMESTAMP_LTZ in its schema -- both scan-node bounds with no vector-path fallback -- would turn a supported query into an error. Both are now pre-flighted and decline instead, keeping the query on an ordinary supported path. Tests: PushDownVectorTopNIntoPaimonScanTest grows from 20 to 27 cases (distance-predicate / unsupported-type / expression filters decline, a simple column filter still rides, file-creation-time declines while a snapshot selector does not, lossy cast declines, Variant projection declines; the two distance-predicate positives flip to the new sound semantics). The ANN regression suite adds the requested differential legs: a distance lower bound, an unconvertible ARRAY predicate and a file-creation-time bound, each compared against the rust-disabled brute force. Co-Authored-By: Claude Opus 5 (1M context) --- .../PushDownVectorTopNIntoPaimonScan.java | 210 +++++++++++++++- .../PushDownVectorTopNIntoPaimonScanTest.java | 227 ++++++++++++++---- .../paimon/test_paimon_pk_vector_ann.groovy | 58 +++++ 3 files changed, 440 insertions(+), 55 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java index 14eb875d331ffe..3b5e04fbbad660 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java @@ -17,19 +17,25 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Type; import org.apache.doris.datasource.credentials.CredentialUtils; import org.apache.doris.datasource.credentials.VendedCredentialsFactory; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonUtil; import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; @@ -47,10 +53,17 @@ import org.apache.doris.thrift.TVectorMetric; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.apache.paimon.CoreOptions; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; import java.util.List; import java.util.Map; @@ -90,6 +103,12 @@ */ public class PushDownVectorTopNIntoPaimonScan implements RewriteRuleFactory { + /** Doris column types the BE paimon-rust predicate converter can push; see + * PaimonRustPredicateConverter._is_supported_slot_type on the BE side. */ + private static final Set VECTOR_FILTER_SUPPORTED_COLUMN_TYPES = ImmutableSet.of( + Type.BOOLEAN, Type.TINYINT, Type.SMALLINT, Type.INT, Type.BIGINT, + Type.STRING, Type.DATE, Type.DATEV2); + @Override public List buildRules() { return ImmutableList.of( @@ -138,7 +157,43 @@ private Plan pushDown( // incremental scan the user requested, with the distance computed in Doris. // (The scan node's vector gate keeps !isIncremental as the authoritative // backstop for any path that produces a vector split regardless.) - if (scan.getScanParams().isPresent() && scan.getScanParams().get().incrementalRead()) { + // + // The same holds for a pinned file-creation-time read state: the ordinary + // path enforces `scan.file-creation-time-millis` by dropping manifest + // entries older than the bound, but the vector search plans its own splits + // and never consults that bound, so old files' nearest neighbors could leak + // into the result. Snapshot/tag/version selectors are fine -- they are + // resolved into the table handle before the vector search -- so only the + // file-level bound forces the decline. + if (scan.getScanParams().isPresent()) { + TableScanParams scanParams = scan.getScanParams().get(); + if (scanParams.incrementalRead()) { + return null; + } + Map rawParams = scanParams.getMapParams(); + if (rawParams.containsKey(CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key())) { + return null; + } + } + + // A bounded vector top-N is only correct when every filter conjunct is + // applied inside the search, before the limit is exhausted. The BE + // paimon-rust predicate converter skips conjuncts it cannot represent (and + // the synthetic distance column is not in its field registry at all), and + // the engine re-applies those residuals on the returned block -- after the + // top-K rows were already selected. A skipped residual can therefore drop + // rows out of a bounded result: with candidate distances 0, 2 and 3, a + // `WHERE dis > 1 ORDER BY dis LIMIT 1` would fetch only the distance-0 row, + // filter it out and return nothing instead of the distance-2 row. So the + // rewrite only fires when each conjunct is provably of a form the converter + // pushes: a simple comparison / IN / null-check on a table column whose + // Doris type is in the converter's supported set (see + // PaimonRustPredicateConverter._is_supported_slot_type on BE). Anything + // else -- an expression predicate, a predicate on the distance itself, an + // unsupported column type -- declines the rewrite and the query runs as an + // ordinary scan with the distance computed in Doris, which is always + // correct. + if (optionalFilter.isPresent() && !conjunctsRideIntoVectorSearch(optionalFilter.get())) { return null; } @@ -169,6 +224,13 @@ private Plan pushDown( return null; } + // The indexed column must be a direct slot reference. A Cast chain over the + // column is NOT stripped: a legal cast can change vector values (for example + // an ARRAY -> ARRAY -> ARRAY round trip loses precision), + // and searching the original vectors while the query means the casted ones + // changes both the distance and the row order. Doris's implicit coercions do + // not produce a cast here for a matching ARRAY column, so anything + // wrapped in a Cast simply declines the rewrite. Expression left; Expression right; if (l2Dist) { @@ -178,9 +240,6 @@ private Plan pushDown( left = ((InnerProductApproximate) orderKeyExpr).left(); right = ((InnerProductApproximate) orderKeyExpr).right(); } - while (left instanceof Cast) { - left = ((Cast) left).child(); - } if (!(left instanceof SlotReference && right.isConstant() && right instanceof ArrayLiteral)) { return null; } @@ -213,6 +272,22 @@ private Plan pushDown( if (!expectedMetric.equals(metric)) { return null; } + + // Pre-flight the projection-dependent bounds the vector split would hit at + // the scan node: a PK-vector split has no JNI fallback (it would return an + // empty distance column), so a query whose projected columns would set + // projectedVariant -- or an ORC table with TIMESTAMP_LTZ in its schema, the + // orc-ltz-shape bound -- must not be rewritten at all. Without the rewrite + // those queries run through the ordinary paths, where they are supported. + try { + if (projectedColumnsContainVariant(project, optionalFilter) + || orcSchemaContainsTimestampLtz(paimonTable)) { + return null; + } + } catch (Throwable t) { + return null; + } + // The thrift-level enum the AnnTopNInfo will carry. expectedMetric is kept as a // string for the comparison above (coreOptions returns a string), but the // AnnTopNInfo / wire contract uses TVectorMetric, so map once here. DOT_PRODUCT @@ -310,4 +385,129 @@ private Plan pushDown( LogicalProject newProject = project.withProjectsAndChild(newProjections, plan); return topN.withChildren(newProject); } + + /** + * Whether every filter conjunct provably rides into the BE vector search (see + * the call site for the bounded-top-N correctness argument). Conservative by + * design: forms outside this whitelist may well be convertible, but the query + * then simply runs un-rewritten, which is always correct. + */ + private static boolean conjunctsRideIntoVectorSearch(LogicalFilter filter) { + for (Expression conjunct : filter.getConjuncts()) { + if (!conjunctRidesIntoVectorSearch(conjunct)) { + return false; + } + } + return true; + } + + private static boolean conjunctRidesIntoVectorSearch(Expression conjunct) { + if (conjunct instanceof ComparisonPredicate) { + ComparisonPredicate compare = (ComparisonPredicate) conjunct; + return (isVectorFilterColumnSlot(compare.left()) && compare.right() instanceof Literal) + || (compare.left() instanceof Literal && isVectorFilterColumnSlot(compare.right())); + } + if (conjunct instanceof InPredicate) { + InPredicate in = (InPredicate) conjunct; + return isVectorFilterColumnSlot(in.getCompareExpr()) + && in.getOptions().stream().allMatch(Literal.class::isInstance); + } + if (conjunct instanceof IsNull) { + return isVectorFilterColumnSlot(((IsNull) conjunct).child()); + } + if (conjunct instanceof Not && ((Not) conjunct).child() instanceof IsNull) { + return isVectorFilterColumnSlot(((IsNull) ((Not) conjunct).child()).child()); + } + return false; + } + + private static boolean isVectorFilterColumnSlot(Expression expr) { + if (!(expr instanceof SlotReference)) { + return false; + } + SlotReference slot = (SlotReference) expr; + return slot.getOriginalColumn().isPresent() + && VECTOR_FILTER_SUPPORTED_COLUMN_TYPES.contains( + slot.getOriginalColumn().get().getType()); + } + + /** + * Mirrors the scan node's projectedVariant bound (PaimonScanNode: any slot whose + * Doris type contains a Variant keeps the query off the rust reader, and a + * PK-vector split has no non-rust fallback): any projected or filtered column + * containing Variant declines the rewrite so the query stays on an ordinary + * supported path. + */ + private static boolean projectedColumnsContainVariant( + LogicalProject project, Optional> optionalFilter) { + List expressions = new java.util.ArrayList<>(); + expressions.addAll((project.getProjects())); + optionalFilter.ifPresent(filter -> expressions.addAll(filter.getConjuncts())); + for (Expression expression : expressions) { + Set slots = expression.collect(SlotReference.class::isInstance); + for (SlotReference slotRef : slots) { + if (!slotRef.getOriginalColumn().isPresent()) { + continue; + } + if (PaimonUtil.containsVariant(slotRef.getOriginalColumn().get().getType())) { + return true; + } + } + } + return false; + } + + /** + * Mirrors the scan node's orc-ltz-schema bound: paimon-rust's ORC + * TIMESTAMP_LTZ decoding is not equivalent to Doris semantics yet, and a vector + * split would materialize rows through that decoder with no fallback. Snapshot + * selection is irrelevant here -- this is a physical-file property. + */ + private static boolean orcSchemaContainsTimestampLtz(PaimonExternalTable paimonTable) { + try { + Table baseTable = paimonTable.getPaimonTable(Optional.empty()); + if (!(baseTable instanceof FileStoreTable)) { + return false; + } + FileStoreTable fileStoreTable = (FileStoreTable) baseTable; + // Paimon's default file format is ORC; only a PARQUET table is known to + // be free of the orc-ltz shape (an unset key falls to the default). + String fileFormat = CoreOptions.fromMap(fileStoreTable.options()).fileFormatString(); + if (!CoreOptions.FILE_FORMAT_PARQUET.equalsIgnoreCase(String.valueOf(fileFormat))) { + return fileStoreTable.schema().fields().stream() + .anyMatch(field -> containsTimestampLtz(field.type())); + } + return false; + } catch (Throwable t) { + // The table might genuinely be unreadable; the ordinary path will surface + // the real error, and declining the rewrite is the safe answer here. + return true; + } + } + + /** Mirrors PaimonScanNode.containsTimestampLtz (package-private there). */ + private static boolean containsTimestampLtz(DataType type) { + if (type == null) { + return false; + } + if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + return true; + } + if (type instanceof ArrayType) { + return containsTimestampLtz(((ArrayType) type).getElementType()); + } + if (type instanceof MapType) { + MapType mapType = (MapType) type; + return containsTimestampLtz(mapType.getKeyType()) + || containsTimestampLtz(mapType.getValueType()); + } + if (type instanceof MultisetType) { + return containsTimestampLtz(((MultisetType) type).getElementType()); + } + if (type instanceof RowType) { + return ((RowType) type).getFields().stream() + .anyMatch(field -> containsTimestampLtz(field.type())); + } + return false; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java index 036a3171dac7b4..f2beb533eb4331 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java @@ -30,7 +30,9 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.LessThan; @@ -273,10 +275,16 @@ public void testOutputColumnNameKeepsTheFunctionCallWithoutAlias() { } @Test - public void testFilterConjunctReusesScore() { + public void testDistancePredicateFilterNotPushedDown() { // WHERE l2_distance_approximate(vec, q) < 10 ORDER BY the same expression. - // Mirroring the internal-table rule, the conjunct is rewritten to reuse the index value - // instead of recomputing the distance from the raw vector. + // A predicate on the distance cannot ride into the BE vector search (the + // synthetic distance column is not in the paimon-rust predicate converter's + // field registry), so it would run as a residual AFTER the bounded top-N + // fetch -- with candidate distances 0, 2 and 3, a `dis > 1 LIMIT 1` would + // fetch only the distance-0 row, filter it out and return nothing instead + // of the distance-2 row. An upper bound happens to be monotone in the + // ASC order, but the rewrite cannot rely on residual monotonicity, so the + // rewrite declines and the query computes the distance in Doris. LogicalFileScan scan = paimonScan(ImmutableMap.of( "pk-vector.index.columns", "vec", "fields.vec.pk-vector.distance.metric", "l2")); @@ -288,21 +296,74 @@ public void testFilterConjunctReusesScore() { LogicalTopN topN = topN( new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + assertNotPushedDown(topN); + } + + @Test + public void testSimpleColumnFilterStillPushedDown() { + // WHERE tag < 3 ORDER BY dis: a comparison on a plain supported-type column + // provably rides into the BE vector search (PaimonRustPredicateConverter + // pushes comparisons on BOOLEAN..BIGINT/STRING/DATE/DATEV2 columns), so the + // bounded top-N sees the predicate and the rewrite stays sound. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + Assertions.assertEquals("tag", tag.getName()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(tag, new IntegerLiteral(3))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + Plan rewritten = applyRule(1, topN); Assertions.assertNotSame(topN, rewritten); - LogicalProject project = (LogicalProject) rewritten.child(0); - LogicalFilter newFilter = (LogicalFilter) project.child(0); - LogicalFileScan newScan = (LogicalFileScan) newFilter.child(0); - Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + LogicalFileScan newScan = (LogicalFileScan) ((LogicalFilter) project.child(0)).child(0); + Assertions.assertTrue(newScan.getAnnTopN().isPresent(), + "the simple column filter must not block the bounded ANN rewrite"); + } - Assertions.assertEquals(1, newFilter.getConjuncts().size()); - Expression conjunct = newFilter.getConjuncts().iterator().next(); - Assertions.assertFalse(containsDistanceFunction(conjunct), - "the filter must not recompute the distance from the raw vector"); - Assertions.assertEquals( - new LessThan(distanceSlotOf(newScan), new IntegerLiteral(10)), - conjunct); + @Test + public void testUnsupportedColumnTypeFilterNotPushedDown() { + // WHERE score < 1.5 with score being a DOUBLE column: the paimon-rust + // predicate converter keeps IEEE-ordering types (DOUBLE/FLOAT/DECIMAL/ + // VARCHAR/DATETIME) residual, and a residual after the bounded top-N is + // unsound, so the rewrite declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference score = (SlotReference) scan.getOutput().get(4); + Assertions.assertEquals("score", score.getName()); + Assertions.assertEquals(Type.DOUBLE, score.getOriginalColumn().get().getType()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(score, new FloatLiteral(1.5f))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + assertNotPushedDown(topN); + } + + @Test + public void testExpressionFilterNotPushedDown() { + // WHERE id + 1 < 10: an arithmetic expression cannot ride into the vector + // search, so the rewrite declines instead of filtering after the limit. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference id = idSlot(scan); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan( + new Add(id, new IntegerLiteral(1)), new IntegerLiteral(10))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + assertNotPushedDown(topN); } @Test @@ -404,11 +465,13 @@ public void testAliasedOutputColumnNameKeepsUserAlias() { } @Test - public void testPredicateOnAliasedDistanceIsRewritten() { + public void testPredicateOnAliasedDistanceNotPushedDown() { // SELECT id, l2_distance_approximate(vec, q) AS dis // FROM xx WHERE l2_distance_approximate(vec, q) < 1 ORDER BY dis - // The filter conjunct uses the bare distance function, which the rule - // replaces with the index-produced distance slot. + // The predicate is on the distance: it cannot ride into the BE vector + // search (the synthetic distance column is outside the paimon-rust + // predicate converter's field registry), and applying it as a residual + // after the bounded top-N fetch is unsound, so the rewrite declines. LogicalFileScan scan = paimonScan(ImmutableMap.of( "pk-vector.index.columns", "vec", "fields.vec.pk-vector.distance.metric", "l2")); @@ -420,49 +483,30 @@ public void testPredicateOnAliasedDistanceIsRewritten() { LogicalTopN topN = topN( new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); - Plan rewritten = applyRule(1, topN); - Assertions.assertNotSame(topN, rewritten); - - LogicalProject project = (LogicalProject) rewritten.child(0); - LogicalFilter newFilter = (LogicalFilter) project.child(0); - LogicalFileScan newScan = (LogicalFileScan) newFilter.child(0); - Assertions.assertTrue(newScan.getAnnTopN().isPresent()); - - // The filter conjunct now references the distance slot, not the function call. - Assertions.assertEquals(1, newFilter.getConjuncts().size()); - Expression conjunct = newFilter.getConjuncts().iterator().next(); - Assertions.assertFalse(containsDistanceFunction(conjunct), - "filter must not recompute the distance from the raw vector"); - Assertions.assertEquals( - new LessThan(distanceSlotOf(newScan), new FloatLiteral(1.0f)), - conjunct); - - // The alias output name stays "dis". - Alias rewrittenDis = (Alias) project.getProjects().get(1); - Assertions.assertEquals("dis", rewrittenDis.getName()); + assertNotPushedDown(topN); } @Test public void testMergedSubqueryRewriteStillTriggers() { // SELECT id, dis FROM ( // SELECT id, l2_distance_approximate(vec, q) AS dis FROM xx - // ) WHERE dis < 1 ORDER BY dis + // ) WHERE tag < 3 ORDER BY dis // // After MergeProjects / MergeFilters runs in the planner, the inner and outer - // projects collapse, and the filter's reference to the `dis` alias is resolved - // back to the underlying L2DistanceApproximate. The rule then sees: + // projects collapse. The rule then sees: // TopN -> Project[id, dis := l2_distance_approximate(vec,q)] - // -> Filter[l2_distance_approximate(vec,q) < 1] + // -> Filter[tag < 3] // -> PaimonScan - // which matches the second rule variant (with filter). + // which matches the second rule variant (with filter). The simple column + // comparison rides into the vector search, so the rewrite stays sound. LogicalFileScan scan = paimonScan(ImmutableMap.of( "pk-vector.index.columns", "vec", "fields.vec.pk-vector.distance.metric", "l2")); SlotReference vec = vecSlot(scan); - Expression distanceExpr = new L2DistanceApproximate(vec, queryVector()); + SlotReference tag = (SlotReference) scan.getOutput().get(3); LogicalFilter filter = new LogicalFilter<>( - ImmutableSet.of(new LessThan(distanceExpr, new FloatLiteral(1.0f))), scan); - Alias dis = new Alias(distanceExpr, "dis"); + ImmutableSet.of(new LessThan(tag, new IntegerLiteral(3))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); LogicalTopN topN = topN( new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); @@ -477,12 +521,10 @@ public void testMergedSubqueryRewriteStillTriggers() { Assertions.assertTrue(newScan.getAnnTopN().isPresent()); Assertions.assertEquals(TVectorMetric.L2, newScan.getAnnTopN().get().getMetric()); - // The filter now uses the distance slot. + // The filter keeps its column predicate (nothing to replace). Expression conjunct = newFilter.getConjuncts().iterator().next(); Assertions.assertFalse(containsDistanceFunction(conjunct)); - Assertions.assertEquals( - new LessThan(distanceSlotOf(newScan), new FloatLiteral(1.0f)), - conjunct); + Assertions.assertEquals(new LessThan(tag, new IntegerLiteral(3)), conjunct); // The alias keeps its user name "dis" and wraps the distance slot. Alias rewrittenDis = (Alias) project.getProjects().get(1); @@ -603,6 +645,82 @@ public void testIncrementalScanNotPushedDown() { new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); } + @Test + public void testFileCreationTimeScanNotPushedDown() { + // t@options('scan.file-creation-time-millis'='...'): the ordinary path drops + // manifest entries older than the bound, but the PK-vector search plans its + // own splits and never consults that bound -- old files' nearest neighbors + // would leak into the bounded result. Snapshot/tag selectors are resolved + // into the table before the vector search and stay eligible; only the + // file-level bound declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("options", + ImmutableMap.of("scan.file-creation-time-millis", "1791793167062"), + ImmutableList.of()))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testSnapshotScanStillPushedDown() { + // t@options('scan.snapshot-id'='1'): snapshot selectors are resolved into + // the table handle before the vector search plans its splits, so they do + // not block the rewrite (contrast with the file-creation-time bound). + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("options", + ImmutableMap.of("scan.snapshot-id", "1"), ImmutableList.of()))); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + Plan rewritten = applyRule(0, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + Assertions.assertNotSame(scan, ((LogicalProject) rewritten.child(0)).child(0)); + Assertions.assertTrue(((LogicalFileScan) ((LogicalProject) rewritten.child(0)) + .child(0)).getAnnTopN().isPresent()); + } + + @Test + public void testCastOnIndexedVectorNotPushedDown() { + // l2_distance_approximate(CAST(vec AS ARRAY), q): a legal cast chain + // can change vector values (an ARRAY -> ARRAY -> ARRAY + // round trip loses precision), so searching the original vectors while the + // query means the casted ones would change the ranking and the distance. + // The rewrite only accepts a direct indexed column slot. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Cast castVec = new Cast(vec, org.apache.doris.nereids.types.ArrayType.of( + org.apache.doris.nereids.types.DoubleType.INSTANCE)); + Alias dis = new Alias(new L2DistanceApproximate(castVec, queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testVariantProjectionNotPushedDown() { + // SELECT payload, dis ... ORDER BY dis with a VARIANT payload column: the + // rust reader has no Variant decoder, so the scan node's projectedVariant + // bound would reject the vector split -- and a vector split has no JNI + // fallback, so the rewrite would turn a supported query into an error. + // Pre-flight the projection and decline instead; the query then scans + // through the ordinary path. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference payload = (SlotReference) scan.getOutput().get(5); + Assertions.assertEquals("payload", payload.getName()); + Assertions.assertTrue(payload.getOriginalColumn().get().getType().isVariantType()); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), payload, dis), scan), dis, true)); + } + // ------------------------------------------------------------------ helpers private Plan applyRule(int ruleIndex, LogicalTopN topN) { @@ -645,7 +763,10 @@ private LogicalFileScan paimonScan(Map options, List schema = ImmutableList.of( new Column("id", Type.INT, true), new Column("vec", new ArrayType(Type.FLOAT), true), - new Column("vec2", new ArrayType(Type.FLOAT), true)); + new Column("vec2", new ArrayType(Type.FLOAT), true), + new Column("tag", Type.INT, true), + new Column("score", Type.DOUBLE, true), + new Column("payload", Type.VARIANT, true)); PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); Mockito.when(table.getName()).thenReturn("paimon_tbl"); @@ -669,6 +790,12 @@ private LogicalFileScan paimonScan(Map options, Mockito.when(paimonTable.schema()).thenReturn(null); Mockito.when(table.getPaimonTable(Mockito.>any())) .thenReturn(paimonTable); + // An options-type TableScanParams binds its schema through + // getFullSchema(TableScanParams) (the relation-option path inside + // PaimonExternalTable, which would otherwise hit the unmocked package + // private helpers). Stub it directly so the options tests get real + // output slots; the rule itself only reads the params' raw map. + Mockito.when(table.getFullSchema(Mockito.any())).thenReturn(schema); PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); Mockito.when(table.getCatalog()).thenReturn(catalog); diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy index 503f70e06c4862..8ba5342dc3dc68 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy @@ -186,6 +186,64 @@ suite("test_paimon_pk_vector_ann", "p0,external") { order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() assertEquals(bruteTop5, noRewrite, "without FileScannerV2 the query must brute-force") + // ------------------------------------------------------------------ + // Bounded-search soundness (review feedback on the rewrite): + // A predicate that cannot ride into the BE vector search must not be + // answered from a top-(LIMIT) fetch, so the rewrite declines for these + // shapes and the query runs with the distance computed in Doris. Every + // leg is a differential against the rust-disabled brute force. + + // A predicate on the distance itself: the synthetic distance column is + // outside the paimon-rust predicate converter's field registry, and a + // residual applied after the bounded fetch would return the wrong rows + // (with candidate distances 0/2/3, `dis > 1 LIMIT 1` must return the + // distance-2 row, not an empty result). + def distanceLowerBound = { + sql "set enable_paimon_rust_reader=0" + def brute = idsOf(sql("""select id from ann_l2 + where l2_distance_approximate(embedding, ${probe}) > 1 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def ann = idsOf(sql("""select id from ann_l2 + where l2_distance_approximate(embedding, ${probe}) > 1 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(brute, ann, "a distance predicate must not be served by a bounded fetch") + } + distanceLowerBound() + + // An unconvertible data predicate (the ARRAY vector column is not + // a paimon-rust predicate type): same soundness rule. + sql "set enable_paimon_rust_reader=0" + def bruteArrayPred = idsOf(sql("""select id from ann_l2 + where embedding is not null + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annArrayPred = idsOf(sql("""select id from ann_l2 + where embedding is not null + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteArrayPred, annArrayPred, + "an unconvertible predicate must not be served by a bounded fetch") + + // A pinned file-creation-time read state: the ordinary path drops manifest + // entries older than the bound, but a PK-vector search plans its own splits + // and never consults the bound -- so the rewrite declines and both legs + // must agree on the file-visible rows. A bound of 1 keeps every file; a + // far-future bound keeps none. + sql "set enable_paimon_rust_reader=0" + def bruteOld = idsOf(sql("""select id from ann_l2@options( + 'scan.file-creation-time-millis'='1') + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + def bruteNone = idsOf(sql("""select count(*) from ann_l2@options( + 'scan.file-creation-time-millis'='1791821967062000')""")) + sql "set enable_paimon_rust_reader=1" + def annOld = idsOf(sql("""select id from ann_l2@options( + 'scan.file-creation-time-millis'='1') + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + def annNone = idsOf(sql("""select count(*) from ann_l2@options( + 'scan.file-creation-time-millis'='1791821967062000')""")) + assertEquals(bruteOld, annOld, "a file-creation-time bound must be honored on the vector path") + assertEquals(bruteNone, annNone, "a far-future file bound must exclude all files on both paths") + // -------------------------------------------------------- inner product compareTopN("ann_ip", true, 5) } finally { From 045fb2ab33260fc54f5380f8735ec80a21a5d24f Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Mon, 19 Oct 2026 15:07:00 +0800 Subject: [PATCH 05/13] [chore](hive-catalog) pin hive-catalog-shade to the released 4.1.0 Replace the 3.1.4-SNAPSHOT dependency with its release build. The 4.1.0 artifact is byte-identical to the snapshot it replaces (same class list and file sizes, same shaded paimon 2.1.0 / hive 3.1.3, identical pom dependency declarations), so this only pins the FE build to a stable release version. Co-Authored-By: Claude Opus 5 (1M context) --- fe/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/pom.xml b/fe/pom.xml index ef0b47a6f48a91..f0cc56a669802c 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -231,7 +231,7 @@ under the License. fe-authentication - 3.1.4-SNAPSHOT + 4.1.0 1.12.1 1.18.0 From 7ab22d894ca599417bedf5378f7dce0f486a4d3b Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Mon, 19 Oct 2026 21:47:38 +0800 Subject: [PATCH 06/13] [fix](paimon) close the bounded-top-N and wire-protocol gaps of the ANN path Static review of this PR surfaced twelve soundness and compatibility gaps on the PK-vector (ANN) search path; every one was verified against the pinned sources before the fix: - Wire protocol: TPaimonFileDesc.vector_payload moves from field 20 to 21 and TFileScanRangeParams.external_search_request from 41 to 42 -- both IDs are already reserved on master (original_file_path, fluss_properties), and reusing them would make a mixed-version cluster drop the payload instead of decoding it. BE/FE must upgrade together. - Vector capability negotiation: a new heartbeat bit supports_paimon_rust_vector_reader (TBackendInfo field 12, free on all branches) gates the one-way rewrite separately from the batch rust bit, so a rolling upgrade keeps ordinary rust reads working and only declines the vector rewrite against older BEs. - Partitioned vector splits now carry the derived partition values (PaimonSplit.setPaimonPartitionValues), which setPaimonParams demands for any table with partition keys. - The rewrite pre-flight now also declines: a query vector containing a NULL element (getDouble() would silently search a zero coordinate while the un-rewritten function raises); scan.creation-time-millis (resolves into the same internal file-creation bound the vector search never consults); a data predicate on a table without deletion vectors or with merge-on-read (paimon-rust rejects it AFTER the one-way rewrite); a table column named __paimon_search_score_to_dis (collides with the synthetic distance slot); and an expression over a complex column (nested pruning reduces the scan tuple while the vector reader projects only root names -- the struct SerDe then binds children by ordinal). The shape whitelist additionally rejects NullLiteral operands: the BE converter rejects a null literal and with it the whole predicate, leaving it in the post-fetch residual. The scan node keeps the nested-projection bound as a loud backstop for vector splits and logs the vector-capability reason. - BE: _convert_score_to_distance maps zero and out-of-range scores to +inf instead of negative sentinels -- a zero score is reachable from a genuine huge distance (the search-mode=full exact fallback computes squared L2 in f32), and -2 sorted ahead of every finite distance, inverting the coordinator's TopN. _build_read_columns skips the synthetic distance name only in vector mode, so an ordinary scan of a real column with that name keeps reading it. The BE test include path is fixed (the file never compiled). - Regression framework: the 0x-token decode pass only substitutes UTF-8-round-trippable bytes and keeps the lossless hex token otherwise -- replacement-character decoding made 0x80 and 0x81 compare equal. Verified: FE UT 36/36 (rewrite) + 96/0 (scan node) + 22/22 (rust compat) + 4/4 (capability); BE UT 30/30; the full ANN regression suite including the four new differential legs (no-deletion-vector first-row table, partitioned table, NULL query vector, IN with a NULL option). Co-Authored-By: Claude Opus 5 (1M context) --- be/src/agent/heartbeat_server.cpp | 4 + .../table/paimon_rust_table_reader.cpp | 39 +++- .../table/paimon_rust_table_reader.h | 22 +- .../table/paimon_rust_table_reader_test.cpp | 72 +++++-- .../paimon/source/PaimonRustEligibility.java | 25 ++- .../source/PaimonRustReaderCapabilities.java | 2 +- .../paimon/source/PaimonScanNode.java | 52 +++++ .../paimon/source/PaimonVectorSearch.java | 9 +- .../PushDownVectorTopNIntoPaimonScan.java | 156 +++++++++++++- .../java/org/apache/doris/system/Backend.java | 14 ++ .../doris/system/BackendHbResponse.java | 12 ++ .../org/apache/doris/system/HeartbeatMgr.java | 3 + .../PushDownVectorTopNIntoPaimonScanTest.java | 202 +++++++++++++++++- .../BackendPaimonRustCapabilityTest.java | 37 ++++ gensrc/thrift/HeartbeatService.thrift | 6 + gensrc/thrift/PlanNodes.thrift | 10 +- .../doris/regression/util/ResultUtils.groovy | 24 ++- .../paimon/test_paimon_pk_vector_ann.groovy | 120 +++++++++++ 18 files changed, 747 insertions(+), 62 deletions(-) diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp index 5cf8c7df01da7b..26fbbec7daa07e 100644 --- a/be/src/agent/heartbeat_server.cpp +++ b/be/src/agent/heartbeat_server.cpp @@ -97,6 +97,10 @@ void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result, heartbeat_result.backend_info.__set_be_mem(MemInfo::physical_mem()); // Reader enum support cannot be inferred from a release version during rolling upgrades. heartbeat_result.backend_info.__set_supports_paimon_rust_reader(true); + // The PK-vector (ANN) protocol is a separate negotiation: an older + // rust-capable BE would reject an empty paimon_split + vector payload as + // "missing paimon_split", so only BEs carrying the vector protocol report it. + heartbeat_result.backend_info.__set_supports_paimon_rust_vector_reader(true); } watch.stop(); if (watch.elapsed_time() > 1000L * 1000L * 1000L) { diff --git a/be/src/format_v2/table/paimon_rust_table_reader.cpp b/be/src/format_v2/table/paimon_rust_table_reader.cpp index d80abeca7a82ba..472f34a20260f2 100644 --- a/be/src/format_v2/table/paimon_rust_table_reader.cpp +++ b/be/src/format_v2/table/paimon_rust_table_reader.cpp @@ -899,9 +899,12 @@ void PaimonRustTableReader::_convert_score_to_distance(Block* block, uint32_t bl // clamp, and computing the inverse on an illegal score would produce a // non-finite distance (score <= 0 -> 1/score is inf; score > 1 -> negative // radicand -> NaN), which makes the coordinator's Top-N comparisons - // ill-defined. Report a distinct negative sentinel per case instead: both - // are unreachable for a genuine Euclidean distance, so they surface as - // visibly bogus rows that also say WHICH way the index is broken. + // ill-defined. Both emit kCorruptScoreDistance instead: +inf sorts last in + // an ASC Top-N, so these rows can never displace a valid closer row. A + // score of exactly 0 is REACHABLE from a genuine huge distance — the exact + // fallback (vector-index.search-mode=full) computes the squared L2 in f32, + // and norms above ~1.8e19 overflow to +inf, so 1/(1+d^2) rounds to 0; + // publishing a sentinel ahead of finite rows would invert the final TopN. // // A score slightly above 1 is NOT corruption: the kernel computes squared // L2 as ||a||^2 + ||b||^2 - 2, which can yield a small negative @@ -911,13 +914,13 @@ void PaimonRustTableReader::_convert_score_to_distance(Block* block, uint32_t bl if (UNLIKELY(score <= 0.0F)) { LOG(WARNING) << "paimon-rust L2 score_to_distance: corrupt score " << score << " (row " << i << ", expect score in (0, 1]), " - << "sentinel " << kCorruptScoreNotPositive; - (*data)[i] = kCorruptScoreNotPositive; + << "distance " << kCorruptScoreDistance; + (*data)[i] = kCorruptScoreDistance; } else if (UNLIKELY(score > 1.0F + kScoreAboveOneTolerance)) { LOG(WARNING) << "paimon-rust L2 score_to_distance: corrupt score " << score << " (row " << i << ", expect score in (0, 1]), " - << "sentinel " << kCorruptScoreAboveOne; - (*data)[i] = kCorruptScoreAboveOne; + << "distance " << kCorruptScoreDistance; + (*data)[i] = kCorruptScoreDistance; } else if (score > 1.0F) { // Numerical artifact of a near-exact hit (see kScoreAboveOneTolerance): // clamp to distance 0 instead of sentineling a legitimate best match. @@ -1301,6 +1304,20 @@ std::optional PaimonRustTableReader::_resolve_branch( return std::nullopt; } +#ifdef BE_TEST +std::vector PaimonRustTableReader::TEST_build_read_columns( + const TFileRangeDesc& range) const { + // _build_read_columns judges vector mode off _current_range; borrow it for the + // test by reading the columns through the same member path. + TFileRangeDesc& current = const_cast(_current_range); + TFileRangeDesc saved = current; + current = range; + std::vector columns = _build_read_columns(); + current = saved; + return columns; +} +#endif + std::vector PaimonRustTableReader::_build_read_columns() const { std::vector columns; columns.reserve(_projected_columns.size()); @@ -1310,9 +1327,11 @@ std::vector PaimonRustTableReader::_build_read_columns() const { } // The PK-vector search distance is reader-produced and auto-appended by // paimon-rust under its own field name; it is never a projectable table - // column, so exclude it from the projection in every mode (in non-vector - // mode it is simply absent). - if (column.name == kPaimonSearchDistanceColumn) { + // column, so exclude it from the projection — but only in vector mode: + // __paimon_search_score_to_dis is also a legal Paimon user column name, and + // skipping it on an ordinary rust scan would drop the real column and + // fill NULLs through _fill_non_arrow_columns. + if (_is_vector_mode(_current_range) && column.name == kPaimonSearchDistanceColumn) { continue; } columns.emplace_back(column.name); diff --git a/be/src/format_v2/table/paimon_rust_table_reader.h b/be/src/format_v2/table/paimon_rust_table_reader.h index 35f769adef2e71..c4d4a2e94e9c40 100644 --- a/be/src/format_v2/table/paimon_rust_table_reader.h +++ b/be/src/format_v2/table/paimon_rust_table_reader.h @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -112,15 +113,15 @@ class PaimonRustTableReader final : public format::TableReader { // paimon-rust FFI handles). // // A score outside l2's legal (0, 1] range is index corruption, and computing - // the inverse anyway would yield a non-finite distance (see the two sentinels - // below). Such rows are emitted as a NEGATIVE distance instead: negative is - // unreachable for a genuine Euclidean distance, so it cannot be confused with - // a real result, it is totally ordered (unlike NaN), and it stays finite, so - // the coordinator's Top-N comparisons remain well-defined. Rows land at the - // very front of an ASC Top-K -- deliberately: a corrupt index should be - // visible in the result, not averaged into it. - static constexpr float kCorruptScoreAboveOne = -1.0F; - static constexpr float kCorruptScoreNotPositive = -2.0F; + // the inverse anyway would yield a non-finite distance (see the two cases + // below). Such rows are emitted as +inf instead: +inf sorts LAST in an ASC + // Top-K, so a corrupt or infinite row can never displace a valid closer row + // from the coordinator's final ordering. A score of exactly 0 is reachable + // from a genuine (huge) distance — the exact fallback in + // vector-index.search-mode=full computes the squared L2 in f32, so norms + // above ~1.8e19 overflow to +inf and score = 1/(1+d^2) rounds to 0 — which + // is why the boundary maps to +inf rather than to an error. + static constexpr float kCorruptScoreDistance = std::numeric_limits::infinity(); // paimon-rust's lumina kernel computes the squared L2 distance as // ||a||^2 + ||b||^2 - 2 (vectorized dot), which suffers catastrophic // cancellation when a ~= b: the three terms nearly cancel and the residual @@ -156,6 +157,9 @@ class PaimonRustTableReader final : public format::TableReader { void TEST_set_projected_columns(std::vector columns) { _projected_columns = std::move(columns); } + // The projection the reader would request for the given range; exposed so the + // synthetic-distance-column exclusion (vector mode only) can be unit-tested. + std::vector TEST_build_read_columns(const TFileRangeDesc& range) const; void TEST_set_partition_values(std::map values) { _partition_values = std::move(values); } diff --git a/be/test/format_v2/table/paimon_rust_table_reader_test.cpp b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp index 24691e79e9daab..c909e219758bf9 100644 --- a/be/test/format_v2/table/paimon_rust_table_reader_test.cpp +++ b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp @@ -40,8 +40,8 @@ #include "core/field.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" -#include "format/table/paimon_rust_predicate_converter.h" #include "format_v2/column_data.h" +#include "format_v2/table/paimon_rust_predicate_converter.h" #include "gen_cpp/PlanNodes_types.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" @@ -274,6 +274,38 @@ TEST_F(PaimonRustTableReaderTest, ValidatesVectorSplit) { EXPECT_TRUE(reader.TEST_validate_rust_split(make_vector_range()).ok()); } +TEST_F(PaimonRustTableReaderTest, OrdinaryScanKeepsSyntheticDistanceNamedColumn) { + // __paimon_search_score_to_dis is also a legal Paimon user column name. The + // synthetic-distance exclusion must therefore apply ONLY to a real vector + // split: an ordinary rust scan of `SELECT id, __paimon_search_score_to_dis` + // over a table that really has such a column must keep reading it (skipping + // it would drop the column from the read projection and fill NULLs through + // _fill_non_arrow_columns). + PaimonRustTableReader reader; + reader.TEST_set_projected_columns( + {make_column("id", std::make_shared()), + make_column(kPaimonSearchDistanceColumn, std::make_shared())}); + + // Ordinary split: the same-named real column rides in the projection. + TFileRangeDesc ordinary = make_rust_range(); + auto ordinaryColumns = reader.TEST_build_read_columns(ordinary); + ASSERT_EQ(2u, ordinaryColumns.size()); + EXPECT_EQ("id", ordinaryColumns[0]); + EXPECT_EQ(kPaimonSearchDistanceColumn, ordinaryColumns[1]); + + // Vector split (empty paimon_split + vector_payload): the synthetic slot is + // reader-produced by paimon-rust itself, so it must NOT be requested. + TFileRangeDesc vectorRange = make_rust_range(); + vectorRange.table_format_params.paimon_params.__set_paimon_split(""); + TPaimonVectorPayload payload; + payload.__set_payload_type(TPaimonVectorPayloadType::BUCKET_SPLIT_BYTES); + payload.__set_bytes("PKVSPLIT-dummy"); + vectorRange.table_format_params.paimon_params.__set_vector_payload(payload); + auto vectorColumns = reader.TEST_build_read_columns(vectorRange); + ASSERT_EQ(1u, vectorColumns.size()); + EXPECT_EQ("id", vectorColumns[0]); +} + // ---------------------------------------------------------------- score -> distance // // paimon-rust encodes the distance as a score per metric (see @@ -332,12 +364,12 @@ TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceClampsCatastrophicCancel // The lumina kernel computes squared L2 as ||a||^2 + ||b||^2 - 2, which can // yield a small negative residual when a ~= b, i.e. score slightly ABOVE 1. That // is a near-exact hit, not corruption: it clamps to distance 0. Only a score past - // kScoreAboveOneTolerance is sentineled. + // kScoreAboveOneTolerance maps to kCorruptScoreDistance. Block block = _score_block({1.0F + 1e-4F, 1.0F + 5e-2F}, /*nullable=*/false); PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 2, L2); EXPECT_FLOAT_EQ(0.0F, _score_at(block, 0)); - EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreAboveOne, _score_at(block, 1)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 1)); } TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceSkipsRowsBeforeOffset) { @@ -377,23 +409,25 @@ TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceNullable) { } TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceFlagsOutOfRangeScores) { - // Unreachable for a well-formed index (d >= 0 implies score in (0, 1]), but the - // distances come out of the closed-source lumina kernel unclamped. Computing the - // inverse anyway would give +inf (1/0) and NaN (sqrt of a negative), either of - // which makes the coordinator's Top-N comparisons ill-defined. Each case maps to - // its own negative sentinel: negative is unreachable for a real Euclidean - // distance, stays finite and totally ordered, and tells which way it broke. - Block block = _score_block({0.0F, -1.0F, 1.5F}, /*nullable=*/false); - PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 3, L2); - - EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreNotPositive, _score_at(block, 0)); - EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreNotPositive, _score_at(block, 1)); - EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreAboveOne, _score_at(block, 2)); + // A well-formed index keeps d >= 0 and score in (0, 1], but the distances + // come out of the closed-source lumina kernel unclamped, and a score of + // exactly 0 is also REACHABLE from a genuine huge distance: the exact + // fallback (vector-index.search-mode=full) computes squared L2 in f32, so + // norms above ~1.8e19 overflow to +inf and 1/(1+d^2) rounds to 0. Every + // out-of-range case maps to kCorruptScoreDistance (+inf): it sorts LAST in + // an ASC Top-N, so it can never displace a valid closer row, and it cannot + // invert the coordinator's final ordering the way a negative sentinel + // (row 0 beating real finite distances) did. + Block block = _score_block({0.0F, -1.0F, 1.5F, 0.5F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 4, L2); + + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 0)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 1)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 2)); + // A legal score between the corrupt rows still converts exactly. + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 3)); for (size_t i = 0; i < 3; ++i) { - const float distance = _score_at(block, i); - EXPECT_TRUE(std::isfinite(distance)) << "row " << i << " = " << distance; - // Must be distinguishable from any genuine distance, which is >= 0. - EXPECT_LT(distance, 0.0F) << "row " << i; + EXPECT_TRUE(std::isinf(_score_at(block, i))) << "row " << i; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java index c194c5ed303a58..95f9f6aa9d300a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java @@ -154,11 +154,18 @@ final class PaimonRustEligibility { // supports_paimon_rust_reader. final boolean allBackendsRustCapable; + // The PK-vector (ANN) protocol is negotiated separately: an older + // rust-capable BE reports the batch bit but would reject an empty + // paimon_split + vector payload ("missing paimon_split"), so the one-way + // vector rewrite must require this bit on every candidate backend too. + final boolean allBackendsVectorCapable; + private PaimonRustEligibility(boolean isFileStoreTable, boolean fallbackReadTable, boolean restTokenTable, boolean queryAuthTable, boolean puAggDeletionVectors, boolean dvMergeOnRead, boolean deduplicateIgnoreDelete, boolean rustUnsupportedMergeOption, boolean scanModeNotDefault, boolean providerModeTranslatable, boolean schemeCapabilityVerified, - boolean hdfsBackendVerified, boolean allBackendsRustCapable) { + boolean hdfsBackendVerified, boolean allBackendsRustCapable, + boolean allBackendsVectorCapable) { this.isFileStoreTable = isFileStoreTable; this.fallbackReadTable = fallbackReadTable; this.restTokenTable = restTokenTable; @@ -172,6 +179,7 @@ private PaimonRustEligibility(boolean isFileStoreTable, boolean fallbackReadTabl this.schemeCapabilityVerified = schemeCapabilityVerified; this.hdfsBackendVerified = hdfsBackendVerified; this.allBackendsRustCapable = allBackendsRustCapable; + this.allBackendsVectorCapable = allBackendsVectorCapable; } /** @@ -280,10 +288,13 @@ static PaimonRustEligibility of(Table table, String tableLocation, tableLocation, backendStorageProperties); boolean allBackendsRustCapable = backends != null && !backends.isEmpty() && backends.stream().allMatch(Backend::isPaimonRustReaderSupported); + boolean allBackendsVectorCapable = backends != null && !backends.isEmpty() + && backends.stream().allMatch(Backend::isPaimonRustVectorReaderSupported); return new PaimonRustEligibility(isFileStoreTable, fallbackReadTable, restTokenTable, queryAuthTable, puAggDeletionVectors, dvMergeOnRead, deduplicateIgnoreDelete, rustUnsupportedMergeOption, scanModeNotDefault, providerModeTranslatable, - schemeCapabilityVerified, hdfsBackendVerified, allBackendsRustCapable); + schemeCapabilityVerified, hdfsBackendVerified, allBackendsRustCapable, + allBackendsVectorCapable); } /** Whether the table, storage and backend shapes all admit the paimon-rust reader. */ @@ -295,4 +306,14 @@ boolean eligible() { && providerModeTranslatable && schemeCapabilityVerified && hdfsBackendVerified && allBackendsRustCapable; } + + /** + * Whether they all admit the PK-vector (ANN) search path on top of the batch + * rust gates: the vector protocol needs every candidate backend to report + * the vector capability bit as well, since a rewritten vector split has no + * JNI fallback and an older rust-capable BE cannot decode it. + */ + boolean vectorEligible() { + return eligible() && allBackendsVectorCapable; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java index f8516f9c0004b8..036ba9fb0558c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java @@ -109,7 +109,7 @@ boolean canRead(DataSplit split) { return true; } - private static boolean hasFullNestedProjection(TupleDescriptor tuple) { + static boolean hasFullNestedProjection(TupleDescriptor tuple) { for (SlotDescriptor slot : tuple.getSlots()) { // The ABI projects only root names, while Arrow struct SerDes bind by ordinal. // A pruned slot must use JNI's recursive read type, even inside arrays or maps. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 1922052aa589f1..18183959ad870e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -662,6 +662,27 @@ private static boolean hasFieldOptionSuffix(String key, String suffix) { return key.startsWith("fields.") && key.endsWith(suffix); } + /** + * Whether the pinned paimon-rust vector search can take a DATA predicate on + * this table. Paimon 2.1 permits a PK-vector index on tables without deletion + * vectors (first-row merge engines especially), but the rust + * {@code PkVectorSearchParams::resolve} rejects every data predicate unless + * deletion vectors are enabled AND {@code deletion-vectors.merge-on-read} is + * disabled: its post-recall per-candidate filtering re-reads each candidate + * file's physical rows, which only agrees with the bucket search when the + * table exposes physical rows directly. The options are read raw from the + * table map BE itself deserializes, so the values cannot diverge from what + * the rust reader sees. + */ + public static boolean vectorSearchSupportsDataPredicate(Table paimonTable) { + Map options = paimonTable.options(); + boolean deletionVectors = "true".equalsIgnoreCase( + options.get(CoreOptions.DELETION_VECTORS_ENABLED.key())); + boolean mergeOnRead = "true".equalsIgnoreCase( + options.get(DELETION_VECTORS_MERGE_ON_READ)); + return deletionVectors && !mergeOnRead; + } + // Whether the schema options carry any option key the pinned paimon-rust // read rejects for this merge engine. @VisibleForTesting @@ -1048,6 +1069,17 @@ && splitHasOrcFile((DataSplit) split) // executes paimon-rust's eager Top-K chain, which applies deletion // vectors and the merge inside the search itself), so it does not // consult them. + // + // The nested-projection bound is NOT DataSplit-specific: the ABI + // projects only root names and the Arrow struct SerDe binds children by + // ordinal, so a pruned complex slot (nested pruning, on by default) + // would return the wrong child on the vector path exactly as it would + // on the batch path. The rewrite pre-flights this and declines; this + // is the loud backstop for shapes the raw-table pre-flight cannot see. + if (vectorSearchSplit) { + canUseRust = canUseRust && rustEligibility.vectorEligible() + && PaimonRustReaderCapabilities.hasFullNestedProjection(desc); + } if (!vectorSearchSplit && canUseRust) { canUseRust = canUseRust && nativeSplit && !splitDvNotMaterialized && !externalFileSplit; @@ -1119,6 +1151,13 @@ && splitHasOrcFile((DataSplit) split) if (!rustEligibility.allBackendsRustCapable) { reasons.add("backend_without_rust_support"); } + if (vectorSearchSplit && !rustEligibility.allBackendsVectorCapable) { + reasons.add("backend_without_vector_support"); + } + if (vectorSearchSplit + && !PaimonRustReaderCapabilities.hasFullNestedProjection(desc)) { + reasons.add("pruned_nested_projection"); + } if (reasons.isEmpty()) { reasons.add(vectorSearchSplit ? "unknown" : "capability_whitelist"); } @@ -1569,6 +1608,12 @@ public List getVectorSearchSplits() throws UserException { List splits = new ArrayList<>(); Set partitions = new HashSet<>(); + // Same partition-metadata requirement as the ordinary split loop below: a + // table with partition keys makes setPaimonParams call setPartitionValues, + // which throws "Missing partition values" on a null map, so every vector + // split over a partitioned table must carry the derived values too. + boolean needPartitionMetadata = !getOrderedPathPartitionKeys().isEmpty(); + Map> partitionInfoMaps = new HashMap<>(); for (VectorSearchSplit vectorSplit : plan.splits()) { if (!(vectorSplit instanceof BucketVectorSearchSplit)) { throw new UserException("Unexpected PK-vector split type: " @@ -1591,6 +1636,13 @@ public List getVectorSearchSplits() throws UserException { DataSplit dataSplit = bucketSplit.dataSplit(); PaimonSplit split = new PaimonSplit(dataSplit); split.setVectorPayloadBytes(payloadBytes); + if (needPartitionMetadata) { + Map partitionInfoMap = + partitionInfoMaps.computeIfAbsent(dataSplit.partition(), k -> PaimonUtil.getPartitionInfoMap( + paimonTable, k, sessionVariable.getTimeZone(), + source.getCatalog().getEnableMappingTimestampTz())); + split.setPaimonPartitionValues(partitionInfoMap); + } splits.add(split); partitions.add(dataSplit.partition()); ++paimonSplitNum; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java index dccb839f0ed661..03fb768c8acf4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java @@ -96,12 +96,15 @@ private static Column createDistanceColumn() { * raw cached table, so a {@code t@options} override can still make the * scan node reject a split the pre-flight accepted — the scan node stays * authoritative and fails loudly. The BE capability negotiation requires - * every candidate backend to report {@code supports_paimon_rust_reader}: - * a PK-vector split is only decodable by a rust-capable BE. + * every candidate backend to report {@code supports_paimon_rust_reader} AND + * {@code supports_paimon_rust_vector_reader}: the vector protocol rides an + * empty {@code paimon_split} plus a vector payload, which an older + * rust-capable BE (rolling upgrade) rejects as "missing paimon_split", so + * the vector bit is negotiated separately from the batch rust bit. */ public static boolean vectorScanEligible(Table paimonTable, String tableLocation, Map backendStorageProperties, Collection backends) { return PaimonRustEligibility.of(paimonTable, tableLocation, - backendStorageProperties, backends).eligible(); + backendStorageProperties, backends).vectorEligible(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java index 3b5e04fbbad660..8756946a5200b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java @@ -25,6 +25,7 @@ import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonUtil; +import org.apache.doris.datasource.paimon.source.PaimonScanNode; import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.rules.Rule; @@ -42,6 +43,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.AnnTopNInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; @@ -65,6 +67,7 @@ import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -165,13 +168,22 @@ private Plan pushDown( // into the result. Snapshot/tag/version selectors are fine -- they are // resolved into the table handle before the vector search -- so only the // file-level bound forces the decline. + // + // `scan.creation-time-millis` must decline as well: when a snapshot exists + // past the bound it resolves into an ordinary snapshot pin (safe), but when + // none does — the exact read state the option exists for — PaimonScanParams + // resolves it into the SAME internal file-creation bound (resolveOptions + // falls through to resolveFileCreationTime), which the vector path again + // never consults. Which case applies depends on the snapshot manager's + // state, so the raw option key is declined unconditionally. if (scan.getScanParams().isPresent()) { TableScanParams scanParams = scan.getScanParams().get(); if (scanParams.incrementalRead()) { return null; } Map rawParams = scanParams.getMapParams(); - if (rawParams.containsKey(CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key())) { + if (rawParams.containsKey(CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key()) + || rawParams.containsKey(CoreOptions.SCAN_CREATION_TIME_MILLIS.key())) { return null; } } @@ -243,6 +255,16 @@ private Plan pushDown( if (!(left instanceof SlotReference && right.isConstant() && right instanceof ArrayLiteral)) { return null; } + // A typed NULL inside the query vector ([CAST(NULL AS FLOAT), 0.0, ...]) is a + // NullLiteral whose getDouble() returns 0, so extracting the coordinates + // would silently search against a zero vector. The ordinary path raises + // instead (FunctionArrayDistance::_validate_no_nulls: "Second argument ... + // cannot have null"), and the rewrite must not replace that error with rows + // for a different input, so any null element declines the rewrite. Both + // supported metrics share this. + if (((ArrayLiteral) right).getValue().stream().anyMatch(NullLiteral.class::isInstance)) { + return null; + } SlotReference vectorSlot = (SlotReference) left; if (!vectorSlot.getOriginalColumn().isPresent() || !vectorSlot.getOriginalTable().isPresent()) { return null; @@ -273,6 +295,43 @@ private Plan pushDown( return null; } + // A data predicate additionally requires the SDK's "physical row read" + // table mode: the predicate itself converts fine, but the pinned + // paimon-rust PkVectorSearchParams::resolve rejects every DATA predicate + // unless deletion vectors are enabled and merge-on-read is disabled — + // so the failure lands AFTER this one-way rewrite replaced the distance + // expression, turning a query that the ordinary path answers into a + // user-facing error. Partition-only conjuncts are enforced by the search + // planning itself and stay fine, so decline only on a non-partition one. + // + // The synthetic distance slot is named after a legal paimon user column + // name (__paimon_search_score_to_dis, see the BE reader). A table that + // actually HAS a column of that name would collide with the synthetic + // slot's block-column matching; decline and run the ordinary path. + // + // A complex-typed column (STRUCT/MAP/nested ARRAY) must ride as a whole + // column: nested pruning rewrites a slot whose fields are accessed — + // Project(struct_element(payload, 'b')) materializes the scan tuple as + // ROW — but the vector reader projects only root names and returns + // the full Arrow value, and the struct SerDe then binds children by + // ORDINAL: the wrong child comes back. Filter conjuncts cannot reach + // this (the whitelist admits only scalar columns); the order-key alias + // is replaced by the synthetic slot and cannot reach it either; so only + // a Project output expression over a complex column has to decline. + try { + Table table = paimonTable.getPaimonTable(Optional.empty()); + if ((optionalFilter.isPresent() + && filterHasDataConjunct(optionalFilter.get(), table) + && !PaimonScanNode.vectorSearchSupportsDataPredicate(table)) + || table.rowType().getFieldNames().stream().anyMatch( + name -> name.equalsIgnoreCase(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)) + || !complexColumnsRideAsWholeColumns(project, orderKeyAlias)) { + return null; + } + } catch (Throwable t) { + return null; + } + // Pre-flight the projection-dependent bounds the vector split would hit at // the scan node: a PK-vector split has no JNI fallback (it would return an // empty distance column), so a query whose projected columns would set @@ -328,11 +387,15 @@ private Plan pushDown( } catch (Throwable t) { return null; } - // NOTE: the FE-side data-predicate interception is intentionally not done. The - // data predicate is handed in full to the BE via the vector search builder's - // residual filter, where the Rust candidate search stage applies it together - // with deletion vectors. If some table shape cannot be handled correctly, it is - // Rust's responsibility to reject it or return correct data, not the planner's. + // NOTE: the FE-side data-predicate interception is intentionally MINIMAL. + // The data predicate is handed in full to the BE via the vector search + // builder's residual filter, where the Rust candidate search stage applies + // it together with deletion vectors. The two planner-side exceptions above + // (unridable conjunct shapes, and the SDK's no-deletion-vector table mode) + // exist because those failures would surface only AFTER the one-way rewrite + // replaced the distance expression. Any other table shape Rust cannot + // handle correctly is Rust's responsibility to reject or answer correctly, + // not the planner's. // paimon-rust's vector index operates on f32, so carry the query as floats // rather than List: it avoids a double→float narrowing on the wire @@ -404,13 +467,13 @@ private static boolean conjunctsRideIntoVectorSearch(LogicalFilter filter) { private static boolean conjunctRidesIntoVectorSearch(Expression conjunct) { if (conjunct instanceof ComparisonPredicate) { ComparisonPredicate compare = (ComparisonPredicate) conjunct; - return (isVectorFilterColumnSlot(compare.left()) && compare.right() instanceof Literal) - || (compare.left() instanceof Literal && isVectorFilterColumnSlot(compare.right())); + return (isVectorFilterColumnSlot(compare.left()) && ridesAsLiteral(compare.right())) + || (ridesAsLiteral(compare.left()) && isVectorFilterColumnSlot(compare.right())); } if (conjunct instanceof InPredicate) { InPredicate in = (InPredicate) conjunct; return isVectorFilterColumnSlot(in.getCompareExpr()) - && in.getOptions().stream().allMatch(Literal.class::isInstance); + && in.getOptions().stream().allMatch(PushDownVectorTopNIntoPaimonScan::ridesAsLiteral); } if (conjunct instanceof IsNull) { return isVectorFilterColumnSlot(((IsNull) conjunct).child()); @@ -421,6 +484,81 @@ private static boolean conjunctRidesIntoVectorSearch(Expression conjunct) { return false; } + /** + * A literal the BE paimon-rust predicate converter accepts. A {@link NullLiteral} + * is a Literal and passes the earlier instanceof check, but + * {@code _convert_literal} rejects it ({@code is_null_at(0)}), which rejects the + * WHOLE predicate — an IN option of {@code tag IN (1, 2, NULL)} silently rejects + * the entire IN, leaving it in the post-fetch residual. That reintroduces the + * bounded-TopK truncation the whitelist exists to prevent, so a null literal + * never rides. + */ + private static boolean ridesAsLiteral(Expression expr) { + return expr instanceof Literal && !(expr instanceof NullLiteral); + } + + /** + * Whether the filter carries a conjunct on a non-partition column — the DATA + * half of the predicate, the half the paimon-rust vector search applies per + * candidate row. Every conjunct already passed the riding whitelist, so each + * is a simple comparison / IN / null-check whose slots map to original table + * columns; a conjunct touching only partition columns is enforced by split + * planning and needs no per-row filtering. + */ + private static boolean filterHasDataConjunct(LogicalFilter filter, Table paimonTable) { + Set partitionKeys = new HashSet<>(paimonTable.partitionKeys()); + for (Expression conjunct : filter.getConjuncts()) { + Set slots = conjunct.collect(SlotReference.class::isInstance); + for (SlotReference slot : slots) { + if (slot.getOriginalColumn().isPresent() + && !partitionKeys.contains(slot.getOriginalColumn().get().getName())) { + return true; + } + } + } + return false; + } + + /** + * Whether every complex-typed column referenced by the projection rides as a + * whole column. Nested pruning materializes the scan tuple with a REDUCED type + * for a column whose fields are accessed ({@code Project(struct_element(payload, + * 'b'))} prunes {@code payload ROW} to {@code ROW}), but the vector + * reader projects only root names and returns the full Arrow value, whose + * struct SerDe binds children by ordinal — the wrong child comes back. A bare + * slot output keeps the full type, and a complex slot consumed ONLY by a + * distance function never reaches the scan at all (this rewrite replaces every + * occurrence of the function with the synthetic distance slot, order-key alias + * included), so only any OTHER expression over a complex column forces the + * decline. Filter conjuncts cannot reach this: the shape whitelist admits only + * scalar columns. + */ + private static boolean complexColumnsRideAsWholeColumns(LogicalProject project, + Alias orderKeyAlias) { + for (NamedExpression projection : project.getProjects()) { + if (!(projection instanceof Alias)) { + continue; + } + Alias alias = (Alias) projection; + if (alias.equals(orderKeyAlias) || alias.child() instanceof SlotReference) { + continue; + } + Set replacedSlots = new HashSet<>(); + Set distanceCalls = alias.child().collect(e -> + e instanceof L2DistanceApproximate || e instanceof InnerProductApproximate); + for (Expression call : distanceCalls) { + replacedSlots.addAll(call.collect(SlotReference.class::isInstance)); + } + Set slots = alias.child().collect(SlotReference.class::isInstance); + for (SlotReference slot : slots) { + if (!replacedSlots.contains(slot) && slot.getDataType().isComplexType()) { + return false; + } + } + } + return true; + } + private static boolean isVectorFilterColumnSlot(Expression expr) { if (!(expr instanceof SlotReference)) { return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java index adb4a57d448385..885e240503ce93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java @@ -147,6 +147,12 @@ public class Backend implements Writable { private long beMemory = 0; @SerializedName("supportsPaimonRustReader") private volatile boolean supportsPaimonRustReader; + // The PK-vector (ANN) search protocol bit, negotiated separately from the + // batch rust reader: an older rust-capable BE reports only the batch bit and + // rejects a vector payload ("missing paimon_split"), so the one-way vector + // rewrite must require this bit specifically. + @SerializedName("supportsPaimonRustVectorReader") + private volatile boolean supportsPaimonRustVectorReader; // from config::pipeline_executor_size , default equal cpuCores @SerializedName("pipelineExecutorSize") private int pipelineExecutorSize = 1; @@ -263,6 +269,10 @@ public boolean isPaimonRustReaderSupported() { return supportsPaimonRustReader; } + public boolean isPaimonRustVectorReaderSupported() { + return supportsPaimonRustVectorReader; + } + public int getBePort() { return bePort; } @@ -882,6 +892,10 @@ public boolean handleHbResponse(BackendHbResponse hbResponse, boolean isReplay) isChanged = true; supportsPaimonRustReader = hbResponse.isPaimonRustReaderSupported(); } + if (supportsPaimonRustVectorReader != hbResponse.isPaimonRustVectorReaderSupported()) { + isChanged = true; + supportsPaimonRustVectorReader = hbResponse.isPaimonRustVectorReaderSupported(); + } if (!this.version.equals(hbResponse.getVersion())) { isChanged = true; this.version = hbResponse.getVersion(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java index 55fb0c9f456a74..42b19395c3efe6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java @@ -53,6 +53,10 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable { private long beMemory = 0; @SerializedName("supportsPaimonRustReader") private boolean supportsPaimonRustReader; + // The PK-vector (ANN) search protocol bit, negotiated separately: an older + // rust-capable BE reports the batch bit but would reject a vector payload. + @SerializedName("supportsPaimonRustVectorReader") + private boolean supportsPaimonRustVectorReader; public BackendHbResponse() { super(HeartbeatResponse.Type.BACKEND); @@ -162,6 +166,14 @@ public void setPaimonRustReaderSupported(boolean supported) { this.supportsPaimonRustReader = supported; } + public boolean isPaimonRustVectorReaderSupported() { + return supportsPaimonRustVectorReader; + } + + public void setPaimonRustVectorReaderSupported(boolean supported) { + this.supportsPaimonRustVectorReader = supported; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index 71e053995c1f07..922b22eae01828 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -379,6 +379,9 @@ private HeartbeatResponse pingOnce() { fragmentNum, lastFragmentUpdateTime, isShutDown, arrowFlightSqlPort, beMemory); response.setPaimonRustReaderSupported(tBackendInfo.isSetSupportsPaimonRustReader() && tBackendInfo.isSupportsPaimonRustReader()); + response.setPaimonRustVectorReaderSupported( + tBackendInfo.isSetSupportsPaimonRustVectorReader() + && tBackendInfo.isSupportsPaimonRustVectorReader()); return response; } else { return new BackendHbResponse(backendId, backend.getHost(), backend.getLastUpdateMs(), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java index f2beb533eb4331..a5508fc00464fb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java @@ -35,18 +35,22 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.AnnTopNInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; @@ -113,9 +117,12 @@ public void setUp() { cascadesContext = Mockito.mock(CascadesContext.class); // The rewrite's pre-flight (PaimonVectorSearch.vectorScanEligible) requires a // rust-capable BE via the all-backend capability negotiation, mirroring - // PaimonScanNodeTest's routing fixture. + // PaimonScanNodeTest's routing fixture — and the VECTOR capability bit on + // top of the batch one, since an older rust-capable BE would reject the + // vector payload ("missing paimon_split"). Backend rustBackend = GsonUtils.GSON.fromJson( - "{\"supportsPaimonRustReader\":true}", Backend.class); + "{\"supportsPaimonRustReader\":true,\"supportsPaimonRustVectorReader\":true}", + Backend.class); systemInfo = Mockito.mock(SystemInfoService.class); Mockito.when(systemInfo.getAllClusterBackends(true)) .thenReturn(ImmutableList.of(rustBackend)); @@ -307,7 +314,8 @@ public void testSimpleColumnFilterStillPushedDown() { // bounded top-N sees the predicate and the rewrite stays sound. LogicalFileScan scan = paimonScan(ImmutableMap.of( "pk-vector.index.columns", "vec", - "fields.vec.pk-vector.distance.metric", "l2")); + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); SlotReference vec = vecSlot(scan); SlotReference tag = (SlotReference) scan.getOutput().get(3); Assertions.assertEquals("tag", tag.getName()); @@ -501,7 +509,8 @@ public void testMergedSubqueryRewriteStillTriggers() { // comparison rides into the vector search, so the rewrite stays sound. LogicalFileScan scan = paimonScan(ImmutableMap.of( "pk-vector.index.columns", "vec", - "fields.vec.pk-vector.distance.metric", "l2")); + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); SlotReference vec = vecSlot(scan); SlotReference tag = (SlotReference) scan.getOutput().get(3); LogicalFilter filter = new LogicalFilter<>( @@ -664,6 +673,168 @@ public void testFileCreationTimeScanNotPushedDown() { new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); } + @Test + public void testCreationTimeScanNotPushedDown() { + // t@options('scan.creation-time-millis'='...'): when a snapshot exists past + // the bound this resolves into an ordinary snapshot pin (safe), but when none + // does it resolves into the SAME internal file-creation bound the vector + // search never consults — which case applies depends on the snapshot + // manager's state, so the raw key declines unconditionally (contrast with + // scan.snapshot-id, which stays eligible). + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("options", + ImmutableMap.of("scan.creation-time-millis", "1791793167062"), + ImmutableList.of()))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testNullQueryVectorElementNotPushedDown() { + // [CAST(NULL AS FLOAT), 0.0]: a NullLiteral's getDouble() is 0, so the + // rewrite would silently search a zero vector and return rows for a + // different input, while the ordinary path raises (the distance functions + // reject NULL elements). Any null element declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + ArrayLiteral nullVector = new ArrayLiteral(ImmutableList.of( + new NullLiteral(), new FloatLiteral(2.0f))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), nullVector), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testInPredicateWithNullOptionNotPushedDown() { + // tag IN (1, 2, NULL): every option is a Literal so the shape whitelist + // passes, but the BE converter rejects the null literal and with it the + // WHOLE predicate, leaving the IN in the post-fetch residual — the bounded + // fetch then returns the wrong rows. A null option never rides. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + LogicalFilter filter = new LogicalFilter<>(ImmutableSet.of( + new InPredicate(tag, ImmutableList.of(new IntegerLiteral(1), new IntegerLiteral(2), + new NullLiteral()))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + } + + @Test + public void testInPredicateStillPushedDown() { + // Control for the NULL-option case: tag IN (1, 2) with every option a + // non-null literal on a supported column rides into the vector search. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + LogicalFilter filter = new LogicalFilter<>(ImmutableSet.of( + new InPredicate(tag, ImmutableList.of(new IntegerLiteral(1), new IntegerLiteral(2)))), + scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + Plan rewritten = applyRule(1, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + Assertions.assertNotSame(scan, ((LogicalFileScan) ((LogicalFilter) + ((LogicalProject) rewritten.child(0)).child(0)).child(0))); + Assertions.assertTrue(((LogicalFileScan) ((LogicalFilter) + ((LogicalProject) rewritten.child(0)).child(0)).child(0)).getAnnTopN().isPresent()); + } + + @Test + public void testFilterWithoutDeletionVectorsNotPushedDown() { + // WHERE tag < 3 on a table without deletion vectors: the predicate + // converts fine, but paimon-rust's PkVectorSearchParams::resolve rejects + // every data predicate unless deletion vectors are enabled and + // merge-on-read is disabled — so the failure would land AFTER the + // one-way rewrite, as a user-facing error on a query the ordinary path + // answers. Decline instead (first-row tables allow the index without DV). + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(tag, new IntegerLiteral(3))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + } + + @Test + public void testSyntheticDistanceColumnCollisionNotPushedDown() { + // __paimon_search_score_to_dis is also a legal paimon user column name; a + // table that HAS such a column collides with the synthetic slot's + // block-column matching, so the rewrite declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.empty(), + ImmutableList.of("id", "vec", PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testPrunedComplexProjectionNotPushedDown() { + // Project(pmap['k'], dis): nested pruning materializes the scan tuple with + // a REDUCED type for pmap, but the vector reader projects only root names + // and returns the full Arrow value, whose SerDe binds children by ordinal — + // the wrong child comes back. Only a whole-column (bare slot) projection + // keeps the rewrite sound. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference pmap = (SlotReference) scan.getOutput().get(6); + Assertions.assertEquals("pmap", pmap.getName()); + Assertions.assertTrue(pmap.getDataType().isComplexType()); + Alias field = new Alias(new ElementAt(pmap, new StringLiteral("k")), "pk"); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), field, dis), scan), dis, true)); + } + + @Test + public void testWholeComplexProjectionStillPushedDown() { + // Control for the pruned case: the complex column rides as a whole column + // (bare slot output), which nested pruning leaves untouched. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference pmap = (SlotReference) scan.getOutput().get(6); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + Plan rewritten = applyRule(0, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), pmap, dis), scan), dis, true)); + Assertions.assertNotSame(scan, ((LogicalFileScan) ((LogicalProject) rewritten.child(0)).child(0))); + Assertions.assertTrue(((LogicalFileScan) ((LogicalProject) rewritten.child(0)).child(0)) + .getAnnTopN().isPresent()); + } + + @Test + public void testBackendWithoutVectorCapabilityNotPushedDown() { + // Rolling upgrade: a BE that reports the batch rust bit but not the vector + // bit would reject the rewritten payload ("missing paimon_split"), so the + // rewrite must decline and leave the query on the ordinary path it can + // still run. Re-stub the shared SystemInfoService with such a BE. + Backend legacyBackend = GsonUtils.GSON.fromJson( + "{\"supportsPaimonRustReader\":true}", Backend.class); + Mockito.when(systemInfo.getAllClusterBackends(true)) + .thenReturn(ImmutableList.of(legacyBackend)); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + @Test public void testSnapshotScanStillPushedDown() { // t@options('scan.snapshot-id'='1'): snapshot selectors are resolved into @@ -760,13 +931,28 @@ private LogicalFileScan paimonScan(Map options) { private LogicalFileScan paimonScan(Map options, Optional scanParams) { + return paimonScan(options, scanParams, null); + } + + /** {@code paimonFieldNames} overrides the schema-derived names the collision + * pre-flight reads off the mocked paimon rowType. */ + private LogicalFileScan paimonScan(Map options, + Optional scanParams, List paimonFieldNames) { List schema = ImmutableList.of( new Column("id", Type.INT, true), new Column("vec", new ArrayType(Type.FLOAT), true), new Column("vec2", new ArrayType(Type.FLOAT), true), new Column("tag", Type.INT, true), new Column("score", Type.DOUBLE, true), - new Column("payload", Type.VARIANT, true)); + new Column("payload", Type.VARIANT, true), + // A complex column the pruned-projection tests access with + // ElementAt (the former struct_element): only the isComplexType() + // verdict matters to the pre-flight, so a Doris MAP stands in for + // a STRUCT this fixture cannot express through catalog types. + new Column("pmap", new org.apache.doris.catalog.MapType(), true)); + if (paimonFieldNames == null) { + paimonFieldNames = schema.stream().map(Column::getName).collect(Collectors.toList()); + } PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); Mockito.when(table.getName()).thenReturn("paimon_tbl"); @@ -785,6 +971,12 @@ private LogicalFileScan paimonScan(Map options, Mockito.when(paimonTable.options()).thenReturn(options); Mockito.when(paimonTable.location()) .thenReturn(new Path("file:///warehouse/wh/paimon_db.db/paimon_tbl")); + // rowType drives the synthetic-distance-column collision pre-flight; its + // field names track the fixture schema (or the test's explicit names). + org.apache.paimon.types.RowType rowType = Mockito.mock( + org.apache.paimon.types.RowType.class); + Mockito.when(rowType.getFieldNames()).thenReturn(paimonFieldNames); + Mockito.when(paimonTable.rowType()).thenReturn(rowType); Mockito.when(paimonTable.fileIO()).thenReturn(null); Mockito.when(paimonTable.coreOptions()).thenReturn(null); Mockito.when(paimonTable.schema()).thenReturn(null); diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java index 9b085b3c7e4a99..ae50e66f89b828 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java @@ -63,6 +63,43 @@ public void testLegacyHeartbeatClearsPreviouslyAdvertisedSupport() { Assert.assertFalse(GsonUtils.GSON.fromJson("{}", Backend.class).isPaimonRustReaderSupported()); } + @Test + public void testVectorCapabilityNegotiatedIndependentlyOfBatchSupport() { + // Rolling upgrade: a BE advertising only the batch rust bit keeps batch + // support and loses the vector protocol — the one-way vector rewrite must + // be gated on this bit specifically (an older rust-capable BE would reject + // the rewritten payload as "missing paimon_split"). + Backend backend = new Backend(1, "127.0.0.1", 9050); + BackendHbResponse batchOnly = heartbeat(); + batchOnly.setPaimonRustReaderSupported(true); + Assert.assertFalse(batchOnly.isPaimonRustVectorReaderSupported()); + backend.handleHbResponse(batchOnly, false); + Assert.assertTrue(backend.isPaimonRustReaderSupported()); + Assert.assertFalse(backend.isPaimonRustVectorReaderSupported()); + + BackendHbResponse bothBits = heartbeat(); + bothBits.setPaimonRustReaderSupported(true); + bothBits.setPaimonRustVectorReaderSupported(true); + BackendHbResponse replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(bothBits), BackendHbResponse.class); + Assert.assertTrue(replay.isPaimonRustVectorReaderSupported()); + backend.handleHbResponse(replay, false); + Assert.assertTrue(backend.isPaimonRustReaderSupported()); + Assert.assertTrue(backend.isPaimonRustVectorReaderSupported()); + Backend restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(backend), Backend.class); + Assert.assertTrue(restored.isPaimonRustVectorReaderSupported()); + + // A later heartbeat without the vector bit (downgrade) clears it again. + BackendHbResponse downgrade = heartbeat(); + downgrade.setPaimonRustReaderSupported(true); + backend.handleHbResponse(downgrade, false); + Assert.assertTrue(backend.isPaimonRustReaderSupported()); + Assert.assertFalse(backend.isPaimonRustVectorReaderSupported()); + + TBackendInfo thrift = new TBackendInfo(); + Assert.assertFalse(thrift.isSetSupportsPaimonRustVectorReader()); + Assert.assertFalse(GsonUtils.GSON.fromJson("{}", Backend.class).isPaimonRustVectorReaderSupported()); + } + private BackendHbResponse heartbeat() { return new BackendHbResponse(1, 9060, 8040, 8060, 1000, 1000, "test-version", Tag.VALUE_MIX, 0, 0, false, 8070); diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift index 8608853b29f81a..b47f84cb3bb0b5 100644 --- a/gensrc/thrift/HeartbeatService.thrift +++ b/gensrc/thrift/HeartbeatService.thrift @@ -64,6 +64,12 @@ struct TBackendInfo { 9: optional Types.TPort arrow_flight_sql_port 10: optional i64 be_mem // The physical memory available for use by BE. 11: optional bool supports_paimon_rust_reader + // The paimon-rust PK-vector (ANN) search protocol: paimon_split is empty and the + // split rides TPaimonFileDesc.vector_payload, which an older rust-capable BE + // rejects as "missing paimon_split". Set only by BEs that implement the vector + // protocol, so a mixed-version cluster keeps ordinary rust reads working and + // only declines the vector rewrite against older BEs. Field 12 is free on master. + 12: optional bool supports_paimon_rust_vector_reader // For cloud 1000: optional i64 fragment_executing_count 1001: optional i64 fragment_last_active_time diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 1ec35acb622a16..806805887d6e6f 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -409,7 +409,10 @@ struct TPaimonFileDesc { // Primary-key vector (ANN) search payload for this split. When set, BE // routes to the paimon-rust vector-search read path instead of the normal // split read; paimon_split is set but empty in that case. - 20: optional TPaimonVectorPayload vector_payload; + // Field 21: id 20 is reserved as `original_file_path` on master and must not + // be reused here — a mixed-version cluster skips mismatched wire types + // instead of receiving either field. + 21: optional TPaimonVectorPayload vector_payload; } struct TTrinoConnectorFileDesc { @@ -770,7 +773,10 @@ struct TFileScanRangeParams { // primary-key vector (ANN) top-N pushed down into the scan. Set at ScanNode level // (shared by all splits); the per-split physical payload stays in // TPaimonFileDesc.vector_payload. Lance requests use lance_scan_params instead. - 41: optional TExternalSearchRequest external_search_request + // Field 42: id 41 is reserved as `fluss_properties` on master and must not be + // reused here — a mixed-version cluster skips mismatched wire types instead + // of receiving either field. + 42: optional TExternalSearchRequest external_search_request } struct TFileRangeDesc { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy index 01f28af668eefa..72b78a36c5d716 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy @@ -119,7 +119,17 @@ class ResultUtils { StringBuffer sb = new StringBuffer() while (matcher.find()) { String hex = matcher.group().substring(2) - matcher.appendReplacement(sb, Matcher.quoteReplacement(decodeHexString(hex))) + String decoded = decodeHexString(hex) + if (decoded == null) { + // The bytes are not valid UTF-8: any String rendering would + // go through a lossy replacement character, collapsing + // distinct values (the single bytes 0x80 and 0x81 both + // become U+FFFD) and making unequal results compare equal. + // Keep the lossless 0x-hex token instead; a rendering that + // cannot round-trip then fails loudly rather than passing. + continue + } + matcher.appendReplacement(sb, Matcher.quoteReplacement(decoded)) } matcher.appendTail(sb) return sb.toString() @@ -142,12 +152,22 @@ class ResultUtils { return value } + /** + * Decodes a hex token to the String the other engine renders for the same + * bytes, or returns null when the bytes are not valid UTF-8: decoding then + * could only go through a lossy replacement character, and the caller keeps + * the lossless 0x-hex form instead. + */ private static String decodeHexString(String hex) { byte[] bytes = new byte[hex.length().intdiv(2)] for (int i = 0; i < bytes.length; i++) { bytes[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16) } - return new String(bytes, StandardCharsets.UTF_8) + String decoded = new String(bytes, StandardCharsets.UTF_8) + if (!Arrays.equals(decoded.getBytes(StandardCharsets.UTF_8), bytes)) { + return null + } + return decoded } static Object normalizeValue(Object value) { diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy index 8ba5342dc3dc68..19a2496a124ebc 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy @@ -244,6 +244,126 @@ suite("test_paimon_pk_vector_ann", "p0,external") { assertEquals(bruteOld, annOld, "a file-creation-time bound must be honored on the vector path") assertEquals(bruteNone, annNone, "a far-future file bound must exclude all files on both paths") + // -------------------------------------------------------- more fixtures + // Two additional shapes the bounded rewrite must handle: a PK-vector table + // whose merge engine permits the index WITHOUT deletion vectors (first-row, + // allowed since paimon 2.1), and a partitioned PK-vector table (whose vector + // splits must carry the derived partition values or every scan fails with + // "Missing partition values"). + spark_paimon_multi """ + CREATE TABLE IF NOT EXISTS paimon.${database}.ann_l2_nodv ( + id BIGINT, tag INT, embedding ARRAY + ) USING paimon TBLPROPERTIES ( + 'primary-key' = 'id', 'bucket' = '2', + 'merge-engine' = 'first-row', + 'changelog-producer' = 'none', + 'vector-field' = 'embedding', + 'field.embedding.vector-dim' = '${dimension}', + 'pk-vector.index.columns' = 'embedding', + 'fields.embedding.pk-vector.index.type' = 'ivf-rq', + 'fields.embedding.pk-vector.distance.metric' = 'l2', + 'fields.embedding.pk-vector.index.options' = '${vectorOptions}', + 'ivf.nprobe' = '4' + ); + INSERT INTO paimon.${database}.ann_l2_nodv ${generatedRows}; + """ + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${database}; + DROP TABLE IF EXISTS paimon.${database}.ann_l2_part; + CREATE TABLE paimon.${database}.ann_l2_part ( + id BIGINT, dt STRING, tag INT, embedding ARRAY + ) USING paimon PARTITIONED BY (dt) TBLPROPERTIES ( + 'primary-key' = 'id', 'bucket' = '2', + 'merge-engine' = 'partial-update', + 'deletion-vectors.enabled' = 'true', + 'deletion-vectors.merge-on-read' = 'false', + 'changelog-producer' = 'none', + 'vector-field' = 'embedding', + 'field.embedding.vector-dim' = '${dimension}', + 'pk-vector.index.columns' = 'embedding', + 'fields.embedding.pk-vector.index.type' = 'ivf-rq', + 'fields.embedding.pk-vector.distance.metric' = 'l2', + 'fields.embedding.pk-vector.index.options' = '${vectorOptions}', + 'ivf.nprobe' = '4' + ); + INSERT INTO paimon.${database}.ann_l2_part + SELECT id, IF(id % 2 = 0, '2026-01-01', '2026-01-02'), CAST(id % 7 AS INT), + transform(sequence(0, 15), + i -> CAST(((id * 17 + i * 13) % 521 - 260) AS FLOAT) / 260.0) + FROM (SELECT explode(sequence(1, ${rows})) AS id); + """ + assertEquals(rows.toLong(), sql("select count(*) from ann_l2_nodv")[0][0]) + assertEquals(rows.toLong(), sql("select count(*) from ann_l2_part")[0][0]) + + // ------------------------------------------------ no-deletion-vector shape + // Unfiltered: paimon-rust's vector search has no data predicate, so the + // rewrite stays sound on the no-DV table. + sql "set enable_paimon_rust_reader=0" + def bruteNoDv = idsOf(sql("""select id from ann_l2_nodv + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + // Filtered: the rust PkVectorSearchParams rejects every data predicate + // unless deletion vectors are enabled without merge-on-read, so the rewrite + // must decline here and both legs agree through the ordinary path. + def bruteNoDvFilter = idsOf(sql("""select id from ann_l2_nodv where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annNoDv = idsOf(sql("""select id from ann_l2_nodv + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + def annNoDvFilter = idsOf(sql("""select id from ann_l2_nodv where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteNoDv, annNoDv, "unfiltered ANN must match brute force on the no-DV table") + assertEquals(bruteNoDvFilter, annNoDvFilter, + "a data predicate on a no-deletion-vector table must not run as a bounded ANN") + + // ------------------------------------------------------------ partitioned + // The vector splits of a partitioned table must carry the derived partition + // values (setPaimonParams throws "Missing partition values" otherwise), and + // the ANN result must still match the ordinary brute-force answer. + sql "set enable_paimon_rust_reader=0" + def brutePart = idsOf(sql("""select id from ann_l2_part + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annPart = idsOf(sql("""select id from ann_l2_part + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(brutePart, annPart, "partitioned ANN must match brute force") + + // ---------------------------------------------- NULL query-vector element + // A NULL element inside the query vector must not become a zero coordinate: + // the ordinary function rejects it, and the rewrite must decline so both + // legs raise the same validation error. + String nullVector = "array(cast(null as float)" + + (1..(dimension.toInteger() - 1)).collect { ", 0.0" }.join() + ")" + def queryWithNulls = """select id from ann_l2 + order by l2_distance_approximate(embedding, ${nullVector}) limit 5""" + def expectValidationError = { String q -> + try { + sql q + return null + } catch (Throwable t) { + return t.getMessage() + } + } + sql "set enable_paimon_rust_reader=0" + def bruteError = expectValidationError(queryWithNulls) + sql "set enable_paimon_rust_reader=1" + def annError = expectValidationError(queryWithNulls) + assertNotNull("the ordinary path must reject a NULL query vector", bruteError) + assertEquals(bruteError, annError, + "the ANN path must reject a NULL query vector the same way, not search a zero vector") + + // ------------------------------------------- IN predicate with a NULL option + // tag IN (0, 1, NULL) passes the shape whitelist (every option is a + // literal) but the BE converter rejects the null option and with it the + // whole predicate, so the rewrite must decline: the IN stays in the + // residual and both legs must agree. + sql "set enable_paimon_rust_reader=0" + def bruteInNull = idsOf(sql("""select id from ann_l2 where tag in (0, 1, null) + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annInNull = idsOf(sql("""select id from ann_l2 where tag in (0, 1, null) + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteInNull, annInNull, "an IN with a NULL option must not run as a bounded ANN") + // -------------------------------------------------------- inner product compareTopN("ann_ip", true, 5) } finally { From 9fb75872d1463048ec9021ba556092a5b2f1ccc3 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Wed, 21 Oct 2026 21:22:06 +0800 Subject: [PATCH 07/13] [fix](paimon) address the second review round on the ANN path Six new findings, all verified and fixed: - Branch identity: the pre-flight resolved the cached main table while the scan reads the relation's bound branch table, so a branch with a different index metric scored and reordered with the wrong metric. All four pre-flight sites now resolve the bound table (relation snapshot, or the options-applied table for t@options); time-travel relations benefit too. - Historical scalar evolution: the vector path skipped the batch reader's per-file schema gate, so a TIMESTAMP(3)->(9) widening would NULL out-of-range values through Arrow's safe cast. The rewrite now declines any rust-incompatible schema history, and the scan node re-checks per split (loud backstop mirroring the pre-flight exactly). - Generic mapping exception: FileQueryScanNode skipped the synthetic distance name for every external catalog, breaking real same-named Hive columns; the skip now applies only when the schema has no such column. - Nonmovable projection: a NoneMovableFunction (assert_true) in the ANN project must keep evaluating below the TopN; any nonmovable expression outside the order-key alias now declines the rewrite. - Pruned bare slot: a bare complex slot whose type was already reduced by nested pruning now declines (slot type must still equal the original column's), instead of tripping the desc-side guard loudly. - IN beyond the converter cap: the whitelist mirrors the BE converter's 16-bit child count, so a raised expr_children_limit no longer drops the predicate into the post-fetch residual. Carried P1 threads closed at their narrowed edges: the pinned-empty generation is fenced in getVectorSearchSplits exactly like the ordinary path (a first commit after binding is no longer readable by vector planning), and the pruned-bare-slot / IN-cap cases above. The stored-NULL differential is kept as a documented limitation: a nullability-based decline cannot ship until PaimonExternalTable.loadSchema stops hardcoding isAllowNull=true (today every paimon column, even PK columns, maps as nullable, so the gate would disable the rewrite for every table). Follow-ups recorded in the review thread. Tests: rewrite rule 36 -> 42 cases, PaimonScanNode +3 (fence twin, loud per-file backstop, schemaId-matching encode); the ANN e2e suite stays green with a new assert_true error-parity leg. BE untouched. Co-Authored-By: Claude Opus 5 (1M context) --- .../doris/datasource/FileQueryScanNode.java | 9 +- .../source/PaimonRustReaderCapabilities.java | 53 ++++ .../paimon/source/PaimonScanNode.java | 30 +++ .../paimon/source/PaimonVectorSearch.java | 13 + .../PushDownVectorTopNIntoPaimonScan.java | 139 ++++++++-- .../paimon/source/PaimonScanNodeTest.java | 125 ++++++++- .../PushDownVectorTopNIntoPaimonScanTest.java | 246 ++++++++++++++++-- .../paimon/test_paimon_pk_vector_ann.groovy | 16 ++ 8 files changed, 590 insertions(+), 41 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index d356f7a8d77d5e..336fdf3266dde7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -382,8 +382,13 @@ private void setColumnPositionMapping() } // Reader-produced Paimon PK-vector search distance: a synthetic output column // filled by the BE reader by name, not present in the table's file schema, - // so it has no Parquet/ORC column position to map. - if (PaimonVectorSearch.SEARCH_DISTANCE_COLUMN.equalsIgnoreCase(colName)) { + // so it has no Parquet/ORC column position to map. Only skip it when the + // schema really does not carry the name: this method serves every external + // catalog, and a REAL column sharing the name (a Hive text table can + // legally define one) must keep its normal position mapping — skipping it + // would emit fewer column_idxs than the file has and fail the BE reader. + if (PaimonVectorSearch.SEARCH_DISTANCE_COLUMN.equalsIgnoreCase(colName) + && columnNames.stream().noneMatch(n -> n.equalsIgnoreCase(colName))) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java index 036ba9fb0558c3..2f42cdc2ddb949 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java @@ -109,6 +109,59 @@ boolean canRead(DataSplit split) { return true; } + /** Vector-path companion to {@link #canRead}. The vector reader materializes the + * projected rows of its Top-K results through the same Arrow decoders as the batch + * path, so a file whose historical schema version decodes differently must stay off + * the vector path too. Only the per-file schema gates apply: the vector path never + * opens a merge reader (its Top-K chain applies deletion vectors inside the search), + * so the merge-retention gates do not judge it. Loud backstop for shapes the rewrite + * pre-flight cannot see (options-selected schemas, mid-statement ALTER races); it + * mirrors {@link #hasRustIncompatibleSchemaHistory} exactly, so it can only fail + * loudly on shapes the pre-flight could not have declined. */ + boolean canVectorRead(DataSplit split) { + if (schema == null || !hasCompatibleSchemaVersion(schema)) { + return false; + } + for (DataFileMeta file : split.dataFiles()) { + if (file.schemaId() != schema.id() && !compatibleFileSchemas.computeIfAbsent( + file.schemaId(), this::hasCompatibleFileSchema)) { + return false; + } + } + return true; + } + + /** Rewrite pre-flight for the vector path: whether ANY historical schema version of + * the table decodes differently from the current one under the rust reader (the + * per-file gate {@link #canRead} runs per split and cannot judge a whole rewrite). + * Example: a TIMESTAMP(3) column widened to TIMESTAMP(9) while old files remain -- + * the vector reader's safe cast NULLs out-of-range values the ordinary path + * preserves, so the rewrite must decline and let the query run ordinary. + * Conservative when the incompatible files are long gone: rare, and correctness-safe. */ + public static boolean hasRustIncompatibleSchemaHistory(FileStoreTable table) { + TableSchema current = table.schema(); + if (current == null || !hasCompatibleSchemaVersion(current)) { + return true; + } + List history; + try { + history = table.schemaManager().listAll(); + } catch (RuntimeException e) { + // Failure to establish compatibility must not opt a table into Rust. + return true; + } + for (TableSchema historical : history) { + if (historical == null || historical.id() == current.id()) { + continue; + } + if (!hasCompatibleSchemaVersion(historical) + || hasIncompatibleEvolution(historical.fields(), current.fields())) { + return true; + } + } + return false; + } + static boolean hasFullNestedProjection(TupleDescriptor tuple) { for (SlotDescriptor slot : tuple.getSlots()) { // The ABI projects only root names, while Arrow struct SerDes bind by ordinal. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 18183959ad870e..7d62605734fe8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -1076,9 +1076,25 @@ && splitHasOrcFile((DataSplit) split) // would return the wrong child on the vector path exactly as it would // on the batch path. The rewrite pre-flights this and declines; this // is the loud backstop for shapes the raw-table pre-flight cannot see. + boolean vectorSchemaIncompatible = false; if (vectorSearchSplit) { canUseRust = canUseRust && rustEligibility.vectorEligible() && PaimonRustReaderCapabilities.hasFullNestedProjection(desc); + if (canUseRust) { + if (rustReaderCapabilities == null) { + rustReaderCapabilities = new PaimonRustReaderCapabilities(paimonFileStoreTable, desc); + } + // Per-file schema-compat backstop mirroring the rewrite's + // hasRustIncompatibleSchemaHistory pre-flight: the vector + // reader materializes projected rows through the same Arrow + // decoders, so a historically incompatible file must stay off + // the vector path. Can only fail for shapes the pre-flight + // could not see (options-selected schemas, mid-statement + // ALTER races) — loud, like every vector backstop. + vectorSchemaIncompatible = + !rustReaderCapabilities.canVectorRead((DataSplit) split); + canUseRust = canUseRust && !vectorSchemaIncompatible; + } } if (!vectorSearchSplit && canUseRust) { canUseRust = canUseRust && nativeSplit && !splitDvNotMaterialized @@ -1158,6 +1174,9 @@ && splitHasOrcFile((DataSplit) split) && !PaimonRustReaderCapabilities.hasFullNestedProjection(desc)) { reasons.add("pruned_nested_projection"); } + if (vectorSearchSplit && vectorSchemaIncompatible) { + reasons.add("historical_schema_evolution"); + } if (reasons.isEmpty()) { reasons.add(vectorSearchSplit ? "unknown" : "capability_whitelist"); } @@ -1579,6 +1598,17 @@ public Map getIncrReadParams() throws UserException { */ @VisibleForTesting public List getVectorSearchSplits() throws UserException { + // The relation may have bound the pinned-empty generation (no commit existed + // at binding). The ordinary path fences that state to an empty result; the + // vector builder would resolve the table's LATEST here and expose a first + // commit that landed after binding, breaking read consistency. Mirror the + // ordinary fence (see getPaimonSplitFromAPI). + Optional relationSnapshot = getRelationSnapshot(); + if (relationSnapshot.isPresent() && relationSnapshot.get() instanceof PaimonMvccSnapshot + && ((PaimonMvccSnapshot) relationSnapshot.get()).getSnapshotCacheValue() + .getSnapshot().getSnapshotId() == PaimonSnapshot.INVALID_SNAPSHOT_ID) { + return Collections.emptyList(); + } Table paimonTable = getProcessedTable(); // Only the partition-half of the predicate is pushed into Paimon planning. // The data (residual) half is left for BE's vector-search filter, and passing diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java index 03fb768c8acf4c..eba89e04536bc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.system.Backend; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import java.util.Collection; @@ -107,4 +108,16 @@ public static boolean vectorScanEligible(Table paimonTable, String tableLocation return PaimonRustEligibility.of(paimonTable, tableLocation, backendStorageProperties, backends).vectorEligible(); } + + /** + * Rewrite pre-flight facade for + * {@link PaimonRustReaderCapabilities#hasRustIncompatibleSchemaHistory}: the + * vector reader materializes projected values through Arrow safe casts, which + * can silently NULL out-of-range values on a historically evolved schema (a + * TIMESTAMP(3) -> TIMESTAMP(9) widening reading year-2300 data); the rewrite + * must decline such tables so the query runs through the ordinary path. + */ + public static boolean hasRustIncompatibleSchemaHistory(FileStoreTable table) { + return PaimonRustReaderCapabilities.hasRustIncompatibleSchemaHistory(table); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java index 8756946a5200b6..c76c152f814ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; @@ -112,6 +113,10 @@ public class PushDownVectorTopNIntoPaimonScan implements RewriteRuleFactory { Type.BOOLEAN, Type.TINYINT, Type.SMALLINT, Type.INT, Type.BIGINT, Type.STRING, Type.DATE, Type.DATEV2); + /** The BE rust predicate converter encodes the IN option list under a 16-bit + * child count; a bigger list is dropped whole into the post-fetch residual. */ + private static final int MAX_IN_PREDICATE_CHILDREN = 0xFFFF; + @Override public List buildRules() { return ImmutableList.of( @@ -271,11 +276,13 @@ private Plan pushDown( } String annColumnName = vectorSlot.getOriginalColumn().get().getName(); - // Validate the PK-vector index and metric on the Paimon table. + // Validate the PK-vector index and metric on the relation's bound Paimon + // table: a branch table carries its own index options and can differ from + // the main table (see boundPaimonTable). PaimonExternalTable paimonTable = (PaimonExternalTable) scan.getTable(); CoreOptions coreOptions; try { - Table basePaimonTable = paimonTable.getPaimonTable(Optional.empty()); + Table basePaimonTable = boundPaimonTable(paimonTable, scan); coreOptions = CoreOptions.fromMap(basePaimonTable.options()); } catch (Throwable t) { return null; @@ -318,14 +325,27 @@ private Plan pushDown( // this (the whitelist admits only scalar columns); the order-key alias // is replaced by the synthetic slot and cannot reach it either; so only // a Project output expression over a complex column has to decline. + // + // A NoneMovableFunction in the projection (assert_true) must keep + // evaluating below the TopN — see projectContainsNonMovableExpression. + // + // A historically evolved schema must stay off the vector reader: its + // materialization decodes projected values through Arrow safe casts, + // which can silently NULL out-of-range values (a TIMESTAMP(3) widened + // to TIMESTAMP(9) reading a year-2300 value) where the ordinary path + // preserves them — see PaimonRustReaderCapabilities.hasRustIncompatibleSchemaHistory. try { - Table table = paimonTable.getPaimonTable(Optional.empty()); + Table table = boundPaimonTable(paimonTable, scan); if ((optionalFilter.isPresent() && filterHasDataConjunct(optionalFilter.get(), table) && !PaimonScanNode.vectorSearchSupportsDataPredicate(table)) || table.rowType().getFieldNames().stream().anyMatch( name -> name.equalsIgnoreCase(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)) - || !complexColumnsRideAsWholeColumns(project, orderKeyAlias)) { + || !complexColumnsRideAsWholeColumns(project, orderKeyAlias) + || projectContainsNonMovableExpression(project, orderKeyAlias) + || (table instanceof FileStoreTable + && PaimonVectorSearch.hasRustIncompatibleSchemaHistory( + (FileStoreTable) table))) { return null; } } catch (Throwable t) { @@ -340,7 +360,7 @@ && filterHasDataConjunct(optionalFilter.get(), table) // those queries run through the ordinary paths, where they are supported. try { if (projectedColumnsContainVariant(project, optionalFilter) - || orcSchemaContainsTimestampLtz(paimonTable)) { + || orcSchemaContainsTimestampLtz(boundPaimonTable(paimonTable, scan))) { return null; } } catch (Throwable t) { @@ -367,7 +387,7 @@ && filterHasDataConjunct(optionalFilter.get(), table) // relation-option-processed table, so it stays authoritative for t@options // overrides this raw-table pre-flight cannot see. try { - Table baseTable = paimonTable.getPaimonTable(Optional.empty()); + Table baseTable = boundPaimonTable(paimonTable, scan); PaimonExternalCatalog catalog = (PaimonExternalCatalog) paimonTable.getCatalog(); Map storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( @@ -472,6 +492,15 @@ private static boolean conjunctRidesIntoVectorSearch(Expression conjunct) { } if (conjunct instanceof InPredicate) { InPredicate in = (InPredicate) conjunct; + // The BE converter encodes the option list under a 16-bit child count + // (see PaimonRustPredicateConverter._convert_in): a bigger list — + // reachable by raising expr_children_limit — is dropped whole and lands + // in the post-fetch residual, re-enabling the bounded-TopK truncation + // this whitelist exists to prevent. Mirror the converter's cap: the + // slot plus the options must fit uint16. + if (in.getOptions().size() + 1 > MAX_IN_PREDICATE_CHILDREN) { + return false; + } return isVectorFilterColumnSlot(in.getCompareExpr()) && in.getOptions().stream().allMatch(PushDownVectorTopNIntoPaimonScan::ridesAsLiteral); } @@ -519,6 +548,50 @@ private static boolean filterHasDataConjunct(LogicalFilter filter, Table paim return false; } + /** + * The relation's bound Paimon table -- branch, time travel, statement fence or + * option overrides included -- instead of the cached main table. A branch is an + * independently configured table: it can carry a different vector-index metric + * (inner-product where main has l2), different deletion-vector options or a + * different schema, and the scan reads the BRANCH's schema through paimon-rust. + * Validating index/metric/eligibility against the main table would accept a + * query whose search then scores and reorders with the branch's metric -- + * wrong results even at full recall. + * + *

Relation options ({@code t@options(...)}) apply over the statement snapshot + * and are resolved through the table's own option-application path; everything + * else rides the relation's bound snapshot captured at binding. + */ + private static Table boundPaimonTable(PaimonExternalTable paimonTable, LogicalFileScan scan) { + if (scan.getScanParams().isPresent() && scan.getScanParams().get().isOptions()) { + return paimonTable.getPaimonTable(scan.getScanParams().get()); + } + return paimonTable.getPaimonTable(scan.getRelationSnapshot()); + } + + /** + * A Project expression whose evaluation must stay BELOW the TopN (a + * NoneMovableFunction, such as {@code assert_true}) still has to raise on rows + * the bounded Top-K would never fetch: on the ordinary path the Project + * evaluates before the TopN and fails for a rejected far row, while + * {@code withVectorTopN} truncates the scan to the near rows first, so the + * assertion never sees them and the query silently succeeds. Any non-movable + * expression outside the order-key alias keeps the ordinary path. This is a + * SELECT expression; the filter residual thread is separate. + */ + private static boolean projectContainsNonMovableExpression( + LogicalProject project, Alias orderKeyAlias) { + for (NamedExpression projection : project.getProjects()) { + if (!(projection instanceof Alias) || projection.equals(orderKeyAlias)) { + continue; + } + if (((Alias) projection).child().anyMatch(e -> e instanceof NoneMovableFunction)) { + return true; + } + } + return false; + } + /** * Whether every complex-typed column referenced by the projection rides as a * whole column. Nested pruning materializes the scan tuple with a REDUCED type @@ -526,21 +599,35 @@ private static boolean filterHasDataConjunct(LogicalFilter filter, Table paim * 'b'))} prunes {@code payload ROW} to {@code ROW}), but the vector * reader projects only root names and returns the full Arrow value, whose * struct SerDe binds children by ordinal — the wrong child comes back. A bare - * slot output keeps the full type, and a complex slot consumed ONLY by a - * distance function never reaches the scan at all (this rewrite replaces every - * occurrence of the function with the synthetic distance slot, order-key alias - * included), so only any OTHER expression over a complex column forces the - * decline. Filter conjuncts cannot reach this: the shape whitelist admits only - * scalar columns. + * slot output keeps the full type — UNLESS it was already pruned (an outer + * field projection can leave an already-pruned bare slot here), whose reduced + * type no longer equals the original column's type; the desc-side backstop + * would fail loudly, so decline instead. A complex slot consumed ONLY by a + * distance function never reaches the scan at all (this rewrite replaces + * every occurrence of the function with the synthetic distance slot, + * order-key alias included), so only any OTHER expression over a complex + * column forces the decline. Filter conjuncts cannot reach this: the shape + * whitelist admits only scalar columns. */ private static boolean complexColumnsRideAsWholeColumns(LogicalProject project, Alias orderKeyAlias) { for (NamedExpression projection : project.getProjects()) { if (!(projection instanceof Alias)) { + // A bare passthrough slot output: whole unless already pruned. + if (projection instanceof SlotReference + && !slotRidesWholeColumn((SlotReference) projection)) { + return false; + } continue; } Alias alias = (Alias) projection; - if (alias.equals(orderKeyAlias) || alias.child() instanceof SlotReference) { + if (alias.equals(orderKeyAlias)) { + continue; + } + if (alias.child() instanceof SlotReference) { + if (!slotRidesWholeColumn((SlotReference) alias.child())) { + return false; + } continue; } Set replacedSlots = new HashSet<>(); @@ -551,14 +638,32 @@ private static boolean complexColumnsRideAsWholeColumns(LogicalProject projec } Set slots = alias.child().collect(SlotReference.class::isInstance); for (SlotReference slot : slots) { - if (!replacedSlots.contains(slot) && slot.getDataType().isComplexType()) { - return false; + if (!replacedSlots.contains(slot)) { + if (slot.getDataType().isComplexType() || !slotRidesWholeColumn(slot)) { + return false; + } } } } return true; } + /** + * A whole-riding slot keeps the type it was created with from the original + * column ({@code SlotReference.fromColumn}: {@code DataType.fromCatalogType}). + * Nested pruning materializes a REDUCED type ({@code ROW} from + * {@code ROW}) while the original column keeps the full one; only a whole + * slot may ride the vector path, matching the desc-side backstop's + * slot-versus-column comparison. + */ + private static boolean slotRidesWholeColumn(SlotReference slot) { + if (!slot.getDataType().isComplexType() || !slot.getOriginalColumn().isPresent()) { + return true; + } + return slot.getDataType().equals(org.apache.doris.nereids.types.DataType.fromCatalogType( + slot.getOriginalColumn().get().getType())); + } + private static boolean isVectorFilterColumnSlot(Expression expr) { if (!(expr instanceof SlotReference)) { return false; @@ -601,9 +706,9 @@ private static boolean projectedColumnsContainVariant( * split would materialize rows through that decoder with no fallback. Snapshot * selection is irrelevant here -- this is a physical-file property. */ - private static boolean orcSchemaContainsTimestampLtz(PaimonExternalTable paimonTable) { + private static boolean orcSchemaContainsTimestampLtz(Table boundTable) { try { - Table baseTable = paimonTable.getPaimonTable(Optional.empty()); + Table baseTable = boundTable; if (!(baseTable instanceof FileStoreTable)) { return false; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 03eda38a32da0d..ea210bb4871d4e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -81,6 +81,7 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.table.AppendOnlyFileStoreTable; @@ -146,8 +147,12 @@ public void saveVariantV2Config() { // Statement-scoped scan task reuse is on by default; the mock's field would otherwise // read false and bypass the cache under test. sv.enableExternalScanTaskReuse = true; - // These routing fixtures bypass scan initialization and use an upgraded eligible BE. - Backend backend = GsonUtils.GSON.fromJson("{\"supportsPaimonRustReader\":true}", Backend.class); + // These routing fixtures bypass scan initialization and use an upgraded eligible + // BE. The vector bit rides along: the vector gates negotiate it independently + // of the batch one, and the vector-split tests below need both. + Backend backend = GsonUtils.GSON.fromJson( + "{\"supportsPaimonRustReader\":true,\"supportsPaimonRustVectorReader\":true}", + Backend.class); new MockUp() { @mockit.Mock public Collection getBackends() { @@ -1174,6 +1179,122 @@ public void testBoundEmptySnapshotDoesNotReadLaterCommit() throws Exception { Mockito.verify(liveTable, Mockito.never()).newReadBuilder(); } + @Test + public void testVectorSearchSplitsFencePinnedEmptyGeneration() throws Exception { + // The vector twin of testBoundEmptySnapshotDoesNotReadLaterCommit: the relation + // bound the pinned-empty generation (no commit existed at binding). The + // ordinary path fences this to empty; without the fence here, the vector + // builder resolves the table's LATEST and exposes a first commit that landed + // after binding, breaking read consistency between the two planning paths. + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Table liveTable = Mockito.mock(Table.class); + node.setSource(source); + node.setRelationSnapshot(Optional.of(new PaimonMvccSnapshot( + new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(PaimonSnapshot.INVALID_SNAPSHOT_ID, 1L, liveTable))))); + + Assert.assertTrue(node.getVectorSearchSplits().isEmpty()); + Mockito.verify(liveTable, Mockito.never()).newBatchVectorSearchBuilder(); + } + + @Test + public void testVectorSplitFailsLoudOnIncompatibleFileSchema() throws Exception { + // A vector split whose file carries a historically incompatible schema id is a + // broken invariant (the rewrite's hasRustIncompatibleSchemaHistory pre-flight + // must decline such a table): the vector reader would materialize projected + // values through the incompatible Arrow decoder with no JNI fallback, so the + // scan node fails loudly instead of silently corrupting values. + PaimonScanNode node = vectorRoutingNode(); + Mockito.when(vectorProcessedTable.schemaManager().schema(1L)) + .thenReturn(incompatibleHistorySchema()); + PaimonSplit split = new PaimonSplit(createDataSplit("vector_hist.parquet")); + split.setVectorPayloadBytes(new byte[] {1}); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + Throwable thrown = null; + try { + invokePrivateMethod(node, "setPaimonParams", + new Class[] {TFileRangeDesc.class, PaimonSplit.class}, rangeDesc, split); + } catch (java.lang.reflect.InvocationTargetException e) { + thrown = e.getCause(); + } + Assert.assertTrue("expected the loud vector-split failure, got " + thrown, + thrown instanceof RuntimeException); + Assert.assertTrue("the failure must name the rust-reader requirement, got " + thrown, + thrown.getMessage().contains("paimon-rust reader")); + } + + @Test + public void testVectorSplitWithMatchingFileSchemaStillEncodes() throws Exception { + // Control: the split's file carries the CURRENT schema id (no history to + // decode differently), so the vector payload is encoded and the split rides + // the rust reader with an empty paimon_split (the DataSplit is already inside + // vector_payload). + PaimonScanNode node = vectorRoutingNode(); + // The current schema id equals the file's schema id (1): the per-file history + // gate never fires. + Mockito.when(vectorProcessedTable.schema()) + .thenReturn(new TableSchema(1, + Collections.singletonList(new DataField(0, "id", new IntType())), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null)); + PaimonSplit split = new PaimonSplit(createDataSplit("vector_ok.parquet")); + split.setVectorPayloadBytes(new byte[] {1}); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + invokePrivateMethod(node, "setPaimonParams", + new Class[] {TFileRangeDesc.class, PaimonSplit.class}, rangeDesc, split); + org.apache.doris.thrift.TPaimonFileDesc fileDesc = + rangeDesc.getTableFormatParams().getPaimonParams(); + Assert.assertEquals( + org.apache.doris.thrift.TPaimonReaderType.PAIMON_RUST, fileDesc.getReaderType()); + Assert.assertEquals(1, fileDesc.getVectorPayload().getBytes().length); + Assert.assertTrue(fileDesc.isSetPaimonSplit() && fileDesc.getPaimonSplit().isEmpty()); + } + + private FileStoreTable vectorProcessedTable; + + /** A routing node for vector splits: rust+V2 on, a processed FileStoreTable with a + * current schema id 5 and per-test history stubs, backend/storage properties open. */ + private PaimonScanNode vectorRoutingNode() throws Exception { + SessionVariable vars = new SessionVariable(); + vars.setEnablePaimonRustReader(true); + vars.enableFileScannerV2 = true; + PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), + new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY); + vectorProcessedTable = Mockito.mock(FileStoreTable.class); + // Lenient stubbings: which are consumed varies between the two vector-split + // tests (the control re-stubs schema() to match the file's schema id). + Mockito.lenient().when(vectorProcessedTable.options()).thenReturn(Collections.emptyMap()); + Mockito.lenient().when(vectorProcessedTable.schema()).thenReturn(new TableSchema(5, + Collections.singletonList(new DataField(0, "id", new IntType())), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null)); + Mockito.lenient().when(vectorProcessedTable.schemaManager()) + .thenReturn(Mockito.mock(SchemaManager.class)); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + Mockito.when(source.getExternalTable()).thenReturn(externalTable); + Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t"); + Mockito.when(externalTable.getDbName()).thenReturn("db"); + Mockito.when(externalTable.getName()).thenReturn("t"); + node.setSource(source); + setField(PaimonScanNode.class, node, "processedTable", vectorProcessedTable); + setField(PaimonScanNode.class, node, "backendStorageProperties", + ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret")); + setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap()); + return node; + } + + /** A schema-1 history whose ts TIMESTAMP(3) cannot decode under the current schema. */ + private TableSchema incompatibleHistorySchema() { + return new TableSchema(1, + Collections.singletonList(new DataField(0, "ts", new TimestampType(3))), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + } + @Test public void testBoundEmptyDataSnapshotStillPlansMetadataSystemTable() throws Exception { // A live statement scope is required: the metadata-system-table path plans through the diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java index a5508fc00464fb..b01b2918ee69c1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Type; +import org.apache.doris.common.Config; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalTable; @@ -35,6 +36,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.Multiply; @@ -42,6 +44,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; @@ -70,13 +73,19 @@ import com.google.common.collect.ImmutableSet; import mockit.MockUp; import org.apache.paimon.fs.Path; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.TimestampType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -892,6 +901,146 @@ public void testVariantProjectionNotPushedDown() { new LogicalProject<>(ImmutableList.of(idSlot(scan), payload, dis), scan), dis, true)); } + @Test + public void testBranchTableWithDifferentMetricNotPushedDown() { + // t@branch(b): the relation's bound table is the branch, which can carry an + // independently configured index metric. The pre-flight must validate the + // BOUND table: an l2 query against an inner-product branch would score with + // the branch's metric and reorder with l2's inverse -- wrong results even + // at full recall. + // field names of the fixture schema (see below): the branch table's rowType + // drives the same synthetic-distance-collision pre-flight as the main table. + List fieldNames = ImmutableList.of("id", "vec", "vec2", "tag", "score", + "payload", "pmap"); + FileStoreTable branch = mockFileStoreTable(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "inner_product"), fieldNames, null); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), Optional.empty(), null, false, + null, branch); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testBranchTableWithMatchingMetricStillPushedDown() { + // Control: the bound branch table carries the same l2 metric, so the + // rewrite must fire exactly as it does against the main table. + List fieldNames = ImmutableList.of("id", "vec", "vec2", "tag", "score", + "payload", "pmap"); + FileStoreTable branch = mockFileStoreTable(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), fieldNames, null); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), Optional.empty(), null, false, + null, branch); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + Plan rewritten = applyRule(0, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + Assertions.assertTrue(((LogicalFileScan) ((LogicalProject) rewritten.child(0)) + .child(0)).getAnnTopN().isPresent()); + } + + @Test + public void testHistoricalSchemaEvolutionNotPushedDown() { + // A TIMESTAMP(3) -> TIMESTAMP(9) widening with old files retained: the + // vector reader's Arrow safe cast NULLs out-of-range values the ordinary + // path preserves, so the rewrite must decline and let the query run + // ordinary (see PaimonRustReaderCapabilities.hasRustIncompatibleSchemaHistory). + TableSchema widened = new TableSchema(4, + Collections.singletonList(new DataField(0, "ts", new TimestampType(9))), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + TableSchema millis = new TableSchema(5, + Collections.singletonList(new DataField(0, "ts", new TimestampType(3))), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), Optional.empty(), null, false, + ImmutableList.of(widened, millis), null); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testNonMovableProjectExpressionNotPushedDown() { + // c := assert_true(id > 0): on the ordinary path the Project evaluates before + // the TopN and raises for a far row the bounded fetch would never deliver; + // withVectorTopN truncates first, so the assertion never fires and the query + // silently succeeds. A NoneMovableFunction in the projection keeps the + // ordinary path. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + Alias guard = new Alias( + new AssertTrue(new GreaterThan(idSlot(scan), new IntegerLiteral(0)), + new StringLiteral("bad id")), + "c"); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN(new LogicalProject<>( + ImmutableList.of(idSlot(scan), guard, dis), scan), dis, true)); + } + + @Test + public void testPrunedBareComplexSlotNotPushedDown() { + // An outer field projection can leave an already-pruned bare slot inside the + // ANN Project (nested pruning materialized its type to the accessed subset). + // The desc-side backstop would fail loudly, so the pre-flight declines: a + // bare slot only rides when its type still equals the original column's. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference pmap = (SlotReference) scan.getOutput().get(6); + Assertions.assertEquals("pmap", pmap.getName()); + SlotReference pruned = new SlotReference( + StatementScopeIdGenerator.getExprIdGenerator().getNextId(), "pmap", + org.apache.doris.nereids.types.ArrayType.of( + org.apache.doris.nereids.types.DoubleType.INSTANCE), + pmap.nullable(), ImmutableList.of(), scan.getTable(), + pmap.getOriginalColumn().get(), scan.getTable(), + pmap.getOriginalColumn().get(), ImmutableList.of()); + Alias prunedAlias = new Alias(pruned, "p"); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN(new LogicalProject<>( + ImmutableList.of(idSlot(scan), prunedAlias, dis), scan), dis, true)); + } + + @Test + public void testInPredicateBeyondConverterCapNotPushedDown() { + // With expr_children_limit raised, an IN can exceed the BE rust predicate + // converter's 16-bit child count: the converter drops the whole predicate, + // and the residual re-applies it AFTER the bounded Top-K — the truncation + // the whitelist exists to prevent. Options whose slot+options children + // exceed the cap keep the ordinary path. + int originalChildrenLimit = Config.expr_children_limit; + Config.expr_children_limit = 0xFFFF + 64; + LogicalFileScan scan; + SlotReference tag; + try { + scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + tag = (SlotReference) scan.getOutput().get(3); + List options = new java.util.ArrayList<>(); + for (int i = 0; i < 0xFFFF; i++) { + options.add(new IntegerLiteral(i)); + } + LogicalFilter filter = + new LogicalFilter<>(ImmutableSet.of(new InPredicate(tag, options)), scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + } finally { + Config.expr_children_limit = originalChildrenLimit; + } + } + // ------------------------------------------------------------------ helpers private Plan applyRule(int ruleIndex, LogicalTopN topN) { @@ -938,10 +1087,31 @@ private LogicalFileScan paimonScan(Map options, * pre-flight reads off the mocked paimon rowType. */ private LogicalFileScan paimonScan(Map options, Optional scanParams, List paimonFieldNames) { + return paimonScan(options, scanParams, paimonFieldNames, false); + } + + /** {@code nullableVector} makes the fixture's vec column nullable (the stored-NULL + * contract tests). */ + private LogicalFileScan paimonScan(Map options, + Optional scanParams, List paimonFieldNames, + boolean nullableVector) { + return paimonScan(options, scanParams, paimonFieldNames, nullableVector, null, null); + } + + /** + * Full fixture. {@code extraHistory} appends rust-incompatible historical schemas + * to the bound table's schema history (evolution tests); {@code boundTable} binds + * a branch-style relation snapshot whose getPaimonTable resolves to that table — + * the main-table stub stays for the anything-else lookups. + */ + private LogicalFileScan paimonScan(Map options, + Optional scanParams, List paimonFieldNames, + boolean nullableVector, List extraHistory, + org.apache.paimon.table.Table boundTable) { List schema = ImmutableList.of( new Column("id", Type.INT, true), - new Column("vec", new ArrayType(Type.FLOAT), true), - new Column("vec2", new ArrayType(Type.FLOAT), true), + new Column("vec", new ArrayType(Type.FLOAT), nullableVector), + new Column("vec2", new ArrayType(Type.FLOAT), nullableVector), new Column("tag", Type.INT, true), new Column("score", Type.DOUBLE, true), new Column("payload", Type.VARIANT, true), @@ -963,31 +1133,29 @@ private LogicalFileScan paimonScan(Map options, Mockito.when(table.initSelectedPartitions(Mockito.any())) .thenReturn(SelectedPartitions.NOT_PRUNED); - // A FileStoreTable (not a bare Table) with a file:// location, no REST FileIO and - // null CoreOptions/schema: the rewrite's pre-flight sees a plain local - // warehouse table whose option gates all skip, so the per-test options map is - // the only input that differs. - FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class); - Mockito.when(paimonTable.options()).thenReturn(options); - Mockito.when(paimonTable.location()) - .thenReturn(new Path("file:///warehouse/wh/paimon_db.db/paimon_tbl")); - // rowType drives the synthetic-distance-column collision pre-flight; its - // field names track the fixture schema (or the test's explicit names). - org.apache.paimon.types.RowType rowType = Mockito.mock( - org.apache.paimon.types.RowType.class); - Mockito.when(rowType.getFieldNames()).thenReturn(paimonFieldNames); - Mockito.when(paimonTable.rowType()).thenReturn(rowType); - Mockito.when(paimonTable.fileIO()).thenReturn(null); - Mockito.when(paimonTable.coreOptions()).thenReturn(null); - Mockito.when(paimonTable.schema()).thenReturn(null); + FileStoreTable paimonTable = mockFileStoreTable(options, paimonFieldNames, extraHistory); Mockito.when(table.getPaimonTable(Mockito.>any())) .thenReturn(paimonTable); + Mockito.when(table.getPaimonTable(Mockito.any())).thenReturn(paimonTable); // An options-type TableScanParams binds its schema through // getFullSchema(TableScanParams) (the relation-option path inside // PaimonExternalTable, which would otherwise hit the unmocked package // private helpers). Stub it directly so the options tests get real // output slots; the rule itself only reads the params' raw map. Mockito.when(table.getFullSchema(Mockito.any())).thenReturn(schema); + + Optional relationSnapshot = Optional.empty(); + if (boundTable != null) { + // Stub AFTER the any() stub: Mockito resolves the last matching stub, + // so this relation's snapshot resolves to the branch/bound table while + // everything else keeps the main table. + org.apache.doris.datasource.mvcc.MvccSnapshot snapshot = + Mockito.mock(org.apache.doris.datasource.mvcc.MvccSnapshot.class); + Mockito.when(table.getPaimonTable(Mockito.eq(Optional.of(snapshot)))) + .thenReturn(boundTable); + relationSnapshot = Optional.of(snapshot); + } + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); Mockito.when(table.getCatalog()).thenReturn(catalog); @@ -1002,7 +1170,45 @@ private LogicalFileScan paimonScan(Map options, return new LogicalFileScan(StatementScopeIdGenerator.newRelationId(), table, ImmutableList.of("paimon_ctl", "paimon_db"), ImmutableList.of(), - Optional.empty(), Optional.empty(), scanParams, Optional.empty()); + Optional.empty(), Optional.empty(), scanParams, Optional.empty(), + relationSnapshot); + } + + /** A FileStoreTable mock: local file:// location, no REST FileIO, and the + * per-test options map as the only input the pre-flight gates see. + * {@code extraHistory} appends historical schemas to the schema-manager's + * listAll, so evolution tests can supply rust-incompatible versions; + * null means an empty history (only the current schema). */ + private FileStoreTable mockFileStoreTable(Map options, + List paimonFieldNames, List extraHistory) { + FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class); + Mockito.when(paimonTable.options()).thenReturn(options); + Mockito.when(paimonTable.location()) + .thenReturn(new Path("file:///warehouse/wh/paimon_db.db/paimon_tbl")); + // The schema-history pre-flight: a current schema whose id is alone in the + // history means no evolution, so the positive cases pass it. + TableSchema currentSchema = new TableSchema( + 5, Collections.singletonList(new DataField(0, "id", new IntType())), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + Mockito.when(paimonTable.schema()).thenReturn(currentSchema); + List history = new java.util.ArrayList<>(); + history.add(currentSchema); + if (extraHistory != null) { + history.addAll(extraHistory); + } + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + Mockito.when(schemaManager.listAll()).thenReturn(history); + Mockito.when(paimonTable.schemaManager()).thenReturn(schemaManager); + // rowType drives the synthetic-distance-column collision pre-flight; its + // field names track the fixture schema (or the test's explicit names). + org.apache.paimon.types.RowType rowType = Mockito.mock( + org.apache.paimon.types.RowType.class); + Mockito.when(rowType.getFieldNames()).thenReturn(paimonFieldNames); + Mockito.when(paimonTable.rowType()).thenReturn(rowType); + Mockito.when(paimonTable.fileIO()).thenReturn(null); + Mockito.when(paimonTable.coreOptions()).thenReturn(null); + return paimonTable; } private SlotReference idSlot(LogicalFileScan scan) { diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy index 19a2496a124ebc..d1716ebb802657 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy @@ -351,6 +351,22 @@ suite("test_paimon_pk_vector_ann", "p0,external") { assertEquals(bruteError, annError, "the ANN path must reject a NULL query vector the same way, not search a zero vector") + + // ------------------------------------------- non-movable project expression + // assert_true in the projection must keep evaluating below the TopN: the + // ordinary path raises for the rejected far rows (the fixture has id%7=0 + // tags), while a bounded fetch that truncates before the Project would + // silently succeed. The rewrite declines, so both legs raise the same error. + def queryWithAssertTrue = """select id, assert_true(tag > 0, 'bad tag') from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""" + sql "set enable_paimon_rust_reader=0" + def bruteAssertError = expectValidationError(queryWithAssertTrue) + sql "set enable_paimon_rust_reader=1" + def annAssertError = expectValidationError(queryWithAssertTrue) + assertNotNull("the ordinary path must reject a failing assert_true", bruteAssertError) + assertEquals(bruteAssertError, annAssertError, + "a non-movable projection must not be truncated by a bounded fetch") + // ------------------------------------------- IN predicate with a NULL option // tag IN (0, 1, NULL) passes the shape whitelist (every option is a // literal) but the BE converter rejects the null option and with it the From 8fd3becda6ec9a4b6b48f8580f8eb125e5fa6ed0 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Thu, 22 Oct 2026 17:06:38 +0800 Subject: [PATCH 08/13] [fix](paimon) adapt the 4.2 porting fallout - Paimon 2.1 turned SchemaManager into an interface; the connector test moves to the FileSystemSchemaManager default implementation. - The LogicalFileScan conflict resolution missed a blank line between constructors (checkstyle). Co-Authored-By: Claude Opus 5 (1M context) --- .../org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java | 4 ++-- .../doris/nereids/trees/plans/logical/LogicalFileScan.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java index ba07f0e2ed5196..a39582969e378c 100644 --- a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java @@ -41,7 +41,7 @@ import org.apache.paimon.data.variant.GenericVariant; import org.apache.paimon.data.variant.Variant; import org.apache.paimon.fs.local.LocalFileIO; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.BatchTableCommit; @@ -289,7 +289,7 @@ public void testPrecisionSevenAndEightKeysRoundTrip(@TempDir Path dir) .column("ts", DataTypes.TIMESTAMP(precision).notNull()) .column("detail", DataTypes.ROW(DataTypes.FIELD(0, "t", DataTypes.TIMESTAMP(precision)))) .primaryKey("ts").option("bucket", "1").option("file.format", format).build(); - new SchemaManager(fileIO, path).createTable(schema); + new FileSystemSchemaManager(fileIO, path).createTable(schema); Table table = FileStoreTableFactory.create(fileIO, path); PaimonWriteSchema writeSchema = PaimonWriteSchema.create(table.rowType(), new String[] {"ts", "detail"}); BatchWriteBuilder builder = table.newBatchWriteBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index 9c8ad170f72a06..c9da479cb0d714 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -150,6 +150,7 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali this.relationSnapshot = relationSnapshot; this.annTopN = annTopN; } + /** * Constructor for LogicalFileScan, kept for subclasses that were not updated with a table alias. */ From 801bea1f62ef06a0d8c6c171edc21f530aaca957 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Fri, 23 Oct 2026 16:39:01 +0800 Subject: [PATCH 09/13] [refactor](paimon) reuse the rust heartbeat bit for the vector protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: drop the separate supports_paimon_rust_vector_reader field and gate the PK-vector rewrite on supports_paimon_rust_reader alone. The rust reader bit has not shipped in any release yet, and the PK-vector protocol ships in the same release, so no deployed BE can ever report the rust bit without the vector protocol — the separate negotiation (and its mixed-version window) cannot occur in practice. Removes the thrift field, the BE heartbeat report, the FE backend-state plumbing, and the vectorEligible layer; the vector gates now reuse the plain eligible() computation. Co-Authored-By: Claude Opus 5.5 (1M context) --- be/src/agent/heartbeat_server.cpp | 4 -- .../paimon/source/PaimonRustEligibility.java | 24 +----------- .../paimon/source/PaimonScanNode.java | 5 +-- .../paimon/source/PaimonVectorSearch.java | 12 +++--- .../java/org/apache/doris/system/Backend.java | 14 ------- .../doris/system/BackendHbResponse.java | 12 ------ .../org/apache/doris/system/HeartbeatMgr.java | 3 -- .../paimon/source/PaimonScanNodeTest.java | 6 +-- .../PushDownVectorTopNIntoPaimonScanTest.java | 25 ++----------- .../BackendPaimonRustCapabilityTest.java | 37 ------------------- gensrc/thrift/HeartbeatService.thrift | 6 --- 11 files changed, 14 insertions(+), 134 deletions(-) diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp index 26fbbec7daa07e..5cf8c7df01da7b 100644 --- a/be/src/agent/heartbeat_server.cpp +++ b/be/src/agent/heartbeat_server.cpp @@ -97,10 +97,6 @@ void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result, heartbeat_result.backend_info.__set_be_mem(MemInfo::physical_mem()); // Reader enum support cannot be inferred from a release version during rolling upgrades. heartbeat_result.backend_info.__set_supports_paimon_rust_reader(true); - // The PK-vector (ANN) protocol is a separate negotiation: an older - // rust-capable BE would reject an empty paimon_split + vector payload as - // "missing paimon_split", so only BEs carrying the vector protocol report it. - heartbeat_result.backend_info.__set_supports_paimon_rust_vector_reader(true); } watch.stop(); if (watch.elapsed_time() > 1000L * 1000L * 1000L) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java index 95f9f6aa9d300a..ed8ba9bd790e6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java @@ -154,18 +154,11 @@ final class PaimonRustEligibility { // supports_paimon_rust_reader. final boolean allBackendsRustCapable; - // The PK-vector (ANN) protocol is negotiated separately: an older - // rust-capable BE reports the batch bit but would reject an empty - // paimon_split + vector payload ("missing paimon_split"), so the one-way - // vector rewrite must require this bit on every candidate backend too. - final boolean allBackendsVectorCapable; - private PaimonRustEligibility(boolean isFileStoreTable, boolean fallbackReadTable, boolean restTokenTable, boolean queryAuthTable, boolean puAggDeletionVectors, boolean dvMergeOnRead, boolean deduplicateIgnoreDelete, boolean rustUnsupportedMergeOption, boolean scanModeNotDefault, boolean providerModeTranslatable, boolean schemeCapabilityVerified, - boolean hdfsBackendVerified, boolean allBackendsRustCapable, - boolean allBackendsVectorCapable) { + boolean hdfsBackendVerified, boolean allBackendsRustCapable) { this.isFileStoreTable = isFileStoreTable; this.fallbackReadTable = fallbackReadTable; this.restTokenTable = restTokenTable; @@ -179,7 +172,6 @@ private PaimonRustEligibility(boolean isFileStoreTable, boolean fallbackReadTabl this.schemeCapabilityVerified = schemeCapabilityVerified; this.hdfsBackendVerified = hdfsBackendVerified; this.allBackendsRustCapable = allBackendsRustCapable; - this.allBackendsVectorCapable = allBackendsVectorCapable; } /** @@ -288,13 +280,10 @@ static PaimonRustEligibility of(Table table, String tableLocation, tableLocation, backendStorageProperties); boolean allBackendsRustCapable = backends != null && !backends.isEmpty() && backends.stream().allMatch(Backend::isPaimonRustReaderSupported); - boolean allBackendsVectorCapable = backends != null && !backends.isEmpty() - && backends.stream().allMatch(Backend::isPaimonRustVectorReaderSupported); return new PaimonRustEligibility(isFileStoreTable, fallbackReadTable, restTokenTable, queryAuthTable, puAggDeletionVectors, dvMergeOnRead, deduplicateIgnoreDelete, rustUnsupportedMergeOption, scanModeNotDefault, providerModeTranslatable, - schemeCapabilityVerified, hdfsBackendVerified, allBackendsRustCapable, - allBackendsVectorCapable); + schemeCapabilityVerified, hdfsBackendVerified, allBackendsRustCapable); } /** Whether the table, storage and backend shapes all admit the paimon-rust reader. */ @@ -307,13 +296,4 @@ boolean eligible() { && allBackendsRustCapable; } - /** - * Whether they all admit the PK-vector (ANN) search path on top of the batch - * rust gates: the vector protocol needs every candidate backend to report - * the vector capability bit as well, since a rewritten vector split has no - * JNI fallback and an older rust-capable BE cannot decode it. - */ - boolean vectorEligible() { - return eligible() && allBackendsVectorCapable; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 7d62605734fe8e..2a32fe65955c10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -1078,7 +1078,7 @@ && splitHasOrcFile((DataSplit) split) // is the loud backstop for shapes the raw-table pre-flight cannot see. boolean vectorSchemaIncompatible = false; if (vectorSearchSplit) { - canUseRust = canUseRust && rustEligibility.vectorEligible() + canUseRust = canUseRust && rustEligibility.eligible() && PaimonRustReaderCapabilities.hasFullNestedProjection(desc); if (canUseRust) { if (rustReaderCapabilities == null) { @@ -1167,9 +1167,6 @@ && splitHasOrcFile((DataSplit) split) if (!rustEligibility.allBackendsRustCapable) { reasons.add("backend_without_rust_support"); } - if (vectorSearchSplit && !rustEligibility.allBackendsVectorCapable) { - reasons.add("backend_without_vector_support"); - } if (vectorSearchSplit && !PaimonRustReaderCapabilities.hasFullNestedProjection(desc)) { reasons.add("pruned_nested_projection"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java index eba89e04536bc2..28b04d77922302 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java @@ -96,17 +96,15 @@ private static Column createDistanceColumn() { * evaluates the relation-option-processed table while the rewrite sees the * raw cached table, so a {@code t@options} override can still make the * scan node reject a split the pre-flight accepted — the scan node stays - * authoritative and fails loudly. The BE capability negotiation requires - * every candidate backend to report {@code supports_paimon_rust_reader} AND - * {@code supports_paimon_rust_vector_reader}: the vector protocol rides an - * empty {@code paimon_split} plus a vector payload, which an older - * rust-capable BE (rolling upgrade) rejects as "missing paimon_split", so - * the vector bit is negotiated separately from the batch rust bit. + * authoritative and fails loudly. The BE capability is the single + * {@code supports_paimon_rust_reader} bit: the rust reader and the + * PK-vector protocol ship in the same release, so every BE that reports + * the bit implements both. */ public static boolean vectorScanEligible(Table paimonTable, String tableLocation, Map backendStorageProperties, Collection backends) { return PaimonRustEligibility.of(paimonTable, tableLocation, - backendStorageProperties, backends).vectorEligible(); + backendStorageProperties, backends).eligible(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java index 885e240503ce93..adb4a57d448385 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java @@ -147,12 +147,6 @@ public class Backend implements Writable { private long beMemory = 0; @SerializedName("supportsPaimonRustReader") private volatile boolean supportsPaimonRustReader; - // The PK-vector (ANN) search protocol bit, negotiated separately from the - // batch rust reader: an older rust-capable BE reports only the batch bit and - // rejects a vector payload ("missing paimon_split"), so the one-way vector - // rewrite must require this bit specifically. - @SerializedName("supportsPaimonRustVectorReader") - private volatile boolean supportsPaimonRustVectorReader; // from config::pipeline_executor_size , default equal cpuCores @SerializedName("pipelineExecutorSize") private int pipelineExecutorSize = 1; @@ -269,10 +263,6 @@ public boolean isPaimonRustReaderSupported() { return supportsPaimonRustReader; } - public boolean isPaimonRustVectorReaderSupported() { - return supportsPaimonRustVectorReader; - } - public int getBePort() { return bePort; } @@ -892,10 +882,6 @@ public boolean handleHbResponse(BackendHbResponse hbResponse, boolean isReplay) isChanged = true; supportsPaimonRustReader = hbResponse.isPaimonRustReaderSupported(); } - if (supportsPaimonRustVectorReader != hbResponse.isPaimonRustVectorReaderSupported()) { - isChanged = true; - supportsPaimonRustVectorReader = hbResponse.isPaimonRustVectorReaderSupported(); - } if (!this.version.equals(hbResponse.getVersion())) { isChanged = true; this.version = hbResponse.getVersion(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java index 42b19395c3efe6..55fb0c9f456a74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java @@ -53,10 +53,6 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable { private long beMemory = 0; @SerializedName("supportsPaimonRustReader") private boolean supportsPaimonRustReader; - // The PK-vector (ANN) search protocol bit, negotiated separately: an older - // rust-capable BE reports the batch bit but would reject a vector payload. - @SerializedName("supportsPaimonRustVectorReader") - private boolean supportsPaimonRustVectorReader; public BackendHbResponse() { super(HeartbeatResponse.Type.BACKEND); @@ -166,14 +162,6 @@ public void setPaimonRustReaderSupported(boolean supported) { this.supportsPaimonRustReader = supported; } - public boolean isPaimonRustVectorReaderSupported() { - return supportsPaimonRustVectorReader; - } - - public void setPaimonRustVectorReaderSupported(boolean supported) { - this.supportsPaimonRustVectorReader = supported; - } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index 922b22eae01828..71e053995c1f07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -379,9 +379,6 @@ private HeartbeatResponse pingOnce() { fragmentNum, lastFragmentUpdateTime, isShutDown, arrowFlightSqlPort, beMemory); response.setPaimonRustReaderSupported(tBackendInfo.isSetSupportsPaimonRustReader() && tBackendInfo.isSupportsPaimonRustReader()); - response.setPaimonRustVectorReaderSupported( - tBackendInfo.isSetSupportsPaimonRustVectorReader() - && tBackendInfo.isSupportsPaimonRustVectorReader()); return response; } else { return new BackendHbResponse(backendId, backend.getHost(), backend.getLastUpdateMs(), diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index ea210bb4871d4e..9f9ab957e56877 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -148,10 +148,10 @@ public void saveVariantV2Config() { // read false and bypass the cache under test. sv.enableExternalScanTaskReuse = true; // These routing fixtures bypass scan initialization and use an upgraded eligible - // BE. The vector bit rides along: the vector gates negotiate it independently - // of the batch one, and the vector-split tests below need both. + // BE. The vector protocol rides the same rust bit — the rust reader and the + // vector protocol ship in the same release. Backend backend = GsonUtils.GSON.fromJson( - "{\"supportsPaimonRustReader\":true,\"supportsPaimonRustVectorReader\":true}", + "{\"supportsPaimonRustReader\":true}", Backend.class); new MockUp() { @mockit.Mock diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java index b01b2918ee69c1..e1f50b44913a71 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java @@ -126,11 +126,10 @@ public void setUp() { cascadesContext = Mockito.mock(CascadesContext.class); // The rewrite's pre-flight (PaimonVectorSearch.vectorScanEligible) requires a // rust-capable BE via the all-backend capability negotiation, mirroring - // PaimonScanNodeTest's routing fixture — and the VECTOR capability bit on - // top of the batch one, since an older rust-capable BE would reject the - // vector payload ("missing paimon_split"). + // PaimonScanNodeTest's routing fixture — the vector protocol rides the + // same bit, since the rust reader and the vector protocol ship together. Backend rustBackend = GsonUtils.GSON.fromJson( - "{\"supportsPaimonRustReader\":true,\"supportsPaimonRustVectorReader\":true}", + "{\"supportsPaimonRustReader\":true}", Backend.class); systemInfo = Mockito.mock(SystemInfoService.class); Mockito.when(systemInfo.getAllClusterBackends(true)) @@ -826,24 +825,6 @@ public void testWholeComplexProjectionStillPushedDown() { .getAnnTopN().isPresent()); } - @Test - public void testBackendWithoutVectorCapabilityNotPushedDown() { - // Rolling upgrade: a BE that reports the batch rust bit but not the vector - // bit would reject the rewritten payload ("missing paimon_split"), so the - // rewrite must decline and leave the query on the ordinary path it can - // still run. Re-stub the shared SystemInfoService with such a BE. - Backend legacyBackend = GsonUtils.GSON.fromJson( - "{\"supportsPaimonRustReader\":true}", Backend.class); - Mockito.when(systemInfo.getAllClusterBackends(true)) - .thenReturn(ImmutableList.of(legacyBackend)); - LogicalFileScan scan = paimonScan(ImmutableMap.of( - "pk-vector.index.columns", "vec", - "fields.vec.pk-vector.distance.metric", "l2")); - Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); - assertNotPushedDown(topN( - new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); - } - @Test public void testSnapshotScanStillPushedDown() { // t@options('scan.snapshot-id'='1'): snapshot selectors are resolved into diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java index ae50e66f89b828..9b085b3c7e4a99 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java @@ -63,43 +63,6 @@ public void testLegacyHeartbeatClearsPreviouslyAdvertisedSupport() { Assert.assertFalse(GsonUtils.GSON.fromJson("{}", Backend.class).isPaimonRustReaderSupported()); } - @Test - public void testVectorCapabilityNegotiatedIndependentlyOfBatchSupport() { - // Rolling upgrade: a BE advertising only the batch rust bit keeps batch - // support and loses the vector protocol — the one-way vector rewrite must - // be gated on this bit specifically (an older rust-capable BE would reject - // the rewritten payload as "missing paimon_split"). - Backend backend = new Backend(1, "127.0.0.1", 9050); - BackendHbResponse batchOnly = heartbeat(); - batchOnly.setPaimonRustReaderSupported(true); - Assert.assertFalse(batchOnly.isPaimonRustVectorReaderSupported()); - backend.handleHbResponse(batchOnly, false); - Assert.assertTrue(backend.isPaimonRustReaderSupported()); - Assert.assertFalse(backend.isPaimonRustVectorReaderSupported()); - - BackendHbResponse bothBits = heartbeat(); - bothBits.setPaimonRustReaderSupported(true); - bothBits.setPaimonRustVectorReaderSupported(true); - BackendHbResponse replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(bothBits), BackendHbResponse.class); - Assert.assertTrue(replay.isPaimonRustVectorReaderSupported()); - backend.handleHbResponse(replay, false); - Assert.assertTrue(backend.isPaimonRustReaderSupported()); - Assert.assertTrue(backend.isPaimonRustVectorReaderSupported()); - Backend restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(backend), Backend.class); - Assert.assertTrue(restored.isPaimonRustVectorReaderSupported()); - - // A later heartbeat without the vector bit (downgrade) clears it again. - BackendHbResponse downgrade = heartbeat(); - downgrade.setPaimonRustReaderSupported(true); - backend.handleHbResponse(downgrade, false); - Assert.assertTrue(backend.isPaimonRustReaderSupported()); - Assert.assertFalse(backend.isPaimonRustVectorReaderSupported()); - - TBackendInfo thrift = new TBackendInfo(); - Assert.assertFalse(thrift.isSetSupportsPaimonRustVectorReader()); - Assert.assertFalse(GsonUtils.GSON.fromJson("{}", Backend.class).isPaimonRustVectorReaderSupported()); - } - private BackendHbResponse heartbeat() { return new BackendHbResponse(1, 9060, 8040, 8060, 1000, 1000, "test-version", Tag.VALUE_MIX, 0, 0, false, 8070); diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift index b47f84cb3bb0b5..8608853b29f81a 100644 --- a/gensrc/thrift/HeartbeatService.thrift +++ b/gensrc/thrift/HeartbeatService.thrift @@ -64,12 +64,6 @@ struct TBackendInfo { 9: optional Types.TPort arrow_flight_sql_port 10: optional i64 be_mem // The physical memory available for use by BE. 11: optional bool supports_paimon_rust_reader - // The paimon-rust PK-vector (ANN) search protocol: paimon_split is empty and the - // split rides TPaimonFileDesc.vector_payload, which an older rust-capable BE - // rejects as "missing paimon_split". Set only by BEs that implement the vector - // protocol, so a mixed-version cluster keeps ordinary rust reads working and - // only declines the vector rewrite against older BEs. Field 12 is free on master. - 12: optional bool supports_paimon_rust_vector_reader // For cloud 1000: optional i64 fragment_executing_count 1001: optional i64 fragment_last_active_time From 8a4495b4732b659ce93d069a19a9482838e82cc9 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Fri, 23 Oct 2026 18:06:41 +0800 Subject: [PATCH 10/13] [fix](paimon) close four remaining vector pre-flight gaps Per the static review round on 8fd3bec: - Nanosecond pre-flight: a TIMESTAMP_NS column in the CURRENT schema (no evolution history) passed the pre-flight but tripped the scan node's nanosecond-reader gate, which a no-fallback vector split turns into a query error. The pre-flight now mirrors PaimonUtil.containsTimestampNs over the slots the Project and Filter reference. - Reserved materializer names: a user column named __paimon_search_score or _PKEY_VECTOR_POSITION makes the rust table open fail after the one-way rewrite; both names join the synthetic-distance collision check. - Distance exemption narrowed: only the call the rewrite actually replaces (the one equal to the order-key expression) may exempt its slots; a distinct distance call keeps its complex-slot whole-column guard instead of shipping a projection rust cannot materialize. - ORC era detection: the orc-ltz pre-flight checked only the CURRENT file format, so a table written ORC with TIMESTAMP_LTZ and later switched to Parquet passed while setPaimonParams still detected the retained ORC files. Any schema era that declared ORC and carried LTZ columns (its own or the current schema's) now declines. Tests: four new rewrite-rule cases (45 total, all green); the ANN e2e suite stays green against the tighter pre-flight. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../PushDownVectorTopNIntoPaimonScan.java | 79 +++++++++++++++++-- .../PushDownVectorTopNIntoPaimonScanTest.java | 78 +++++++++++++++++- 2 files changed, 148 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java index c76c152f814ba6..85e388c34f3932 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java @@ -59,6 +59,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.apache.paimon.CoreOptions; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.types.ArrayType; @@ -340,12 +341,15 @@ private Plan pushDown( && filterHasDataConjunct(optionalFilter.get(), table) && !PaimonScanNode.vectorSearchSupportsDataPredicate(table)) || table.rowType().getFieldNames().stream().anyMatch( - name -> name.equalsIgnoreCase(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)) - || !complexColumnsRideAsWholeColumns(project, orderKeyAlias) + name -> name.equalsIgnoreCase(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN) + || name.equalsIgnoreCase("__paimon_search_score") + || name.equalsIgnoreCase("_PKEY_VECTOR_POSITION")) + || !complexColumnsRideAsWholeColumns(project, orderKeyAlias, orderKeyExpr) || projectContainsNonMovableExpression(project, orderKeyAlias) || (table instanceof FileStoreTable && PaimonVectorSearch.hasRustIncompatibleSchemaHistory( - (FileStoreTable) table))) { + (FileStoreTable) table)) + || projectionNeedsNanosecondReader(project, optionalFilter)) { return null; } } catch (Throwable t) { @@ -526,6 +530,32 @@ private static boolean ridesAsLiteral(Expression expr) { return expr instanceof Literal && !(expr instanceof NullLiteral); } + /** + * Mirror of the scan node's nanosecond-reader gate: a slot whose type + * recursively contains a TIMESTAMP_NS column stays on JNI (the reader + * family that preserves INT96/ORC nanos, nested values included), and a + * rewritten vector split has no JNI fallback — so a projection or filter + * referencing one must decline the rewrite. The scan node checks the desc + * slots; here the same slots are the ones the Project and Filter reference. + * The schema-history gate does not subsume this: a table can carry + * TIMESTAMP(9) columns in its CURRENT schema with no evolution history. + */ + private static boolean projectionNeedsNanosecondReader( + LogicalProject project, Optional> optionalFilter) { + Set slots = new HashSet<>(); + project.getProjects().forEach( + p -> slots.addAll(p.collect(SlotReference.class::isInstance))); + optionalFilter.ifPresent(filter -> filter.getConjuncts().forEach( + conjunct -> slots.addAll(conjunct.collect(SlotReference.class::isInstance)))); + for (SlotReference slot : slots) { + if (slot.getOriginalColumn().isPresent() + && PaimonUtil.containsTimestampNs(slot.getOriginalColumn().get().getType())) { + return true; + } + } + return false; + } + /** * Whether the filter carries a conjunct on a non-partition column — the DATA * half of the predicate, the half the paimon-rust vector search applies per @@ -610,7 +640,7 @@ private static boolean projectContainsNonMovableExpression( * whitelist admits only scalar columns. */ private static boolean complexColumnsRideAsWholeColumns(LogicalProject project, - Alias orderKeyAlias) { + Alias orderKeyAlias, Expression orderKeyExpr) { for (NamedExpression projection : project.getProjects()) { if (!(projection instanceof Alias)) { // A bare passthrough slot output: whole unless already pruned. @@ -633,8 +663,14 @@ private static boolean complexColumnsRideAsWholeColumns(LogicalProject projec Set replacedSlots = new HashSet<>(); Set distanceCalls = alias.child().collect(e -> e instanceof L2DistanceApproximate || e instanceof InnerProductApproximate); + // Only the call the rewrite actually replaces (the one equal to the + // order-key expression) may exempt its slots: a distinct distance call + // over a complex column stays in the projection after the rewrite, so + // its complex slots must keep the whole-column guard. for (Expression call : distanceCalls) { - replacedSlots.addAll(call.collect(SlotReference.class::isInstance)); + if (call.equals(orderKeyExpr)) { + replacedSlots.addAll(call.collect(SlotReference.class::isInstance)); + } } Set slots = alias.child().collect(SlotReference.class::isInstance); for (SlotReference slot : slots) { @@ -705,6 +741,12 @@ private static boolean projectedColumnsContainVariant( * TIMESTAMP_LTZ decoding is not equivalent to Doris semantics yet, and a vector * split would materialize rows through that decoder with no fallback. Snapshot * selection is irrelevant here -- this is a physical-file property. + * + *

The check is per PHYSICAL FILE, not per current format: a table written + * as ORC with TIMESTAMP_LTZ and later switched to Parquet (retaining the ORC + * rows) still exposes those rows to setPaimonParams' ORC detection, so a + * currently-Parquet table declines when any schema era declared ORC and any + * schema era (its own fields, or the current ones) carried LTZ columns. */ private static boolean orcSchemaContainsTimestampLtz(Table boundTable) { try { @@ -713,13 +755,34 @@ private static boolean orcSchemaContainsTimestampLtz(Table boundTable) { return false; } FileStoreTable fileStoreTable = (FileStoreTable) baseTable; - // Paimon's default file format is ORC; only a PARQUET table is known to - // be free of the orc-ltz shape (an unset key falls to the default). + // Paimon's default file format is ORC; only a PARQUET declaration is + // known to be free of the orc shape (an unset key falls to the default). String fileFormat = CoreOptions.fromMap(fileStoreTable.options()).fileFormatString(); - if (!CoreOptions.FILE_FORMAT_PARQUET.equalsIgnoreCase(String.valueOf(fileFormat))) { + boolean currentOrc = !CoreOptions.FILE_FORMAT_PARQUET.equalsIgnoreCase( + String.valueOf(fileFormat)); + if (currentOrc) { return fileStoreTable.schema().fields().stream() .anyMatch(field -> containsTimestampLtz(field.type())); } + // Currently Parquet: files are shaped by the schema era that wrote + // them, so an ORC era's retained files still reach the reader. Any + // historical schema that declared ORC AND carried LTZ columns (in its + // own fields or in the current schema, which those rows must also + // decode under) forces the decline. + for (TableSchema historical : fileStoreTable.schemaManager().listAll()) { + if (historical == null || historical.id() == fileStoreTable.schema().id()) { + continue; + } + String eraFormat = CoreOptions.fromMap(historical.options()).fileFormatString(); + if (CoreOptions.FILE_FORMAT_PARQUET.equalsIgnoreCase(String.valueOf(eraFormat))) { + continue; + } + if (historical.fields().stream().anyMatch(field -> containsTimestampLtz(field.type())) + || fileStoreTable.schema().fields().stream() + .anyMatch(field -> containsTimestampLtz(field.type()))) { + return true; + } + } return false; } catch (Throwable t) { // The table might genuinely be unreadable; the ordinary path will surface diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java index e1f50b44913a71..7769b1d527e560 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java @@ -21,6 +21,8 @@ import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; import org.apache.doris.datasource.CatalogProperty; @@ -78,6 +80,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.types.DataField; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.TimestampType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -1022,6 +1025,76 @@ public void testInPredicateBeyondConverterCapNotPushedDown() { } } + @Test + public void testReservedMaterializerNamesNotPushedDown() { + // __paimon_search_score and _PKEY_VECTOR_POSITION are reserved by the rust + // materializer: a user column named either makes the rust table open fail + // after the one-way rewrite, so the pre-flight declines for both. + for (String reserved : ImmutableList.of("__paimon_search_score", "_PKEY_VECTOR_POSITION")) { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.empty(), + ImmutableList.of("id", "vec", reserved)); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + } + + @Test + public void testNanosecondColumnNotPushedDown() { + // A TIMESTAMP_NS column in the projection keeps the reader family that + // preserves nanoseconds (JNI); a vector split has no fallback, so the + // rewrite declines. The schema-history gate does not subsume this — + // the table carries the column in its CURRENT schema, no evolution. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference tsns = (SlotReference) scan.getOutput().get(7); + Assertions.assertEquals("tsns", tsns.getName()); + Assertions.assertTrue(tsns.getOriginalColumn().get().getType().isTimeStampNs()); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN(new LogicalProject<>( + ImmutableList.of(idSlot(scan), new Alias(tsns, "t"), dis), scan), dis, true)); + } + + @Test + public void testDistinctDistanceCallNotExempted() { + // extra := inner_product(vec2, q): only the order-key call is replaced by + // the synthetic slot; a DISTINCT distance call stays in the projection and + // keeps its complex (ARRAY) slot, so the whole-column guard applies + // to it — the rewrite declines instead of shipping a vector split whose + // projection rust cannot materialize. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec2 = (SlotReference) scan.getOutput().get(2); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + Alias extra = new Alias(new InnerProductApproximate(vec2, queryVector()), "extra"); + assertNotPushedDown(topN(new LogicalProject<>( + ImmutableList.of(idSlot(scan), extra, dis), scan), dis, true)); + } + + @Test + public void testOrcEraRetainedFilesNotPushedDown() { + // The table is Parquet NOW, but an ORC schema era with TIMESTAMP_LTZ left + // retained files: setPaimonParams still detects ORC files, so the rewrite + // must decline even though the current format is Parquet. + TableSchema orcEra = new TableSchema(4, + Collections.singletonList(new DataField(0, "t", new LocalZonedTimestampType())), + 0, Collections.emptyList(), Collections.emptyList(), + ImmutableMap.of("file.format", "orc"), null); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "file.format", "parquet"), Optional.empty(), null, false, + ImmutableList.of(orcEra), null); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + // ------------------------------------------------------------------ helpers private Plan applyRule(int ruleIndex, LogicalTopN topN) { @@ -1100,7 +1173,10 @@ private LogicalFileScan paimonScan(Map options, // ElementAt (the former struct_element): only the isComplexType() // verdict matters to the pre-flight, so a Doris MAP stands in for // a STRUCT this fixture cannot express through catalog types. - new Column("pmap", new org.apache.doris.catalog.MapType(), true)); + new Column("pmap", new org.apache.doris.catalog.MapType(), true), + // A nanosecond TIMESTAMP column: the scan node's nanosecond-reader + // gate (JNI-only family) must be pre-flighted by the rewrite too. + new Column("tsns", ScalarType.createType(PrimitiveType.TIMESTAMP_NS), true)); if (paimonFieldNames == null) { paimonFieldNames = schema.stream().map(Column::getName).collect(Collectors.toList()); } From 199ad37bee174d4faee9cfac222f8c640b21c2c0 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Fri, 23 Oct 2026 18:07:43 +0800 Subject: [PATCH 11/13] [fix](regression) keep the nested-binary token decode at value boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 0x-token substitution ran as a substring replacement on every normalized string, so a literal-text fragment shaped like a token could collapse distinct values across engines (Spark "prefixA" vs Doris "prefix0x41" comparing equal after the decode), and quoted strings and map keys were equally exposed. The token pattern now requires a value/element boundary (negative lookbehind on alphanumerics, quotes and backslash), so only shapes Doris can only have produced as a binary element ("[0x616C, ", "{0x41: ", "ROW(0x61") decode; the quantifier also stops matching a bare "0x" with zero byte pairs. A literal string whose whole value coincides with a bare token remains the one documented ambiguity — distinguishing it needs column types the comparison does not have. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../doris/regression/util/ResultUtils.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy index 72b78a36c5d716..2b038b839012b7 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy @@ -47,9 +47,21 @@ class ResultUtils { private static final Pattern DATETIME_PATTERN = Pattern.compile( "(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2})(?::(\\d{2})(?:\\.(\\d{1,9}))?)?") private static final Pattern UNICODE_ESCAPE_PATTERN = Pattern.compile("[0-9a-fA-F]{4}") - /** A "0x" token as Doris renders a VARBINARY nested in arrays/maps/structs. */ + /** + * A "0x" token as Doris renders a VARBINARY nested in arrays/maps/structs. + * The negative lookbehind keeps the match at a VALUE/ELEMENT boundary: a + * token preceded by an alphanumeric ("prefix0x41"), a quote (a nested + * STRING the engine renders quoted) or a backslash is a literal-string + * fragment, not a binary rendering — substituting it could make unequal + * rows compare equal (Spark "prefixA" vs Doris "prefix0x41"). Only a token + * that Doris could only have produced as a binary element ("[0x616C, " + * / "{0x41: 0x42}" / "ROW(0x61)") is decoded. A literal string whose whole + * value coincides with a bare token ("0x41") is indistinguishable from a + * binary rendering without column types and remains the one known + * ambiguity. + */ private static final Pattern NESTED_BINARY_TOKEN = Pattern.compile( - "(?i)0x(?:[0-9a-f]{2})*(?![0-9a-fA-F])") + "(?i)(?> normalizeRows(List> rows) { List> normalizedRows = new ArrayList<>() From 792aa85f0503e0b84482bfcf1537e9d08d17b0b3 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Fri, 23 Oct 2026 19:51:33 +0800 Subject: [PATCH 12/13] [fix](regression) quote the nested-binary token pattern correctly in Groovy The lookbehind's escaped quote terminated the double-quoted string literal mid-pattern, so the shared framework fails to compile and no suite can start. The pattern moves to a slashy string literal, where neither the backslash nor either quote character needs escaping. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../groovy/org/apache/doris/regression/util/ResultUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy index 2b038b839012b7..68cedf4694abcd 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy @@ -61,7 +61,7 @@ class ResultUtils { * ambiguity. */ private static final Pattern NESTED_BINARY_TOKEN = Pattern.compile( - "(?i)(?> normalizeRows(List> rows) { List> normalizedRows = new ArrayList<>() From af5c7cce5af8d84e85a76864a59e4550b49de785 Mon Sep 17 00:00:00 2001 From: dengfangyuan Date: Fri, 23 Oct 2026 19:51:33 +0800 Subject: [PATCH 13/13] [fix](paimon) do not let the Hadoop S3 mirror overwrite explicit options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mirrorHadoopS3OptionsToCatalogOptions loop set every fs.s3a.* entry from the generated Hadoop configuration unconditionally, so for a catalog with static s3 keys plus an explicit assumed-role credentials provider the Hadoop side's SimpleAWSCredentialsProvider replaced the requested provider in the catalog options — the warehouse then re-opened under the base identity and failed with AccessDenied. The mirror now fills only options the catalog has not already set: the user's explicit fs.s3a.* entries keep precedence over the generated defaults. This affects the filesystem, HMS and JDBC paimon catalog variants alike. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../metastore/AbstractPaimonProperties.java | 8 ++++- .../AbstractPaimonPropertiesTest.java | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index bddddf4e05e2df..07a1b8e0ab4724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -146,7 +146,13 @@ protected void mirrorHadoopS3OptionsToCatalogOptions(Configuration conf) { } conf.forEach(entry -> { String key = entry.getKey(); - if (key.startsWith(HADOOP_S3A_PREFIX) && StringUtils.isNotBlank(entry.getValue())) { + // Fill only ABSENT options: the user's explicit fs.s3a.* catalog options + // (e.g. an assumed-role credentials provider set via a paimon.fs.s3a.* + // property) take precedence over the Hadoop-generated defaults (the + // SimpleAWSCredentialsProvider the static S3 keys produce); an + // unconditional set would re-open the warehouse under the base identity. + if (key.startsWith(HADOOP_S3A_PREFIX) && StringUtils.isNotBlank(entry.getValue()) + && catalogOptions.get(key) == null) { catalogOptions.set(key, entry.getValue()); } }); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java index 24b825560358a5..417c0fad154465 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java @@ -250,6 +250,37 @@ void testForwardTableDefaultOptionsToPaimonCatalog() { "7", testProps.getCatalogOptionsMap().get("table-default.scan.snapshot-id")); } + @Test + public void testHadoopS3MirrorDoesNotOverwriteExplicitProvider() { + // Static S3 keys plus an explicit assumed-role credentials provider: the + // Hadoop configuration carries the SimpleAWSCredentialsProvider the static + // keys produce, but the user's provider entry is already in the catalog + // options — the mirror must fill only absent options, or the warehouse + // re-opens under the base identity and fails AccessDenied. + Map input = new HashMap<>(); + input.put("warehouse", "s3://tmp/warehouse"); + input.put("paimon.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + testProps.buildCatalogOptions(); + + // The Hadoop side only knows the Simple provider generated from the static + // keys, plus defaults the catalog has not set yet. Mirror BEFORE the first + // getCatalogOptionsMap() call: the map is memoized once read. + org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(false); + conf.set("fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + conf.set("fs.s3a.connection.maximum", "100"); + testProps.mirrorHadoopS3OptionsToCatalogOptions(conf); + + Map options = testProps.getCatalogOptionsMap(); + Assertions.assertEquals("org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider", + options.get("fs.s3a.aws.credentials.provider"), + "the explicit user provider must survive the Hadoop mirror"); + Assertions.assertEquals("100", options.get("fs.s3a.connection.maximum"), + "absent defaults are still filled"); + } + @Test public void testWeightGovernanceDisablesSdkMetadataCacheUnlessUserConfigured() { // Doris weight governance owns retention: Paimon's own CachingCatalog would retain