From f42db79d4c86391efd444170d6ffd756083c7dd7 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 12 Oct 2026 14:35:27 +0800 Subject: [PATCH 01/13] Fix CASE expression caches leaking across TsBlocks --- .../it/db/it/IoTDBCaseWhenThenTableIT.java | 28 ++ .../process/FilterAndProjectOperator.java | 11 + ...AbstractCaseWhenThenColumnTransformer.java | 11 + .../process/FilterAndProjectOperatorTest.java | 430 ++++++++++++++++++ 4 files changed, 480 insertions(+) create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index 50053867636bd..ba56a57371ee8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -695,4 +695,32 @@ public void testThenWithBinarySameConstant() { String[] retArray = new String[] {"0,", "11,", "22,", "33,"}; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); } + + @Test + public void testCaseWhenReusesAggregationAcrossBlocks() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_aggregation_cache (dev STRING TAG, c DOUBLE FIELD, q INT32 FIELD)", + "INSERT INTO case_aggregation_cache(time, dev, c, q) VALUES " + + "(0, 'd0', 1.0, 0), (0, 'd1', 1.0, 0), (3600000, 'd1', 1.0, 0)" + }); + + // The non-pushdown predicate and the different device/window shapes make the aggregation + // produce multiple blocks. CASE clears the shared aggregate before the next output uses it. + String suffix = " FROM case_aggregation_cache WHERE q % 2 < 1 GROUP BY dev, date_bin(1h, time)"; + for (String projection : + new String[] { + "CASE WHEN sum(c) > 0 THEN 1.0 END AS case_result, CAST(sum(c) AS DOUBLE) AS total", + "CASE WHEN sum(c) > 0 THEN 1.0 END AS case_result, sum(c) + 0 AS total", + "CASE WHEN sum(c) > 0 THEN CAST(sum(c) AS DOUBLE) / CAST(sum(c) AS DOUBLE) END " + + "AS case_result, CAST(sum(c) AS DOUBLE) AS total" + }) { + tableResultSetEqualTest( + "SELECT " + projection + suffix, + new String[] {"case_result", "total"}, + new String[] {"1.0,1.0,", "1.0,1.0,", "1.0,1.0,"}, + DATABASE); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 5ab7e58a6962f..9742f1cea15a3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -152,6 +152,13 @@ public TsBlock next() throws Exception { private TsBlock getFilterTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); + // Short-circuit evaluation can reset cache reference counts when a shared expression is + // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate + // expressions whose caches may still have unconsumed references from the previous block. + filterOutputTransformer.clearCache(); + for (ColumnTransformer columnTransformer : commonTransformerList) { + columnTransformer.clearCache(); + } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { leafColumnTransformer.initFromTsBlock(input); @@ -263,6 +270,10 @@ public static boolean satisfy(Column filterColumn, int rowIndex) { private TsBlock getTransformedTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); + // Clear all output DAGs before evaluating any output, preserving reuse within this block. + for (ColumnTransformer columnTransformer : projectOutputTransformerList) { + columnTransformer.clearCache(); + } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { leafColumnTransformer.initFromTsBlock(input); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index 5603bde031d1a..dd65cc12f5dbf 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -234,6 +234,17 @@ protected void checkType() { // do nothing } + @Override + public void clearCache() { + super.clearCache(); + for (Pair whenThenColumnTransformer : + whenThenTransformers) { + whenThenColumnTransformer.left.clearCache(); + whenThenColumnTransformer.right.clearCache(); + } + elseTransformer.clearCache(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java new file mode 100644 index 0000000000000..e15152b0a43e5 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.process; + +import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; +import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.FailFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareGreaterThanColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FilterAndProjectOperatorTest { + + @Test + public void testProjectionDoesNotReuseColumnsAcrossDifferentSizedBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {null, 1.0}, new Double[] {-2.0, 3.0}); + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {4.0}); + } + } + + @Test + public void testProjectionDoesNotReuseColumnsAcrossEqualSizedBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1, 2), block(-3, 4)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}, new Double[] {1.0, 2.0}); + assertBlock(operator.next(), new Double[] {null, 1.0}, new Double[] {-3.0, 4.0}); + } + } + + @Test + public void testProjectionHandlesEmptyBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(), block(-2)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[0], new Double[0]); + assertBlock(operator.next(), new Double[] {null}, new Double[] {-2.0}); + } + } + + @Test + public void testProjectionClearsSharedIntermediateExpressions() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(shared, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2)}, caseWhen, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertBlock(operator.next(), new Double[] {null}, new Double[] {-1.0}); + } + } + + @Test + public void testFilterDoesNotReuseColumnsAcrossBlocks() throws Exception { + Graph filter = new Graph(); + ColumnTransformer input = filter.input(0); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(input, 0), filter.constant(1), filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(filter.add(caseWhen, input), 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1), block(-2, 3), block(-4)}, + filter, + predicate, + Collections.emptyList(), + projection, + projection.input(0))) { + assertBlock(operator.next(), new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {3.0}); + assertBlock(operator.next(), new Double[0]); + } + } + + @Test + public void testFilterCommonExpressionsDoNotCrossBlocks() throws Exception { + for (boolean runLengthEncodedTime : new boolean[] {false, true}) { + Graph filter = new Graph(); + ColumnTransformer shared = filter.add(filter.input(0), 0); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(shared, 0), filter.constant(1), filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(caseWhen, 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] { + block(runLengthEncodedTime, 1), + block(runLengthEncodedTime, -2, 3), + block(runLengthEncodedTime, 4, -5) + }, + filter, + predicate, + Collections.singletonList(shared), + projection, + projection.input(0), + projection.input(1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {3.0}, new Double[] {3.0}); + assertBlock(operator.next(), new Double[] {4.0}, new Double[] {4.0}); + } + } + } + + @Test + public void testProjectionClearsCachedDescendantsOfNestedCase() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 0); + ColumnTransformer innerCase = + graph.caseWhen(graph.greaterThan(shared, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer outerCase = + graph.caseWhen(graph.greaterThan(innerCase, 0), graph.constant(2), graph.constant(0)); + // Simulate an intermediate cache retained from the previous input block. Clearing only the + // outer CASE itself must not allow this cached value to reach the new block's WHEN condition. + shared.initializeColumnCache(doubleColumn(1)); + try (Operator operator = project(graph, new TsBlock[] {block(-2)}, outerCase)) { + assertBlock(operator.next(), new Double[] {0.0}); + } + } + + @Test + public void testThenExpressionIsRecomputedWithoutSelectionForLaterOutput() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer shared = graph.add(input, 1); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), shared, graph.constant(0)); + try (Operator operator = + project(graph, new TsBlock[] {block(-2, 3), block(-4, 5)}, caseWhen, shared)) { + assertBlock(operator.next(), new Double[] {0.0, 4.0}, new Double[] {-1.0, 4.0}); + assertBlock(operator.next(), new Double[] {0.0, 6.0}, new Double[] {-3.0, 6.0}); + } + } + + @Test + public void testCaseStillSkipsUnusedFailBranch() throws Exception { + Graph graph = new Graph(); + ColumnTransformer caseWhen = + graph.caseWhen( + graph.greaterThan(graph.input(0), 0), + graph.constant(1), + new FailFunctionColumnTransformer(DOUBLE, "The unused ELSE branch was evaluated")); + try (Operator operator = project(graph, new TsBlock[] {block(1, 2), block(3)}, caseWhen)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}); + assertBlock(operator.next(), new Double[] {1.0}); + } + } + + @Test + public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + ColumnTransformer shared = graph.add(input, 0); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertEquals(1, input.evaluationCount); + assertBlock(operator.next(), new Double[] {2.0}, new Double[] {3.0}); + assertEquals(2, input.evaluationCount); + } + } + + private static FilterAndProjectOperator project( + Graph graph, TsBlock[] blocks, ColumnTransformer... outputs) { + return operator(blocks, null, null, Collections.emptyList(), graph, outputs); + } + + private static FilterAndProjectOperator operator( + TsBlock[] blocks, + Graph filter, + ColumnTransformer predicate, + List common, + Graph projection, + ColumnTransformer... outputs) { + List projectOutputs = new ArrayList<>(); + for (ColumnTransformer output : outputs) { + projectOutputs.add(reference(output)); + } + for (ColumnTransformer transformer : common) { + reference(transformer); + } + return new FilterAndProjectOperator( + new TestOperatorContext(), + new TsBlockSourceOperator(blocks), + Collections.nCopies(1 + common.size(), TSDataType.DOUBLE), + filter == null ? Collections.emptyList() : filter.leaves, + predicate == null ? null : reference(predicate), + common, + projection.leaves, + projectOutputs, + false, + predicate != null); + } + + private static ColumnTransformer reference(ColumnTransformer transformer) { + transformer.addReferenceCount(); + return transformer; + } + + private static DoubleColumn doubleColumn(double... values) { + return new DoubleColumn(values.length, Optional.empty(), values); + } + + private static TsBlock block(double... values) { + return block(false, values); + } + + private static TsBlock block(boolean runLengthEncodedTime, double... values) { + Column timeColumn = + runLengthEncodedTime + ? new RunLengthEncodedColumn(new TimeColumn(1, new long[] {0}), values.length) + : new TimeColumn(values.length, new long[values.length]); + return new TsBlock(timeColumn, doubleColumn(values)); + } + + private static void assertBlock(TsBlock actual, Double[]... expectedColumns) { + assertEquals(expectedColumns.length, actual.getValueColumnCount()); + assertEquals(expectedColumns[0].length, actual.getPositionCount()); + for (int columnIndex = 0; columnIndex < expectedColumns.length; columnIndex++) { + Column column = actual.getColumn(columnIndex); + Double[] expected = expectedColumns[columnIndex]; + assertEquals(expected.length, column.getPositionCount()); + for (int row = 0; row < expected.length; row++) { + if (expected[row] == null) { + assertTrue(column.isNull(row)); + } else { + assertFalse(column.isNull(row)); + assertEquals(expected[row], column.getDouble(row), 0.0); + } + } + } + } + + private static class Graph { + private final List leaves = new ArrayList<>(); + + private ColumnTransformer input(int index) { + return leaf(new IdentityColumnTransformer(DOUBLE, index)); + } + + private ColumnTransformer constant(double value) { + return leaf(new ConstantColumnTransformer(DOUBLE, doubleColumn(value))); + } + + private ColumnTransformer nullValue() { + return leaf(new NullColumnTransformer()); + } + + private ColumnTransformer leaf(LeafColumnTransformer transformer) { + leaves.add(transformer); + return transformer; + } + + private ColumnTransformer add(ColumnTransformer input, double value) { + return add(input, constant(value)); + } + + private ColumnTransformer add(ColumnTransformer left, ColumnTransformer right) { + return new ArithmeticAdditionColumnTransformer(DOUBLE, reference(left), reference(right)); + } + + private ColumnTransformer greaterThan(ColumnTransformer input, double value) { + return new CompareGreaterThanColumnTransformer( + BOOLEAN, reference(input), reference(constant(value))); + } + + private ColumnTransformer caseWhen( + ColumnTransformer when, ColumnTransformer then, ColumnTransformer otherwise) { + return new TableCaseWhenThenColumnTransformer( + DOUBLE, + Collections.singletonList(reference(when)), + Collections.singletonList(reference(then)), + reference(otherwise)); + } + } + + private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { + private int evaluationCount; + + private CountingIdentityColumnTransformer() { + super(DOUBLE, 0); + } + + @Override + protected void evaluate() { + evaluationCount++; + super.evaluate(); + } + } + + private static class TestOperatorContext extends CommonOperatorContext { + private TestOperatorContext() { + super(0, new PlanNodeId("test"), "test"); + } + + @Override + public MemoryReservationManager getMemoryReservationContext() { + return null; + } + + @Override + public int getFragmentId() { + return 0; + } + + @Override + public int getPipelineId() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } + + private static class TsBlockSourceOperator implements Operator { + private final Iterator blocks; + private final CommonOperatorContext context = new TestOperatorContext(); + + private TsBlockSourceOperator(TsBlock[] blocks) { + this.blocks = Arrays.asList(blocks).iterator(); + } + + @Override + public CommonOperatorContext getOperatorContext() { + return context; + } + + @Override + public TsBlock next() { + return blocks.next(); + } + + @Override + public boolean hasNext() { + return blocks.hasNext(); + } + + @Override + public boolean isFinished() { + return !hasNext(); + } + + @Override + public void close() { + // No resources to release. + } + + @Override + public long calculateMaxPeekMemory() { + return 0; + } + + @Override + public long calculateMaxReturnSize() { + return 0; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } +} From b863e7b6c20664e086eb574c2f995b8b80c2a102 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 12 Oct 2026 18:27:42 +0800 Subject: [PATCH 02/13] Separate block-boundary cache clearing from CASE cleanup --- .../it/db/it/IoTDBCaseWhenThenTableIT.java | 22 +++++++++++++++++++ .../process/FilterAndProjectOperator.java | 6 ++--- ...AbstractCaseWhenThenColumnTransformer.java | 10 ++++----- .../dag/column/ColumnTransformer.java | 9 ++++++++ .../binary/BinaryColumnTransformer.java | 7 ++++++ .../multi/MappableUDFColumnTransformer.java | 8 +++++++ .../column/multi/MultiColumnTransformer.java | 8 +++++++ .../ternary/TernaryColumnTransformer.java | 8 +++++++ .../column/unary/UnaryColumnTransformer.java | 6 +++++ .../process/FilterAndProjectOperatorTest.java | 21 ++++++++++++++++++ 10 files changed, 97 insertions(+), 8 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index ba56a57371ee8..169ebd3084ae3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -723,4 +723,26 @@ public void testCaseWhenReusesAggregationAcrossBlocks() { DATABASE); } } + + @Test + public void testReusingCasePreservesPendingSiblingColumn() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_sibling_cache (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_sibling_cache(time, dev, x) VALUES " + + "(0, 'd0', -3.0), (1, 'd0', 0.0), (2, 'd0', 2.0)" + }); + + String sharedCase = "CASE WHEN x > 0 THEN 1.0 ELSE 0.0 END"; + tableResultSetEqualTest( + "SELECT " + + sharedCase + + " AS a, x + CASE WHEN (" + + sharedCase + + ") > 0 THEN 2.0 ELSE 3.0 END AS b FROM case_sibling_cache ORDER BY time", + new String[] {"a", "b"}, + new String[] {"0.0,0.0,", "0.0,3.0,", "1.0,4.0,"}, + DATABASE); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 9742f1cea15a3..29b51ac5cb663 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -155,9 +155,9 @@ private TsBlock getFilterTsBlock(TsBlock input) { // Short-circuit evaluation can reset cache reference counts when a shared expression is // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate // expressions whose caches may still have unconsumed references from the previous block. - filterOutputTransformer.clearCache(); + filterOutputTransformer.clearCacheForNewInput(); for (ColumnTransformer columnTransformer : commonTransformerList) { - columnTransformer.clearCache(); + columnTransformer.clearCacheForNewInput(); } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { @@ -272,7 +272,7 @@ private TsBlock getTransformedTsBlock(TsBlock input) { final int positionCount = originTimeColumn.getPositionCount(); // Clear all output DAGs before evaluating any output, preserving reuse within this block. for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.clearCache(); + columnTransformer.clearCacheForNewInput(); } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index dd65cc12f5dbf..bcf4bdf7722cb 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -235,14 +235,14 @@ protected void checkType() { } @Override - public void clearCache() { - super.clearCache(); + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); for (Pair whenThenColumnTransformer : whenThenTransformers) { - whenThenColumnTransformer.left.clearCache(); - whenThenColumnTransformer.right.clearCache(); + whenThenColumnTransformer.left.clearCacheForNewInput(); + whenThenColumnTransformer.right.clearCacheForNewInput(); } - elseTransformer.clearCache(); + elseTransformer.clearCacheForNewInput(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index d10d728cd2db5..11407afbdd227 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -115,4 +115,13 @@ public void close() { public void clearCache() { this.columnCache.clear(); } + + /** + * Clears cached columns throughout this DAG before initializing a new input block. Unlike {@link + * #clearCache()}, this also visits CASE descendants. Only call this before evaluating any + * expressions for the new block, when no sibling expression can still need a cached column. + */ + public void clearCacheForNewInput() { + this.columnCache.clear(); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index 5403e9f995f13..d494f652c8079 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -96,6 +96,13 @@ public void clearCache() { this.rightTransformer.clearCache(); } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + leftTransformer.clearCacheForNewInput(); + rightTransformer.clearCacheForNewInput(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index f619f5692119c..75ed45ac64278 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -100,4 +100,12 @@ public void clearCache() { columnTransformer.clearCache(); } } + + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + for (ColumnTransformer columnTransformer : inputColumnTransformers) { + columnTransformer.clearCacheForNewInput(); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index 09cb1530bc475..0cfecdd87e5f3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -97,6 +97,14 @@ public void clearCache() { } } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + for (ColumnTransformer columnTransformer : columnTransformerList) { + columnTransformer.clearCacheForNewInput(); + } + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index 51648f797e3f7..f68cc612da875 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -111,6 +111,14 @@ public void clearCache() { thirdColumnTransformer.clearCache(); } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + firstColumnTransformer.clearCacheForNewInput(); + secondColumnTransformer.clearCacheForNewInput(); + thirdColumnTransformer.clearCacheForNewInput(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 2a88aea340a95..1a8943357cd15 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -73,6 +73,12 @@ public void clearCache() { childColumnTransformer.clearCache(); } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + childColumnTransformer.clearCacheForNewInput(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index e15152b0a43e5..92d4931ea5674 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -220,6 +220,27 @@ public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Except } } + @Test + public void testReusingCasePreservesPendingSiblingColumn() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer sharedCase = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer nestedCase = + graph.caseWhen(graph.greaterThan(sharedCase, 0), graph.constant(2), graph.constant(3)); + // The first output leaves sharedCase cached. In the second output, input is evaluated before + // nestedCase reuses sharedCase. Its cleanup must preserve input until the addition reads it. + try (Operator operator = + project( + graph, + new TsBlock[] {block(-3, 0, 2), block(4, -1)}, + sharedCase, + graph.add(input, nestedCase))) { + assertBlock(operator.next(), new Double[] {0.0, 0.0, 1.0}, new Double[] {0.0, 3.0, 4.0}); + assertBlock(operator.next(), new Double[] {1.0, 0.0}, new Double[] {6.0, 2.0}); + } + } + private static FilterAndProjectOperator project( Graph graph, TsBlock[] blocks, ColumnTransformer... outputs) { return operator(blocks, null, null, Collections.emptyList(), graph, outputs); From dc2ff7dbcfca907944ccf71a1d85bc41aa11fd70 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 10:19:13 +0800 Subject: [PATCH 03/13] Deduplicate transformer cache clearing across shared DAGs --- .../process/FilterAndProjectOperator.java | 13 +++-- ...AbstractCaseWhenThenColumnTransformer.java | 10 ++-- .../dag/column/ColumnTransformer.java | 16 +++++- .../binary/BinaryColumnTransformer.java | 9 ++-- .../multi/MappableUDFColumnTransformer.java | 7 +-- .../column/multi/MultiColumnTransformer.java | 6 +-- .../ternary/TernaryColumnTransformer.java | 11 ++-- .../column/unary/UnaryColumnTransformer.java | 7 +-- .../process/FilterAndProjectOperatorTest.java | 51 +++++++++++++++++++ 9 files changed, 101 insertions(+), 29 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 29b51ac5cb663..d764114496767 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -47,8 +47,11 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.OptionalInt; +import java.util.Set; public class FilterAndProjectOperator implements ProcessOperator { @@ -155,9 +158,10 @@ private TsBlock getFilterTsBlock(TsBlock input) { // Short-circuit evaluation can reset cache reference counts when a shared expression is // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate // expressions whose caches may still have unconsumed references from the previous block. - filterOutputTransformer.clearCacheForNewInput(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + filterOutputTransformer.clearCacheForNewInput(visited); for (ColumnTransformer columnTransformer : commonTransformerList) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { @@ -270,9 +274,10 @@ public static boolean satisfy(Column filterColumn, int rowIndex) { private TsBlock getTransformedTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); - // Clear all output DAGs before evaluating any output, preserving reuse within this block. + // Clear all output DAGs before evaluating any output, visiting shared subexpressions only once. + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index bcf4bdf7722cb..bc083f65b3b45 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; public abstract class AbstractCaseWhenThenColumnTransformer extends ColumnTransformer { @@ -235,14 +236,13 @@ protected void checkType() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { for (Pair whenThenColumnTransformer : whenThenTransformers) { - whenThenColumnTransformer.left.clearCacheForNewInput(); - whenThenColumnTransformer.right.clearCacheForNewInput(); + whenThenColumnTransformer.left.clearCacheForNewInput(visited); + whenThenColumnTransformer.right.clearCacheForNewInput(visited); } - elseTransformer.clearCacheForNewInput(); + elseTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index 11407afbdd227..8e12b18e720c0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; +import java.util.Set; + import static org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils.isNumericType; public abstract class ColumnTransformer { @@ -119,9 +121,19 @@ public void clearCache() { /** * Clears cached columns throughout this DAG before initializing a new input block. Unlike {@link * #clearCache()}, this also visits CASE descendants. Only call this before evaluating any - * expressions for the new block, when no sibling expression can still need a cached column. + * expressions for the new block, when no sibling expression can still need a cached column. All + * roots in the same evaluation phase must share an identity-based visited set so shared + * subexpressions are cleared only once. */ - public void clearCacheForNewInput() { + public final void clearCacheForNewInput(Set visited) { + if (!visited.add(this)) { + return; + } this.columnCache.clear(); + clearChildrenCacheForNewInput(visited); + } + + protected void clearChildrenCacheForNewInput(Set visited) { + // Leaf transformers have no children. } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index d494f652c8079..1483e96416e67 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public abstract class BinaryColumnTransformer extends ColumnTransformer { protected final ColumnTransformer leftTransformer; @@ -97,10 +99,9 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); - leftTransformer.clearCacheForNewInput(); - rightTransformer.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { + leftTransformer.clearCacheForNewInput(visited); + rightTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 75ed45ac64278..0d7576025c5e8 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -26,6 +26,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public class MappableUDFColumnTransformer extends ColumnTransformer { private final ColumnTransformer[] inputColumnTransformers; @@ -102,10 +104,9 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { for (ColumnTransformer columnTransformer : inputColumnTransformers) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index 0cfecdd87e5f3..5d8be08896d91 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; public abstract class MultiColumnTransformer extends ColumnTransformer { @@ -98,10 +99,9 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { for (ColumnTransformer columnTransformer : columnTransformerList) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index f68cc612da875..1c8dfce914a8d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public abstract class TernaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer firstColumnTransformer; @@ -112,11 +114,10 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); - firstColumnTransformer.clearCacheForNewInput(); - secondColumnTransformer.clearCacheForNewInput(); - thirdColumnTransformer.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { + firstColumnTransformer.clearCacheForNewInput(visited); + secondColumnTransformer.clearCacheForNewInput(visited); + thirdColumnTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 1a8943357cd15..3e28d3cfe874a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public abstract class UnaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer childColumnTransformer; @@ -74,9 +76,8 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); - childColumnTransformer.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { + childColumnTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 92d4931ea5674..d50587fdea757 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -47,6 +47,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; @@ -220,6 +221,50 @@ public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Except } } + @Test + public void testProjectionClearsSharedDagOnlyOncePerBlock() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + ColumnTransformer shared = input; + // Each level references the same child twice. A traversal without deduplication would visit + // the input 4096 times per root, although this DAG contains only 13 distinct transformers. + for (int i = 0; i < 12; i++) { + shared = graph.add(shared, shared); + } + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {4096.0}, new Double[] {4097.0}); + assertEquals(1, input.cacheClearCount); + assertBlock(operator.next(), new Double[] {8192.0}, new Double[] {8193.0}); + assertEquals(2, input.cacheClearCount); + } + } + + @Test + public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws Exception { + Graph filter = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + filter.leaves.add(input); + ColumnTransformer shared = filter.add(input, 1); + ColumnTransformer predicate = filter.greaterThan(shared, 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1), block(-2, 3)}, + filter, + predicate, + Collections.singletonList(shared), + projection, + projection.input(0), + projection.input(1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertEquals(1, input.cacheClearCount); + assertBlock(operator.next(), new Double[] {3.0}, new Double[] {4.0}); + assertEquals(2, input.cacheClearCount); + } + } + @Test public void testReusingCasePreservesPendingSiblingColumn() throws Exception { Graph graph = new Graph(); @@ -357,6 +402,7 @@ private ColumnTransformer caseWhen( private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { private int evaluationCount; + private int cacheClearCount; private CountingIdentityColumnTransformer() { super(DOUBLE, 0); @@ -367,6 +413,11 @@ protected void evaluate() { evaluationCount++; super.evaluate(); } + + @Override + protected void clearChildrenCacheForNewInput(Set visited) { + cacheClearCount++; + } } private static class TestOperatorContext extends CommonOperatorContext { From 3d10a44e2f2af096b0b4e18a707ef5c37f6f9acd Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 10:52:10 +0800 Subject: [PATCH 04/13] Clear CASE caches for schema batches and unify DAG traversal --- .../it/schema/IoTDBCaseDeviceCacheIT.java | 136 ++++++++ .../process/FilterAndProjectOperator.java | 4 +- ...AbstractCaseWhenThenColumnTransformer.java | 16 +- .../dag/column/ColumnTransformer.java | 15 +- .../column/FailFunctionColumnTransformer.java | 7 + .../binary/BinaryColumnTransformer.java | 15 +- .../column/leaf/LeafColumnTransformer.java | 7 + .../multi/MappableUDFColumnTransformer.java | 14 +- .../column/multi/MultiColumnTransformer.java | 16 +- .../ternary/TernaryColumnTransformer.java | 18 +- .../column/unary/UnaryColumnTransformer.java | 12 +- .../process/FilterAndProjectOperatorTest.java | 39 ++- .../schema/source/DeviceAttributeUpdater.java | 19 +- .../schema/source/DevicePredicateHandler.java | 9 + .../source/DevicePredicateHandlerTest.java | 323 ++++++++++++++++++ 15 files changed, 578 insertions(+), 72 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java new file mode 100644 index 0000000000000..3ef2464181106 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; + +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqual; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCaseDeviceCacheIT { + private static final String DATABASE = "case_device_cache"; + private static final String CASE_RESULT = "CASE WHEN src = 'match' THEN 'match' ELSE 'other' END"; + + @BeforeClass + public static void setUp() throws Exception { + // Keep every device in the same schema region and force a new batch for each device. + EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultSchemaRegionGroupNumPerDatabase(1); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockLineNumber(1); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(new String[] {"CREATE DATABASE " + DATABASE}); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDevicePredicatesAcrossBatches() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "predicate_cache"); + String predicate = CASE_RESULT + " = src"; + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM predicate_cache WHERE " + predicate), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,null,null,", + "d1,match,null,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + assertResultSetEqual( + statement.executeQuery("COUNT DEVICES FROM predicate_cache WHERE " + predicate), + "count(devices),", + Collections.singleton("4,")); + } + } + + @Test + public void testAttributeProjectionAcrossBatches() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "projection_cache"); + statement.execute("UPDATE projection_cache SET a = " + CASE_RESULT + ", b = src"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM projection_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match,match,", + "d1,match,match,match,", + "d2,other,other,other,", + "d3,other,other,other,"))); + } + } + + @Test + public void testAttributeCommonExpressionClearedByCase() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "common_cache"); + statement.execute( + "UPDATE common_cache SET a = src " + + "WHERE CASE WHEN src = 'match' THEN true ELSE false END"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM common_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match,null,", + "d1,match,match,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + } + } + + private static void createTable(Statement statement, String table) throws SQLException { + statement.execute("USE " + DATABASE); + statement.execute( + "CREATE TABLE " + + table + + " (id STRING TAG, src STRING ATTRIBUTE, a STRING ATTRIBUTE, b STRING ATTRIBUTE, v DOUBLE FIELD)"); + statement.execute( + "INSERT INTO " + + table + + "(time, id, src, v) VALUES (1, 'd0', 'match', 1), (2, 'd1', 'match', 1), " + + "(3, 'd2', 'other', 1), (4, 'd3', 'other', 1)"); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index d764114496767..7598a0c2196b8 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -183,8 +183,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { if (!hasNonMappableUDF) { // get result of calculated common sub expressions for (ColumnTransformer columnTransformer : commonTransformerList) { - // CASE WHEN clause would clear all its cache - // evaluate again to acquire cache + // CASE evaluation can clear a common expression after the filter used it. Recompute it + // before materializing the column for projection within this same input block. columnTransformer.tryEvaluate(); resultColumns.add(columnTransformer.getColumn()); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index bc083f65b3b45..e110b7331b120 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Set; +import java.util.function.Consumer; public abstract class AbstractCaseWhenThenColumnTransformer extends ColumnTransformer { @@ -236,13 +236,19 @@ protected void checkType() { } @Override - protected void clearChildrenCacheForNewInput(Set visited) { + protected boolean shouldClearChildrenCache() { + // Clearing a cached CASE during evaluation must preserve pending sibling columns. + return false; + } + + @Override + protected void forEachChild(Consumer action) { for (Pair whenThenColumnTransformer : whenThenTransformers) { - whenThenColumnTransformer.left.clearCacheForNewInput(visited); - whenThenColumnTransformer.right.clearCacheForNewInput(visited); + action.accept(whenThenColumnTransformer.left); + action.accept(whenThenColumnTransformer.right); } - elseTransformer.clearCacheForNewInput(visited); + action.accept(elseTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index 8e12b18e720c0..e8dcbee6fb0fb 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -26,6 +26,7 @@ import org.apache.tsfile.read.common.type.TypeEnum; import java.util.Set; +import java.util.function.Consumer; import static org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils.isNumericType; @@ -114,8 +115,11 @@ public void close() { // do nothing } - public void clearCache() { + public final void clearCache() { this.columnCache.clear(); + if (shouldClearChildrenCache()) { + forEachChild(ColumnTransformer::clearCache); + } } /** @@ -130,10 +134,13 @@ public final void clearCacheForNewInput(Set visited) { return; } this.columnCache.clear(); - clearChildrenCacheForNewInput(visited); + forEachChild(child -> child.clearCacheForNewInput(visited)); } - protected void clearChildrenCacheForNewInput(Set visited) { - // Leaf transformers have no children. + protected boolean shouldClearChildrenCache() { + return true; } + + /** Defines the child edges used by both evaluation-time and new-input cache clearing. */ + protected abstract void forEachChild(Consumer action); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java index 351c4396c0fc3..07b484deff0d4 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.common.block.column.NullColumn; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + /** * when evaluate of FailFunctionColumnTransformer is called, throw Exception to inform the user with * specified errorMsg @@ -59,4 +61,9 @@ public void evaluateWithSelection(boolean[] selection) { protected void checkType() { // do nothing } + + @Override + protected void forEachChild(Consumer action) { + // The fail function has no children. + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index 1483e96416e67..d2ca5270dee0f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -25,7 +25,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public abstract class BinaryColumnTransformer extends ColumnTransformer { @@ -92,16 +92,9 @@ public ColumnTransformer getRightTransformer() { } @Override - public void clearCache() { - super.clearCache(); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - leftTransformer.clearCacheForNewInput(visited); - rightTransformer.clearCacheForNewInput(visited); + protected void forEachChild(Consumer action) { + action.accept(leftTransformer); + action.accept(rightTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java index e9f69657a5405..ce16518ce245d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + public abstract class LeafColumnTransformer extends ColumnTransformer { protected TsBlock input; @@ -39,4 +41,9 @@ public void checkType() { public void initFromTsBlock(TsBlock input) { this.input = input; } + + @Override + protected void forEachChild(Consumer action) { + // Leaf transformers have no children. + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 0d7576025c5e8..3712a74468b26 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -26,7 +26,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public class MappableUDFColumnTransformer extends ColumnTransformer { @@ -96,17 +96,9 @@ public void close() { } @Override - public void clearCache() { - super.clearCache(); + protected void forEachChild(Consumer action) { for (ColumnTransformer columnTransformer : inputColumnTransformers) { - columnTransformer.clearCache(); - } - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - for (ColumnTransformer columnTransformer : inputColumnTransformers) { - columnTransformer.clearCacheForNewInput(visited); + action.accept(columnTransformer); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index 5d8be08896d91..afcd309069e87 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; -import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; public abstract class MultiColumnTransformer extends ColumnTransformer { @@ -91,18 +91,8 @@ public List getChildren() { } @Override - public void clearCache() { - super.clearCache(); - for (ColumnTransformer columnTransformer : columnTransformerList) { - columnTransformer.clearCache(); - } - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - for (ColumnTransformer columnTransformer : columnTransformerList) { - columnTransformer.clearCacheForNewInput(visited); - } + protected void forEachChild(Consumer action) { + columnTransformerList.forEach(action); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index 1c8dfce914a8d..a92d59292a6e9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -25,7 +25,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public abstract class TernaryColumnTransformer extends ColumnTransformer { @@ -106,18 +106,10 @@ public ColumnTransformer getThirdColumnTransformer() { } @Override - public void clearCache() { - super.clearCache(); - firstColumnTransformer.clearCache(); - secondColumnTransformer.clearCache(); - thirdColumnTransformer.clearCache(); - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - firstColumnTransformer.clearCacheForNewInput(visited); - secondColumnTransformer.clearCacheForNewInput(visited); - thirdColumnTransformer.clearCacheForNewInput(visited); + protected void forEachChild(Consumer action) { + action.accept(firstColumnTransformer); + action.accept(secondColumnTransformer); + action.accept(thirdColumnTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 3e28d3cfe874a..1dcc6303534b6 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -25,7 +25,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public abstract class UnaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer childColumnTransformer; @@ -70,14 +70,8 @@ protected abstract void doTransform( Column column, ColumnBuilder columnBuilder, boolean[] selection); @Override - public void clearCache() { - super.clearCache(); - childColumnTransformer.clearCache(); - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - childColumnTransformer.clearCacheForNewInput(visited); + protected void forEachChild(Consumer action) { + action.accept(childColumnTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index d50587fdea757..e21252dd87105 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -31,9 +31,12 @@ import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.DoubleColumn; @@ -41,13 +44,14 @@ import org.apache.tsfile.read.common.block.column.TimeColumn; import org.junit.Test; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Optional; -import java.util.Set; +import java.util.function.Consumer; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; @@ -265,6 +269,37 @@ public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws E } } + @Test + public void testMappableUdfClearsCaseDescendantsAcrossBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer udf = + new MappableUDFColumnTransformer( + DOUBLE, + new ColumnTransformer[] {reference(caseWhen), reference(input)}, + new UDTFExecutor("test", ZoneId.of("UTC")) { + @Override + public void execute(Column[] columns, ColumnBuilder builder) { + for (int i = 0; i < columns[0].getPositionCount(); i++) { + builder.writeDouble(columns[0].getDouble(i) + columns[1].getDouble(i)); + } + } + + @Override + public void beforeDestroy() { + // No UDF resources are allocated by this test executor. + } + }); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, udf)) { + assertBlock(operator.next(), new Double[] {2.0}); + assertBlock(operator.next(), new Double[] {-2.0, 4.0}); + assertBlock(operator.next(), new Double[] {5.0}); + } + } + @Test public void testReusingCasePreservesPendingSiblingColumn() throws Exception { Graph graph = new Graph(); @@ -415,7 +450,7 @@ protected void evaluate() { } @Override - protected void clearChildrenCacheForNewInput(Set visited) { + protected void forEachChild(Consumer action) { cacheClearCount++; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 9120444ba0d23..98e0e66b50bfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -40,8 +40,11 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -97,6 +100,8 @@ public void handleDeviceNode(final IDeviceMNode node) throws Metadata protected void update() throws MetadataException { final TsBlock block = getFilterTsBlock(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + projectOutputTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); projectLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(block)); @@ -145,9 +150,13 @@ private TsBlock getFilterTsBlock() { final List filterResultColumns = new ArrayList<>(Arrays.asList(curBlock.getValueColumns())); - // Get result of calculated common sub expressions + // CASE evaluation can clear a common expression after the filter used it. Recompute it before + // materializing the column for projection within this same batch. commonTransformerList.forEach( - columnTransformer -> filterResultColumns.add(columnTransformer.getColumn())); + columnTransformer -> { + columnTransformer.tryEvaluate(); + filterResultColumns.add(columnTransformer.getColumn()); + }); final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); final int rowCount = @@ -158,6 +167,12 @@ private TsBlock getFilterTsBlock() { return filterTsBlockBuilder.build(new TimeColumn(rowCount, new long[rowCount])); } + @Override + protected void clearFilterCacheForNewInput(Set visited) { + super.clearFilterCacheForNewInput(visited); + commonTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); + } + @Override public void close() throws MetadataException { super.close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java index db7d0a2b374bf..600e6a2a9aaaf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java @@ -35,8 +35,11 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.satisfy; @@ -106,6 +109,8 @@ public void prepareBatchResult() { return; } + // A CASE can leave shared expressions cached after the previous batch was consumed. + clearFilterCacheForNewInput(Collections.newSetFromMap(new IdentityHashMap<>())); // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant filterLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(curBlock)); @@ -120,6 +125,10 @@ public void prepareBatchResult() { curFilterColumn = filterColumn; } + protected void clearFilterCacheForNewInput(Set visited) { + filterOutputTransformer.clearCacheForNewInput(visited); + } + protected boolean withoutFilter() { return Objects.isNull(filterOutputTransformer); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java new file mode 100644 index 0000000000000..08b56188b2125 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareEqualToColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl.ShowDevicesResult; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.StringType.STRING; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DevicePredicateHandlerTest { + + private static final List COLUMNS = + Collections.singletonList(new TagColumnSchema("tag", TSDataType.STRING)); + + @Test + public void testPredicateFilterDoesNotReuseFullBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + try (DevicePredicateFilter filter = predicateFilter(graph, predicate)) { + int batchSize = DevicePredicateHandler.DEFAULT_MAX_TS_BLOCK_LINE_NUMBER; + for (String value : new String[] {"match", "other"}) { + for (int i = 0; i < batchSize; i++) { + filter.addBatch(device(value)); + } + // addBatch evaluates full batches automatically; exhausting the iterator resets the batch. + List result = consume(filter); + assertEquals(batchSize, result.size()); + for (String actual : result) { + assertEquals(value, actual); + } + } + } + } + + @Test + public void testPredicateFilterDoesNotReuseDifferentSizedBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + try (DevicePredicateFilter filter = predicateFilter(graph, predicate)) { + assertEquals(Collections.singletonList("match"), filterBatch(filter, "match")); + assertEquals(Arrays.asList("other", "match"), filterBatch(filter, "other", "match")); + assertEquals(Collections.emptyList(), filterBatch(filter, "skip")); + } + } + + @Test + public void testBlackListConstructorDoesNotReusePredicateAcrossBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + List> nodes = + Arrays.asList(node("match", 0), node("other", 1), node("match", 2)); + try (DeviceBlackListConstructor constructor = + new DeviceBlackListConstructor( + graph.leaves, + reference(predicate), + "table", + COLUMNS, + (pointer, name) -> null, + mock(MemSchemaRegionStatistics.class), + "test", + new TsTable("table"))) { + constructor.handleDeviceNode(nodes.get(0)); + constructor.prepareBatchResult(); + constructor.update(); + constructor.handleDeviceNode(nodes.get(1)); + constructor.handleDeviceNode(nodes.get(2)); + constructor.prepareBatchResult(); + constructor.update(); + assertEquals(3, constructor.getPreDeletedNum()); + for (IDeviceMNode node : nodes) { + verify(node).preDeactivateSelfOrTemplate(); + } + } + } + + @Test + public void testAttributeUpdaterDoesNotReuseFilterAcrossBatches() throws Exception { + Graph filter = new Graph(); + ColumnTransformer predicate = filter.matchesCase(filter.input(0)); + Graph projection = new Graph(); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + filter, predicate, Collections.emptyList(), projection, updates, projection.input(0))) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList( + Arrays.asList("match", "match"), + Arrays.asList("other", "other"), + Arrays.asList("match", "match")), + updates); + } + } + + @Test + public void testAttributeUpdaterDoesNotReuseProjectionAcrossBatches() throws Exception { + Graph projection = new Graph(); + ColumnTransformer input = projection.input(0); + ColumnTransformer caseWhen = projection.caseResult(input); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + null, null, Collections.emptyList(), projection, updates, caseWhen, input)) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList( + Arrays.asList("match", "match", "match"), + Arrays.asList("other", "other", "other"), + Arrays.asList("match", "match", "match")), + updates); + } + } + + @Test + public void testAttributeUpdaterRecomputesCommonExpressionClearedByCase() throws Exception { + Graph filter = new Graph(); + ColumnTransformer shared = filter.input(0); + ColumnTransformer predicate = + filter.caseWhen( + filter.equal(shared, filter.constant("match")), + filter.constant(true), + filter.constant(false)); + Graph projection = new Graph(); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + filter, + predicate, + Collections.singletonList(shared), + projection, + updates, + projection.input(1))) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList(Arrays.asList("match", "match"), Arrays.asList("match", "match")), updates); + } + } + + private static DevicePredicateFilter predicateFilter(Graph graph, ColumnTransformer predicate) { + return new DevicePredicateFilter( + graph.leaves, reference(predicate), COLUMNS, "test", new TsTable("table")); + } + + private static List filterBatch(DevicePredicateFilter filter, String... values) { + for (String value : values) { + filter.addBatch(device(value)); + } + filter.prepareBatchResult(); + return consume(filter); + } + + private static List consume(DevicePredicateFilter filter) { + List result = new ArrayList<>(); + while (filter.hasNext()) { + result.add(filter.next().getRawNodes()[3]); + } + return result; + } + + private static DeviceAttributeUpdater attributeUpdater( + Graph filter, + ColumnTransformer predicate, + List common, + Graph projection, + List> updates, + ColumnTransformer... outputs) { + List projectOutputs = new ArrayList<>(); + for (ColumnTransformer output : outputs) { + projectOutputs.add(reference(output)); + } + common.forEach(ColumnTransformer::addReferenceCount); + return new DeviceAttributeUpdater( + Collections.nCopies(1 + common.size(), TSDataType.STRING), + filter == null ? Collections.emptyList() : filter.leaves, + predicate == null ? null : reference(predicate), + common, + COLUMNS, + projection.leaves, + projectOutputs, + (pointer, name) -> null, + (deviceId, pointer, values) -> { + List row = new ArrayList<>(); + row.add(deviceId[0]); + for (Object value : values) { + row.add(((Binary) value).getStringValue(TSFileConfig.STRING_CHARSET)); + } + updates.add(row); + }, + Collections.nCopies(outputs.length, "attribute"), + "test", + new TsTable("table")); + } + + private static void updateBatch(DeviceAttributeUpdater updater, String... values) + throws Exception { + for (int i = 0; i < values.length; i++) { + updater.handleDeviceNode(node(values[i], i)); + } + updater.prepareBatchResult(); + updater.update(); + } + + private static ShowDevicesResult device(String value) { + return new ShowDevicesResult(null, true, 0, new String[] {"root", "test", "table", value}); + } + + @SuppressWarnings("unchecked") + private static IDeviceMNode node(String value, int pointer) { + IDeviceMNode node = mock(IDeviceMNode.class); + TableDeviceInfo info = new TableDeviceInfo<>(); + info.setAttributePointer(pointer); + when(node.getDeviceInfo()).thenReturn(info); + when(node.getPartialPath()).thenReturn(new PartialPath(device(value).getRawNodes())); + when(node.isAlignedNullable()).thenReturn(true); + return node; + } + + private static ColumnTransformer reference(ColumnTransformer transformer) { + transformer.addReferenceCount(); + return transformer; + } + + private static class Graph { + private final List leaves = new ArrayList<>(); + + private ColumnTransformer input(int index) { + return leaf(new IdentityColumnTransformer(STRING, index)); + } + + private ColumnTransformer constant(String value) { + return leaf( + new ConstantColumnTransformer( + STRING, + new BinaryColumn( + 1, + Optional.empty(), + new Binary[] {new Binary(value, TSFileConfig.STRING_CHARSET)}))); + } + + private ColumnTransformer constant(boolean value) { + return leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {value}))); + } + + private ColumnTransformer leaf(LeafColumnTransformer transformer) { + leaves.add(transformer); + return transformer; + } + + private ColumnTransformer equal(ColumnTransformer left, ColumnTransformer right) { + return new CompareEqualToColumnTransformer(BOOLEAN, reference(left), reference(right)); + } + + private ColumnTransformer caseWhen( + ColumnTransformer when, ColumnTransformer then, ColumnTransformer otherwise) { + return new TableCaseWhenThenColumnTransformer( + then.getType(), + Collections.singletonList(reference(when)), + Collections.singletonList(reference(then)), + reference(otherwise)); + } + + private ColumnTransformer caseResult(ColumnTransformer input) { + return caseWhen(equal(input, constant("match")), constant("match"), constant("other")); + } + + private ColumnTransformer matchesCase(ColumnTransformer input) { + return equal(caseResult(input), input); + } + } +} From 50c38c16ddf29032a6d1cf1c89b2bcf67627e957 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 11:27:51 +0800 Subject: [PATCH 05/13] Preserve Mockito compatibility for legacy cache clearing --- .../calc/transformation/dag/column/ColumnTransformer.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index e8dcbee6fb0fb..5189693d30e18 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -115,7 +115,13 @@ public void close() { // do nothing } - public final void clearCache() { + /** + * Clears caches during evaluation. Subclasses should define child edges via {@link + * #forEachChild(Consumer)} instead of overriding this method. + * + *

This legacy entry point remains non-final for compatibility with existing test doubles. + */ + public void clearCache() { this.columnCache.clear(); if (shouldClearChildrenCache()) { forEachChild(ColumnTransformer::clearCache); From 5e0cb1c8a98345af99bb1115eebad1be7ea12ffc Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 15:57:15 +0800 Subject: [PATCH 06/13] Precompute transformer cache reset nodes for each input phase --- .../process/FilterAndProjectOperator.java | 32 ++++--- .../dag/column/ColumnTransformer.java | 40 ++++++-- .../process/FilterAndProjectOperatorTest.java | 34 +++++-- .../dag/column/ColumnTransformerTest.java | 91 +++++++++++++++++++ .../schema/source/DeviceAttributeUpdater.java | 18 ++-- .../source/DeviceBlackListConstructor.java | 2 + .../schema/source/DevicePredicateFilter.java | 2 + .../schema/source/DevicePredicateHandler.java | 18 ++-- .../operator/schema/source/DeviceUpdater.java | 2 + 9 files changed, 191 insertions(+), 48 deletions(-) create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 7598a0c2196b8..03524b7ce6344 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -47,11 +47,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.List; import java.util.OptionalInt; -import java.util.Set; public class FilterAndProjectOperator implements ProcessOperator { @@ -70,6 +67,10 @@ public class FilterAndProjectOperator implements ProcessOperator { private final List projectOutputTransformerList; + private final ColumnTransformer[] filterCacheTransformers; + + private final ColumnTransformer[] projectCacheTransformers; + private final TsBlockBuilder filterTsBlockBuilder; private final boolean hasNonMappableUDF; @@ -103,6 +104,13 @@ public FilterAndProjectOperator( this.hasNonMappableUDF = hasNonMappableUDF; this.filterTsBlockBuilder = new TsBlockBuilder(8, filterOutputDataTypes); this.hasFilter = hasFilter; + List filterRoots = new ArrayList<>(commonTransformerList); + if (filterOutputTransformer != null) { + filterRoots.add(filterOutputTransformer); + } + this.filterCacheTransformers = ColumnTransformer.collectTransformers(filterRoots); + this.projectCacheTransformers = + ColumnTransformer.collectTransformers(projectOutputTransformerList); } public FilterAndProjectOperator( @@ -114,6 +122,8 @@ public FilterAndProjectOperator( this.projectLeafColumnTransformerList = filterAndProjectOperator.projectLeafColumnTransformerList; this.projectOutputTransformerList = filterAndProjectOperator.projectOutputTransformerList; + this.filterCacheTransformers = filterAndProjectOperator.filterCacheTransformers; + this.projectCacheTransformers = filterAndProjectOperator.projectCacheTransformers; this.hasNonMappableUDF = filterAndProjectOperator.hasNonMappableUDF; this.hasFilter = filterAndProjectOperator.hasFilter; this.filterTsBlockBuilder = filterAndProjectOperator.filterTsBlockBuilder; @@ -158,10 +168,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { // Short-circuit evaluation can reset cache reference counts when a shared expression is // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate // expressions whose caches may still have unconsumed references from the previous block. - Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); - filterOutputTransformer.clearCacheForNewInput(visited); - for (ColumnTransformer columnTransformer : commonTransformerList) { - columnTransformer.clearCacheForNewInput(visited); + for (ColumnTransformer columnTransformer : filterCacheTransformers) { + columnTransformer.clearCacheForNewInput(); } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { @@ -274,10 +282,10 @@ public static boolean satisfy(Column filterColumn, int rowIndex) { private TsBlock getTransformedTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); - // Clear all output DAGs before evaluating any output, visiting shared subexpressions only once. - Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); - for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.clearCacheForNewInput(visited); + // The flat array was deduplicated at construction; clear each node before evaluating any + // output. + for (ColumnTransformer columnTransformer : projectCacheTransformers) { + columnTransformer.clearCacheForNewInput(); } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { @@ -455,6 +463,8 @@ public long ramBytesUsed() { return INSTANCE_SIZE + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(inputOperator) + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + RamUsageEstimator.shallowSizeOf(filterCacheTransformers) + + RamUsageEstimator.shallowSizeOf(projectCacheTransformers) + filterTsBlockBuilder.getRetainedSizeInBytes(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index 5189693d30e18..cb1d121c55c4f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -25,6 +25,10 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; import java.util.Set; import java.util.function.Consumer; @@ -129,18 +133,34 @@ public void clearCache() { } /** - * Clears cached columns throughout this DAG before initializing a new input block. Unlike {@link - * #clearCache()}, this also visits CASE descendants. Only call this before evaluating any - * expressions for the new block, when no sibling expression can still need a cached column. All - * roots in the same evaluation phase must share an identity-based visited set so shared - * subexpressions are cleared only once. + * Clears only this node's cached column before initializing a new input block. Call this on every + * node collected by {@link #collectTransformers(Iterable)} before evaluating any expressions for + * the new block, when no sibling expression can still need a cached column. */ - public final void clearCacheForNewInput(Set visited) { - if (!visited.add(this)) { - return; - } + public final void clearCacheForNewInput() { this.columnCache.clear(); - forEachChild(child -> child.clearCacheForNewInput(visited)); + } + + /** + * Collects all nodes reachable from the evaluation phase's roots, including CASE descendants, + * deduplicating by identity. Collect once after constructing the DAG and retain the flat array so + * new-input cache clearing needs no graph traversal or visited-set allocation. + */ + public static ColumnTransformer[] collectTransformers(Iterable roots) { + List transformers = new ArrayList<>(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + for (ColumnTransformer root : roots) { + root.collectTransformers(transformers, visited); + } + return transformers.toArray(new ColumnTransformer[0]); + } + + private void collectTransformers( + List transformers, Set visited) { + if (visited.add(this)) { + transformers.add(this); + forEachChild(child -> child.collectTransformers(transformers, visited)); + } } protected boolean shouldClearChildrenCache() { diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index e21252dd87105..4b35a94c912b5 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -226,7 +226,23 @@ public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Except } @Test - public void testProjectionClearsSharedDagOnlyOncePerBlock() throws Exception { + public void testCopiedOperatorReusesCollectedTransformers() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + try (FilterAndProjectOperator original = project(graph, new TsBlock[0], graph.add(input, 1)); + FilterAndProjectOperator copy = + new FilterAndProjectOperator( + original, new TsBlockSourceOperator(new TsBlock[] {block(1), block(2)}))) { + assertEquals(1, input.childTraversalCount); + assertBlock(copy.next(), new Double[] {2.0}); + assertBlock(copy.next(), new Double[] {3.0}); + assertEquals(1, input.childTraversalCount); + } + } + + @Test + public void testProjectionCollectsSharedDagOnlyOnce() throws Exception { Graph graph = new Graph(); CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); graph.leaves.add(input); @@ -238,15 +254,16 @@ public void testProjectionClearsSharedDagOnlyOncePerBlock() throws Exception { } try (Operator operator = project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {4096.0}, new Double[] {4097.0}); - assertEquals(1, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {8192.0}, new Double[] {8193.0}); - assertEquals(2, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); } } @Test - public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws Exception { + public void testFilterAndCommonExpressionsCollectSharedDagOnlyOnce() throws Exception { Graph filter = new Graph(); CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); filter.leaves.add(input); @@ -262,10 +279,11 @@ public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws E projection, projection.input(0), projection.input(1))) { + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); - assertEquals(1, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {3.0}, new Double[] {4.0}); - assertEquals(2, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); } } @@ -437,7 +455,7 @@ private ColumnTransformer caseWhen( private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { private int evaluationCount; - private int cacheClearCount; + private int childTraversalCount; private CountingIdentityColumnTransformer() { super(DOUBLE, 0); @@ -451,7 +469,7 @@ protected void evaluate() { @Override protected void forEachChild(Consumer action) { - cacheClearCount++; + childTraversalCount++; } } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java new file mode 100644 index 0000000000000..8c81d6f5ad8b0 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +public class ColumnTransformerTest { + + @Test + public void testCollectionDeduplicatesByIdentity() { + ColumnTransformer first = new EqualIdentityColumnTransformer(0); + ColumnTransformer second = new EqualIdentityColumnTransformer(1); + ColumnTransformer root = new ArithmeticAdditionColumnTransformer(DOUBLE, first, second); + + ColumnTransformer[] collected = + ColumnTransformer.collectTransformers(Arrays.asList(root, first, root)); + + assertEquals(3, collected.length); + assertSame(root, collected[0]); + assertSame(first, collected[1]); + assertSame(second, collected[2]); + } + + @Test + public void testNewInputClearingOnlyClearsOwnCache() { + ColumnTransformer left = new IdentityColumnTransformer(DOUBLE, 0); + ColumnTransformer right = new IdentityColumnTransformer(DOUBLE, 1); + ColumnTransformer root = new ArithmeticAdditionColumnTransformer(DOUBLE, left, right); + left.addReferenceCount(); + right.addReferenceCount(); + root.addReferenceCount(); + left.initializeColumnCache(column(1)); + right.initializeColumnCache(column(2)); + root.initializeColumnCache(column(3)); + + root.clearCacheForNewInput(); + + assertEquals(0, root.getColumnCachePositionCount()); + assertEquals(1.0, left.getColumn().getDouble(0), 0.0); + assertEquals(2.0, right.getColumn().getDouble(0), 0.0); + } + + private static Column column(double value) { + return new DoubleColumn(1, Optional.empty(), new double[] {value}); + } + + private static class EqualIdentityColumnTransformer extends IdentityColumnTransformer { + private EqualIdentityColumnTransformer(int index) { + super(DOUBLE, index); + } + + @Override + public boolean equals(Object other) { + return other instanceof EqualIdentityColumnTransformer; + } + + @Override + public int hashCode() { + return 1; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 98e0e66b50bfc..04e08c9d133b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -40,11 +40,8 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -54,6 +51,7 @@ public class DeviceAttributeUpdater extends DeviceUpdater { private final List commonTransformerList; private final List projectLeafColumnTransformerList; private final List projectOutputTransformerList; + private final ColumnTransformer[] projectCacheTransformers; private final TriConsumer attributeUpdater; private final TsBlockBuilder filterTsBlockBuilder; private final List attributePointers = new ArrayList<>(); @@ -78,6 +76,7 @@ public DeviceAttributeUpdater( super( filterLeafColumnTransformerList, filterOutputTransformer, + commonTransformerList, columnSchemaList, attributeProvider, database, @@ -85,6 +84,8 @@ public DeviceAttributeUpdater( this.commonTransformerList = commonTransformerList; this.projectLeafColumnTransformerList = projectLeafColumnTransformerList; this.projectOutputTransformerList = projectOutputTransformerList; + this.projectCacheTransformers = + ColumnTransformer.collectTransformers(projectOutputTransformerList); this.attributeUpdater = attributeUpdater; this.filterTsBlockBuilder = new TsBlockBuilder(filterOutputDataTypes); this.attributeNames = attributeNames; @@ -100,8 +101,9 @@ public void handleDeviceNode(final IDeviceMNode node) throws Metadata protected void update() throws MetadataException { final TsBlock block = getFilterTsBlock(); - Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); - projectOutputTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); + for (ColumnTransformer transformer : projectCacheTransformers) { + transformer.clearCacheForNewInput(); + } projectLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(block)); @@ -167,12 +169,6 @@ private TsBlock getFilterTsBlock() { return filterTsBlockBuilder.build(new TimeColumn(rowCount, new long[rowCount])); } - @Override - protected void clearFilterCacheForNewInput(Set visited) { - super.clearFilterCacheForNewInput(visited); - commonTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); - } - @Override public void close() throws MetadataException { super.close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java index 3391f3e4080d8..4d3793dd44637 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java @@ -31,6 +31,7 @@ import org.apache.tsfile.utils.Binary; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.function.BiFunction; @@ -54,6 +55,7 @@ public DeviceBlackListConstructor( super( filterLeafColumnTransformerList, filterOutputTransformer, + Collections.emptyList(), columnSchemaList, attributeProvider, database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java index 589f119c3ae14..a9db01b879a49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -44,6 +45,7 @@ public DevicePredicateFilter( super( filterLeafColumnTransformerList, filterOutputTransformer, + Collections.emptyList(), columnSchemaList, database, table); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java index 600e6a2a9aaaf..dd933d8403e75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java @@ -35,11 +35,8 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.satisfy; @@ -48,6 +45,7 @@ public abstract class DevicePredicateHandler implements AutoCloseable { private final List filterLeafColumnTransformerList; protected final ColumnTransformer filterOutputTransformer; + private final ColumnTransformer[] filterCacheTransformers; private final List inputDataTypes; private final List columnSchemaList; private final int tagIndex; @@ -65,11 +63,17 @@ public abstract class DevicePredicateHandler implements AutoCloseable { protected DevicePredicateHandler( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, + final List commonTransformerList, final List columnSchemaList, final String database, final TsTable table) { this.filterLeafColumnTransformerList = filterLeafColumnTransformerList; this.filterOutputTransformer = filterOutputTransformer; + List filterRoots = new ArrayList<>(commonTransformerList); + if (filterOutputTransformer != null) { + filterRoots.add(filterOutputTransformer); + } + this.filterCacheTransformers = ColumnTransformer.collectTransformers(filterRoots); this.columnSchemaList = columnSchemaList; this.inputDataTypes = columnSchemaList.stream() @@ -110,7 +114,9 @@ public void prepareBatchResult() { } // A CASE can leave shared expressions cached after the previous batch was consumed. - clearFilterCacheForNewInput(Collections.newSetFromMap(new IdentityHashMap<>())); + for (ColumnTransformer transformer : filterCacheTransformers) { + transformer.clearCacheForNewInput(); + } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant filterLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(curBlock)); @@ -125,10 +131,6 @@ public void prepareBatchResult() { curFilterColumn = filterColumn; } - protected void clearFilterCacheForNewInput(Set visited) { - filterOutputTransformer.clearCacheForNewInput(visited); - } - protected boolean withoutFilter() { return Objects.isNull(filterOutputTransformer); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java index 2679f501dd060..5debad67608e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java @@ -41,6 +41,7 @@ public abstract class DeviceUpdater extends DevicePredicateHandler { protected DeviceUpdater( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, + final List commonTransformerList, final List columnSchemaList, final BiFunction attributeProvider, final String database, @@ -48,6 +49,7 @@ protected DeviceUpdater( super( filterLeafColumnTransformerList, filterOutputTransformer, + commonTransformerList, columnSchemaList, database, table); From d7239384fd3a6200617de426fbf3d57930e6af87 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 10:19:55 +0800 Subject: [PATCH 07/13] Respect filter selection when rebuilding common expression caches Share masked common-column materialization between query and schema UPDATE paths. Retain sibling columns before later evaluations can clear their caches, and read multi-argument sizes from retained columns. --- .../it/db/it/IoTDBCaseWhenThenTableIT.java | 66 +++++ .../it/schema/IoTDBCaseDeviceCacheIT.java | 79 ++++-- .../process/FilterAndProjectOperator.java | 31 +- .../dag/column/ColumnTransformer.java | 9 +- .../binary/BinaryColumnTransformer.java | 3 +- .../multi/MappableUDFColumnTransformer.java | 24 +- .../column/multi/MultiColumnTransformer.java | 17 +- .../ternary/TernaryColumnTransformer.java | 9 +- .../process/FilterAndProjectOperatorTest.java | 264 +++++++++++++----- .../schema/source/DeviceAttributeUpdater.java | 10 +- .../source/DevicePredicateHandlerTest.java | 7 +- 11 files changed, 382 insertions(+), 137 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index 169ebd3084ae3..0839c7e1f3415 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -724,6 +724,72 @@ public void testCaseWhenReusesAggregationAcrossBlocks() { } } + @Test + public void testCaseOperandsAndMultiArgumentFunctions() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_operands (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_operands(time, dev, x) VALUES (0, 'd0', 1.0), (1, 'd0', -2.0)" + }); + String caseWhen = "CASE WHEN x > 0 THEN 2.0 ELSE 3.0 END"; + String[] expressions = { + "x + " + caseWhen, + "CASE WHEN x > -100 THEN coalesce(x, 0.0) END", + "greatest(x, " + caseWhen + ")", + "CASE WHEN x BETWEEN -5.0 AND " + caseWhen + " THEN 1.0 ELSE 0.0 END", + "CASE WHEN x > -100 THEN CASE WHEN x BETWEEN -5.0 AND " + + caseWhen + + " THEN 1.0 ELSE 0.0 END END", + "coalesce(x, " + caseWhen + ")", + "CASE WHEN x > -100 THEN coalesce(x, " + caseWhen + ") END" + }; + String[][] results = { + {"3.0,", "1.0,"}, + {"1.0,", "-2.0,"}, + {"2.0,", "3.0,"}, + {"1.0,", "1.0,"}, + {"1.0,", "1.0,"}, + {"1.0,", "-2.0,"}, + {"1.0,", "-2.0,"} + }; + for (int i = 0; i < expressions.length; i++) { + tableResultSetEqualTest( + "SELECT " + expressions[i] + " AS result FROM case_operands ORDER BY time", + new String[] {"result"}, + results[i], + DATABASE); + } + tableResultSetEqualTest( + "SELECT coalesce(x, 0.0) AS result FROM case_operands " + + "WHERE CASE WHEN coalesce(x, 0.0) > 0 THEN 1.0 ELSE 0.0 END > 0", + new String[] {"result"}, + new String[] {"1.0,"}, + DATABASE); + } + + @Test + public void testCommonCastRespectsCaseFilter() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE guarded_common_cast (dev STRING TAG, src STRING FIELD)", + "INSERT INTO guarded_common_cast(time, dev, src) VALUES " + + "(0, 'd0', 'n/a'), (1, 'd0', '5')" + }); + String query = + "SELECT CAST(src AS INT32) + 1 AS result FROM guarded_common_cast " + + "WHERE CASE WHEN src <> 'n/a' THEN CAST(src AS INT32) > 10 ELSE false END"; + tableResultSetEqualTest(query, new String[] {"result"}, new String[0], DATABASE); + + prepareTableData( + new String[] { + "USE " + DATABASE, + "INSERT INTO guarded_common_cast(time, dev, src) VALUES (2, 'd0', '50')" + }); + tableResultSetEqualTest(query, new String[] {"result"}, new String[] {"51,"}, DATABASE); + } + @Test public void testReusingCasePreservesPendingSiblingColumn() { prepareTableData( diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java index 3ef2464181106..6596f17f4274e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java @@ -66,20 +66,21 @@ public void testDevicePredicatesAcrossBatches() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { createTable(statement, "predicate_cache"); - String predicate = CASE_RESULT + " = src"; - assertResultSetEqual( - statement.executeQuery("SHOW DEVICES FROM predicate_cache WHERE " + predicate), - "id,src,a,b,", - new HashSet<>( - Arrays.asList( - "d0,match,null,null,", - "d1,match,null,null,", - "d2,other,null,null,", - "d3,other,null,null,"))); - assertResultSetEqual( - statement.executeQuery("COUNT DEVICES FROM predicate_cache WHERE " + predicate), - "count(devices),", - Collections.singleton("4,")); + for (String predicate : new String[] {CASE_RESULT + " = src", "src = " + CASE_RESULT}) { + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM predicate_cache WHERE " + predicate), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,null,null,", + "d1,match,null,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + assertResultSetEqual( + statement.executeQuery("COUNT DEVICES FROM predicate_cache WHERE " + predicate), + "count(devices),", + Collections.singleton("4,")); + } } } @@ -98,6 +99,18 @@ public void testAttributeProjectionAcrossBatches() throws Exception { "d1,match,match,match,", "d2,other,other,other,", "d3,other,other,other,"))); + statement.execute( + "UPDATE projection_cache SET a = " + + "concat(src, CASE WHEN src = 'match' THEN '-m' ELSE '-o' END)"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM projection_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match-m,match,", + "d1,match,match-m,match,", + "d2,other,other-o,other,", + "d3,other,other-o,other,"))); } } @@ -107,20 +120,50 @@ public void testAttributeCommonExpressionClearedByCase() throws Exception { Statement statement = connection.createStatement()) { createTable(statement, "common_cache"); statement.execute( - "UPDATE common_cache SET a = src " - + "WHERE CASE WHEN src = 'match' THEN true ELSE false END"); + "UPDATE common_cache SET a = upper(src) " + + "WHERE CASE WHEN upper(src) = 'MATCH' THEN true ELSE false END"); assertResultSetEqual( statement.executeQuery("SHOW DEVICES FROM common_cache"), "id,src,a,b,", new HashSet<>( Arrays.asList( - "d0,match,match,null,", - "d1,match,match,null,", + "d0,match,MATCH,null,", + "d1,match,MATCH,null,", "d2,other,null,null,", "d3,other,null,null,"))); } } + @Test + public void testGuardedCommonCastInAttributeUpdate() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + statement.execute( + "CREATE TABLE guarded_cast " + + "(id STRING TAG, src STRING ATTRIBUTE, a STRING ATTRIBUTE, v DOUBLE FIELD)"); + statement.execute( + "INSERT INTO guarded_cast(time, id, src, a, v) VALUES " + + "(1, 'd0', 'n/a', 'seed', 1), (2, 'd1', '5', 'seed', 1)"); + String update = + "UPDATE guarded_cast SET a = CAST(CAST(src AS INT32) + 1 AS STRING) " + + "WHERE CASE WHEN src <> 'n/a' THEN CAST(src AS INT32) > 10 ELSE false END"; + + statement.execute(update); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM guarded_cast"), + "id,src,a,", + new HashSet<>(Arrays.asList("d0,n/a,seed,", "d1,5,seed,"))); + + statement.execute("UPDATE guarded_cast SET src = '50' WHERE id = 'd1'"); + statement.execute(update); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM guarded_cast"), + "id,src,a,", + new HashSet<>(Arrays.asList("d0,n/a,seed,", "d1,50,51,"))); + } + } + private static void createTable(Statement statement, String table) throws SQLException { statement.execute("USE " + DATABASE); statement.execute( diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 03524b7ce6344..b258ee71ebf1f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -189,13 +189,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { } if (!hasNonMappableUDF) { - // get result of calculated common sub expressions - for (ColumnTransformer columnTransformer : commonTransformerList) { - // CASE evaluation can clear a common expression after the filter used it. Recompute it - // before materializing the column for projection within this same input block. - columnTransformer.tryEvaluate(); - resultColumns.add(columnTransformer.getColumn()); - } + appendCommonExpressionColumns( + resultColumns, commonTransformerList, filterColumn, positionCount); } final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); @@ -222,6 +217,28 @@ private TsBlock getFilterTsBlock(TsBlock input) { } } + /** + * Materializes common expressions for projection after filtering. CASE may have invalidated their + * caches, so any recomputation must preserve the filter mask and skip rejected input rows. + */ + public static void appendCommonExpressionColumns( + List resultColumns, + List commonTransformers, + Column filterColumn, + int positionCount) { + if (commonTransformers.isEmpty()) { + return; + } + boolean[] selection = new boolean[positionCount]; + for (int i = 0; i < positionCount; i++) { + selection[i] = satisfy(filterColumn, i); + } + for (ColumnTransformer transformer : commonTransformers) { + transformer.tryEvaluateWithSelection(selection); + resultColumns.add(transformer.getColumn()); + } + } + private static int constructFilteredTsBlock( List resultColumns, TimeColumnBuilder timeBuilder, diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index cb1d121c55c4f..d21b89b783db9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -54,6 +54,13 @@ public void tryEvaluate() { } } + /** Reuses the cached column, or evaluates with the selection if it was invalidated. */ + public void tryEvaluateWithSelection(boolean[] selection) { + if (!columnCache.hasCached()) { + evaluateWithSelection(selection); + } + } + public Column getColumn() { return columnCache.getColumn(); } @@ -123,7 +130,7 @@ public void close() { * Clears caches during evaluation. Subclasses should define child edges via {@link * #forEachChild(Consumer)} instead of overriding this method. * - *

This legacy entry point remains non-final for compatibility with existing test doubles. + *

Kept non-final so existing Mockito-based tests can stub it. */ public void clearCache() { this.columnCache.clear(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index d2ca5270dee0f..d1715b12c3c8a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -44,10 +44,11 @@ protected BinaryColumnTransformer( @Override public void evaluate() { leftTransformer.tryEvaluate(); - rightTransformer.tryEvaluate(); // attention: get positionCount before calling getColumn int positionCount = leftTransformer.getColumnCachePositionCount(); Column leftColumn = leftTransformer.getColumn(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + rightTransformer.tryEvaluate(); Column rightColumn = rightTransformer.getColumn(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 3712a74468b26..9826c6a9b0428 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -44,30 +44,30 @@ public MappableUDFColumnTransformer( @Override public void evaluate() { // pull columns from previous transformers - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.tryEvaluate(); + Column[] columns = new Column[inputColumnTransformers.length]; + for (int i = 0; i < inputColumnTransformers.length; i++) { + inputColumnTransformers[i].tryEvaluate(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + columns[i] = inputColumnTransformers[i].getColumn(); } - doTransform(); + doTransform(columns); } @Override public void evaluateWithSelection(boolean[] selection) { - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.evaluateWithSelection(selection); + Column[] columns = new Column[inputColumnTransformers.length]; + for (int i = 0; i < inputColumnTransformers.length; i++) { + inputColumnTransformers[i].evaluateWithSelection(selection); + columns[i] = inputColumnTransformers[i].getColumn(); } - doTransform(); + doTransform(columns); // clear cache for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { inputColumnTransformer.clearCache(); } } - private void doTransform() { - int size = inputColumnTransformers.length; - Column[] columns = new Column[size]; - for (int i = 0; i < size; i++) { - columns[i] = inputColumnTransformers[i].getColumn(); - } + private void doTransform(Column[] columns) { // construct input TsBlock with columns int positionCount = columns[0].getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index afcd309069e87..cfd9c8913f24b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; -import java.util.stream.Collectors; public abstract class MultiColumnTransformer extends ColumnTransformer { @@ -43,20 +42,17 @@ protected MultiColumnTransformer(Type returnType, List column @Override public void evaluate() { + List childrenColumns = new ArrayList<>(columnTransformerList.size()); for (ColumnTransformer child : columnTransformerList) { child.tryEvaluate(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + childrenColumns.add(child.getColumn()); } - // attention: get positionCount before calling getColumn - int positionCount = columnTransformerList.get(0).getColumnCachePositionCount(); + int positionCount = childrenColumns.get(0).getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); - doTransform( - columnTransformerList.stream() - .map(ColumnTransformer::getColumn) - .collect(Collectors.toList()), - builder, - positionCount); + doTransform(childrenColumns, builder, positionCount); initializeColumnCache(builder.build()); } @@ -69,7 +65,8 @@ public void evaluateWithSelection(boolean[] selection) { childrenColumns.add(child.getColumn()); } - int positionCount = columnTransformerList.get(0).getColumnCachePositionCount(); + // getColumn() can release the first child's cache; read the retained column's size instead. + int positionCount = childrenColumns.get(0).getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); doTransform(childrenColumns, builder, positionCount, selection); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index a92d59292a6e9..a1b7c91357cd3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -50,11 +50,12 @@ protected TernaryColumnTransformer( @Override public void evaluate() { firstColumnTransformer.tryEvaluate(); - secondColumnTransformer.tryEvaluate(); - thirdColumnTransformer.tryEvaluate(); int positionCount = firstColumnTransformer.getColumnCachePositionCount(); Column firstColumn = firstColumnTransformer.getColumn(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + secondColumnTransformer.tryEvaluate(); Column secondColumn = secondColumnTransformer.getColumn(); + thirdColumnTransformer.tryEvaluate(); Column thirdColumn = thirdColumnTransformer.getColumn(); ColumnBuilder columnBuilder = returnType.createColumnBuilder(positionCount); doTransform(firstColumn, secondColumn, thirdColumn, columnBuilder, positionCount); @@ -64,11 +65,11 @@ public void evaluate() { @Override public void evaluateWithSelection(boolean[] selection) { firstColumnTransformer.evaluateWithSelection(selection); - secondColumnTransformer.evaluateWithSelection(selection); - thirdColumnTransformer.evaluateWithSelection(selection); int positionCount = firstColumnTransformer.getColumnCachePositionCount(); Column firstColumn = firstColumnTransformer.getColumn(); + secondColumnTransformer.evaluateWithSelection(selection); Column secondColumn = secondColumnTransformer.getColumn(); + thirdColumnTransformer.evaluateWithSelection(selection); Column thirdColumn = thirdColumnTransformer.getColumn(); ColumnBuilder columnBuilder = returnType.createColumnBuilder(positionCount); doTransform(firstColumn, secondColumn, thirdColumn, columnBuilder, positionCount, selection); diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 4b35a94c912b5..0ba2c4d7ef50c 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -27,34 +27,43 @@ import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.binary.CompareGreaterThanColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareNonEqualColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.CoalesceColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.ternary.BetweenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.CastFunctionColumnTransformer; import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.utils.Binary; import org.junit.Test; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.function.Consumer; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.apache.tsfile.read.common.type.IntType.INT32; +import static org.apache.tsfile.read.common.type.StringType.STRING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -166,6 +175,153 @@ public void testFilterCommonExpressionsDoNotCrossBlocks() throws Exception { } } + @Test + public void testCommonCastOnlyEvaluatesRowsPassingCaseFilter() throws Exception { + for (boolean runLengthEncodedTime : new boolean[] {false, true}) { + Graph filter = new Graph(); + ColumnTransformer input = filter.leaf(new IdentityColumnTransformer(STRING, 0)); + ColumnTransformer invalid = + filter.leaf( + new ConstantColumnTransformer( + STRING, + new BinaryColumn( + 1, + Optional.empty(), + new Binary[] {new Binary("n/a", TSFileConfig.STRING_CHARSET)}))); + ColumnTransformer guard = + new CompareNonEqualColumnTransformer(BOOLEAN, reference(input), reference(invalid)); + ColumnTransformer common = + new CastFunctionColumnTransformer(INT32, reference(input), ZoneId.of("UTC")); + ColumnTransformer otherwise = + filter.leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {false}))); + ColumnTransformer predicate = + new TableCaseWhenThenColumnTransformer( + BOOLEAN, + Collections.singletonList(reference(guard)), + Collections.singletonList(reference(filter.greaterThan(common, 10))), + reference(otherwise)); + IdentityColumnTransformer output = new IdentityColumnTransformer(INT32, 1); + reference(common); + try (Operator operator = + new FilterAndProjectOperator( + new TestOperatorContext(), + new ValuesOperator( + new TestOperatorContext(), + Arrays.asList( + stringBlock(runLengthEncodedTime, "n/a", "5"), + stringBlock(runLengthEncodedTime, "n/a", "50", "5"), + stringBlock(runLengthEncodedTime, "n/a", "60"))), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + filter.leaves, + reference(predicate), + Collections.singletonList(common), + Collections.singletonList(output), + Collections.singletonList(reference(output)), + false, + true)) { + assertEquals(0, operator.next().getPositionCount()); + TsBlock firstMatch = operator.next(); + assertEquals(1, firstMatch.getPositionCount()); + assertEquals(50, firstMatch.getColumn(0).getInt(0)); + TsBlock secondMatch = operator.next(); + assertEquals(1, secondMatch.getPositionCount()); + assertEquals(60, secondMatch.getColumn(0).getInt(0)); + } + } + } + + @Test + public void testCommonMultiArgumentExpressionRespectsCaseFilter() throws Exception { + Graph filter = new Graph(); + ColumnTransformer common = + new CoalesceColumnTransformer( + DOUBLE, Arrays.asList(reference(filter.input(0)), reference(filter.constant(0)))); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(common, 0), filter.constant(1), filter.constant(0)); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(-2, 3), block(4, -5)}, + filter, + filter.greaterThan(caseWhen, 0), + Collections.singletonList(common), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {3.0}); + assertBlock(operator.next(), new Double[] {4.0}); + } + } + + @Test + public void testBinaryPreservesSiblingColumnBeforeCase() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + try (Operator operator = + project(graph, new TsBlock[] {block(1, -2)}, graph.add(input, caseWhen))) { + assertBlock(operator.next(), new Double[] {3.0, 1.0}); + } + } + + @Test + public void testMultiArgumentFunctionPreservesSiblingColumnBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer output = + new CoalesceColumnTransformer( + DOUBLE, Arrays.asList(reference(input), reference(caseWhen))); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {1.0, -2.0}); + } + } + } + + @Test + public void testTernaryPreservesSiblingColumnsBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer between = + new BetweenColumnTransformer( + BOOLEAN, reference(input), reference(graph.constant(-5)), reference(caseWhen), false); + ColumnTransformer output = graph.caseWhen(between, graph.constant(1), graph.constant(0)); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}); + } + } + } + + @Test + public void testMappableUdfPreservesSiblingColumnsBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer output = sumUdf(input, caseWhen); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {3.0, 1.0}); + } + } + } + @Test public void testProjectionClearsCachedDescendantsOfNestedCase() throws Exception { Graph graph = new Graph(); @@ -233,7 +389,8 @@ public void testCopiedOperatorReusesCollectedTransformers() throws Exception { try (FilterAndProjectOperator original = project(graph, new TsBlock[0], graph.add(input, 1)); FilterAndProjectOperator copy = new FilterAndProjectOperator( - original, new TsBlockSourceOperator(new TsBlock[] {block(1), block(2)}))) { + original, + new ValuesOperator(new TestOperatorContext(), Arrays.asList(block(1), block(2))))) { assertEquals(1, input.childTraversalCount); assertBlock(copy.next(), new Double[] {2.0}); assertBlock(copy.next(), new Double[] {3.0}); @@ -293,23 +450,7 @@ public void testMappableUdfClearsCaseDescendantsAcrossBlocks() throws Exception ColumnTransformer input = graph.input(0); ColumnTransformer caseWhen = graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); - ColumnTransformer udf = - new MappableUDFColumnTransformer( - DOUBLE, - new ColumnTransformer[] {reference(caseWhen), reference(input)}, - new UDTFExecutor("test", ZoneId.of("UTC")) { - @Override - public void execute(Column[] columns, ColumnBuilder builder) { - for (int i = 0; i < columns[0].getPositionCount(); i++) { - builder.writeDouble(columns[0].getDouble(i) + columns[1].getDouble(i)); - } - } - - @Override - public void beforeDestroy() { - // No UDF resources are allocated by this test executor. - } - }); + ColumnTransformer udf = sumUdf(caseWhen, input); try (Operator operator = project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, udf)) { assertBlock(operator.next(), new Double[] {2.0}); @@ -318,6 +459,25 @@ public void beforeDestroy() { } } + private static ColumnTransformer sumUdf(ColumnTransformer left, ColumnTransformer right) { + return new MappableUDFColumnTransformer( + DOUBLE, + new ColumnTransformer[] {reference(left), reference(right)}, + new UDTFExecutor("test", ZoneId.of("UTC")) { + @Override + public void execute(Column[] columns, ColumnBuilder builder) { + for (int i = 0; i < columns[0].getPositionCount(); i++) { + builder.writeDouble(columns[0].getDouble(i) + columns[1].getDouble(i)); + } + } + + @Override + public void beforeDestroy() { + // No UDF resources are allocated by this test executor. + } + }); + } + @Test public void testReusingCasePreservesPendingSiblingColumn() throws Exception { Graph graph = new Graph(); @@ -360,7 +520,7 @@ private static FilterAndProjectOperator operator( } return new FilterAndProjectOperator( new TestOperatorContext(), - new TsBlockSourceOperator(blocks), + new ValuesOperator(new TestOperatorContext(), Arrays.asList(blocks)), Collections.nCopies(1 + common.size(), TSDataType.DOUBLE), filter == null ? Collections.emptyList() : filter.leaves, predicate == null ? null : reference(predicate), @@ -384,6 +544,16 @@ private static TsBlock block(double... values) { return block(false, values); } + private static TsBlock stringBlock(boolean runLengthEncodedTime, String... values) { + Binary[] binaries = + Arrays.stream(values) + .map(value -> new Binary(value, TSFileConfig.STRING_CHARSET)) + .toArray(Binary[]::new); + return new TsBlock( + block(runLengthEncodedTime, new double[values.length]).getTimeColumn(), + new BinaryColumn(values.length, Optional.empty(), binaries)); + } + private static TsBlock block(boolean runLengthEncodedTime, double... values) { Column timeColumn = runLengthEncodedTime @@ -498,58 +668,4 @@ public long ramBytesUsed() { return 0; } } - - private static class TsBlockSourceOperator implements Operator { - private final Iterator blocks; - private final CommonOperatorContext context = new TestOperatorContext(); - - private TsBlockSourceOperator(TsBlock[] blocks) { - this.blocks = Arrays.asList(blocks).iterator(); - } - - @Override - public CommonOperatorContext getOperatorContext() { - return context; - } - - @Override - public TsBlock next() { - return blocks.next(); - } - - @Override - public boolean hasNext() { - return blocks.hasNext(); - } - - @Override - public boolean isFinished() { - return !hasNext(); - } - - @Override - public void close() { - // No resources to release. - } - - @Override - public long calculateMaxPeekMemory() { - return 0; - } - - @Override - public long calculateMaxReturnSize() { - return 0; - } - - @Override - public long calculateRetainedSizeAfterCallingNext() { - return 0; - } - - @Override - public long ramBytesUsed() { - return 0; - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 04e08c9d133b7..8d42d07b2a374 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -45,6 +45,7 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; +import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.appendCommonExpressionColumns; import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.constructFilteredTsBlock; public class DeviceAttributeUpdater extends DeviceUpdater { @@ -152,13 +153,8 @@ private TsBlock getFilterTsBlock() { final List filterResultColumns = new ArrayList<>(Arrays.asList(curBlock.getValueColumns())); - // CASE evaluation can clear a common expression after the filter used it. Recompute it before - // materializing the column for projection within this same batch. - commonTransformerList.forEach( - columnTransformer -> { - columnTransformer.tryEvaluate(); - filterResultColumns.add(columnTransformer.getColumn()); - }); + appendCommonExpressionColumns( + filterResultColumns, commonTransformerList, curFilterColumn, deviceSchemaBatch.size()); final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); final int rowCount = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java index 08b56188b2125..149062fa53e48 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.UpperColumnTransformer; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.table.TsTable; @@ -164,10 +165,10 @@ public void testAttributeUpdaterDoesNotReuseProjectionAcrossBatches() throws Exc @Test public void testAttributeUpdaterRecomputesCommonExpressionClearedByCase() throws Exception { Graph filter = new Graph(); - ColumnTransformer shared = filter.input(0); + ColumnTransformer shared = new UpperColumnTransformer(STRING, reference(filter.input(0))); ColumnTransformer predicate = filter.caseWhen( - filter.equal(shared, filter.constant("match")), + filter.equal(shared, filter.constant("MATCH")), filter.constant(true), filter.constant(false)); Graph projection = new Graph(); @@ -183,7 +184,7 @@ public void testAttributeUpdaterRecomputesCommonExpressionClearedByCase() throws updateBatch(updater, "match"); updateBatch(updater, "other", "match"); assertEquals( - Arrays.asList(Arrays.asList("match", "match"), Arrays.asList("match", "match")), updates); + Arrays.asList(Arrays.asList("match", "MATCH"), Arrays.asList("match", "MATCH")), updates); } } From aecaa05e84d4ab3bc794c6b2df1cb1abb559d793 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 12:39:10 +0800 Subject: [PATCH 08/13] Document cache follow-ups and preserve schema constructor compatibility Track the known DIFF failure with a correct-result reproducer. Guard common-expression evaluation counts, document helper side effects and shallow array accounting, and retain constructor descriptors used by precompiled subclasses. --- .../process/FilterAndProjectOperator.java | 11 +++ .../process/FilterAndProjectOperatorTest.java | 80 +++++++++++++++++++ .../schema/source/DevicePredicateHandler.java | 17 ++++ .../operator/schema/source/DeviceUpdater.java | 19 +++++ 4 files changed, 127 insertions(+) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index b258ee71ebf1f..5307cde136075 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -220,6 +220,15 @@ private TsBlock getFilterTsBlock(TsBlock input) { /** * Materializes common expressions for projection after filtering. CASE may have invalidated their * caches, so any recomputation must preserve the filter mask and skip rejected input rows. + * + *

This mutates both the destination list and transformer cache state: uncached expressions are + * evaluated with selection, which can replace or clear descendant caches, and each appended + * column is consumed through {@link ColumnTransformer#getColumn()}, decrementing its cache + * reference count and possibly releasing the cached reference. Existing cached columns are + * reused; this method does not reset the DAG for a new input block. + * + *

Appended columns retain the original input row indexes. The caller must use the same filter + * to construct the compacted output block. */ public static void appendCommonExpressionColumns( List resultColumns, @@ -477,6 +486,8 @@ private int getMaxLevelOfColumnTransformerTree(ColumnTransformer columnTransform @Override public long ramBytesUsed() { + // Shallow array sizes include the header and every reference slot. This estimate does not + // include the transformer objects referenced by the arrays. return INSTANCE_SIZE + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(inputOperator) + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 0ba2c4d7ef50c..c9ac3ad328f41 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -36,6 +36,7 @@ import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.ternary.BetweenColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.CastFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -50,6 +51,7 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.Binary; +import org.junit.Ignore; import org.junit.Test; import java.time.ZoneId; @@ -175,6 +177,62 @@ public void testFilterCommonExpressionsDoNotCrossBlocks() throws Exception { } } + @Test + public void testProjectionReadsMaterializedCommonExpression() throws Exception { + for (boolean withCase : new boolean[] {false, true}) { + Graph filter = new Graph(); + CountingAdditionColumnTransformer common = + new CountingAdditionColumnTransformer( + reference(filter.input(0)), reference(filter.constant(1))); + ColumnTransformer predicate = filter.greaterThan(common, 0); + if (withCase) { + predicate = + filter.greaterThan( + filter.caseWhen(predicate, filter.constant(1), filter.constant(0)), 0); + } + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(-2, 3), block(4, -5)}, + filter, + predicate, + Collections.singletonList(common), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {4.0}); + assertEquals(1, common.fullEvaluationCount); + assertEquals(withCase ? 1 : 0, common.selectedEvaluationCount); + assertBlock(operator.next(), new Double[] {5.0}); + assertEquals(2, common.fullEvaluationCount); + assertEquals(withCase ? 2 : 0, common.selectedEvaluationCount); + } + } + } + + @Ignore("Known incorrect DIFF state advancement: https://github.com/apache/iotdb/issues/18863") + @Test + public void testStatefulDiffIsNotReevaluatedAfterCaseFilter() throws Exception { + Graph filter = new Graph(); + ColumnTransformer diff = + new DiffFunctionColumnTransformer(DOUBLE, reference(filter.input(0)), true); + ColumnTransformer predicate = + filter.greaterThan( + filter.caseWhen(filter.greaterThan(diff, 1), filter.constant(1), filter.constant(0)), + 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1, 3, 6), block(10, 11)}, + filter, + predicate, + Collections.singletonList(diff), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {2.0, 3.0}); + assertBlock(operator.next(), new Double[] {4.0}); + } + } + @Test public void testCommonCastOnlyEvaluatesRowsPassingCaseFilter() throws Exception { for (boolean runLengthEncodedTime : new boolean[] {false, true}) { @@ -623,6 +681,28 @@ private ColumnTransformer caseWhen( } } + private static class CountingAdditionColumnTransformer + extends ArithmeticAdditionColumnTransformer { + private int fullEvaluationCount; + private int selectedEvaluationCount; + + private CountingAdditionColumnTransformer(ColumnTransformer left, ColumnTransformer right) { + super(DOUBLE, left, right); + } + + @Override + public void evaluate() { + fullEvaluationCount++; + super.evaluate(); + } + + @Override + public void evaluateWithSelection(boolean[] selection) { + selectedEvaluationCount++; + super.evaluateWithSelection(selection); + } + } + private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { private int evaluationCount; private int childTraversalCount; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java index dd933d8403e75..b3cf75f120cc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java @@ -35,6 +35,7 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -60,6 +61,22 @@ public abstract class DevicePredicateHandler implements AutoCloseable { protected TsBlock curBlock; protected Column curFilterColumn; + // Retain the original constructor descriptor for precompiled subclasses without common roots. + protected DevicePredicateHandler( + final List filterLeafColumnTransformerList, + final ColumnTransformer filterOutputTransformer, + final List columnSchemaList, + final String database, + final TsTable table) { + this( + filterLeafColumnTransformerList, + filterOutputTransformer, + Collections.emptyList(), + columnSchemaList, + database, + table); + } + protected DevicePredicateHandler( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java index 5debad67608e3..c6690634e91cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java @@ -31,6 +31,7 @@ import org.apache.tsfile.utils.Binary; +import java.util.Collections; import java.util.List; import java.util.function.BiFunction; @@ -38,6 +39,24 @@ public abstract class DeviceUpdater extends DevicePredicateHandler { private final BiFunction attributeProvider; + // Retain the original constructor descriptor for precompiled subclasses without common roots. + protected DeviceUpdater( + final List filterLeafColumnTransformerList, + final ColumnTransformer filterOutputTransformer, + final List columnSchemaList, + final BiFunction attributeProvider, + final String database, + final TsTable table) { + this( + filterLeafColumnTransformerList, + filterOutputTransformer, + Collections.emptyList(), + columnSchemaList, + attributeProvider, + database, + table); + } + protected DeviceUpdater( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, From 2fb7ad284e04134790b8b1065932611e27f99d1a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 14:52:56 +0800 Subject: [PATCH 09/13] Isolate selected columns and preserve DIFF state per input --- .../db/it/query/IoTDBCaseWhenThenIT.java | 16 +++ .../it/db/it/IoTDBCaseWhenThenTableIT.java | 40 +++++++ .../process/FilterAndProjectOperator.java | 25 ++--- ...AbstractCaseWhenThenColumnTransformer.java | 45 +++----- .../dag/column/ColumnCache.java | 4 + .../dag/column/ColumnTransformer.java | 63 ++++++++--- .../column/FailFunctionColumnTransformer.java | 5 +- .../binary/BinaryColumnTransformer.java | 19 ++-- .../binary/LogicAndColumnTransformer.java | 14 +-- .../binary/LogicOrColumnTransformer.java | 14 +-- .../leaf/ConstantColumnTransformer.java | 4 +- .../leaf/IdentityColumnTransformer.java | 4 +- .../column/leaf/NullColumnTransformer.java | 5 +- .../column/leaf/TimeColumnTransformer.java | 4 +- .../LogicalAndMultiColumnTransformer.java | 12 +-- .../LogicalOrMultiColumnTransformer.java | 11 +- .../multi/MappableUDFColumnTransformer.java | 43 ++++++-- .../column/multi/MultiColumnTransformer.java | 13 +-- .../ternary/TernaryColumnTransformer.java | 20 ++-- .../UserDefineScalarFunctionTransformer.java | 2 +- .../column/unary/UnaryColumnTransformer.java | 8 +- .../scalar/DiffFunctionColumnTransformer.java | 45 +++++--- .../process/FilterAndProjectOperatorTest.java | 101 ++++++++++++++++-- .../dag/column/ColumnTransformerTest.java | 90 ++++++++++++++++ .../DiffFunctionColumnTransformerTest.java | 89 +++++++++++++++ .../schema/source/DeviceAttributeUpdater.java | 12 ++- ...neScalarFunctionColumnTransformerTest.java | 89 +++++++++++++++ .../hmac/HmacMd5ColumnTransformerTest.java | 1 + .../hmac/HmacSha1ColumnTransformerTest.java | 1 + .../hmac/HmacSha256ColumnTransformerTest.java | 1 + .../hmac/HmacSha512ColumnTransformerTest.java | 1 + .../ternary/LpadColumnTransformerTest.java | 3 +- .../ternary/RpadColumnTransformerTest.java | 3 +- .../BlobLengthColumnTransformerTest.java | 3 +- .../FromBigEndian32ColumnTransformerTest.java | 3 +- .../FromBigEndian64ColumnTransformerTest.java | 3 +- ...E754_32BigEndianColumnTransformerTest.java | 3 +- ...E754_64BigEndianColumnTransformerTest.java | 3 +- ...omLittleEndian32ColumnTransformerTest.java | 3 +- ...omLittleEndian64ColumnTransformerTest.java | 3 +- .../ToBigEndian32ColumnTransformerTest.java | 3 +- .../ToBigEndian64ColumnTransformerTest.java | 3 +- ...E754_32BigEndianColumnTransformerTest.java | 3 +- ...E754_64BigEndianColumnTransformerTest.java | 3 +- ...ToLittleEndian32ColumnTransformerTest.java | 3 +- ...ToLittleEndian64ColumnTransformerTest.java | 3 +- .../scalar/ReverseColumnTransformerTest.java | 3 +- .../FromBase32ColumnTransformerTest.java | 3 +- .../FromBase64ColumnTransformerTest.java | 3 +- .../FromBase64UrlColumnTransformerTest.java | 3 +- .../ToBase32ColumnTransformerTest.java | 3 +- .../ToBase64ColumnTransformerTest.java | 3 +- .../ToBase64UrlColumnTransformerTest.java | 3 +- .../hashing/Crc32ColumnTransformerTest.java | 3 +- .../hashing/Md5ColumnTransformerTest.java | 3 +- .../hashing/Murmur3ColumnTransformerTest.java | 3 +- .../hashing/Sha1ColumnTransformerTest.java | 3 +- .../hashing/Sha256ColumnTransformerTest.java | 3 +- .../hashing/Sha512ColumnTransformerTest.java | 3 +- .../SpookyHashV2_32ColumnTransformerTest.java | 3 +- .../SpookyHashV2_64ColumnTransformerTest.java | 3 +- .../XxHash64ColumnTransformerTest.java | 3 +- .../FromHexColumnTransformerTest.java | 3 +- .../ToHexColumnTransformerTest.java | 3 +- 64 files changed, 685 insertions(+), 215 deletions(-) create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/UserDefineScalarFunctionColumnTransformerTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java index a88b89e90b5da..b467f0139e156 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java @@ -928,4 +928,20 @@ public void testThenWithBinarySameConstant() { new String[] {"0,0.0,", "1000000,11.0,", "20000000,22.0,", "210000000,33.0,"}; resultSetEqualTest(sql, expectedHeader, retArray); } + + @Test + public void testNullableAndInLaterCaseBranch() { + prepareData( + new String[] { + "CREATE TIMESERIES root.sg.case_null.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.case_null.s2 WITH DATATYPE=INT32, ENCODING=PLAIN", + "INSERT INTO root.sg.case_null(time, s2) VALUES (1, 1)", + "INSERT INTO root.sg.case_null(time, s1, s2) VALUES (2, 1, 1), (3, 0, 1), (4, 101, 1)" + }); + resultSetEqualTest( + "SELECT CASE WHEN s1 > 100 THEN 1 WHEN s1 > 0 AND s2 > 0 THEN 2 ELSE 3 END AS result " + + "FROM root.sg.case_null", + new String[] {TIMESTAMP_STR, "result"}, + new String[] {"1,3.0,", "2,2.0,", "3,3.0,", "4,1.0,"}); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index 0839c7e1f3415..4cbc77697abb5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -72,6 +72,7 @@ public class IoTDBCaseWhenThenTableIT { @BeforeClass public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockLineNumber(1); EnvFactory.getEnv().initClusterEnvironment(); prepareTableData(SQLs); List moreSQLs = new ArrayList<>(); @@ -811,4 +812,43 @@ public void testReusingCasePreservesPendingSiblingColumn() { new String[] {"0.0,0.0,", "0.0,3.0,", "1.0,4.0,"}, DATABASE); } + + @Test + public void testSharedDiffAcrossCaseFilterAndProjection() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_shared_diff (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_shared_diff(time, dev, x) VALUES " + + "(1, 'd0', 1.0), (2, 'd0', 3.0), (3, 'd0', 6.0)", + "FLUSH", + "INSERT INTO case_shared_diff(time, dev, x) VALUES (4, 'd0', 10.0), (5, 'd0', 11.0)", + "FLUSH" + }); + tableResultSetEqualTest( + "SELECT time, diff(x) AS d FROM case_shared_diff " + + "WHERE CASE WHEN diff(x) > 1.0 THEN 1.0 ELSE 0.0 END > 0.0", + new String[] {"time", "d"}, + new String[] { + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + "1970-01-01T00:00:00.004Z,4.0," + }, + DATABASE); + tableResultSetEqualTest( + "SELECT diff(x) AS d FROM case_shared_diff " + + "WHERE CASE WHEN diff(x) > 1.0 THEN true ELSE true END", + new String[] {"d"}, + new String[] {"null,", "2.0,", "3.0,", "4.0,", "1.0,"}, + DATABASE); + tableResultSetEqualTest( + "SELECT CASE WHEN x > 5 THEN diff(x) ELSE 100.0 END AS a, " + + "CASE WHEN x > 5 THEN 200.0 ELSE diff(x) END AS b, diff(x) AS d " + + "FROM case_shared_diff", + new String[] {"a", "b", "d"}, + new String[] { + "100.0,null,null,", "100.0,2.0,2.0,", "3.0,200.0,3.0,", "4.0,200.0,4.0,", "1.0,200.0,1.0," + }, + DATABASE); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 5307cde136075..148c5eca93190 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -165,9 +165,9 @@ public TsBlock next() throws Exception { private TsBlock getFilterTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); - // Short-circuit evaluation can reset cache reference counts when a shared expression is - // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate - // expressions whose caches may still have unconsumed references from the previous block. + // A selected consumer does not consume full-cache references. Release all results from the + // preceding input, including stateful transformers' per-input results, before feeding this + // block. for (ColumnTransformer columnTransformer : filterCacheTransformers) { columnTransformer.clearCacheForNewInput(); } @@ -217,20 +217,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { } } - /** - * Materializes common expressions for projection after filtering. CASE may have invalidated their - * caches, so any recomputation must preserve the filter mask and skip rejected input rows. - * - *

This mutates both the destination list and transformer cache state: uncached expressions are - * evaluated with selection, which can replace or clear descendant caches, and each appended - * column is consumed through {@link ColumnTransformer#getColumn()}, decrementing its cache - * reference count and possibly releasing the cached reference. Existing cached columns are - * reused; this method does not reset the DAG for a new input block. - * - *

Appended columns retain the original input row indexes. The caller must use the same filter - * to construct the compacted output block. - */ - public static void appendCommonExpressionColumns( + // Selected results belong to this materialization, not the shared full-column caches. + private static void appendCommonExpressionColumns( List resultColumns, List commonTransformers, Column filterColumn, @@ -243,8 +231,7 @@ public static void appendCommonExpressionColumns( selection[i] = satisfy(filterColumn, i); } for (ColumnTransformer transformer : commonTransformers) { - transformer.tryEvaluateWithSelection(selection); - resultColumns.add(transformer.getColumn()); + resultColumns.add(transformer.getColumnWithSelection(selection)); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index e110b7331b120..a2f9af534bd9d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -69,7 +69,7 @@ public void evaluate() { // region evaluate first when ColumnTransformer firstWhenColumnTransformer = whenThenTransformers.get(0).left; - firstWhenColumnTransformer.evaluate(); + firstWhenColumnTransformer.tryEvaluate(); Column firstWhenColumn = firstWhenColumnTransformer.getColumn(); int positionCount = firstWhenColumn.getPositionCount(); @@ -97,8 +97,7 @@ public void evaluate() { } ColumnTransformer firstThenColumnTransformer = whenThenTransformers.get(0).right; - firstThenColumnTransformer.evaluateWithSelection(selectionForThen); - Column firstThenColumn = firstThenColumnTransformer.getColumn(); + Column firstThenColumn = firstThenColumnTransformer.getColumnWithSelection(selectionForThen); thenColumnList.add(firstThenColumn); // endregion @@ -106,8 +105,7 @@ public void evaluate() { // when and then columns for (int i = 1; i < whenThenTransformers.size(); i++) { ColumnTransformer whenColumnTransformer = whenThenTransformers.get(i).left; - whenColumnTransformer.evaluateWithSelection(selection); - Column whenColumn = whenColumnTransformer.getColumn(); + Column whenColumn = whenColumnTransformer.getColumnWithSelection(selection); selectionForThen = selection.clone(); @@ -123,17 +121,16 @@ public void evaluate() { } ColumnTransformer thenColumnTransformer = whenThenTransformers.get(i).right; - thenColumnTransformer.evaluateWithSelection(selectionForThen); - Column thenColumn = thenColumnTransformer.getColumn(); + Column thenColumn = thenColumnTransformer.getColumnWithSelection(selectionForThen); thenColumnList.add(thenColumn); } // elseColumn - doTransform(branchIndexForEachRow, positionCount, thenColumnList); + initializeColumnCache(doTransform(branchIndexForEachRow, positionCount, thenColumnList)); } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { // region initialize branchIndexForEachRow. // branchIndexForEachRow indicates the index of the WhenTransformer matched by each row. @@ -158,8 +155,7 @@ public void evaluateWithSelection(boolean[] selection) { // when and then columns for (int i = 0; i < whenThenTransformers.size(); i++) { ColumnTransformer whenColumnTransformer = whenThenTransformers.get(i).left; - whenColumnTransformer.evaluateWithSelection(selectionForWhen); - Column whenColumn = whenColumnTransformer.getColumn(); + Column whenColumn = whenColumnTransformer.getColumnWithSelection(selectionForWhen); boolean[] selectionForThen = selectionForWhen.clone(); @@ -175,17 +171,16 @@ public void evaluateWithSelection(boolean[] selection) { } ColumnTransformer thenColumnTransformer = whenThenTransformers.get(i).right; - thenColumnTransformer.evaluateWithSelection(selectionForThen); - Column thenColumn = thenColumnTransformer.getColumn(); + Column thenColumn = thenColumnTransformer.getColumnWithSelection(selectionForThen); thenColumnList.add(thenColumn); } // elseColumn // when selectionForElse[i] is false represent the rows that do not need to evaluate - doTransform(branchIndexForEachRow, positionCount, thenColumnList); + return doTransform(branchIndexForEachRow, positionCount, thenColumnList); } - private void doTransform( + private Column doTransform( int[] branchIndexForEachRow, int positionCount, List thenColumnList) { boolean[] selectionForElse = new boolean[positionCount]; for (int i = 0; i < branchIndexForEachRow.length; i++) { @@ -193,10 +188,8 @@ private void doTransform( selectionForElse[i] = true; } } - elseTransformer.evaluateWithSelection(selectionForElse); - + Column elseColumn = elseTransformer.getColumnWithSelection(selectionForElse); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); - Column elseColumn = elseTransformer.getColumn(); for (int i = 0; i < positionCount; i++) { Column resultColumn = null; @@ -216,15 +209,7 @@ private void doTransform( } } - initializeColumnCache(builder.build()); - // Because of short-circuit evaluation, CaseWhen does not calculate all Then phrases, so its - // calculation results cannot be reused. - for (Pair whenThenColumnTransformer : - whenThenTransformers) { - whenThenColumnTransformer.left.clearCache(); - whenThenColumnTransformer.right.clearCache(); - } - elseTransformer.clearCache(); + return builder.build(); } protected abstract void writeToColumnBuilder( @@ -235,12 +220,6 @@ protected void checkType() { // do nothing } - @Override - protected boolean shouldClearChildrenCache() { - // Clearing a cached CASE during evaluation must preserve pending sibling columns. - return false; - } - @Override protected void forEachChild(Consumer action) { for (Pair whenThenColumnTransformer : diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java index 6ada45d6d10e4..c7cf716c1fcaf 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java @@ -62,4 +62,8 @@ public void cacheColumn(Column column, int referenceCount) { public boolean hasCached() { return column != null; } + + public Column peekColumn() { + return column; + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index d21b89b783db9..b93248fa8234d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -40,6 +40,9 @@ public abstract class ColumnTransformer { protected final ColumnCache columnCache; + // Compatibility with the evaluateWithSelection()/getColumn() API. Never used as a full cache. + private ColumnCache selectedColumnCache; + protected int referenceCount; protected ColumnTransformer(Type returnType) { @@ -49,31 +52,54 @@ protected ColumnTransformer(Type returnType) { } public void tryEvaluate() { + selectedColumnCache = null; if (!columnCache.hasCached()) { evaluate(); } } - /** Reuses the cached column, or evaluates with the selection if it was invalidated. */ + /** + * @deprecated Use {@link #getColumnWithSelection(boolean[])} to retain the result directly. + */ + @Deprecated public void tryEvaluateWithSelection(boolean[] selection) { + selectedColumnCache = null; if (!columnCache.hasCached()) { evaluateWithSelection(selection); } } public Column getColumn() { + if (selectedColumnCache != null) { + return selectedColumnCache.getColumn(); + } return columnCache.getColumn(); } + /** + * Returns a column whose selected rows are evaluated, without consuming or replacing the shared + * full-column cache. Unselected rows are unspecified. Callers must retain the returned column and + * only read selected rows. Partial results belong to the caller, so different CASE branches can + * use different masks without invalidating one another's columns. + */ + public Column getColumnWithSelection(boolean[] selection) { + Column fullColumn = columnCache.peekColumn(); + return fullColumn != null ? fullColumn : evaluateSelection(selection); + } + public void addReferenceCount() { referenceCount++; } public void initializeColumnCache(Column column) { + selectedColumnCache = null; columnCache.cacheColumn(column, referenceCount); } public int getColumnCachePositionCount() { + if (selectedColumnCache != null) { + return selectedColumnCache.getPositionCount(); + } return columnCache.getPositionCount(); } @@ -118,25 +144,35 @@ public boolean typeNotEquals(TypeEnum typeEnum) { /** Responsible for the calculation. */ protected abstract void evaluate(); - public abstract void evaluateWithSelection(boolean[] selection); + /** + * @deprecated Use {@link #getColumnWithSelection(boolean[])} to retain the result directly. + */ + @Deprecated + public void evaluateWithSelection(boolean[] selection) { + Column result = getColumnWithSelection(selection); + selectedColumnCache = new ColumnCache(); + selectedColumnCache.cacheColumn(result, referenceCount); + } + + protected abstract Column evaluateSelection(boolean[] selection); protected abstract void checkType(); public void close() { - // do nothing + columnCache.clear(); + selectedColumnCache = null; } /** - * Clears caches during evaluation. Subclasses should define child edges via {@link + * Clears cached results in this subgraph. Subclasses should define child edges via {@link * #forEachChild(Consumer)} instead of overriding this method. * *

Kept non-final so existing Mockito-based tests can stub it. */ public void clearCache() { + selectedColumnCache = null; this.columnCache.clear(); - if (shouldClearChildrenCache()) { - forEachChild(ColumnTransformer::clearCache); - } + forEachChild(ColumnTransformer::clearCache); } /** @@ -145,7 +181,14 @@ public void clearCache() { * the new block, when no sibling expression can still need a cached column. */ public final void clearCacheForNewInput() { + selectedColumnCache = null; this.columnCache.clear(); + resetForNewInput(); + } + + /** Releases input-specific state while preserving state carried between blocks, such as DIFF. */ + protected void resetForNewInput() { + // Stateless transformers only need their shared cache cleared. } /** @@ -170,10 +213,6 @@ private void collectTransformers( } } - protected boolean shouldClearChildrenCache() { - return true; - } - - /** Defines the child edges used by both evaluation-time and new-input cache clearing. */ + /** Defines the child edges used for cache clearing and input lifecycle management. */ protected abstract void forEachChild(Consumer action); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java index 07b484deff0d4..af52965ab742d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.read.common.block.column.NullColumn; import org.apache.tsfile.read.common.type.Type; @@ -46,7 +47,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { // if there is true value in selection, throw exception for (boolean b : selection) { if (b) { @@ -54,7 +55,7 @@ public void evaluateWithSelection(boolean[] selection) { } } // Result of fail function should be ignored, we just fake the output here. - initializeColumnCache(new NullColumn(1)); + return new NullColumn(selection.length); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index d1715b12c3c8a..4390626276761 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -47,7 +47,7 @@ public void evaluate() { // attention: get positionCount before calling getColumn int positionCount = leftTransformer.getColumnCachePositionCount(); Column leftColumn = leftTransformer.getColumn(); - // A later child can clear shared caches; retain each column before evaluating its sibling. + // Retain each input column before evaluating the next sibling. rightTransformer.tryEvaluate(); Column rightColumn = rightTransformer.getColumn(); @@ -57,21 +57,14 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - leftTransformer.evaluateWithSelection(selection); - // attention: get positionCount before calling getColumn - int positionCount = leftTransformer.getColumnCachePositionCount(); - Column leftColumn = leftTransformer.getColumn(); - // When the rightTransformer has the same constant as the leftTransformer, the leftTransformer's - // cache will be cleared, so need to getColumn before evaluate rightTransformer. - rightTransformer.evaluateWithSelection(selection); - Column rightColumn = rightTransformer.getColumn(); + protected Column evaluateSelection(boolean[] selection) { + Column leftColumn = leftTransformer.getColumnWithSelection(selection); + Column rightColumn = rightTransformer.getColumnWithSelection(selection); + int positionCount = leftColumn.getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); doTransform(leftColumn, rightColumn, builder, positionCount, selection); - initializeColumnCache(builder.build()); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); + return builder.build(); } protected abstract void doTransform( diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java index 5e7197eaab893..3516784149137 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java @@ -32,21 +32,19 @@ public LogicAndColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); - leftTransformer.evaluateWithSelection(selectionCopy); - int positionCount = leftTransformer.getColumnCachePositionCount(); - Column leftColumn = leftTransformer.getColumn(); + Column leftColumn = leftTransformer.getColumnWithSelection(selectionCopy); + int positionCount = leftColumn.getPositionCount(); for (int i = 0; i < positionCount; i++) { if (!leftColumn.isNull(i) && !leftColumn.getBoolean(i)) { selectionCopy[i] = false; } } - rightTransformer.evaluateWithSelection(selectionCopy); - Column rightColumn = rightTransformer.getColumn(); + Column rightColumn = rightTransformer.getColumnWithSelection(selectionCopy); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); for (int i = 0; i < positionCount; i++) { @@ -77,9 +75,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(rightColumn); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java index 5f2437238b7f6..fa7ca37519a15 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java @@ -32,21 +32,19 @@ public LogicOrColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); - leftTransformer.evaluateWithSelection(selectionCopy); - int positionCount = leftTransformer.getColumnCachePositionCount(); - Column leftColumn = leftTransformer.getColumn(); + Column leftColumn = leftTransformer.getColumnWithSelection(selectionCopy); + int positionCount = leftColumn.getPositionCount(); for (int i = 0; i < positionCount; i++) { if (!leftColumn.isNull(i) && leftColumn.getBoolean(i)) { selectionCopy[i] = false; } } - rightTransformer.evaluateWithSelection(selectionCopy); - Column rightColumn = rightTransformer.getColumn(); + Column rightColumn = rightTransformer.getColumnWithSelection(selectionCopy); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); @@ -78,9 +76,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(builder.build()); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java index d16072d729838..aa919caf19acf 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java @@ -39,7 +39,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { int positionCount = input.getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); for (int i = 0; i < positionCount; i++) { @@ -49,6 +49,6 @@ public void evaluateWithSelection(boolean[] selection) { builder.write(value, 0); } } - initializeColumnCache(builder.build()); + return builder.build(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java index 38dfd063ac19c..14a03983f2158 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java @@ -43,7 +43,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { ColumnBuilder builder = returnType.createColumnBuilder(selection.length); Column inputColumn = input.getColumn(inputIndex); for (int i = 0; i < selection.length; i++) { @@ -53,6 +53,6 @@ public void evaluateWithSelection(boolean[] selection) { builder.write(inputColumn, i); } } - initializeColumnCache(builder.build()); + return builder.build(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java index 31a01f67f3047..db1a2430371a0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.calc.transformation.dag.column.leaf; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.read.common.block.column.NullColumn; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; @@ -35,7 +36,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - initializeColumnCache(new NullColumn(input.getPositionCount())); + protected Column evaluateSelection(boolean[] selection) { + return new NullColumn(input.getPositionCount()); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java index 8d359a0904a6e..2e220ba2216bb 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java @@ -36,7 +36,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { ColumnBuilder builder = TIMESTAMP.createColumnBuilder(selection.length); Column timeColumn = input.getTimeColumn(); for (int i = 0; i < selection.length; i++) { @@ -46,6 +46,6 @@ public void evaluateWithSelection(boolean[] selection) { builder.write(timeColumn, i); } } - initializeColumnCache(builder.build()); + return builder.build(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java index 7e8dce8502e6c..2309d8f92e51a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java @@ -36,14 +36,13 @@ public LogicalAndMultiColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); boolean[] hasNull = new boolean[selection.length]; List childrenColumns = new ArrayList<>(); for (ColumnTransformer child : columnTransformerList) { - child.evaluateWithSelection(selectionCopy); - Column childColumn = child.getColumn(); + Column childColumn = child.getColumnWithSelection(selectionCopy); childrenColumns.add(childColumn); for (int i = 0; i < childColumn.getPositionCount(); i++) { if (childColumn.isNull(i)) { @@ -77,12 +76,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(builder.build()); - - // clear children's cache - for (ColumnTransformer child : columnTransformerList) { - child.clearCache(); - } + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java index 326948200130c..b91ae096a2bb3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java @@ -36,14 +36,13 @@ public LogicalOrMultiColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); boolean[] hasNull = new boolean[selection.length]; List childColumns = new ArrayList<>(); for (ColumnTransformer child : columnTransformerList) { - child.evaluateWithSelection(selectionCopy); - Column childColumn = child.getColumn(); + Column childColumn = child.getColumnWithSelection(selectionCopy); childColumns.add(childColumn); for (int i = 0; i < childColumn.getPositionCount(); i++) { @@ -77,11 +76,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(builder.build()); - - for (ColumnTransformer child : columnTransformerList) { - child.clearCache(); - } + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 9826c6a9b0428..40f9090467520 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -47,33 +47,54 @@ public void evaluate() { Column[] columns = new Column[inputColumnTransformers.length]; for (int i = 0; i < inputColumnTransformers.length; i++) { inputColumnTransformers[i].tryEvaluate(); - // A later child can clear shared caches; retain each column before evaluating its sibling. + // Retain each input column before evaluating the next sibling. columns[i] = inputColumnTransformers[i].getColumn(); } - doTransform(columns); + initializeColumnCache(doTransform(columns)); } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { + int[] positions = new int[selection.length]; + int selectedCount = 0; + for (int i = 0; i < selection.length; i++) { + if (selection[i]) { + positions[selectedCount++] = i; + } + } Column[] columns = new Column[inputColumnTransformers.length]; for (int i = 0; i < inputColumnTransformers.length; i++) { - inputColumnTransformers[i].evaluateWithSelection(selection); - columns[i] = inputColumnTransformers[i].getColumn(); + columns[i] = + inputColumnTransformers[i] + .getColumnWithSelection(selection) + .getPositions(positions, 0, selectedCount); } - doTransform(columns); - // clear cache - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.clearCache(); + ColumnBuilder builder = returnType.createColumnBuilder(selection.length); + // A mappable UDF only sees selected rows, even when its inputs reuse full cached columns. + Column result = selectedCount == 0 ? null : doTransform(columns); + int resultIndex = 0; + for (boolean selected : selection) { + if (selected) { + if (result.isNull(resultIndex)) { + builder.appendNull(); + } else { + builder.write(result, resultIndex); + } + resultIndex++; + } else { + builder.appendNull(); + } } + return builder.build(); } - private void doTransform(Column[] columns) { + private Column doTransform(Column[] columns) { // construct input TsBlock with columns int positionCount = columns[0].getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); // executor UDF and cache result executor.execute(columns, builder); - initializeColumnCache(builder.build()); + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index cfd9c8913f24b..6d45a0f2221b2 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -45,7 +45,7 @@ public void evaluate() { List childrenColumns = new ArrayList<>(columnTransformerList.size()); for (ColumnTransformer child : columnTransformerList) { child.tryEvaluate(); - // A later child can clear shared caches; retain each column before evaluating its sibling. + // Retain each input column before evaluating the next sibling. childrenColumns.add(child.getColumn()); } @@ -57,12 +57,11 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { List childrenColumns = new ArrayList<>(); for (ColumnTransformer child : columnTransformerList) { - child.evaluateWithSelection(selection); - childrenColumns.add(child.getColumn()); + childrenColumns.add(child.getColumnWithSelection(selection)); } // getColumn() can release the first child's cache; read the retained column's size instead. @@ -70,11 +69,7 @@ public void evaluateWithSelection(boolean[] selection) { ColumnBuilder builder = returnType.createColumnBuilder(positionCount); doTransform(childrenColumns, builder, positionCount, selection); - initializeColumnCache(builder.build()); - - for (ColumnTransformer child : columnTransformerList) { - child.clearCache(); - } + return builder.build(); } protected abstract void doTransform( diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index a1b7c91357cd3..03c56d7d90900 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -52,7 +52,7 @@ public void evaluate() { firstColumnTransformer.tryEvaluate(); int positionCount = firstColumnTransformer.getColumnCachePositionCount(); Column firstColumn = firstColumnTransformer.getColumn(); - // A later child can clear shared caches; retain each column before evaluating its sibling. + // Retain each input column before evaluating the next sibling. secondColumnTransformer.tryEvaluate(); Column secondColumn = secondColumnTransformer.getColumn(); thirdColumnTransformer.tryEvaluate(); @@ -63,20 +63,14 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - firstColumnTransformer.evaluateWithSelection(selection); - int positionCount = firstColumnTransformer.getColumnCachePositionCount(); - Column firstColumn = firstColumnTransformer.getColumn(); - secondColumnTransformer.evaluateWithSelection(selection); - Column secondColumn = secondColumnTransformer.getColumn(); - thirdColumnTransformer.evaluateWithSelection(selection); - Column thirdColumn = thirdColumnTransformer.getColumn(); + protected Column evaluateSelection(boolean[] selection) { + Column firstColumn = firstColumnTransformer.getColumnWithSelection(selection); + Column secondColumn = secondColumnTransformer.getColumnWithSelection(selection); + Column thirdColumn = thirdColumnTransformer.getColumnWithSelection(selection); + int positionCount = firstColumn.getPositionCount(); ColumnBuilder columnBuilder = returnType.createColumnBuilder(positionCount); doTransform(firstColumn, secondColumn, thirdColumn, columnBuilder, positionCount, selection); - initializeColumnCache(columnBuilder.build()); - firstColumnTransformer.clearCache(); - secondColumnTransformer.clearCache(); - thirdColumnTransformer.clearCache(); + return columnBuilder.build(); } protected abstract void doTransform( diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java index 6ef156a6b707c..cddc145fb5ec9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java @@ -112,7 +112,7 @@ protected void doTransform( while (iterator.hasNext()) { try { Record input = iterator.next(); - if (selection[i++]) { + if (!selection[i++]) { builder.appendNull(); continue; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 1dcc6303534b6..496e8e8494e4c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -46,13 +46,11 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - childColumnTransformer.evaluateWithSelection(selection); - Column column = childColumnTransformer.getColumn(); + protected Column evaluateSelection(boolean[] selection) { + Column column = childColumnTransformer.getColumnWithSelection(selection); ColumnBuilder columnBuilder = returnType.createColumnBuilder(column.getPositionCount()); doTransform(column, columnBuilder, selection); - initializeColumnCache(columnBuilder.build()); - childColumnTransformer.clearCache(); + return columnBuilder.build(); } public ColumnTransformer getChildColumnTransformer() { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java index 04d65c594b9c5..dfa9d0808c16f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java @@ -26,8 +26,6 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Arrays; - public class DiffFunctionColumnTransformer extends UnaryColumnTransformer { // default is true @@ -39,6 +37,9 @@ public class DiffFunctionColumnTransformer extends UnaryColumnTransformer { // indicate whether lastValue is null private boolean lastValueIsNull = true; + // DIFF must advance its state once per input, even when several masks/consumers request it. + private Column currentInputResult; + public DiffFunctionColumnTransformer( Type returnType, ColumnTransformer childColumnTransformer, boolean ignoreNull) { super(returnType, childColumnTransformer); @@ -46,16 +47,36 @@ public DiffFunctionColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { - // The DIFF function depends on forward and backward rows and cannot be short-circuited. - boolean[] selectionCopy = new boolean[selection.length]; - Arrays.fill(selectionCopy, true); - childColumnTransformer.evaluateWithSelection(selectionCopy); - Column column = childColumnTransformer.getColumn(); - ColumnBuilder columnBuilder = returnType.createColumnBuilder(column.getPositionCount()); - doTransform(column, columnBuilder, selection); - initializeColumnCache(columnBuilder.build()); - childColumnTransformer.clearCache(); + public void evaluate() { + initializeColumnCache(evaluateCurrentInput()); + } + + @Override + protected Column evaluateSelection(boolean[] selection) { + // DIFF depends on all preceding rows, including rows rejected by a CASE/filter mask. + return evaluateCurrentInput(); + } + + private Column evaluateCurrentInput() { + if (currentInputResult == null) { + childColumnTransformer.tryEvaluate(); + Column column = childColumnTransformer.getColumn(); + ColumnBuilder builder = returnType.createColumnBuilder(column.getPositionCount()); + doTransform(column, builder); + currentInputResult = builder.build(); + } + return currentInputResult; + } + + @Override + protected void resetForNewInput() { + currentInputResult = null; + } + + @Override + public void close() { + currentInputResult = null; + super.close(); } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index c9ac3ad328f41..562ef33ce6d57 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -51,7 +51,6 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.Binary; -import org.junit.Ignore; import org.junit.Test; import java.time.ZoneId; @@ -201,15 +200,14 @@ public void testProjectionReadsMaterializedCommonExpression() throws Exception { projection.input(1))) { assertBlock(operator.next(), new Double[] {4.0}); assertEquals(1, common.fullEvaluationCount); - assertEquals(withCase ? 1 : 0, common.selectedEvaluationCount); + assertEquals(0, common.selectedEvaluationCount); assertBlock(operator.next(), new Double[] {5.0}); assertEquals(2, common.fullEvaluationCount); - assertEquals(withCase ? 2 : 0, common.selectedEvaluationCount); + assertEquals(0, common.selectedEvaluationCount); } } } - @Ignore("Known incorrect DIFF state advancement: https://github.com/apache/iotdb/issues/18863") @Test public void testStatefulDiffIsNotReevaluatedAfterCaseFilter() throws Exception { Graph filter = new Graph(); @@ -380,6 +378,35 @@ public void testMappableUdfPreservesSiblingColumnsBeforeCase() throws Exception } } + @Test + public void testSelectedMappableUdfSkipsRejectedRowsWithFullCachedInput() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer udf = + new MappableUDFColumnTransformer( + DOUBLE, + new ColumnTransformer[] {reference(input)}, + new UDTFExecutor("test", ZoneId.of("UTC")) { + @Override + public void execute(Column[] columns, ColumnBuilder builder) { + for (int i = 0; i < columns[0].getPositionCount(); i++) { + double value = columns[0].getDouble(i); + org.junit.Assert.assertTrue(value > 0); + builder.writeDouble(value + 1); + } + } + + @Override + public void beforeDestroy() {} + }); + ColumnTransformer output = graph.caseWhen(graph.greaterThan(input, 0), udf, graph.constant(0)); + try (Operator operator = + project(graph, new TsBlock[] {block(-2, 3), block(-4)}, input, output)) { + assertBlock(operator.next(), new Double[] {-2.0, 3.0}, new Double[] {0.0, 4.0}); + assertBlock(operator.next(), new Double[] {-4.0}, new Double[] {0.0}); + } + } + @Test public void testProjectionClearsCachedDescendantsOfNestedCase() throws Exception { Graph graph = new Graph(); @@ -410,6 +437,68 @@ public void testThenExpressionIsRecomputedWithoutSelectionForLaterOutput() throw } } + @Test + public void testTwoCasesUseIndependentMasksForSharedExpression() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer shared = graph.add(input, 1); + ColumnTransformer positive = + graph.caseWhen(graph.greaterThan(input, 0), shared, graph.constant(100)); + ColumnTransformer negative = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(200), shared); + try (Operator operator = + project(graph, new TsBlock[] {block(-2, 3), block(4, -5)}, positive, negative, shared)) { + assertBlock( + operator.next(), + new Double[] {100.0, 4.0}, + new Double[] {-1.0, 200.0}, + new Double[] {-1.0, 4.0}); + assertBlock( + operator.next(), + new Double[] {5.0, 100.0}, + new Double[] {200.0, -4.0}, + new Double[] {5.0, -4.0}); + } + } + + @Test + public void testTwoCasesAndFullOutputShareStatefulDiff() throws Exception { + for (boolean reverseOutputs : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer diff = new DiffFunctionColumnTransformer(DOUBLE, reference(input), true); + ColumnTransformer positive = + graph.caseWhen(graph.greaterThan(input, 5), diff, graph.constant(100)); + ColumnTransformer negative = + graph.caseWhen(graph.greaterThan(input, 5), graph.constant(200), diff); + ColumnTransformer[] outputs = + reverseOutputs + ? new ColumnTransformer[] {diff, positive, negative} + : new ColumnTransformer[] {positive, negative, diff}; + try (Operator operator = + project(graph, new TsBlock[] {block(1, 3, 6), block(10, 11)}, outputs)) { + Double[] firstDiff = {null, 2.0, 3.0}; + Double[] firstPositive = {100.0, 100.0, 3.0}; + Double[] firstNegative = {null, 2.0, 200.0}; + if (reverseOutputs) { + assertBlock(operator.next(), firstDiff, firstPositive, firstNegative); + assertBlock( + operator.next(), + new Double[] {4.0, 1.0}, + new Double[] {4.0, 1.0}, + new Double[] {200.0, 200.0}); + } else { + assertBlock(operator.next(), firstPositive, firstNegative, firstDiff); + assertBlock( + operator.next(), + new Double[] {4.0, 1.0}, + new Double[] {200.0, 200.0}, + new Double[] {4.0, 1.0}); + } + } + } + } + @Test public void testCaseStillSkipsUnusedFailBranch() throws Exception { Graph graph = new Graph(); @@ -697,9 +786,9 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { selectedEvaluationCount++; - super.evaluateWithSelection(selection); + return super.evaluateSelection(selection); } } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java index 8c81d6f5ad8b0..e797d5c124687 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java @@ -20,15 +20,21 @@ package org.apache.iotdb.calc.transformation.dag.column; import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.LogicAndColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.LogicOrColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; import org.junit.Test; import java.util.Arrays; import java.util.Optional; +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; @@ -69,6 +75,90 @@ public void testNewInputClearingOnlyClearsOwnCache() { assertEquals(2.0, right.getColumn().getDouble(0), 0.0); } + @Test + public void testSelectedResultDoesNotBecomeFullCache() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + Column values = new DoubleColumn(2, Optional.empty(), new double[] {1, 2}); + input.initFromTsBlock(new TsBlock(new TimeColumn(2, new long[2]), values)); + + Column selected = input.getColumnWithSelection(new boolean[] {true, false}); + assertEquals(1.0, selected.getDouble(0), 0.0); + org.junit.Assert.assertTrue(selected.isNull(1)); + assertEquals(0, input.getColumnCachePositionCount()); + + input.tryEvaluate(); + // Borrowing a full cached column for a selected consumer must not consume its reference. + assertSame(values, input.getColumnWithSelection(new boolean[] {false, true})); + assertSame(values, input.getColumn()); + } + + @Test + public void testLegacySelectedEntryDoesNotReplacePendingFullColumn() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + Column values = new DoubleColumn(2, Optional.empty(), new double[] {1, 2}); + input.initFromTsBlock(new TsBlock(new TimeColumn(2, new long[2]), values)); + input.tryEvaluate(); + input.evaluateWithSelection(new boolean[] {true, false}); + input.getColumn(); + input.tryEvaluate(); + assertSame(values, input.getColumn()); + } + + @Test + public void testSelectedLogicalOperatorsPreserveThreeValuedLogicWithCachedInputs() { + Boolean[] values = {null, false, true}; + ColumnBuilder left = BOOLEAN.createColumnBuilder(9); + ColumnBuilder right = BOOLEAN.createColumnBuilder(9); + Boolean[] expectedAnd = {null, false, null, false, false, false, null, false, true}; + Boolean[] expectedOr = {null, null, true, null, false, true, true, true, true}; + for (Boolean a : values) { + for (Boolean b : values) { + if (a == null) { + left.appendNull(); + } else { + left.writeBoolean(a); + } + if (b == null) { + right.appendNull(); + } else { + right.writeBoolean(b); + } + } + } + ColumnTransformer a = new IdentityColumnTransformer(BOOLEAN, 0); + ColumnTransformer b = new IdentityColumnTransformer(BOOLEAN, 1); + a.addReferenceCount(); + b.addReferenceCount(); + a.initializeColumnCache(left.build()); + b.initializeColumnCache(right.build()); + ColumnTransformer[] operators = { + new LogicAndColumnTransformer(BOOLEAN, a, b), new LogicOrColumnTransformer(BOOLEAN, a, b) + }; + for (int i = 0; i < operators.length; i++) { + Boolean[] expected = i == 0 ? expectedAnd : expectedOr; + for (boolean partial : new boolean[] {false, true}) { + boolean[] mask = new boolean[9]; + for (int j = 0; j < mask.length; j++) { + mask[j] = !partial || j % 2 == 0; + } + Column result = operators[i].getColumnWithSelection(mask); + assertEquals(9, result.getPositionCount()); + for (int j = 0; j < mask.length; j++) { + if (!mask[j] || expected[j] == null) { + org.junit.Assert.assertTrue(result.isNull(j)); + } else { + assertEquals(expected[j], result.getBoolean(j)); + } + } + } + } + // Selected consumers did not consume the two pending full-column references. + assertEquals(9, a.getColumn().getPositionCount()); + assertEquals(9, b.getColumn().getPositionCount()); + } + private static Column column(double value) { return new DoubleColumn(1, Optional.empty(), new double[] {value}); } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java new file mode 100644 index 0000000000000..db1e569f8a1bd --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class DiffFunctionColumnTransformerTest { + @Test + public void testMasksAndConsumedFullCacheAdvanceStateOncePerInput() { + for (boolean ignoreNull : new boolean[] {true, false}) { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + DiffFunctionColumnTransformer diff = + new DiffFunctionColumnTransformer(DOUBLE, input, ignoreNull); + diff.addReferenceCount(); + ColumnTransformer[] nodes = ColumnTransformer.collectTransformers(Arrays.asList(diff)); + TsBlock[] blocks = { + block(new double[] {1, 3, 0}, new boolean[] {false, false, true}), + block(new double[] {10, 11}, null) + }; + Double[][] expected = {{null, 2.0, null}, {ignoreNull ? 7.0 : null, 1.0}}; + for (int i = 0; i < blocks.length; i++) { + for (ColumnTransformer node : nodes) { + node.clearCacheForNewInput(); + } + input.initFromTsBlock(blocks[i]); + boolean[] mask = new boolean[blocks[i].getPositionCount()]; + // Even an all-false mask must advance DIFF across all input rows exactly once. + assertValues(diff.getColumnWithSelection(mask), expected[i]); + Arrays.fill(mask, true); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + // The ordinary cache has been consumed; re-requesting it must not advance state again. + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + } + } + } + + private static TsBlock block(double[] values, boolean[] nulls) { + return new TsBlock( + new TimeColumn(values.length, new long[values.length]), + new DoubleColumn(values.length, Optional.ofNullable(nulls), values)); + } + + private static void assertValues(Column column, Double[] expected) { + assertEquals(expected.length, column.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(column.isNull(i)); + } else { + assertEquals(expected[i], column.getDouble(i), 0.0); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 8d42d07b2a374..60486568f00b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -45,7 +45,6 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; -import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.appendCommonExpressionColumns; import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.constructFilteredTsBlock; public class DeviceAttributeUpdater extends DeviceUpdater { @@ -153,8 +152,15 @@ private TsBlock getFilterTsBlock() { final List filterResultColumns = new ArrayList<>(Arrays.asList(curBlock.getValueColumns())); - appendCommonExpressionColumns( - filterResultColumns, commonTransformerList, curFilterColumn, deviceSchemaBatch.size()); + if (!commonTransformerList.isEmpty()) { + boolean[] selection = new boolean[deviceSchemaBatch.size()]; + for (int i = 0; i < selection.length; i++) { + selection[i] = !curFilterColumn.isNull(i) && curFilterColumn.getBoolean(i); + } + for (ColumnTransformer transformer : commonTransformerList) { + filterResultColumns.add(transformer.getColumnWithSelection(selection)); + } + } final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); final int rowCount = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/UserDefineScalarFunctionColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/UserDefineScalarFunctionColumnTransformerTest.java new file mode 100644 index 0000000000000..fda1cf8232838 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/UserDefineScalarFunctionColumnTransformerTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column; + +import org.apache.iotdb.calc.execution.relational.ColumnTransformerBuilder; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.udf.UserDefineScalarFunctionTransformer; +import org.apache.iotdb.udf.api.IoTDBLocal; +import org.apache.iotdb.udf.api.relational.ScalarFunction; +import org.apache.iotdb.udf.api.relational.access.Record; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class UserDefineScalarFunctionColumnTransformerTest { + @Test + public void testSelectionSkipsRejectedRowsWithCachedAndUncachedInputs() throws Exception { + for (boolean cached : new boolean[] {false, true}) { + IoTDBLocal local = Mockito.mock(IoTDBLocal.class); + ColumnTransformerBuilder.Context context = + Mockito.mock(ColumnTransformerBuilder.Context.class); + Mockito.when(context.getIoTDBLocalFactory()).thenReturn((s, f, q, d) -> local); + Mockito.when(context.getFragmentInstanceId()).thenReturn("test-fragment"); + Mockito.when(context.getOuterGlobalQueryId()).thenReturn("test-query"); + Mockito.when(context.getOuterQueryDeadlineMs()).thenReturn(Long.MAX_VALUE); + ScalarFunction function = Mockito.mock(ScalarFunction.class); + Mockito.when(function.evaluate(Mockito.any(Record.class), Mockito.eq(local))) + .thenAnswer( + invocation -> { + double value = ((Record) invocation.getArgument(0)).getDouble(0); + assertTrue(value > 0); + return value + 1; + }); + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + input.initFromTsBlock( + new TsBlock( + new TimeColumn(2, new long[2]), + new DoubleColumn(2, Optional.empty(), new double[] {-2, 3}))); + if (cached) { + input.tryEvaluate(); + } + UserDefineScalarFunctionTransformer transformer = + new UserDefineScalarFunctionTransformer( + DOUBLE, function, Collections.singletonList(input), null, context); + try { + Column result = transformer.getColumnWithSelection(new boolean[] {false, true}); + assertEquals(2, result.getPositionCount()); + assertTrue(result.isNull(0)); + assertEquals(4.0, result.getDouble(1), 0.0); + Column empty = transformer.getColumnWithSelection(new boolean[] {false, false}); + assertTrue(empty.isNull(0)); + assertTrue(empty.isNull(1)); + Mockito.verify(function, Mockito.times(1)) + .evaluate(Mockito.any(Record.class), Mockito.eq(local)); + } finally { + transformer.close(); + } + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java index 65bc3234ff5e6..dd381a0747b20 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacMd5(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java index e37ad94329cf0..c260856d5440c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacSha1(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java index a73d248012868..472a270f2c1c2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacSha256(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java index 541faceeebef6..da0e7110a4a86 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacSha512(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java index 0bca3e93575b5..44755d61821fa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java @@ -44,7 +44,8 @@ private ColumnTransformer mockColumnTransformer(Column column) { // Common mocking for evaluation methods Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); - Mockito.doNothing().when(mockTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockTransformer.getColumn()); Mockito.when(mockTransformer.getColumnCachePositionCount()) .thenReturn(column.getPositionCount()); return mockTransformer; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java index 7b1250a2491b0..71cadc4f21098 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java @@ -44,7 +44,8 @@ private ColumnTransformer mockColumnTransformer(Column column) { // Common mocking for evaluation methods Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); - Mockito.doNothing().when(mockTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockTransformer.getColumn()); Mockito.when(mockTransformer.getColumnCachePositionCount()) .thenReturn(column.getPositionCount()); return mockTransformer; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java index 6b50738c7184f..09fb5a2cbc984 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java @@ -41,7 +41,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java index dfbc215a88f8b..a14402023ba30 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java index 4cf2a63440b76..25e3a03f6d92b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java index ebd1d4943352f..42acc65b422e8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java index 0d1514e2af82f..a3fcab6fd4436 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java index fb99ce5d34dcd..4d07413a79017 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java @@ -46,7 +46,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java index 533484563e706..7ff069309fdf0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java @@ -46,7 +46,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java index e13563e41df23..92a8fab86ea4d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java index 2d15cf4a5c362..24962167a42e2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java index ffda49dec3bdc..dd767459dbecd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java index ded4c9480c8de..c7b017626effd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java index bcee3901b6140..79bab3a3226af 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java @@ -43,7 +43,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java index 420eb8ef5673b..764e1b31be897 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java @@ -43,7 +43,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java index 03811972f5d19..a018396e92665 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java index 068654977deca..f6b9ef933fe1d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java @@ -52,7 +52,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java index 1135633bb200f..e10cb989a4514 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java index 83bbe84f4d0b1..f171e7004188b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java @@ -48,7 +48,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java index d000b22571119..21a1ec62fdf4c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java @@ -50,7 +50,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java index d02f9765b802e..e4fee39dca4d2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java index d83fdd7239ffc..5427d60b4ddd1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java index c3829eb04693e..a47b15ec398e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java @@ -51,7 +51,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java index 7135f334a182e..3c77de2f50ce3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java index 2ce9dba7d8805..6eb78c01111c9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java index ef9ae8e222cf4..7b6191071f58e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java index 902693f87afc1..e72de9e6090f7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java index 488b67d02d52d..644af479b381b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java index 258b2db8aad03..19f035ea8facf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java index 5c39e064a4c0c..8aec8f3f19656 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java index e8a0202919d9e..39af38e18576f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java @@ -52,7 +52,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java index 2b636b9095a8b..2933519c5989c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java @@ -46,7 +46,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java index f846ddce17c57..d2419d027755a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } From c2f0b6ea7efd586017dc6277460ba911b44c3e12 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 12:41:30 +0800 Subject: [PATCH 10/13] Retain binary DIFF results for each input block --- .../it/db/it/IoTDBCaseWhenThenTableIT.java | 54 ++++++ .../dag/column/ColumnTransformer.java | 8 +- .../unary/scalar/DiffColumnTransformer.java | 39 +++++ .../dag/column/DiffColumnTransformerTest.java | 156 ++++++++++++++++++ .../DiffFunctionColumnTransformerTest.java | 4 + 5 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffColumnTransformerTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index 4cbc77697abb5..05ae9b227b3aa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -851,4 +851,58 @@ public void testSharedDiffAcrossCaseFilterAndProjection() { }, DATABASE); } + + @Test + public void testDynamicIgnoreNullDiffAcrossCaseFilterAndProjection() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_dynamic_diff " + + "(dev STRING TAG, x DOUBLE FIELD, ignore_null BOOLEAN FIELD)", + "INSERT INTO case_dynamic_diff(time, dev, x, ignore_null) VALUES " + + "(1, 'd0', 1.0, true), (2, 'd0', 3.0, false), (3, 'd0', null, true), " + + "(4, 'd0', 10.0, null), (5, 'd0', null, false)", + "FLUSH", + "INSERT INTO case_dynamic_diff(time, dev, x, ignore_null) VALUES " + + "(6, 'd0', 20.0, true), (7, 'd0', 25.0, false), " + + "(8, 'd0', null, true), (9, 'd0', 30.0, true)", + "FLUSH" + }); + String diff = "diff(x, ignore_null)"; + tableResultSetEqualTest( + "SELECT " + diff + " AS d FROM case_dynamic_diff ORDER BY time", + new String[] {"d"}, + new String[] {"null,", "2.0,", "null,", "7.0,", "null,", "null,", "5.0,", "null,", "5.0,"}, + DATABASE); + tableResultSetEqualTest( + "SELECT CASE WHEN x > 5 THEN " + + diff + + " ELSE 100.0 END AS a, CASE WHEN x > 5 THEN 200.0 ELSE " + + diff + + " END AS b, " + + diff + + " AS d FROM case_dynamic_diff ORDER BY time", + new String[] {"a", "b", "d"}, + new String[] { + "100.0,null,null,", + "100.0,2.0,2.0,", + "100.0,null,null,", + "7.0,200.0,7.0,", + "100.0,null,null,", + "null,200.0,null,", + "5.0,200.0,5.0,", + "100.0,null,null,", + "5.0,200.0,5.0," + }, + DATABASE); + tableResultSetEqualTest( + "SELECT " + + diff + + " AS d FROM case_dynamic_diff WHERE CASE WHEN x IS NULL THEN false WHEN " + + diff + + " > 1.0 THEN true ELSE false END ORDER BY time", + new String[] {"d"}, + new String[] {"2.0,", "7.0,", "5.0,", "5.0,"}, + DATABASE); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index b93248fa8234d..3cb10044886cd 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -164,8 +164,12 @@ public void close() { } /** - * Clears cached results in this subgraph. Subclasses should define child edges via {@link - * #forEachChild(Consumer)} instead of overriding this method. + * Clears ordinary column caches in this subgraph. Input-specific retained results, such as DIFF's + * complete result, remain valid so another request for the same input cannot advance state again. + * This method does not mark a new input: before replacing leaf inputs, call {@link + * #clearCacheForNewInput()} on every node collected by {@link #collectTransformers(Iterable)}. + * Subclasses should define child edges via {@link #forEachChild(Consumer)} instead of overriding + * this method. * *

Kept non-final so existing Mockito-based tests can stub it. */ diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java index 5aa14fb6f245d..7437c119fdd3e 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java @@ -34,6 +34,9 @@ public class DiffColumnTransformer extends BinaryColumnTransformer { // indicate whether lastValue is null private boolean lastValueIsNull = true; + // DIFF must advance its state once per input, even when several masks/consumers request it. + private Column currentInputResult; + public DiffColumnTransformer( Type returnType, ColumnTransformer leftTransformer, ColumnTransformer rightTransformer) { super(returnType, leftTransformer, rightTransformer); @@ -44,6 +47,42 @@ protected void checkType() { // do nothing } + @Override + public void evaluate() { + initializeColumnCache(evaluateCurrentInput()); + } + + @Override + protected Column evaluateSelection(boolean[] selection) { + // DIFF depends on all preceding rows and their ignoreNull flags, even outside the mask. + return evaluateCurrentInput(); + } + + private Column evaluateCurrentInput() { + if (currentInputResult == null) { + leftTransformer.tryEvaluate(); + Column leftColumn = leftTransformer.getColumn(); + rightTransformer.tryEvaluate(); + Column rightColumn = rightTransformer.getColumn(); + int positionCount = leftColumn.getPositionCount(); + ColumnBuilder builder = returnType.createColumnBuilder(positionCount); + doTransform(leftColumn, rightColumn, builder, positionCount); + currentInputResult = builder.build(); + } + return currentInputResult; + } + + @Override + protected void resetForNewInput() { + currentInputResult = null; + } + + @Override + public void close() { + currentInputResult = null; + super.close(); + } + @Override protected void doTransform( Column leftColumn, Column rightColumn, ColumnBuilder builder, int positionCount) { diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffColumnTransformerTest.java new file mode 100644 index 0000000000000..26b72ad844169 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffColumnTransformerTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; + +import java.util.Arrays; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DiffColumnTransformerTest { + @Test + public void testMasksAndConsumedFullCacheAdvanceStateOncePerInput() { + for (Boolean ignoreNull : new Boolean[] {true, false, null}) { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + IdentityColumnTransformer flags = new IdentityColumnTransformer(BOOLEAN, 1); + input.addReferenceCount(); + flags.addReferenceCount(); + DiffColumnTransformer diff = new DiffColumnTransformer(DOUBLE, input, flags); + diff.addReferenceCount(); + ColumnTransformer[] nodes = ColumnTransformer.collectTransformers(Arrays.asList(diff)); + TsBlock[] blocks = { + block(new Double[] {1.0, 3.0, null}, new Boolean[] {false, null, ignoreNull}), + block(new Double[] {10.0, null, 20.0, null}, new Boolean[] {false, true, false, null}), + block(new Double[] {30.0, 32.0}, new Boolean[] {true, false}), + block(new Double[] {40.0, 41.0}, new Boolean[] {false, true}) + }; + Double[][] expected = { + {null, 2.0, null}, + {Boolean.TRUE.equals(ignoreNull) ? 7.0 : null, null, 10.0, null}, + {null, 2.0}, + {8.0, 1.0} + }; + for (int i = 0; i < blocks.length; i++) { + for (ColumnTransformer node : nodes) { + node.clearCacheForNewInput(); + } + input.initFromTsBlock(blocks[i]); + flags.initFromTsBlock(blocks[i]); + boolean[] mask = new boolean[blocks[i].getPositionCount()]; + // Even an all-false mask must advance DIFF across the complete input exactly once. + assertValues(diff.getColumnWithSelection(mask), expected[i]); + Arrays.fill(mask, true); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + // Clearing ordinary caches within the same input must not advance state again. + diff.clearCache(); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + } + diff.close(); + } + } + + @Test + public void testPartialSelectionDoesNotMaskInputs() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + IdentityColumnTransformer flags = new IdentityColumnTransformer(BOOLEAN, 1); + input.addReferenceCount(); + flags.addReferenceCount(); + TsBlock block = block(new Double[] {1.0, 3.0, 6.0}, new Boolean[] {true, false, true}); + input.initFromTsBlock(block); + flags.initFromTsBlock(block); + DiffColumnTransformer diff = new DiffColumnTransformer(DOUBLE, input, flags); + diff.addReferenceCount(); + assertValues( + diff.getColumnWithSelection(new boolean[] {false, true, false}), + new Double[] {null, 2.0, 3.0}); + diff.tryEvaluate(); + assertValues(diff.getColumn(), new Double[] {null, 2.0, 3.0}); + diff.close(); + } + + @Test + public void testConsumedFullCacheDoesNotAdvanceStateAgain() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + IdentityColumnTransformer flags = new IdentityColumnTransformer(BOOLEAN, 1); + input.addReferenceCount(); + flags.addReferenceCount(); + TsBlock block = block(new Double[] {1.0, 3.0, 6.0}, new Boolean[] {true, false, true}); + input.initFromTsBlock(block); + flags.initFromTsBlock(block); + DiffColumnTransformer diff = new DiffColumnTransformer(DOUBLE, input, flags); + diff.addReferenceCount(); + diff.tryEvaluate(); + assertValues(diff.getColumn(), new Double[] {null, 2.0, 3.0}); + diff.tryEvaluate(); + assertValues(diff.getColumn(), new Double[] {null, 2.0, 3.0}); + diff.close(); + } + + private static TsBlock block(Double[] values, Boolean[] flags) { + ColumnBuilder valueBuilder = DOUBLE.createColumnBuilder(values.length); + ColumnBuilder flagBuilder = BOOLEAN.createColumnBuilder(values.length); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + valueBuilder.appendNull(); + } else { + valueBuilder.writeDouble(values[i]); + } + if (flags[i] == null) { + flagBuilder.appendNull(); + } else { + flagBuilder.writeBoolean(flags[i]); + } + } + return new TsBlock( + new TimeColumn(values.length, new long[values.length]), + valueBuilder.build(), + flagBuilder.build()); + } + + private static void assertValues(Column column, Double[] expected) { + assertEquals(expected.length, column.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(column.isNull(i)); + } else { + assertFalse(column.isNull(i)); + assertEquals(expected[i], column.getDouble(i), 0.0); + } + } + } +} diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java index db1e569f8a1bd..0210d4ea16a2d 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java @@ -66,6 +66,10 @@ public void testMasksAndConsumedFullCacheAdvanceStateOncePerInput() { diff.tryEvaluate(); assertValues(diff.getColumn(), expected[i]); assertValues(diff.getColumnWithSelection(mask), expected[i]); + // Ordinary cache clearing does not mark the start of a new input. + diff.clearCache(); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); } } } From 4620d0f2234b45231e341e6aae86067545c5c1da Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 12:41:30 +0800 Subject: [PATCH 11/13] Account for retained DIFF columns in operator memory estimates --- .../process/FilterAndProjectOperator.java | 34 ++++++++++- .../process/FilterAndProjectOperatorTest.java | 59 +++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 4678cd031fd1b..006b9a201d250 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -32,6 +32,8 @@ import org.apache.iotdb.calc.transformation.dag.column.multi.MultiColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.ternary.TernaryColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.UnaryColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; import org.apache.iotdb.commons.queryengine.execution.MemoryEstimationHelper; import com.google.common.util.concurrent.ListenableFuture; @@ -47,8 +49,11 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.OptionalInt; +import java.util.Set; public class FilterAndProjectOperator implements ProcessOperator { @@ -71,6 +76,8 @@ public class FilterAndProjectOperator implements ProcessOperator { private final ColumnTransformer[] projectCacheTransformers; + private final int retainedInputColumnCount; + private final TsBlockBuilder filterTsBlockBuilder; private final boolean hasNonMappableUDF; @@ -111,6 +118,8 @@ public FilterAndProjectOperator( this.filterCacheTransformers = ColumnTransformer.collectTransformers(filterRoots); this.projectCacheTransformers = ColumnTransformer.collectTransformers(projectOutputTransformerList); + this.retainedInputColumnCount = + countRetainedInputColumns(filterCacheTransformers, projectCacheTransformers); } public FilterAndProjectOperator( @@ -124,12 +133,26 @@ public FilterAndProjectOperator( this.projectOutputTransformerList = filterAndProjectOperator.projectOutputTransformerList; this.filterCacheTransformers = filterAndProjectOperator.filterCacheTransformers; this.projectCacheTransformers = filterAndProjectOperator.projectCacheTransformers; + this.retainedInputColumnCount = filterAndProjectOperator.retainedInputColumnCount; this.hasNonMappableUDF = filterAndProjectOperator.hasNonMappableUDF; this.hasFilter = filterAndProjectOperator.hasFilter; this.filterTsBlockBuilder = filterAndProjectOperator.filterTsBlockBuilder; this.inputOperator = inputOperator; } + private static int countRetainedInputColumns(ColumnTransformer[]... phases) { + Set retained = Collections.newSetFromMap(new IdentityHashMap<>()); + for (ColumnTransformer[] transformers : phases) { + for (ColumnTransformer transformer : transformers) { + if (transformer instanceof DiffFunctionColumnTransformer + || transformer instanceof DiffColumnTransformer) { + retained.add(transformer); + } + } + } + return retained.size(); + } + @Override public CommonOperatorContext getOperatorContext() { return operatorContext; @@ -356,7 +379,7 @@ public long calculateMaxPeekMemory() { maxPeekMemory, (long) maxCachedColumn * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()) - + inputOperator.calculateRetainedSizeAfterCallingNext(); + + calculateRetainedSizeAfterCallingNext(); } // has Filter @@ -373,7 +396,7 @@ public long calculateMaxPeekMemory() { return Math.max( maxPeekMemory, (long) maxCachedColumn * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()) - + inputOperator.calculateRetainedSizeAfterCallingNext(); + + calculateRetainedSizeAfterCallingNext(); } @Override @@ -390,7 +413,12 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { - return inputOperator.calculateRetainedSizeAfterCallingNext(); + // DIFF retains a complete result until the next input even after its ordinary cache is + // consumed. + // Filter and projection results can coexist, so count distinct nodes across both phases. + return inputOperator.calculateRetainedSizeAfterCallingNext() + + (long) retainedInputColumnCount + * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); } private int getMaxLevelOfColumnTransformerTree(ColumnTransformer columnTransformer) { diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 562ef33ce6d57..463c674bac3af 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -36,6 +36,7 @@ import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.ternary.BetweenColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.CastFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -43,6 +44,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.BinaryColumn; @@ -499,6 +501,63 @@ public void testTwoCasesAndFullOutputShareStatefulDiff() throws Exception { } } + @Test + public void testNestedAndIndependentDiffResultsAreIncludedInMemoryEstimates() throws Exception { + int diffCount = 8; + long pageSize = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + for (boolean nested : new boolean[] {true, false}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer output = nested ? input : graph.constant(0); + for (int i = 0; i < diffCount; i++) { + ColumnTransformer argument = nested ? output : graph.add(input, i); + ColumnTransformer diff = + new DiffFunctionColumnTransformer(DOUBLE, reference(argument), true); + output = nested ? diff : graph.add(output, diff); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, 3, 6)}, output)) { + // Retained DIFF columns coexist with the input/scratch columns during evaluation. + assertTrue(operator.calculateMaxPeekMemory() >= (diffCount + 1) * pageSize); + operator.next(); + // The child has returned its only block, but every DIFF still retains its result. + assertEquals(diffCount * pageSize, operator.calculateRetainedSizeAfterCallingNext()); + } + } + } + + @Test + public void testDiffRetainedMemoryDeduplicatesSharedNodesAcrossPhases() throws Exception { + Graph filter = new Graph(); + ColumnTransformer input = filter.input(0); + ColumnTransformer unary = new DiffFunctionColumnTransformer(DOUBLE, reference(input), true); + ColumnTransformer flags = + filter.leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {true}))); + ColumnTransformer binary = + new DiffColumnTransformer(DOUBLE, reference(input), reference(flags)); + ColumnTransformer predicate = filter.greaterThan(filter.add(unary, binary), 0); + long pageSize = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + try (FilterAndProjectOperator original = + operator( + new TsBlock[0], + filter, + predicate, + Arrays.asList(unary, binary), + new Graph(), + unary, + binary, + unary); + FilterAndProjectOperator copy = + new FilterAndProjectOperator( + original, new ValuesOperator(new TestOperatorContext(), Collections.emptyList()))) { + assertEquals(2 * pageSize, original.calculateRetainedSizeAfterCallingNext()); + assertTrue(original.calculateMaxPeekMemory() >= 6 * pageSize); + assertEquals(2 * pageSize, copy.calculateRetainedSizeAfterCallingNext()); + assertEquals(original.calculateMaxPeekMemory(), copy.calculateMaxPeekMemory()); + } + } + @Test public void testCaseStillSkipsUnusedFailBranch() throws Exception { Graph graph = new Graph(); From a0e87a6528a471306ae77601b8446e07e951aac4 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 14:55:31 +0800 Subject: [PATCH 12/13] Release completed phase caches and bound retained DIFF memory --- .../process/FilterAndProjectOperator.java | 50 +++++++-- .../dag/column/ColumnTransformer.java | 16 ++- .../process/FilterAndProjectOperatorTest.java | 105 ++++++++++++++++++ 3 files changed, 157 insertions(+), 14 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 006b9a201d250..cf352b680c20b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -43,6 +43,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.utils.Pair; @@ -55,11 +56,17 @@ import java.util.OptionalInt; import java.util.Set; +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfDoubleArray; + public class FilterAndProjectOperator implements ProcessOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(FilterAndProjectOperator.class); + private static final long DIFF_COLUMN_INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(DoubleColumn.class); + private final Operator inputOperator; private final List filterLeafColumnTransformerList; @@ -170,7 +177,13 @@ public TsBlock next() throws Exception { } long inputRowCount = input.getPositionCount(); - TsBlock filterResult = getFilterTsBlock(input); + TsBlock filterResult; + try { + filterResult = getFilterTsBlock(input); + } finally { + // Materialization and compaction have consumed every filter result before projection starts. + clearColumnCaches(filterCacheTransformers); + } filteredRowCount += inputRowCount - filterResult.getPositionCount(); operatorContext.recordSpecifiedInfo("Filtered Rows", Long.toString(filteredRowCount)); @@ -181,6 +194,12 @@ public TsBlock next() throws Exception { return getTransformedTsBlock(filterResult); } + private static void clearColumnCaches(ColumnTransformer[] transformers) { + for (ColumnTransformer transformer : transformers) { + transformer.clearColumnCache(); + } + } + /** * Return the TsBlock that contains both initial input columns and columns of common * subexpressions after filtering. @@ -330,13 +349,18 @@ private TsBlock getTransformedTsBlock(TsBlock input) { leafColumnTransformer.initFromTsBlock(input); } - List resultColumns = new ArrayList<>(projectOutputTransformerList.size()); - for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.tryEvaluate(); - resultColumns.add(columnTransformer.getColumn()); + try { + List resultColumns = new ArrayList<>(projectOutputTransformerList.size()); + for (ColumnTransformer columnTransformer : projectOutputTransformerList) { + columnTransformer.tryEvaluate(); + resultColumns.add(columnTransformer.getColumn()); + } + return TsBlock.wrapBlocksWithoutCopy( + positionCount, originTimeColumn, resultColumns.toArray(new Column[0])); + } finally { + // Returned columns are retained by the TsBlock, not by ordinary transformer caches. + clearColumnCaches(projectCacheTransformers); } - return TsBlock.wrapBlocksWithoutCopy( - positionCount, originTimeColumn, resultColumns.toArray(new Column[0])); } @Override @@ -417,8 +441,16 @@ public long calculateRetainedSizeAfterCallingNext() { // consumed. // Filter and projection results can coexist, so count distinct nodes across both phases. return inputOperator.calculateRetainedSizeAfterCallingNext() - + (long) retainedInputColumnCount - * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + + (long) retainedInputColumnCount * getMaxRetainedDiffColumnSize(); + } + + private static long getMaxRetainedDiffColumnSize() { + int maxRows = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); + // DIFF always builds a DoubleColumn. A larger TsBlock can exceed a storage page, so account + // for the value array, NULL flags, and object/array headers using the column's sizing rules. + return Math.max( + TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(), + DIFF_COLUMN_INSTANCE_SIZE + sizeOfBooleanArray(maxRows) + sizeOfDoubleArray(maxRows)); } private int getMaxLevelOfColumnTransformerTree(ColumnTransformer columnTransformer) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index 3cb10044886cd..af2319ff0a51d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -159,8 +159,16 @@ public void evaluateWithSelection(boolean[] selection) { protected abstract void checkType(); public void close() { - columnCache.clear(); + clearColumnCache(); + } + + /** + * Releases only this node's ordinary cached columns after all consumers in an evaluation phase + * finish. Stateful per-input results remain valid until {@link #clearCacheForNewInput()}. + */ + public final void clearColumnCache() { selectedColumnCache = null; + columnCache.clear(); } /** @@ -174,8 +182,7 @@ public void close() { *

Kept non-final so existing Mockito-based tests can stub it. */ public void clearCache() { - selectedColumnCache = null; - this.columnCache.clear(); + clearColumnCache(); forEachChild(ColumnTransformer::clearCache); } @@ -185,8 +192,7 @@ public void clearCache() { * the new block, when no sibling expression can still need a cached column. */ public final void clearCacheForNewInput() { - selectedColumnCache = null; - this.columnCache.clear(); + clearColumnCache(); resetForNewInput(); } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 463c674bac3af..71f0251a210e2 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -525,6 +525,111 @@ public void testNestedAndIndependentDiffResultsAreIncludedInMemoryEstimates() th } } + @Test + public void testFilterReleasesBorrowedColumnsBeforeProjection() throws Exception { + Graph filter = new Graph(); + ColumnTransformer shared = filter.add(filter.input(0), 1); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(shared, 0), shared, filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(caseWhen, 1); + Graph projection = new Graph(); + ColumnTransformer output = + projection.leaf( + new IdentityColumnTransformer(DOUBLE, 0) { + @Override + protected void evaluate() { + assertEquals(0, shared.getColumnCachePositionCount()); + super.evaluate(); + } + }); + try (Operator operator = + operator( + new TsBlock[] {block(1, 2, 3), block(-2, 0, 5)}, + filter, + predicate, + Collections.emptyList(), + projection, + output)) { + assertBlock(operator.next(), new Double[] {1.0, 2.0, 3.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertBlock(operator.next(), new Double[] {5.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertEquals(0, operator.calculateRetainedSizeAfterCallingNext()); + } + } + + @Test + public void testProjectionReleasesBorrowedColumnsAfterReturningOutput() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 1); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(shared, 0), shared, graph.constant(0)); + try (Operator operator = + project(graph, new TsBlock[] {block(1, 2, 3), block(-2, 0, 5)}, caseWhen)) { + TsBlock first = operator.next(); + assertBlock(first, new Double[] {2.0, 3.0, 4.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertBlock(operator.next(), new Double[] {0.0, 1.0, 6.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertEquals(0, operator.calculateRetainedSizeAfterCallingNext()); + // Releasing transformer caches does not invalidate columns already returned to callers. + assertBlock(first, new Double[] {2.0, 3.0, 4.0}); + } + } + + @Test + public void testLargeDiffColumnsAreIncludedInMemoryEstimates() throws Exception { + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + int oldMaxRows = config.getMaxTsBlockLineNumber(); + int oldMaxBytes = config.getMaxTsBlockSizeInBytes(); + int rowCount = 20_000; + try { + config.setMaxTsBlockLineNumber(rowCount); + config.setMaxTsBlockSizeInBytes(1024 * 1024); + double[] values = new double[rowCount]; + for (int i = 0; i < rowCount; i++) { + values[i] = i; + } + boolean[] ignoreNull = new boolean[rowCount]; + Arrays.fill(ignoreNull, true); + TsBlock input = + new TsBlock( + new TimeColumn(rowCount, new long[rowCount]), + doubleColumn(values), + new BooleanColumn(rowCount, Optional.empty(), ignoreNull)); + Graph graph = new Graph(); + ColumnTransformer output = graph.input(0); + ColumnTransformer flags = graph.leaf(new IdentityColumnTransformer(BOOLEAN, 1)); + List diffs = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + output = + i % 2 == 0 + ? new DiffFunctionColumnTransformer(DOUBLE, reference(output), true) + : new DiffColumnTransformer(DOUBLE, reference(output), reference(flags)); + diffs.add(output); + } + try (Operator operator = project(graph, new TsBlock[] {input}, output)) { + long estimatedPeak = operator.calculateMaxPeekMemory(); + TsBlock result = operator.next(); + assertEquals(rowCount, result.getPositionCount()); + long retainedBytes = 0; + boolean[] mask = new boolean[rowCount]; + for (ColumnTransformer diff : diffs) { + Column retained = diff.getColumnWithSelection(mask); + // Phase cleanup must retain DIFF's original result rather than advance state again. + assertTrue(retained.isNull(0)); + retainedBytes += retained.getRetainedSizeInBytes(); + } + assertTrue(retainedBytes > 8L * config.getPageSizeInByte()); + assertTrue(operator.calculateRetainedSizeAfterCallingNext() >= retainedBytes); + assertTrue(estimatedPeak >= retainedBytes + input.getRetainedSizeInBytes()); + } + } finally { + config.setMaxTsBlockLineNumber(oldMaxRows); + config.setMaxTsBlockSizeInBytes(oldMaxBytes); + } + } + @Test public void testDiffRetainedMemoryDeduplicatesSharedNodesAcrossPhases() throws Exception { Graph filter = new Graph(); From fbb60942a83f162009e619c62952ae134c983761 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 15:58:20 +0800 Subject: [PATCH 13/13] Fix array access for selected mappable UDF inputs --- .../db/it/query/IoTDBCaseWhenThenIT.java | 62 ++++ .../multi/MappableUDFColumnTransformer.java | 18 +- .../MappableUDFColumnTransformerTest.java | 331 ++++++++++++++++++ 3 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/MappableUDFColumnTransformerTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java index b467f0139e156..7b3aacca9f0fa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java @@ -64,6 +64,13 @@ public class IoTDBCaseWhenThenIT { "INSERT INTO root.sg.d2(timestamp,s4) values(1000000, 55)", "INSERT INTO root.sg.d2(timestamp,s4) values(20000000, 66)", "INSERT INTO root.sg.d2(timestamp,s4) values(210000000, 77)", + "CREATE TIMESERIES root.sg.case_udf.x WITH DATATYPE=DOUBLE, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.case_udf.txt WITH DATATYPE=TEXT, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.case_udf.selector WITH DATATYPE=INT32, ENCODING=PLAIN", + "INSERT INTO root.sg.case_udf(time, x, txt, selector) VALUES " + + "(1, -1.0, 'sKiP', 0), (2, -2.0, 'mIxEd', 1), " + + "(3, -3.0, 'ThIrD', 2), (5, -5.0, 'lAsT', 0)", + "INSERT INTO root.sg.case_udf(time, selector) VALUES (4, 3)", }; @BeforeClass @@ -944,4 +951,59 @@ public void testNullableAndInLaterCaseBranch() { new String[] {TIMESTAMP_STR, "result"}, new String[] {"1,3.0,", "2,2.0,", "3,3.0,", "4,1.0,"}); } + + @Test + public void testMappableUdfInCaseMultipleSelectedRows() { + resultSetEqualTest( + "SELECT CASE WHEN selector > 0 THEN abs(x) ELSE 0.0 END AS abs_result, " + + "CASE WHEN selector > 0 THEN upper(txt) ELSE 'skipped' END AS upper_result, " + + "CASE WHEN selector > 0 THEN lower(txt) ELSE 'skipped' END AS lower_result " + + "FROM root.sg.case_udf", + new String[] {TIMESTAMP_STR, "abs_result", "upper_result", "lower_result"}, + new String[] { + "1,0.0,skipped,skipped,", + "2,2.0,MIXED,mixed,", + "3,3.0,THIRD,third,", + "4,null,null,null,", + "5,0.0,skipped,skipped," + }); + } + + @Test + public void testMappableUdfInCaseSingleNonFirstSelectedRow() { + resultSetEqualTest( + "SELECT CASE WHEN selector = 2 THEN abs(x) ELSE 0.0 END AS abs_result, " + + "CASE WHEN selector = 2 THEN upper(txt) ELSE 'skipped' END AS upper_result, " + + "CASE WHEN selector = 2 THEN lower(txt) ELSE 'skipped' END AS lower_result " + + "FROM root.sg.case_udf", + new String[] {TIMESTAMP_STR, "abs_result", "upper_result", "lower_result"}, + new String[] { + "1,0.0,skipped,skipped,", + "2,0.0,skipped,skipped,", + "3,3.0,THIRD,third,", + "4,0.0,skipped,skipped,", + "5,0.0,skipped,skipped," + }); + } + + @Test + public void testMappableUdfInCaseWithCachedNullableInputs() { + // Project the raw inputs first so the CASE branches borrow their complete cached columns. + resultSetEqualTest( + "SELECT x AS raw_x, txt AS raw_txt, " + + "CASE WHEN selector > 0 THEN abs(x) ELSE 0.0 END AS abs_result, " + + "CASE WHEN selector > 0 THEN upper(txt) ELSE 'skipped' END AS upper_result, " + + "CASE WHEN selector > 0 THEN lower(txt) ELSE 'skipped' END AS lower_result " + + "FROM root.sg.case_udf", + new String[] { + TIMESTAMP_STR, "raw_x", "raw_txt", "abs_result", "upper_result", "lower_result" + }, + new String[] { + "1,-1.0,sKiP,0.0,skipped,skipped,", + "2,-2.0,mIxEd,2.0,MIXED,mixed,", + "3,-3.0,ThIrD,3.0,THIRD,third,", + "4,null,null,null,null,null,", + "5,-5.0,lAsT,0.0,skipped,skipped," + }); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 40f9090467520..3b545a2a44c89 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -64,10 +64,20 @@ protected Column evaluateSelection(boolean[] selection) { } Column[] columns = new Column[inputColumnTransformers.length]; for (int i = 0; i < inputColumnTransformers.length; i++) { - columns[i] = - inputColumnTransformers[i] - .getColumnWithSelection(selection) - .getPositions(positions, 0, selectedCount); + ColumnTransformer inputTransformer = inputColumnTransformers[i]; + Column inputColumn = inputTransformer.getColumnWithSelection(selection); + // UDFs can read backing arrays directly. Materialize selected positions instead of passing + // dictionary or offset views whose arrays do not match their logical positions. + ColumnBuilder inputBuilder = inputTransformer.getType().createColumnBuilder(selectedCount); + for (int j = 0; j < selectedCount; j++) { + int position = positions[j]; + if (inputColumn.isNull(position)) { + inputBuilder.appendNull(); + } else { + inputBuilder.write(inputColumn, position); + } + } + columns[i] = inputBuilder.build(); } ColumnBuilder builder = returnType.createColumnBuilder(selection.length); // A mappable UDF only sees selected rows, even when its inputs reuse full cached columns. diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/MappableUDFColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/MappableUDFColumnTransformerTest.java new file mode 100644 index 0000000000000..6d0d3c208589b --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/MappableUDFColumnTransformerTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.TimeColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; +import org.apache.iotdb.commons.udf.builtin.String.UDTFLower; +import org.apache.iotdb.commons.udf.builtin.String.UDTFUpper; +import org.apache.iotdb.commons.udf.builtin.UDTFAbs; +import org.apache.iotdb.udf.api.UDTF; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DictionaryColumn; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.tsfile.read.common.type.BinaryType.TEXT; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.apache.tsfile.read.common.type.LongType.INT64; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class MappableUDFColumnTransformerTest { + + @Test + public void testAbsWithSelectedRowsAndNulls() throws Exception { + Column values = doubles(-10.0, -20.0, null, -40.0); + for (boolean cached : new boolean[] {false, true}) { + assertDoubles( + transform( + new UDTFAbs(), DOUBLE, values, cached, new boolean[] {false, true, false, true}), + null, + 20.0, + null, + 40.0); + assertDoubles( + transform( + new UDTFAbs(), DOUBLE, values, cached, new boolean[] {false, true, false, false}), + null, + 20.0, + null, + null); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, values, cached, new boolean[] {false, true, true, true}), + null, + 20.0, + null, + 40.0); + } + } + + @Test + public void testAbsWithDictionaryOffsetAndRunLengthEncodedInputs() throws Exception { + Column dictionary = DictionaryColumn.create(4, doubles(-10.0, -20.0), new int[] {1, 0, 1, 0}); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, dictionary, true, new boolean[] {false, true, true, true}), + null, + 10.0, + 20.0, + 10.0); + + Column offset = doubles(-100.0, -10.0, -20.0, -30.0, -40.0).getRegion(1, 4); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, offset, true, new boolean[] {false, true, true, false}), + null, + 20.0, + 30.0, + null); + + Column repeated = new RunLengthEncodedColumn(doubles(-7.0), 4); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, repeated, true, new boolean[] {false, true, true, false}), + null, + 7.0, + 7.0, + null); + assertDoubles( + transform( + new UDTFAbs(), + DOUBLE, + doubles(null, null, null), + true, + new boolean[] {true, true, true}), + null, + null, + null); + } + + @Test + public void testStringBuiltinsWithSelectedRows() throws Exception { + ColumnBuilder builder = TEXT.createColumnBuilder(4); + builder.writeBinary(new Binary("FIRST", TSFileConfig.STRING_CHARSET)); + builder.writeBinary(new Binary("MiXeD", TSFileConfig.STRING_CHARSET)); + builder.appendNull(); + builder.writeBinary(new Binary("LAST", TSFileConfig.STRING_CHARSET)); + Column values = builder.build(); + for (boolean cached : new boolean[] {false, true}) { + assertStrings( + transform(new UDTFLower(), TEXT, values, cached, new boolean[] {false, true, true, true}), + null, + "mixed", + null, + "last"); + assertStrings( + transform( + new UDTFUpper(), TEXT, values, cached, new boolean[] {false, true, false, false}), + null, + "MIXED", + null, + null); + } + } + + @Test + public void testAllAndNoSelectedRows() throws Exception { + RecordingAbs udf = new RecordingAbs(); + Column values = doubles(-10.0, -20.0, -30.0); + assertDoubles( + transform(udf, DOUBLE, values, true, new boolean[] {false, false, false}), + null, + null, + null); + assertEquals(0, udf.invocations); + + assertDoubles( + transform(udf, DOUBLE, values, true, new boolean[] {true, true, true}), 10.0, 20.0, 30.0); + assertEquals(1, udf.invocations); + assertArrayEquals(new long[] {10, 20, 30}, udf.times); + } + + @Test + public void testTimeColumnOnlyContainsSelectedPositions() throws Exception { + RecordingAbs udf = new RecordingAbs(); + assertDoubles( + transform( + udf, + DOUBLE, + doubles(-10.0, -20.0, -30.0, -40.0), + true, + new boolean[] {false, true, false, true}), + null, + 20.0, + null, + 40.0); + assertArrayEquals(new long[] {20, 40}, udf.times); + + assertDoubles( + transform( + udf, DOUBLE, doubles(-10.0, -20.0, -30.0), false, new boolean[] {false, false, true}), + null, + null, + 30.0); + assertArrayEquals(new long[] {30}, udf.times); + } + + @Test + public void testNoSelectedRowsStillAdvanceDiffInput() throws Exception { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + TimeColumnTransformer time = new TimeColumnTransformer(INT64); + time.addReferenceCount(); + DiffFunctionColumnTransformer diff = new DiffFunctionColumnTransformer(DOUBLE, input, true); + diff.addReferenceCount(); + RecordingAbs udf = new RecordingAbs(); + MappableUDFColumnTransformer transformer = + new MappableUDFColumnTransformer( + DOUBLE, new ColumnTransformer[] {diff, time}, new TestExecutor(udf, DOUBLE)); + try { + TsBlock first = block(doubles(-10.0, -20.0, -30.0)); + input.initFromTsBlock(first); + time.initFromTsBlock(first); + assertDoubles( + transformer.getColumnWithSelection(new boolean[] {false, false, false}), + null, + null, + null); + assertEquals(0, udf.invocations); + + for (ColumnTransformer node : + ColumnTransformer.collectTransformers(Collections.singletonList(transformer))) { + node.clearCacheForNewInput(); + } + TsBlock second = block(doubles(-50.0, -70.0)); + input.initFromTsBlock(second); + time.initFromTsBlock(second); + assertDoubles(transformer.getColumnWithSelection(new boolean[] {true, true}), 20.0, 20.0); + assertEquals(1, udf.invocations); + } finally { + transformer.close(); + } + } + + private static Column transform( + UDTF udf, Type type, Column values, boolean cached, boolean[] selection) throws Exception { + IdentityColumnTransformer input = new IdentityColumnTransformer(type, 0); + input.addReferenceCount(); + TimeColumnTransformer time = new TimeColumnTransformer(INT64); + time.addReferenceCount(); + TsBlock block = block(values); + input.initFromTsBlock(block); + time.initFromTsBlock(block); + if (cached) { + input.tryEvaluate(); + time.tryEvaluate(); + } + MappableUDFColumnTransformer transformer = + new MappableUDFColumnTransformer( + type, new ColumnTransformer[] {input, time}, new TestExecutor(udf, type)); + try { + Column result = transformer.getColumnWithSelection(selection); + if (cached) { + // Selected evaluation must not consume either pending full-column reference. + assertSame(values, input.getColumn()); + assertSame(block.getTimeColumn(), time.getColumn()); + } + return result; + } finally { + transformer.close(); + } + } + + private static TsBlock block(Column values) { + long[] times = new long[values.getPositionCount()]; + for (int i = 0; i < times.length; i++) { + times[i] = (i + 1) * 10L; + } + return new TsBlock(new TimeColumn(times.length, times), values); + } + + private static Column doubles(Double... values) { + ColumnBuilder builder = DOUBLE.createColumnBuilder(values.length); + for (Double value : values) { + if (value == null) { + builder.appendNull(); + } else { + builder.writeDouble(value); + } + } + return builder.build(); + } + + private static void assertDoubles(Column result, Double... expected) { + assertEquals(expected.length, result.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(result.isNull(i)); + } else { + assertFalse(result.isNull(i)); + assertEquals(expected[i], result.getDouble(i), 0.0); + } + } + } + + private static void assertStrings(Column result, String... expected) { + assertEquals(expected.length, result.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(result.isNull(i)); + } else { + assertFalse(result.isNull(i)); + assertEquals(expected[i], result.getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET)); + } + } + } + + private static class TestExecutor extends UDTFExecutor { + private TestExecutor(UDTF udf, Type type) throws Exception { + super("selection_test", ZoneId.of("UTC")); + this.udtf = udf; + UDFParameters parameters = + new UDFParameters( + Collections.singletonList("input"), + Collections.singletonList( + type == DOUBLE + ? org.apache.iotdb.udf.api.type.Type.DOUBLE + : org.apache.iotdb.udf.api.type.Type.TEXT), + Collections.emptyMap()); + udf.validate(new UDFParameterValidator(parameters)); + udf.beforeStart(parameters, configurations); + configurations.check(); + } + } + + private static class RecordingAbs extends UDTFAbs { + private int invocations; + private long[] times; + + @Override + public void transform(Column[] columns, ColumnBuilder builder) throws Exception { + invocations++; + Column time = columns[columns.length - 1]; + times = Arrays.copyOf(time.getLongs(), time.getPositionCount()); + super.transform(columns, builder); + } + } +}