Skip to content

Commit 1bc2dfb

Browse files
AMashenkovAMashenkov
authored
IGNITE-28389 SQL. Eliminate table scans with always_false predicate (#7959)
1 parent 652a4d5 commit 1bc2dfb

7 files changed

Lines changed: 201 additions & 53 deletions

File tree

‎modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItDmlTest.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.apache.ignite.lang.IgniteException;
5656
import org.apache.ignite.tx.Transaction;
5757
import org.apache.ignite.tx.TransactionOptions;
58+
import org.hamcrest.Matchers;
5859
import org.jetbrains.annotations.Nullable;
5960
import org.junit.jupiter.api.AfterEach;
6061
import org.junit.jupiter.api.Disabled;
@@ -1179,6 +1180,17 @@ public void rejectInvalidColumnNumberOnInsert() {
11791180
);
11801181
}
11811182

1183+
@Test
1184+
public void insertFromSelectWithAlwaysFalseCondition() {
1185+
sql("CREATE TABLE test (id INT PRIMARY KEY, val REAL)");
1186+
sql("CREATE TABLE test2 (id INT PRIMARY KEY, val REAL)");
1187+
1188+
assertQuery("INSERT INTO test2 SELECT id, val FROM test WHERE val > 1 AND val < 0")
1189+
.matches(Matchers.not(containsSubPlan("TableModify")))
1190+
.returns(0L)
1191+
.check();
1192+
}
1193+
11821194
private static Stream<Arguments> decimalLimits() {
11831195
return Stream.of(
11841196
arguments(SqlTypeName.BIGINT.getName(), Long.MAX_VALUE, Long.MIN_VALUE),

‎modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,8 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner) {
182182

183183
rel = planner.transform(PlannerPhase.HEP_PROJECT_PUSH_DOWN, rel.getTraitSet(), rel);
184184

185+
rel = planner.transform(PlannerPhase.HEP_EMPTY_NODES_ELIMINATION, rel.getTraitSet(), rel);
186+
185187
if (fastQueryOptimizationEnabled()) {
186188
// the sole purpose of this code block is to limit scope of `simpleOperation` variable.
187189
// The result of `HEP_TO_SIMPLE_KEY_VALUE_OPERATION` phase MUST NOT be passed to next stage,

‎modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerPhase.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
import org.apache.ignite.internal.sql.engine.rule.logical.IgniteProjectCorrelateTransposeRule;
7979
import org.apache.ignite.internal.sql.engine.rule.logical.LogicalOrToUnionRule;
8080
import org.apache.ignite.internal.sql.engine.rule.logical.ProjectScanMergeRule;
81+
import org.apache.ignite.internal.sql.engine.rule.logical.PruneTableModifyRule;
8182
import org.apache.ignite.internal.sql.engine.util.Commons;
8283

8384
/**
@@ -161,6 +162,22 @@ public Program getProgram(PlanningContext ctx) {
161162
}
162163
},
163164

165+
HEP_EMPTY_NODES_ELIMINATION(
166+
"Heuristic phase to eliminate empty nodes",
167+
PruneEmptyRules.PROJECT_INSTANCE,
168+
PruneEmptyRules.FILTER_INSTANCE,
169+
PruneEmptyRules.SORT_INSTANCE,
170+
PruneEmptyRules.AGGREGATE_INSTANCE,
171+
PruneEmptyRules.JOIN_LEFT_INSTANCE,
172+
PruneEmptyRules.JOIN_RIGHT_INSTANCE
173+
) {
174+
/** {@inheritDoc} */
175+
@Override
176+
public Program getProgram(PlanningContext ctx) {
177+
return hep(getRules(ctx));
178+
}
179+
},
180+
164181
HEP_OPTIMIZE_JOIN_ORDER(
165182
"Heuristic phase to optimize join order"
166183
) {
@@ -207,6 +224,13 @@ public Program getProgram(PlanningContext ctx) {
207224
IgniteJoinConditionPushRule.INSTANCE,
208225
CoreRules.JOIN_PUSH_TRANSITIVE_PREDICATES,
209226

227+
PruneEmptyRules.PROJECT_INSTANCE,
228+
PruneEmptyRules.FILTER_INSTANCE,
229+
PruneEmptyRules.SORT_INSTANCE,
230+
PruneEmptyRules.AGGREGATE_INSTANCE,
231+
PruneEmptyRules.JOIN_LEFT_INSTANCE,
232+
PruneEmptyRules.JOIN_RIGHT_INSTANCE,
233+
210234
FilterIntoJoinRule.FilterIntoJoinRuleConfig.DEFAULT
211235
.withOperandSupplier(b0 ->
212236
b0.operand(LogicalFilter.class).oneInput(b1 ->
@@ -255,6 +279,7 @@ public Program getProgram(PlanningContext ctx) {
255279

256280
PruneEmptyRules.CORRELATE_LEFT_INSTANCE,
257281
PruneEmptyRules.CORRELATE_RIGHT_INSTANCE,
282+
PruneTableModifyRule.INSTANCE,
258283

259284
// Useful of this rule is not clear now.
260285
// CoreRules.AGGREGATE_REDUCE_FUNCTIONS,

‎modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/FilterScanMergeRule.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.calcite.plan.RelTraitSet;
2727
import org.apache.calcite.rel.RelNode;
2828
import org.apache.calcite.rel.logical.LogicalFilter;
29+
import org.apache.calcite.rel.logical.LogicalValues;
2930
import org.apache.calcite.rex.RexBuilder;
3031
import org.apache.calcite.rex.RexInputRef;
3132
import org.apache.calcite.rex.RexNode;
@@ -99,6 +100,21 @@ public RexNode visitInputRef(RexInputRef ref) {
99100
// We need to replace RexInputRef with RexLocalRef because TableScan doesn't have inputs.
100101
condition = RexUtils.replaceInputRefs(condition);
101102

103+
// Eliminate scan if always false condition found.
104+
if (condition.isAlwaysFalse()) {
105+
call.transformTo(LogicalValues.createEmpty(cluster, scan.getRowType()));
106+
call.getPlanner().prune(filter);
107+
call.getPlanner().prune(scan);
108+
return;
109+
}
110+
111+
// Eliminate always true condition.
112+
if (condition.isAlwaysTrue()) {
113+
call.transformTo(scan);
114+
call.getPlanner().prune(filter);
115+
return;
116+
}
117+
102118
// Set default traits, real traits will be calculated for physical node.
103119
RelTraitSet trait = cluster.traitSet();
104120

‎modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/PruneTableModifyRule.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.ignite.internal.sql.engine.rule.logical;
19+
20+
import java.util.Collections;
21+
import java.util.List;
22+
import org.apache.calcite.plan.RelOptRule;
23+
import org.apache.calcite.plan.RelOptRuleCall;
24+
import org.apache.calcite.plan.RelRule;
25+
import org.apache.calcite.rel.RelNode;
26+
import org.apache.calcite.rel.core.TableModify;
27+
import org.apache.calcite.rel.core.Values;
28+
import org.apache.calcite.rel.rules.SubstitutionRule;
29+
import org.apache.calcite.rex.RexLiteral;
30+
import org.apache.ignite.internal.sql.engine.rex.IgniteRexBuilder;
31+
import org.apache.ignite.internal.sql.engine.rule.logical.PruneTableModifyRule.Config;
32+
import org.immutables.value.Value;
33+
34+
/**
35+
* Rule that eliminates table modify node if it doesn't have any source rows.
36+
*/
37+
@Value.Enclosing
38+
public class PruneTableModifyRule extends RelRule<Config> implements SubstitutionRule {
39+
public static final RelOptRule INSTANCE = Config.DEFAULT.toRule();
40+
41+
/**
42+
* Constructor.
43+
*
44+
* @param config Rule configuration.
45+
*/
46+
private PruneTableModifyRule(PruneTableModifyRule.Config config) {
47+
super(config);
48+
}
49+
50+
@Override public void onMatch(RelOptRuleCall call) {
51+
TableModify singleRel = call.rel(0);
52+
53+
// TODO https://issues.apache.org/jira/browse/IGNITE-23512: Default Calcite RexBuilder ignores field type and extract type from
54+
// the given value. E.g. for zero value RexBuilder creates INT literal. Use simple way create `singleValue` after fixing the issue.
55+
// RelNode singleValue = call.builder().values(singleRel.getRowType(), 0L).build();
56+
RexLiteral zeroLiteral = IgniteRexBuilder.INSTANCE.makeLiteral(0L, singleRel.getRowType().getFieldList().get(0).getType());
57+
RelNode singleValue = call.builder().values(List.of(List.of(zeroLiteral)), singleRel.getRowType()).build();
58+
59+
singleValue = singleValue.copy(singleRel.getCluster().traitSet(), Collections.emptyList());
60+
call.transformTo(singleValue);
61+
}
62+
63+
/** Rule configuration. */
64+
@Value.Immutable(singleton = false)
65+
public interface Config extends RuleFactoryConfig<Config> {
66+
Config DEFAULT = ImmutablePruneTableModifyRule.Config.builder()
67+
.withDescription("PruneTableModify")
68+
.withRuleFactory(PruneTableModifyRule::new)
69+
.withOperandSupplier(b0 ->
70+
b0.operand(TableModify.class).oneInput(b1 ->
71+
b1.operand(Values.class).predicate(Values::isEmpty).noInputs()))
72+
.build();
73+
}
74+
}

‎modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/ProjectFilterScanMergePlannerTest.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,18 @@
1919

2020
import java.util.List;
2121
import java.util.Objects;
22+
import java.util.function.Predicate;
2223
import java.util.function.UnaryOperator;
2324
import java.util.stream.Collectors;
25+
import org.apache.calcite.rex.RexLiteral;
2426
import org.apache.calcite.rex.RexNode;
2527
import org.apache.calcite.util.ImmutableIntList;
2628
import org.apache.ignite.internal.sql.engine.framework.TestBuilders.TableBuilder;
2729
import org.apache.ignite.internal.sql.engine.prepare.bounds.SearchBounds;
2830
import org.apache.ignite.internal.sql.engine.rel.IgniteAggregate;
2931
import org.apache.ignite.internal.sql.engine.rel.IgniteIndexScan;
3032
import org.apache.ignite.internal.sql.engine.rel.IgniteTableScan;
33+
import org.apache.ignite.internal.sql.engine.rel.IgniteValues;
3134
import org.apache.ignite.internal.sql.engine.schema.IgniteSchema;
3235
import org.apache.ignite.internal.sql.engine.trait.IgniteDistributions;
3336
import org.apache.ignite.internal.type.NativeTypes;
@@ -275,6 +278,75 @@ public void testFilterIdentityFilterMerge() throws Exception {
275278
"ProjectFilterTransposeRule", "FilterProjectTransposeRule");
276279
}
277280

281+
@Test
282+
public void testAlwaysTrueFilterPruning() throws Exception {
283+
String sql = "SELECT a, c FROM tbl WHERE a > 1 OR a < 3 OR a IS NULL";
284+
285+
assertPlan(sql, publicSchema, isInstanceOf(IgniteTableScan.class)
286+
.and(scan -> scan.projects() == null)
287+
.and(scan -> scan.condition() == null)
288+
.and(scan -> ImmutableIntList.of(0, 2).equals(scan.requiredColumns())),
289+
"ProjectFilterTransposeRule", "FilterProjectTransposeRule");
290+
}
291+
292+
@Test
293+
public void testAlwaysFalseFilterPruning() throws Exception {
294+
Predicate<IgniteValues> hasEmptyValuesOnly = hasEmptyValuesOnlyPredicate();
295+
296+
// Table scan elimination.
297+
String sql = "SELECT a, c FROM tbl WHERE a > 1 AND a < 0";
298+
assertPlan(sql, publicSchema, hasEmptyValuesOnly);
299+
300+
sql = "SELECT a, c FROM (SELECT a, c FROM tbl WHERE a > 1) WHERE c = 1 AND c IS NULL";
301+
assertPlan(sql, publicSchema, hasEmptyValuesOnly,
302+
"ProjectFilterTransposeRule", "FilterProjectTransposeRule");
303+
304+
sql = "SELECT a, c FROM (SELECT a, c FROM tbl WHERE a > 1) WHERE a < 0";
305+
assertPlan(sql, publicSchema, hasEmptyValuesOnly,
306+
"ProjectFilterTransposeRule", "FilterProjectTransposeRule");
307+
308+
// JOIN branch elimination.
309+
sql = "SELECT t1.a, t2.a, t1.c FROM tbl AS t1 LEFT JOIN tbl AS t2 ON t1.a = t2.a WHERE t2.a = 1 AND t2.a IS NULL AND t1.c = 1";
310+
assertPlan(sql, publicSchema, hasEmptyValuesOnly);
311+
312+
sql = "SELECT t1.a, t2.a, t1.c FROM tbl AS t1 INNER JOIN tbl AS t2 ON t1.a = t2.a WHERE t2.a = 1 AND t2.a IS NULL";
313+
assertPlan(sql, publicSchema, hasEmptyValuesOnly);
314+
315+
sql = "SELECT t1.a, t2.a, t1.c FROM tbl AS t1 INNER JOIN tbl AS t2 ON t1.a = t2.a WHERE t1.a = 1 AND t2.a = 2";
316+
assertPlan(sql, publicSchema, hasEmptyValuesOnlyPredicate());
317+
}
318+
319+
@Test
320+
public void testJoinWithAlwaysFalseConditionPruning() throws Exception {
321+
String sql = "SELECT t1.a, t2.a, t1.c FROM tbl AS t1 LEFT JOIN tbl AS t2 ON (t1.a = t2.a AND t2.a = 1 AND t2.a = 2) WHERE t1.c = 1";
322+
assertPlan(sql, publicSchema, isInstanceOf(IgniteTableScan.class)
323+
.and(scan -> scan.projects() != null)
324+
.and(scan -> scan.condition() != null)
325+
.and(scan -> "=($t1, 1)".equals(scan.condition().toString()))
326+
);
327+
328+
sql = "SELECT t1.a, t2.a, t1.c FROM tbl AS t1 INNER JOIN tbl AS t2 ON t1.a = t2.a AND t2.a = 1 AND t2.a = 2";
329+
assertPlan(sql, publicSchema, hasEmptyValuesOnlyPredicate());
330+
}
331+
332+
@Test
333+
public void testAlwaysFalseFilterPruningWithDml() throws Exception {
334+
Predicate<IgniteValues> zeroDmlResultPredicate = isInstanceOf(IgniteValues.class)
335+
.and(values -> values.getTuples().size() == 1) // single row
336+
.and(values -> values.getTuples().get(0).size() == 1) // row of single column
337+
.and(values -> RexLiteral.longValue(values.getTuples().get(0).get(0)) == 0L);
338+
339+
String sql = "INSERT INTO tbl (a, c) SELECT a, b FROM tbl WHERE a > 1 AND a < 0";
340+
assertPlan(sql, publicSchema, zeroDmlResultPredicate);
341+
342+
sql = "INSERT INTO tbl (a, c) (SELECT a, c FROM (SELECT a, c FROM tbl WHERE a > 1) WHERE a < 0)";
343+
assertPlan(sql, publicSchema, zeroDmlResultPredicate);
344+
}
345+
346+
private Predicate<IgniteValues> hasEmptyValuesOnlyPredicate() {
347+
return isInstanceOf(IgniteValues.class).and(values -> values.getTuples().isEmpty());
348+
}
349+
278350
/**
279351
* Convert search bounds to RexNodes.
280352
*/

‎modules/sql-engine/src/test/resources/mapping/test_partition_pruning.test‎

Lines changed: 0 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -163,59 +163,6 @@ Fragment#4
163163
fieldNames: [ID, C1, C2]
164164
est: (rows=1)
165165
---
166-
# Self join, different predicates that produce disjoint set of partitions
167-
# TODO https://issues.apache.org/jira/browse/IGNITE-28389: Fix the test. We expect the mapper should eliminate all the disjoined parts.
168-
N1
169-
SELECT /*+ DISABLE_RULE('NestedLoopJoinConverter', 'HashJoinConverter', 'CorrelatedNestedLoopJoin') */ *
170-
FROM t1_n1n2n3 as t1, t1_n1n2n3 as t2
171-
WHERE t1.id = t2.id and t1.id IN (1, 3) and t2.id IN (42, 44)
172-
---
173-
Fragment#2 root
174-
distribution: single
175-
executionNodes: [N1]
176-
exchangeSourceNodes: {3=[N1, N2, N3]}
177-
colocationGroup[-1]: {nodes=[N1], sourceIds=[-1, 3], assignments={}, partitionsWithConsistencyTokens={N1=[]}}
178-
colocationGroup[3]: {nodes=[N1], sourceIds=[-1, 3], assignments={}, partitionsWithConsistencyTokens={N1=[]}}
179-
tree:
180-
Receiver
181-
fieldNames: [ID, C1, C2, ID$0, C1$0, C2$0]
182-
sourceFragmentId: 3
183-
est: (rows=1)
184-
185-
Fragment#3
186-
distribution: table PUBLIC.T1_N1N2N3 in zone ZONE_1
187-
executionNodes: [N1, N2, N3]
188-
targetNodes: [N1]
189-
colocationGroup[0]: {nodes=[N1, N2, N3], sourceIds=[0, 1], assignments={part_0=N1:3, part_1=N2:3, part_2=N3:3}, partitionsWithConsistencyTokens={N1=[part_0:3], N2=[part_1:3], N3=[part_2:3]}}
190-
colocationGroup[1]: {nodes=[N1, N2, N3], sourceIds=[0, 1], assignments={part_0=N1:3, part_1=N2:3, part_2=N3:3}, partitionsWithConsistencyTokens={N1=[part_0:3], N2=[part_1:3], N3=[part_2:3]}}
191-
partitions: [T1_N1N2N3=[N1={0}, N2={1}, N3={2}]]
192-
tree:
193-
Sender
194-
distribution: single
195-
targetFragmentId: 2
196-
est: (rows=6250)
197-
MergeJoin
198-
predicate: =(ID, ID$0)
199-
fieldNames: [ID, C1, C2, ID$0, C1$0, C2$0]
200-
type: inner
201-
est: (rows=6250)
202-
Sort
203-
collation: [ID ASC]
204-
est: (rows=25000)
205-
TableScan
206-
table: PUBLIC.T1_N1N2N3
207-
predicate: false
208-
fieldNames: [ID, C1, C2]
209-
est: (rows=25000)
210-
Sort
211-
collation: [ID ASC]
212-
est: (rows=25000)
213-
TableScan
214-
table: PUBLIC.T1_N1N2N3
215-
predicate: false
216-
fieldNames: [ID, C1, C2]
217-
est: (rows=25000)
218-
---
219166
# Correlated
220167
# Prune partitions from left arm statically, and pass meta to the right arm.
221168
# Same set of nodes.

0 commit comments

Comments
 (0)