Skip to content

Commit 666753c

Browse files
yuancuyuancu
committed
Support semi and anti join in the converted SQL (1860/2027)
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent 738129d commit 666753c

4 files changed

Lines changed: 170 additions & 1 deletion

File tree

‎core/src/main/java/org/opensearch/sql/calcite/validate/OpenSearchSparkSqlDialect.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77

88
import com.google.common.collect.ImmutableMap;
99
import java.util.Map;
10+
import lombok.experimental.Delegate;
1011
import org.apache.calcite.sql.SqlCall;
1112
import org.apache.calcite.sql.SqlWriter;
1213
import org.apache.calcite.sql.dialect.SparkSqlDialect;
14+
import org.apache.calcite.sql.validate.SqlConformance;
1315

1416
/**
1517
* Custom Spark SQL dialect that extends Calcite's SparkSqlDialect to handle OpenSearch-specific
@@ -68,4 +70,24 @@ private void unparseFunction(
6870
}
6971
writer.endList(frame);
7072
}
73+
74+
@Override
75+
public SqlConformance getConformance() {
76+
return new ConformanceDelegate(super.getConformance());
77+
}
78+
79+
/** SqlConformance delegator that enables liberal mode for LEFT SEMI/ANTI JOIN support. */
80+
private static class ConformanceDelegate implements SqlConformance {
81+
@Delegate private final SqlConformance delegate;
82+
83+
ConformanceDelegate(SqlConformance delegate) {
84+
this.delegate = delegate;
85+
}
86+
87+
@Override
88+
public boolean isLiberal() {
89+
// This allows SQL feature LEFT ANTI JOIN & LEFT SEMI JOIN
90+
return true;
91+
}
92+
}
7193
}

‎core/src/main/java/org/opensearch/sql/calcite/validate/PplRelToSqlNodeConverter.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package org.opensearch.sql.calcite.validate;
77

88
import org.apache.calcite.rel.core.Correlate;
9+
import org.apache.calcite.rel.core.Join;
910
import org.apache.calcite.rel.core.JoinRelType;
1011
import org.apache.calcite.rel.rel2sql.RelToSqlConverter;
1112
import org.apache.calcite.sql.JoinConditionType;
@@ -53,4 +54,57 @@ public Result visit(Correlate e) {
5354
}
5455
return result;
5556
}
57+
58+
/**
59+
* Override to convert ANTI and SEMI joins to Spark SQL's native LEFT ANTI JOIN and LEFT SEMI JOIN
60+
* syntax, instead of using NOT EXISTS / EXISTS subqueries.
61+
*
62+
* <p>The default implementation in {@link RelToSqlConverter#visitAntiOrSemiJoin} converts
63+
* ANTI/SEMI joins to standard SQL using NOT EXISTS / EXISTS subqueries. However, a subtle bug in
64+
* calcite (as of Calcite 1.41) leads to incorrect results after the conversion: correlation
65+
* variables in the subquery are generated as unqualified identifiers.
66+
*
67+
* <p>For example:
68+
*
69+
* <pre>{@code
70+
* -- Base implementation generates:
71+
* SELECT ... FROM table1 AS t0
72+
* WHERE ... AND NOT EXISTS (
73+
* SELECT 1 FROM table2 AS t2
74+
* WHERE name = t2.name -- 'name' is unqualified!
75+
* )
76+
* }</pre>
77+
*
78+
* <p>The unqualified {@code name} is resolved to the inner scope (t2.name) instead of the outer
79+
* scope (t0.name), resulting in incorrect results.
80+
*
81+
* <p>The override implementation uses ANTI / SEMI join syntax:
82+
*
83+
* <pre>{@code
84+
* SELECT ... FROM table1 AS t0
85+
* LEFT ANTI JOIN table2 AS t2 ON t0.name = t2.name
86+
* }</pre>
87+
*/
88+
@Override
89+
protected Result visitAntiOrSemiJoin(Join e) {
90+
final Result leftResult = visitInput(e, 0).resetAlias();
91+
final Result rightResult = visitInput(e, 1).resetAlias();
92+
final Context leftContext = leftResult.qualifiedContext();
93+
final Context rightContext = rightResult.qualifiedContext();
94+
95+
JoinType joinType =
96+
e.getJoinType() == JoinRelType.ANTI ? JoinType.LEFT_ANTI_JOIN : JoinType.LEFT_SEMI_JOIN;
97+
SqlNode sqlCondition = convertConditionToSqlNode(e.getCondition(), leftContext, rightContext);
98+
SqlNode join =
99+
new SqlJoin(
100+
POS,
101+
leftResult.asFrom(),
102+
SqlLiteral.createBoolean(false, POS),
103+
joinType.symbol(POS), // LEFT ANTI JOIN or LEFT SEMI JOIN
104+
rightResult.asFrom(),
105+
JoinConditionType.ON.symbol(POS),
106+
sqlCondition);
107+
108+
return result(join, leftResult, rightResult);
109+
}
56110
}

‎core/src/main/java/org/opensearch/sql/calcite/validate/PplSqlToRelConverter.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.validate;
7+
8+
import java.util.List;
9+
import org.apache.calcite.plan.RelOptCluster;
10+
import org.apache.calcite.plan.RelOptTable;
11+
import org.apache.calcite.prepare.Prepare;
12+
import org.apache.calcite.rel.RelNode;
13+
import org.apache.calcite.rel.core.JoinRelType;
14+
import org.apache.calcite.rel.logical.LogicalJoin;
15+
import org.apache.calcite.rel.logical.LogicalProject;
16+
import org.apache.calcite.sql.JoinType;
17+
import org.apache.calcite.sql.SqlJoin;
18+
import org.apache.calcite.sql.SqlNode;
19+
import org.apache.calcite.sql.validate.SqlValidator;
20+
import org.apache.calcite.sql2rel.SqlRexConvertletTable;
21+
import org.apache.calcite.sql2rel.SqlToRelConverter;
22+
import org.checkerframework.checker.nullness.qual.Nullable;
23+
24+
public class PplSqlToRelConverter extends SqlToRelConverter {
25+
public PplSqlToRelConverter(
26+
RelOptTable.ViewExpander viewExpander,
27+
@Nullable SqlValidator validator,
28+
Prepare.CatalogReader catalogReader,
29+
RelOptCluster cluster,
30+
SqlRexConvertletTable convertletTable,
31+
Config config) {
32+
super(viewExpander, validator, catalogReader, cluster, convertletTable, config);
33+
}
34+
35+
/**
36+
* Override to support Spark SQL's LEFT ANTI JOIN and LEFT SEMI JOIN conversion to RelNode.
37+
*
38+
* <p>The default implementation in {@link SqlToRelConverter#convertJoinType} does not expect
39+
* LEFT_ANTI_JOIN and LEFT_SEMI_JOIN. This override works around the limitation by first
40+
* temporarily changing LEFT_ANTI_JOIN/LEFT_SEMI_JOIN to LEFT join in the SqlJoin node, then
41+
* calling {@code super.convertFrom()} to perform normal conversion, finally substituting the join
42+
* type in the resulting RelNode to ANTI/SEMI.
43+
*
44+
* @param bb Scope within which to resolve identifiers
45+
* @param from FROM clause of a query.
46+
* @param fieldNames Field aliases, usually come from AS clause, or null
47+
*/
48+
@Override
49+
protected void convertFrom(
50+
Blackboard bb, @Nullable SqlNode from, @Nullable List<String> fieldNames) {
51+
JoinType originalJoinType = null;
52+
if (from instanceof SqlJoin join) {
53+
JoinType joinType = join.getJoinType();
54+
if (joinType == JoinType.LEFT_SEMI_JOIN || joinType == JoinType.LEFT_ANTI_JOIN) {
55+
join.setOperand(2, JoinType.LEFT.symbol(from.getParserPosition()));
56+
originalJoinType = joinType;
57+
}
58+
}
59+
super.convertFrom(bb, from, fieldNames);
60+
if (originalJoinType != null) {
61+
RelNode root = bb.root();
62+
if (root != null) {
63+
JoinRelType correctJoinType =
64+
originalJoinType == JoinType.LEFT_SEMI_JOIN ? JoinRelType.SEMI : JoinRelType.ANTI;
65+
RelNode fixedRoot = modifyJoinType(root, correctJoinType);
66+
bb.setRoot(fixedRoot, false);
67+
}
68+
}
69+
}
70+
71+
private RelNode modifyJoinType(RelNode root, JoinRelType correctJoinType) {
72+
if (root instanceof LogicalProject project) {
73+
RelNode input = project.getInput();
74+
RelNode fixedInput = modifyJoinType(input, correctJoinType);
75+
if (fixedInput != input) {
76+
return project.copy(
77+
project.getTraitSet(), fixedInput, project.getProjects(), project.getRowType());
78+
}
79+
} else if (root instanceof LogicalJoin join) {
80+
if (join.getJoinType() == JoinRelType.LEFT) {
81+
return join.copy(
82+
join.getTraitSet(),
83+
join.getCondition(),
84+
join.getLeft(),
85+
join.getRight(),
86+
correctJoinType,
87+
join.isSemiJoinDone());
88+
}
89+
}
90+
return root;
91+
}
92+
}

‎core/src/main/java/org/opensearch/sql/executor/QueryService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import org.opensearch.sql.calcite.validate.OpenSearchSparkSqlDialect;
6666
import org.opensearch.sql.calcite.validate.PplConvertletTable;
6767
import org.opensearch.sql.calcite.validate.PplRelToSqlNodeConverter;
68+
import org.opensearch.sql.calcite.validate.PplSqlToRelConverter;
6869
import org.opensearch.sql.calcite.validate.shuttles.PplRelToSqlRelShuttle;
6970
import org.opensearch.sql.calcite.validate.shuttles.SkipRelValidationShuttle;
7071
import org.opensearch.sql.common.response.ResponseListener;
@@ -391,7 +392,7 @@ public SqlNode visit(SqlIdentifier id) {
391392
SqlToRelConverter.Config sql2relConfig =
392393
SqlToRelConverter.config().withRemoveSortInSubQuery(false);
393394
SqlToRelConverter sql2rel =
394-
new SqlToRelConverter(
395+
new PplSqlToRelConverter(
395396
viewExpander,
396397
validator,
397398
catalogReader,

0 commit comments

Comments
 (0)