[CALCITE-5390] RelDecorrelator throws NullPointerException

Author: iwanttobepowerful

core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java

@@ -918,7 +918,7 @@ protected RexNode removeCorrelationExpr(
    *                     CASE WHEN cnt0 IS NOT NULL THEN cnt0 ELSE 0 END AS cnt
    *              FROM (SELECT deptno FROM dept GROUP BY deptno) d2
    *              LEFT JOIN (
-   *                  SELECT deptno, COUNT(e.empno) cnt0
+   *                  SELECT deptno, COUNT(emp.empno) cnt0
    *                  FROM emp
    *                  WHERE deptno IS NOT NULL
    *                  GROUP BY deptno) e
@@ -1421,7 +1421,8 @@ private static void shiftMapping(Map<Integer, Integer> mapping, int startIndex,
     for (CorRef corVar : correlations) {
       final int oldCorVarOffset = corVar.field;
 
-      final RelNode oldInput = requireNonNull(getCorRel(corVar));
+      final RelNode oldInput = findInputRel(corVar);
+
       final Frame frame = requireNonNull(getOrCreateFrame(oldInput));
       final RelNode newInput = frame.r;
 
@@ -1453,7 +1454,7 @@ private static void shiftMapping(Map<Integer, Integer> mapping, int startIndex,
 
     RelNode r = null;
     for (CorRef corVar : correlations) {
-      final RelNode oldInput = requireNonNull(getCorRel(corVar));
+      final RelNode oldInput = findInputRel(corVar);
       final RelNode newInput = requireNonNull(getOrCreateFrame(oldInput).r);
 
       if (!joinedInputs.contains(newInput)) {
@@ -1487,7 +1488,7 @@ private static void shiftMapping(Map<Integer, Integer> mapping, int startIndex,
     for (CorRef corRef : correlations) {
       // The first input of a Correlate is always the rel defining
       // the correlated variables.
-      final RelNode oldInput = requireNonNull(getCorRel(corRef));
+      final RelNode oldInput = findInputRel(corRef);
       final Frame frame = getOrCreateFrame(oldInput);
       final RelNode newInput = requireNonNull(frame.r);
 
@@ -1533,6 +1534,39 @@ private RelNode getCorRel(CorRef corVar) {
         () -> "r.getInput(0) is null for " + r);
   }
 
+  /**
+   * Finds the RelNode that produces the given correlation variable.
+   *
+   * <p>This method resolves correlation variables by inspecting the {@link #frameStack},
+   * which maintains the active correlation contexts during the top-down traversal.
+   *
+   * <p>The lookup logic implements <b>Lexical Scoping</b> (with Shadowing):
+   * <ul>
+   *   <li>The {@code frameStack} is traversed from top to bottom (most recently pushed to
+   *       least recently pushed). This ensures that if multiple nested queries use the same
+   *       {@link CorrelationId}, the innermost definition takes precedence, shadowing outer ones.
+   *   </li>
+   * </ul>
+   *
+   * <p>If the variable is not found in the {@code frameStack} (e.g., it might be defined outside
+   * the current traversal path or in a global context), the method falls back to looking it up
+   * in the global {@link #cm} (CorelMap).
+   *
+   * @param corVar The correlation variable reference to resolve.
+   * @return The {@link RelNode} that produces the correlation variable.
+   */
+  private RelNode findInputRel(CorRef corVar) {
+    final int oldCorVarOffset = corVar.field;
+    for (Pair<CorrelationId, Frame> pair : frameStack) {
+      if (pair.left.equals(corVar.corr)) {
+        if (oldCorVarOffset < pair.right.oldRel.getRowType().getFieldCount()) {
+          return pair.right.oldRel;
+        }
+      }
+    }
+    return getCorRel(corVar);
+  }
+
   /** Adds a value generator to satisfy the correlating variables used by
    * a relational expression, if those variables are not already provided by
    * its input. */
@@ -3766,12 +3800,16 @@ private RexVisitorImpl<Void> rexVisitor(final RelNode rel) {
    * and where to find the output fields and correlation variables
    * among its output fields. */
   static class Frame {
+    // The original relational expression before decorrelation
+    final RelNode oldRel;
+    // The decorrelated relational expression
     final RelNode r;
     final ImmutableSortedMap<CorDef, Integer> corDefOutputs;
     final ImmutableSortedMap<Integer, Integer> oldToNewOutputs;
 
     Frame(RelNode oldRel, RelNode r, NavigableMap<CorDef, Integer> corDefOutputs,
         Map<Integer, Integer> oldToNewOutputs) {
+      this.oldRel = requireNonNull(oldRel, "oldRel");
       this.r = requireNonNull(r, "r");
       this.corDefOutputs = ImmutableSortedMap.copyOf(corDefOutputs);
       this.oldToNewOutputs = ImmutableSortedMap.copyOf(oldToNewOutputs);

core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java

@@ -973,8 +973,6 @@ public TopDownGeneralDecorrelator getVisitor() {
    * Unnesting information.
    */
   static class UnnestedQuery extends Frame {
-    final RelNode oldRel;
-
     /**
      * Creates a UnnestedQuery.
      *
@@ -986,7 +984,6 @@ static class UnnestedQuery extends Frame {
     UnnestedQuery(RelNode oldRel, RelNode r, NavigableMap<CorDef, Integer> corDefOutputs,
         Map<Integer, Integer> oldToNewOutputs) {
       super(oldRel, r, corDefOutputs, oldToNewOutputs);
-      this.oldRel = oldRel;
     }
 
     /**

core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java

@@ -1669,6 +1669,95 @@ public static Frameworks.ConfigBuilder config() {
     assertThat(after, hasTree(planAfter));
   }
 
+  /** Test case for <a href="https://issues.apache.org/jira/browse/CALCITE-5390">[CALCITE-5390]
+   * RelDecorrelator throws NullPointerException</a>. */
+  @Test void testCorrelationLexicalScoping() {
+    final FrameworkConfig frameworkConfig = config().build();
+    final RelBuilder builder = RelBuilder.create(frameworkConfig);
+    final RelOptCluster cluster = builder.getCluster();
+    final Planner planner = Frameworks.getPlanner(frameworkConfig);
+    final String sql = ""
+        + "select deptno,\n"
+        + "  (select min(1) from emp where empno > d.deptno) as i0,\n"
+        + "  (select min(0) from emp where deptno = d.deptno and "
+        + "ename = 'SMITH' and d.deptno > 0) as i1\n"
+        + "from dept as d";
+    final RelNode originalRel;
+    try {
+      final SqlNode parse = planner.parse(sql);
+      final SqlNode validate = planner.validate(parse);
+      originalRel = planner.rel(validate).rel;
+    } catch (Exception e) {
+      throw TestUtil.rethrow(e);
+    }
+
+    final HepProgram hepProgram = HepProgram.builder()
+        .addRuleCollection(
+            ImmutableList.of(
+                // SubQuery program rules
+                CoreRules.FILTER_SUB_QUERY_TO_CORRELATE,
+                CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE,
+                CoreRules.JOIN_SUB_QUERY_TO_CORRELATE))
+        .build();
+    final Program program =
+        Programs.of(hepProgram, true,
+            requireNonNull(cluster.getMetadataProvider()));
+    final RelNode before =
+        program.run(cluster.getPlanner(), originalRel, cluster.traitSet(),
+            Collections.emptyList(), Collections.emptyList());
+    final String planBefore = ""
+        + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$4])\n"
+        + "  LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n"
+        + "    LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n"
+        + "      LogicalTableScan(table=[[scott, DEPT]])\n"
+        + "      LogicalAggregate(group=[{}], EXPR$0=[MIN($0)])\n"
+        + "        LogicalProject($f0=[1])\n"
+        + "          LogicalFilter(condition=[>($0, CAST($cor0.DEPTNO):SMALLINT NOT NULL)])\n"
+        + "            LogicalTableScan(table=[[scott, EMP]])\n"
+        + "    LogicalAggregate(group=[{}], EXPR$0=[MIN($0)])\n"
+        + "      LogicalProject($f0=[0])\n"
+        + "        LogicalFilter(condition=[AND(=($7, $cor0.DEPTNO), =($1, 'SMITH'), >(CAST($cor0.DEPTNO):INTEGER NOT NULL, 0))])\n"
+        + "          LogicalTableScan(table=[[scott, EMP]])\n";
+    assertThat(before, hasTree(planBefore));
+
+    // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator
+    final RelNode after =
+        RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()),
+            RuleSets.ofList(Collections.emptyList()));
+    final String planAfter = ""
+        + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$8])\n"
+        + "  LogicalJoin(condition=[AND(=($0, $6), =($5, $7))], joinType=[left])\n"
+        + "    LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], DEPTNO0=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n"
+        + "      LogicalJoin(condition=[=($3, $4)], joinType=[left])\n"
+        + "        LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n"
+        + "          LogicalTableScan(table=[[scott, DEPT]])\n"
+        + "        LogicalAggregate(group=[{0}], EXPR$0=[MIN($1)])\n"
+        + "          LogicalProject(DEPTNO0=[$8], $f0=[1])\n"
+        + "            LogicalJoin(condition=[>($0, $8)], joinType=[inner])\n"
+        + "              LogicalTableScan(table=[[scott, EMP]])\n"
+        + "              LogicalAggregate(group=[{0}])\n"
+        + "                LogicalProject(DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n"
+        + "                  LogicalTableScan(table=[[scott, DEPT]])\n"
+        + "    LogicalAggregate(group=[{0, 1}], EXPR$0=[MIN($2)])\n"
+        + "      LogicalProject(DEPTNO0=[$8], $f5=[$9], $f0=[0])\n"
+        + "        LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n"
+        + "          LogicalFilter(condition=[=($1, 'SMITH')])\n"
+        + "            LogicalTableScan(table=[[scott, EMP]])\n"
+        + "          LogicalFilter(condition=[$1])\n"
+        + "            LogicalProject(DEPTNO=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n"
+        + "              LogicalJoin(condition=[=($3, $4)], joinType=[left])\n"
+        + "                LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n"
+        + "                  LogicalTableScan(table=[[scott, DEPT]])\n"
+        + "                LogicalAggregate(group=[{0}], EXPR$0=[MIN($1)])\n"
+        + "                  LogicalProject(DEPTNO0=[$8], $f0=[1])\n"
+        + "                    LogicalJoin(condition=[>($0, $8)], joinType=[inner])\n"
+        + "                      LogicalTableScan(table=[[scott, EMP]])\n"
+        + "                      LogicalAggregate(group=[{0}])\n"
+        + "                        LogicalProject(DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n"
+        + "                          LogicalTableScan(table=[[scott, DEPT]])\n";
+    assertThat(after, hasTree(planAfter));
+  }
+
   /** Test case for <a href="https://issues.apache.org/jira/browse/CALCITE-7320">[CALCITE-7320]
    * AggregateProjectMergeRule throws AssertionError when Project maps multiple grouping keys
    * to the same field</a>. */

core/src/test/resources/sql/sub-query.iq

@@ -8967,4 +8967,79 @@ where e.deptno = 10;
 
 !ok
 
+# [CALCITE-5390] RelDecorrelator throws NullPointerException
+# Verified against PostgreSQL.
+select deptno,
+  (select min(1) from emp where empno > d.deptno) as i0,
+  (select min(0) from emp where deptno = d.deptno and ename = 'SMITH' and d.deptno > 0) as i1
+from dept as d;
++--------+----+----+
+| DEPTNO | I0 | I1 |
++--------+----+----+
+|     10 |  1 |    |
+|     20 |  1 |  0 |
+|     30 |  1 |    |
+|     40 |  1 |    |
++--------+----+----+
+(4 rows)
+
+!ok
+
+# [CALCITE-5390] RelDecorrelator throws NullPointerException
+# Verified against PostgreSQL.
+SELECT
+  (SELECT 1 FROM emp d WHERE d.job = a.job LIMIT 1) AS t1,
+  (SELECT a.job = 'PRESIDENT' FROM emp s LIMIT 1) as t2
+FROM emp a;
++----+-------+
+| T1 | T2    |
++----+-------+
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | false |
+|  1 | true  |
++----+-------+
+(14 rows)
+
+!ok
+
+# [CALCITE-5390] RelDecorrelator throws NullPointerException
+# Verified against PostgreSQL.
+SELECT *
+FROM emp e
+WHERE e.ename NOT IN (
+    SELECT d.dname
+    FROM dept d
+    WHERE e.deptno = d.deptno OR e.sal > 2000.0);
++-------+--------+-----------+------+------------+---------+---------+--------+
+| EMPNO | ENAME  | JOB       | MGR  | HIREDATE   | SAL     | COMM    | DEPTNO |
++-------+--------+-----------+------+------------+---------+---------+--------+
+|  7369 | SMITH  | CLERK     | 7902 | 1980-12-17 |  800.00 |         |     20 |
+|  7499 | ALLEN  | SALESMAN  | 7698 | 1981-02-20 | 1600.00 |  300.00 |     30 |
+|  7521 | WARD   | SALESMAN  | 7698 | 1981-02-22 | 1250.00 |  500.00 |     30 |
+|  7566 | JONES  | MANAGER   | 7839 | 1981-02-04 | 2975.00 |         |     20 |
+|  7654 | MARTIN | SALESMAN  | 7698 | 1981-09-28 | 1250.00 | 1400.00 |     30 |
+|  7698 | BLAKE  | MANAGER   | 7839 | 1981-01-05 | 2850.00 |         |     30 |
+|  7782 | CLARK  | MANAGER   | 7839 | 1981-06-09 | 2450.00 |         |     10 |
+|  7788 | SCOTT  | ANALYST   | 7566 | 1987-04-19 | 3000.00 |         |     20 |
+|  7839 | KING   | PRESIDENT |      | 1981-11-17 | 5000.00 |         |     10 |
+|  7844 | TURNER | SALESMAN  | 7698 | 1981-09-08 | 1500.00 |    0.00 |     30 |
+|  7876 | ADAMS  | CLERK     | 7788 | 1987-05-23 | 1100.00 |         |     20 |
+|  7900 | JAMES  | CLERK     | 7698 | 1981-12-03 |  950.00 |         |     30 |
+|  7902 | FORD   | ANALYST   | 7566 | 1981-12-03 | 3000.00 |         |     20 |
+|  7934 | MILLER | CLERK     | 7782 | 1982-01-23 | 1300.00 |         |     10 |
++-------+--------+-----------+------+------------+---------+---------+--------+
+(14 rows)
+
+!ok
 # End sub-query.iq