From e2fa21b3659777901609b89e8050bc9d6abdfa7d Mon Sep 17 00:00:00 2001 From: Mohammad Naqvi <81446894+mohammadnaqvi04@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:53:36 -0400 Subject: [PATCH 1/4] reject correlated IN/NOT IN over a groupless aggregate --- .../src/decorrelate_predicate_subquery.rs | 30 +++++++++++++++++++ .../sqllogictest/test_files/subquery.slt | 26 ++++++++++++---- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs index febbd3a6c84b5..45564c3eb5603 100644 --- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs +++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs @@ -854,6 +854,36 @@ fn build_join( if !pull_up.can_pull_up { return Ok(None); } + // Stopgap until a follow-up adds count-bug compensation for `IN`/`NOT IN`. + // Reject unless an outer row with no match gets the right answer anyway. + if !exists_sub_query { + let null_reads_as_false = match join_type { + JoinType::LeftSemi => true, + JoinType::LeftMark => !needs_null_aware_mark, + _ => false, + }; + let count_bug_is_harmless = match rewrite(true) { + Ok((plan, pull_up)) => pull_up + .collected_count_expr_map + .get(&plan) + .filter(|m| !m.is_empty()) + .is_none_or(|empty_input_values| { + null_reads_as_false + && empty_input_values + .get(subquery.schema().field(0).name()) + .is_some_and(|value| { + matches!(value, Expr::Literal(v, _) if v.is_null()) + }) + }), + // Count-bug handling errors on some shapes it cannot compensate. + Err(_) => false, + }; + if !count_bug_is_harmless { + return not_impl_err!( + "Correlated IN/NOT IN subquery with an aggregate and no GROUP BY is not supported: an outer row with no match gets no subquery row instead of the aggregate's value on empty input" + ); + } + } let sub_query_alias = LogicalPlanBuilder::from(new_plan) .alias(alias.to_string())? diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index d4a42cca0c801..53ee85b0ae539 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2124,14 +2124,28 @@ WHERE t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) ---- 11 -# correlated_not_in_groupless_count_agg_is_not_count_bug_compensated -query I rowsort +# correlated_in_groupless_count_agg_count_bug_not_implemented +query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +SELECT t1_id FROM t1 +WHERE 0 IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id) + +# correlated_not_in_groupless_count_agg_count_bug_not_implemented +query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY SELECT t1_id FROM t1 WHERE t1_int NOT IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id) ----- -22 -33 -44 + +# correlated_in_groupless_count_agg_in_projection_count_bug_not_implemented +query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +SELECT t1_id, 0 IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id) FROM t1 + +# correlated_not_in_groupless_max_agg_count_bug_not_implemented +query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +SELECT t1_id FROM t1 +WHERE t1_int + 2 NOT IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) + +# correlated_in_groupless_max_agg_in_projection_count_bug_not_implemented +query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +SELECT t1_id, t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) FROM t1 statement ok DROP TABLE limit_zero_outer; From e8f91c3fc4a65ced681d186b221aea9c896995a3 Mon Sep 17 00:00:00 2001 From: Mohammad Naqvi <81446894+mohammadnaqvi04@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:01:24 -0400 Subject: [PATCH 2/4] chore: tidy up error message --- datafusion/optimizer/src/decorrelate_predicate_subquery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs index 45564c3eb5603..0da855af31a41 100644 --- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs +++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs @@ -880,7 +880,7 @@ fn build_join( }; if !count_bug_is_harmless { return not_impl_err!( - "Correlated IN/NOT IN subquery with an aggregate and no GROUP BY is not supported: an outer row with no match gets no subquery row instead of the aggregate's value on empty input" + "Correlated IN/NOT IN subquery with an aggregate and no GROUP BY is not supported" ); } } From 87405286031d72d35b7a0107213f369edd5b67ed Mon Sep 17 00:00:00 2001 From: Mohammad Naqvi <81446894+mohammadnaqvi04@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:40:58 -0400 Subject: [PATCH 3/4] chore: keep the subquery correlated --- .../src/decorrelate_predicate_subquery.rs | 7 ++--- .../sqllogictest/test_files/subquery.slt | 29 ++++++++++++------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs index 0da855af31a41..da17dd3e7c789 100644 --- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs +++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs @@ -855,7 +855,8 @@ fn build_join( return Ok(None); } // Stopgap until a follow-up adds count-bug compensation for `IN`/`NOT IN`. - // Reject unless an outer row with no match gets the right answer anyway. + // The subquery stays correlated unless an outer row with no match gets the + // right answer anyway. if !exists_sub_query { let null_reads_as_false = match join_type { JoinType::LeftSemi => true, @@ -879,9 +880,7 @@ fn build_join( Err(_) => false, }; if !count_bug_is_harmless { - return not_impl_err!( - "Correlated IN/NOT IN subquery with an aggregate and no GROUP BY is not supported" - ); + return Ok(None); } } diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 53ee85b0ae539..3ba806b749c97 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2124,27 +2124,34 @@ WHERE t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) ---- 11 -# correlated_in_groupless_count_agg_count_bug_not_implemented -query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +# correlated_in_groupless_count_agg_stays_correlated +# Errors rather than returning wrong rows. Expected `33`. +query error Physical plan does not support logical expression InSubquery SELECT t1_id FROM t1 WHERE 0 IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id) -# correlated_not_in_groupless_count_agg_count_bug_not_implemented -query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +# correlated_not_in_groupless_count_agg_stays_correlated +# Errors rather than returning wrong rows. Expected `11, 22, 44`. +query error Physical plan does not support logical expression InSubquery SELECT t1_id FROM t1 -WHERE t1_int NOT IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id) +WHERE 0 NOT IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id) -# correlated_in_groupless_count_agg_in_projection_count_bug_not_implemented -query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +# correlated_in_groupless_count_agg_in_projection_stays_correlated +# Errors rather than returning wrong rows. +# Expected `(11, false), (22, false), (33, true), (44, false)`. +query error Physical plan does not support logical expression InSubquery SELECT t1_id, 0 IN (SELECT count(*) FROM t2 WHERE t2.t2_id = t1.t1_id) FROM t1 -# correlated_not_in_groupless_max_agg_count_bug_not_implemented -query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +# correlated_not_in_groupless_max_agg_stays_correlated +# Errors rather than returning wrong rows. Expected `22, 44`. +query error Physical plan does not support logical expression InSubquery SELECT t1_id FROM t1 WHERE t1_int + 2 NOT IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) -# correlated_in_groupless_max_agg_in_projection_count_bug_not_implemented -query error This feature is not implemented: Correlated IN/NOT IN subquery with an aggregate and no GROUP BY +# correlated_in_groupless_max_agg_in_projection_stays_correlated +# Errors rather than returning wrong rows. +# Expected `(11, true), (22, false), (33, NULL), (44, false)`. +query error Physical plan does not support logical expression InSubquery SELECT t1_id, t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) FROM t1 statement ok From 0f5c13271d1f2c31ed956452d33c359de3852e71 Mon Sep 17 00:00:00 2001 From: Mohammad Naqvi <81446894+mohammadnaqvi04@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:21:15 -0400 Subject: [PATCH 4/4] chore: address feedback --- .../optimizer/src/decorrelate_predicate_subquery.rs | 4 ++-- datafusion/sqllogictest/test_files/subquery.slt | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs index da17dd3e7c789..858c6baac60d3 100644 --- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs +++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs @@ -854,10 +854,10 @@ fn build_join( if !pull_up.can_pull_up { return Ok(None); } - // Stopgap until a follow-up adds count-bug compensation for `IN`/`NOT IN`. + // TODO(#24960): stopgap until count-bug compensation covers `IN`/`NOT IN`. // The subquery stays correlated unless an outer row with no match gets the // right answer anyway. - if !exists_sub_query { + if !exists_sub_query && pull_up.pulled_up_scalar_agg { let null_reads_as_false = match join_type { JoinType::LeftSemi => true, JoinType::LeftMark => !needs_null_aware_mark, diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 3ba806b749c97..565d3c481c014 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2124,6 +2124,15 @@ WHERE t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) ---- 11 +# correlated_in_groupless_max_agg_embedded_in_or_is_not_count_bug_compensated +query I +SELECT t1_id FROM t1 +WHERE t1_int + 2 IN (SELECT max(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id) OR t1_id = 33 +ORDER BY t1_id +---- +11 +33 + # correlated_in_groupless_count_agg_stays_correlated # Errors rather than returning wrong rows. Expected `33`. query error Physical plan does not support logical expression InSubquery