From 3a614fd798e2149e4935cdea1c7d2035015e6b88 Mon Sep 17 00:00:00 2001 From: Qiwei Huang Date: Tue, 1 Sep 2026 23:01:20 +0800 Subject: [PATCH] test: cover terminal hash aggregate drain --- .../physical-plan/src/aggregates/mod.rs | 99 ++++++++++++++++++- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index f00f1a160e27a..8a052624b5b90 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -3252,8 +3252,8 @@ mod tests { use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::{ - Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, - Signature, Volatility, + Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupSelection, + GroupsAccumulator, Signature, Volatility, }; use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf; use datafusion_functions_aggregate::array_agg::array_agg_udaf; @@ -4338,6 +4338,72 @@ mod tests { Ok(()) } + /// Single and Final hash aggregation share the same terminal drain helper. Single + /// exercises it directly from raw input without coupling this test to partial state. + #[tokio::test] + async fn single_grouped_aggregate_avoids_destructive_terminal_drain() -> Result<()> { + const BATCH_SIZE: usize = 2; + const NUM_GROUPS: usize = 3; + + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + ])); + let input_batches = vec![RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 1, 2, 3])), + Arc::new(Int32Array::from(vec![10, 20, 30, 40, 50, 60])), + ], + )?]; + let input = + TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?; + let group_by = + PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]); + let udaf = Arc::new(AggregateUDF::from(NoFirstEmitUdaf::new())); + let aggregates: Vec> = vec![Arc::new( + AggregateExprBuilder::new(udaf, vec![col("value", &schema)?]) + .schema(Arc::clone(&schema)) + .alias("no_first_emit(value)") + .build()?, + )]; + let aggregate = Arc::new(AggregateExec::try_new( + AggregateMode::Single, + group_by, + aggregates, + vec![None], + input, + Arc::clone(&schema), + )?); + let task_ctx = new_migrated_hash_ctx(BATCH_SIZE); + + let stream = aggregate.execute_typed(0, &task_ctx)?; + assert!(matches!(stream, StreamType::SingleHash(_))); + + let stream: SendableRecordBatchStream = stream.into(); + let batches = collect(stream).await?; + assert!(batches.len() > 1, "expected terminal output to be chunked"); + assert!( + batches.iter().all(|batch| batch.num_rows() <= BATCH_SIZE), + "terminal output exceeded the configured batch size" + ); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + NUM_GROUPS + ); + assert_snapshot!(batches_to_sort_string(&batches), @r" + +-----+----------------------+ + | key | no_first_emit(value) | + +-----+----------------------+ + | 1 | 2 | + | 2 | 2 | + | 3 | 2 | + +-----+----------------------+ + "); + + Ok(()) + } + #[tokio::test] async fn limited_distinct_aggregate_uses_migrated_hash_streams() -> Result<()> { let schema = @@ -8105,7 +8171,7 @@ mod tests { Ok(Arc::new(Int64Array::from(counts))) } EmitTo::First(_) => internal_err!( - "partial grouped aggregate output must materialize with EmitTo::All before slicing" + "grouped aggregate terminal output must not use EmitTo::First" ), } } @@ -8130,10 +8196,37 @@ mod tests { self.emit_counts(emit_to) } + fn evaluate_preserving( + &mut self, + selection: GroupSelection<'_>, + ) -> Result { + selection.validate_num_groups(self.counts.len())?; + let counts = selection + .iter() + .map(|index| self.counts[index]) + .collect::>(); + Ok(Arc::new(Int64Array::from(counts))) + } + + fn supports_evaluate_preserving(&self) -> bool { + true + } + fn state(&mut self, emit_to: EmitTo) -> Result> { Ok(vec![self.emit_counts(emit_to)?]) } + fn state_preserving( + &mut self, + selection: GroupSelection<'_>, + ) -> Result> { + self.evaluate_preserving(selection).map(|array| vec![array]) + } + + fn supports_state_preserving(&self) -> bool { + true + } + fn convert_to_state( &self, values: &[ArrayRef],