Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 96 additions & 3 deletions datafusion/physical-plan/src/aggregates/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3252,8 +3252,8 @@ mod tests {
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::{
Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator,
Signature, Volatility,
Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupSelection,
GroupsAccumulator, Signature, Volatility,
};
use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
use datafusion_functions_aggregate::array_agg::array_agg_udaf;
Expand Down Expand Up @@ -4338,6 +4338,72 @@ mod tests {
Ok(())
}

/// Single and Final hash aggregation share the same terminal drain helper. Single
/// exercises it directly from raw input without coupling this test to partial state.
#[tokio::test]
async fn single_grouped_aggregate_avoids_destructive_terminal_drain() -> Result<()> {
const BATCH_SIZE: usize = 2;
const NUM_GROUPS: usize = 3;

let schema = Arc::new(Schema::new(vec![
Field::new("key", DataType::Int32, false),
Field::new("value", DataType::Int32, false),
]));
let input_batches = vec![RecordBatch::try_new(
Arc::clone(&schema),
vec![
Arc::new(Int32Array::from(vec![1, 2, 3, 1, 2, 3])),
Arc::new(Int32Array::from(vec![10, 20, 30, 40, 50, 60])),
],
)?];
let input =
TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;
let group_by =
PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);
let udaf = Arc::new(AggregateUDF::from(NoFirstEmitUdaf::new()));
let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(
AggregateExprBuilder::new(udaf, vec![col("value", &schema)?])
.schema(Arc::clone(&schema))
.alias("no_first_emit(value)")
.build()?,
)];
let aggregate = Arc::new(AggregateExec::try_new(
AggregateMode::Single,
group_by,
aggregates,
vec![None],
input,
Arc::clone(&schema),
)?);
let task_ctx = new_migrated_hash_ctx(BATCH_SIZE);

let stream = aggregate.execute_typed(0, &task_ctx)?;
assert!(matches!(stream, StreamType::SingleHash(_)));

let stream: SendableRecordBatchStream = stream.into();
let batches = collect(stream).await?;
assert!(batches.len() > 1, "expected terminal output to be chunked");
assert!(
batches.iter().all(|batch| batch.num_rows() <= BATCH_SIZE),
"terminal output exceeded the configured batch size"
);
assert_eq!(
batches.iter().map(RecordBatch::num_rows).sum::<usize>(),
NUM_GROUPS
);
assert_snapshot!(batches_to_sort_string(&batches), @r"
+-----+----------------------+
| key | no_first_emit(value) |
+-----+----------------------+
| 1 | 2 |
| 2 | 2 |
| 3 | 2 |
+-----+----------------------+
");

Ok(())
}

#[tokio::test]
async fn limited_distinct_aggregate_uses_migrated_hash_streams() -> Result<()> {
let schema =
Expand Down Expand Up @@ -8105,7 +8171,7 @@ mod tests {
Ok(Arc::new(Int64Array::from(counts)))
}
EmitTo::First(_) => internal_err!(
"partial grouped aggregate output must materialize with EmitTo::All before slicing"
"grouped aggregate terminal output must not use EmitTo::First"
),
}
}
Expand All @@ -8130,10 +8196,37 @@ mod tests {
self.emit_counts(emit_to)
}

fn evaluate_preserving(
&mut self,
selection: GroupSelection<'_>,
) -> Result<ArrayRef> {
selection.validate_num_groups(self.counts.len())?;
let counts = selection
.iter()
.map(|index| self.counts[index])
.collect::<Vec<_>>();
Ok(Arc::new(Int64Array::from(counts)))
}

fn supports_evaluate_preserving(&self) -> bool {
true
}

fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
Ok(vec![self.emit_counts(emit_to)?])
}

fn state_preserving(
&mut self,
selection: GroupSelection<'_>,
) -> Result<Vec<ArrayRef>> {
self.evaluate_preserving(selection).map(|array| vec![array])
}

fn supports_state_preserving(&self) -> bool {
true
}

fn convert_to_state(
&self,
values: &[ArrayRef],
Expand Down