From a700462ffa129c54018dcd7798bed7ccb056faa1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 23 Nov 2026 04:59:02 +1100 Subject: [PATCH 1/2] drm/asahi: m3: let queued compute submissions share a firmware batch The M3 runtime publishes one compute batch at a time and publishes the next one only after the previous one fully retired. Later pending packets could join render batches only, so every queued compute submission paid its own firmware round trip. On a J615 (M3, 27.2 firmware) that round trip is about 260 us for a 182 ns shader, so a light compute job costs ~390 us per vkQueueSubmit even at queue depth 8. Let later packets of the same VM and urgency join a compute batch from their first command, as render batches already do, and bound compute batches by the same GPU-time budget as render batches, using a per-VM measured GPU time per compute command. The compute batch size now defaults to the compute storage's slot count; the budget and the pending queue bound the batch in practice. The runtime knob m3_compute_batch_override still wins when set. On the J615 (one-shot kernel, in-boot ABAB of the batch size 1/8/1/8): compute.light at depth 8 goes from 394 to 174 us per submit, a light render clear from 363 to 173 us, heavy ALU compute from 1180 to 857 us, with 0 firmware error events. gpubench-stress-vk passes every test that does not inject a GPU fault. Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 6 ++++-- drivers/gpu/drm/asahi/m3/m3_params.rs | 13 ++++++------ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 22 ++++++++++++--------- tools/asahi/m3-batching-tests.rs | 9 ++++----- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index 27295660640ae..cb0c877edf667 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -48,8 +48,10 @@ pub(crate) fn urgent_waiting(urgencies: impl Iterator, active:u8) -> bo urgencies.into_iter().any(|p|p>active) } -pub(crate) const fn may_join(render:bool, next:usize, same_vm:bool, same_urgency:bool) -> bool { - render && next==0 && same_vm && same_urgency +/// A later pending packet may extend a batch of either engine from its first command, for the +/// batch's VM and at its urgency. The batch limit and GPU-time budget still bound the batch. +pub(crate) const fn may_join(next:usize, same_vm:bool, same_urgency:bool) -> bool { + next==0 && same_vm && same_urgency } pub(crate) const fn default_depth(t8122:bool) -> usize {if t8122 {2} else {1}} diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index 85033c88e4192..6a2e1c74d7d70 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -208,8 +208,9 @@ pub(crate) fn render_internal_timestamps()->bool { /// `asahi.m3_render_batch_size`: bounded ordered render batch size (1..16). static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64); -/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). -static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). The default is the +/// compute storage's slot count; the batch's GPU-time budget and the pending queue bound it. +static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(crate::m3::m3_compute_storage::SLOTS as u64); m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64); /// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies. static M3_EARLY_TILING: AtomicU64 = AtomicU64::new(0); @@ -483,10 +484,10 @@ pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize { } } -/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render batch takes in later packets -/// of its VM only while the VM's measured GPU time per pass, times the passes already in the -/// batch, stays below this budget (4000 µs on experimental T8122, disabled on Pro). -/// Coalescing a client's heavy frames +/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render or compute batch takes in +/// later packets of its VM only while the VM's measured GPU time per pass (or per compute +/// command), times the passes already in the batch, stays below this budget (4000 µs on +/// experimental T8122, disabled on Pro). Coalescing a client's heavy frames or dispatches /// otherwise makes every other client's work, the compositor's included, wait for all of them. /// 0 disables the budget; u64::MAX restores the SoC default. static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 8bd242a483432..65bcae740a709 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -72,6 +72,8 @@ struct Inner { /// Recent measured render GPU time per pass by VM. Smoothed over batches; /// replacing the least recently measured VM does not retain its mappings. pass_cost: policy::PassCosts, + /// Measured GPU time per compute command, per VM (the compute twin of `pass_cost`). + compute_cost: policy::PassCosts, /// Latched only after the isolated InitBM draw fully retired. render_initialized: bool, config: Config, @@ -231,8 +233,8 @@ impl Inner { for (packet,_) in entries {packet.finish(result);} } /// Choose the next batch from the front of `pending`: the front packet's next commands of - /// one engine, and for render also the leading render commands of following packets of the - /// same VM, up to the ordered-batch limit. + /// one engine, and the leading commands of that engine of following packets of the same VM, + /// up to the ordered-batch limit and the batch's GPU-time budget. fn next_batch(&self,max:usize)->Result,usize)>> { let mut entries=KVec::new(); let Some((packet,next))=self.pending.first() else {return Ok(entries);}; @@ -256,17 +258,17 @@ impl Inner { .clamp(1,crate::m3::m3_compute_storage::SLOTS) }; let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..}); - let per_pass=self.pass_cost.get(packet.vm.id()); + let per_pass=if render {self.pass_cost.get(packet.vm.id())} else {self.compute_cost.get(packet.vm.id())}; let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc()); let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); for (position,(candidate,start)) in self.pending.iter().enumerate() { - // Later packets join only render batches, only from their first command, and only - // for the VM the batch is bound to, while the batch's expected GPU time is within - // the budget. - if position>0 && !policy::may_join(render,*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;} + // Later packets join only from their first command, only for the VM the batch is + // bound to and at its urgency, while the batch's expected GPU time is within the + // budget. Each queued submission otherwise pays a full firmware round trip. + if position>0 && !policy::may_join(*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;} let mut index=*start; while index driver::AsahiDevRef { self.inner.drm.clone() } diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index d9a0a35c04dd0..18b7775e0aa37 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -21,11 +21,10 @@ fn priority_fifo_and_partial_packet_yields() { // An unfinished low-priority packet gets a high-priority predecessor next batch. let low_continuation=vec![0,0]; assert_eq!(pending_position(low_continuation.into_iter(),3),0); - assert!(!may_join(true,0,true,false)); - assert!(!may_join(true,1,true,true)); - assert!(!may_join(true,0,false,true)); - assert!(!may_join(false,0,true,true)); - assert!(may_join(true,0,true,true)); + assert!(!may_join(0,true,false)); + assert!(!may_join(1,true,true)); + assert!(!may_join(0,false,true)); + assert!(may_join(0,true,true)); // A sorted pending head cannot detect its own urgency advantage. Compare // with the active batch so same-VM urgent work also stops extending it. assert!(urgent_waiting([3,0].into_iter(),0)); From 8eaf76139b6b55af3b869565ea1ded41b1ebbf23 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 23 Nov 2026 05:38:54 +1100 Subject: [PATCH 2/2] drm/asahi: m3: bound joined compute batches by budget and deadline Review follow-up for the compute batching change: - Each batched command keeps the 2 s completion deadline it had when it was published alone: the batch deadline is now 2 s per command. Before, a batch of several long dispatches could exceed one 2 s deadline and fail the GPU although every command was in time. - A compute batch grows past one command only under a GPU-time budget and once the VM's compute cost is measured, whether the commands come from the front packet or from later packets. Without a budget every compute batch is one command, as before. An explicit m3_compute_batch_override still bounds the front packet's own commands alone, as before; it never lets other packets join without the budget. - If a batch that joined later packets fails before anything reached the firmware, the joined packets are marked unpublished again and the front packet is retried alone before it is failed, so it does not take the blame for a joined packet. Host tests cover the admission rule and the deadline. Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 17 ++++++++++ drivers/gpu/drm/asahi/m3/m3_params.rs | 11 ++++-- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 23 ++++++++++--- drivers/gpu/drm/asahi/m3/m3_submit.rs | 3 ++ tools/asahi/m3-batching-tests.rs | 37 +++++++++++++++++++++ 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index cb0c877edf667..2558ea6ae6ab2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -70,6 +70,23 @@ pub(crate) fn pipeline_room(depth: usize, batches: usize, passes: usize, Some(((last_base + last_count) % SLOTS, SLOTS - passes)) } +/// Whether the next command may extend the batch. Render passes always need the budget. A compute +/// batch always takes one command; further commands need a budget and the VM's measured compute +/// cost, so that without a budget every compute batch is one command, as before. Only an +/// explicit `m3_compute_batch_override` lets the front packet's own commands go unbudgeted. +pub(crate) fn admits(render:bool, joined:bool, explicit:bool, cost:Option, budget:Option, + passes:usize) -> bool { + if render || passes == 0 { return within_budget(cost, budget, passes); } + if explicit && !joined { return true; } + budget.is_some() && within_budget(cost, budget, passes) +} + +/// Seconds a published batch may run before it is failed: each command keeps the two seconds it +/// had when it was published alone, so batching never shortens a command's deadline. +pub(crate) const fn batch_deadline_secs(commands:usize) -> i64 { + 2 * (if commands == 0 { 1 } else { commands }) as i64 +} + /// Always admit one pass; subsequent passes must fit the measured budget. /// Unknown cost permits one pass until a measurement is available. pub(crate) fn within_budget(cost: Option, budget: Option, passes: usize) -> bool { diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index 6a2e1c74d7d70..628e927bc42a1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -209,7 +209,8 @@ pub(crate) fn render_internal_timestamps()->bool { static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64); /// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). The default is the -/// compute storage's slot count; the batch's GPU-time budget and the pending queue bound it. +/// compute storage's slot count. A compute batch grows past one command only under the GPU-time +/// budget (`m3_render_batch_budget_us`); without a budget each compute batch is one command. static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(crate::m3::m3_compute_storage::SLOTS as u64); m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64); /// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies. @@ -264,6 +265,11 @@ static M3_COMPUTE_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, parse_compute_batch_override, 0o644, Some(get_atomic_param)); +/// Whether `m3_compute_batch_override` is set (it then bounds a packet's own commands alone). +pub(crate) fn compute_batch_overridden() -> bool { + M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed) != 0 +} + /// Snapshot once per packet, so a live write never changes an in-flight batch. pub(crate) fn compute_batch_size() -> usize { let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); @@ -487,7 +493,8 @@ pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize { /// `asahi.m3_render_batch_budget_us`: writable at runtime. A render or compute batch takes in /// later packets of its VM only while the VM's measured GPU time per pass (or per compute /// command), times the passes already in the batch, stays below this budget (4000 µs on -/// experimental T8122, disabled on Pro). Coalescing a client's heavy frames or dispatches +/// experimental T8122, disabled on Pro, where compute packets then never join). Each batched +/// command keeps its own completion deadline. Coalescing a client's heavy frames or dispatches /// otherwise makes every other client's work, the compositor's included, wait for all of them. /// 0 disables the budget; u64::MAX restores the SoC default. static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 65bcae740a709..81cdd4dbdac8b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -63,6 +63,9 @@ struct Inner { /// Whether a T8122 job verdict line was already emitted for the current job, so the /// execute-path fallback does not add a second one. T8122 only. t8122_verdict: bool, + /// Publish the front packet alone once: a batch that joined later packets failed before + /// anything reached the firmware, and the front packet must not take the blame for them. + solo_next: bool, /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. boot_step: crate::m3::t8122_start::BootStep, /// Packets that joined a batch started by an earlier packet (debug summary). @@ -223,6 +226,7 @@ impl Inner { self.packets.retain(|p| self.active.iter().any(|b|b.entries.iter().any(|(q,_)|Arc::ptr_eq(p,q)))); } fn fail_front(&mut self,error:Error) { + self.solo_next=false; if let Ok((packet,_))=self.pending.remove(0) { if !self.packet_active(&packet) && !self.fault_captured {packet.release_guard();} packet.finish(Err(error)); @@ -260,15 +264,16 @@ impl Inner { let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..}); let per_pass=if render {self.pass_cost.get(packet.vm.id())} else {self.compute_cost.get(packet.vm.id())}; let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc()); + let explicit=crate::m3::m3_params::compute_batch_overridden(); let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); for (position,(candidate,start)) in self.pending.iter().enumerate() { // Later packets join only from their first command, only for the VM the batch is // bound to and at its urgency, while the batch's expected GPU time is within the // budget. Each queued submission otherwise pays a full firmware round trip. - if position>0 && !policy::may_join(*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;} + if position>0 && (self.solo_next || !policy::may_join(*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency)) {break;} let mut index=*start; while index0,explicit,per_pass,budget,in_flight+entries.len()) {return Ok(entries);} entries.push((candidate.clone(),index),GFP_KERNEL)?; index+=1; } @@ -340,6 +345,7 @@ impl Inner { }; match self.publish(&entries,base,overlap) { Ok(batch)=>{ + self.solo_next=false; // Consume the published commands from `pending`. let Some((last,last_index))=entries.last().map(|(p,i)|(p.clone(),*i)) else {break;}; while let Some((packet,_))=self.pending.first() { @@ -374,6 +380,13 @@ impl Inner { // Not publishable behind the running batch (new VM binding or storage // growth): wait for it to retire. break; + } else if !self.solo_next && entries.iter().any(|(p,_)|!Arc::ptr_eq(p,&entries[0].0)) { + // Nothing reached the firmware, but later packets joined: they joined from + // their first command and were never published before, so they are + // unpublished again. Retry the front packet alone before failing it. + for (p,_) in entries.iter().filter(|(p,_)|!Arc::ptr_eq(p,&entries[0].0)) {p.clear_published();} + self.solo_next=true; + progress=true; } else { // Nothing reached the firmware: fail only the front packet, as a failed // execute() did before publication. @@ -734,7 +747,7 @@ impl Inner { return Ok(true); } Ok(false)=>{ - let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(2),b.previous_events)).ok_or(EIO)?; + let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(policy::batch_deadline_secs(b.entries.len())),b.previous_events)).ok_or(EIO)?; if expired { dev_err!(self.drm.as_ref(),"M3 pipelined completion events={} previous={}\n",self.config.completed_events,previous); self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); @@ -810,7 +823,7 @@ impl Inner { if evented && batch.events_ns<0 {batch.events_ns=at;} } if let Some(t)=poll_start {batch.polling_ns+=t.elapsed().as_nanos();batch.polls+=1;} - (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) + (evented,batch.start.elapsed()>=Delta::from_secs(policy::batch_deadline_secs(batch.entries.len())),batch.previous_events) }; if stamped && evented { let retire_mmio=crate::m3::m3_params::retire_mmio(self.device.soc()); @@ -1124,7 +1137,7 @@ impl Runtime { }; Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, window, drm, device, jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3::m3_pass_layout::SLOTS], - geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),compute_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, + geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),compute_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false,solo_next:false, boot_step:crate::m3::t8122_start::BootStep::Publish })) }) } pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 155bda714e9d9..4c9dffdd7891a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -85,6 +85,9 @@ impl Packet { pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} /// Called before the first firmware-visible change of a batch that includes this packet. pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} + /// Undo `mark_published` for a packet that joined a batch from its first command when that + /// batch failed before any firmware-visible change. + pub(crate) fn clear_published(&self) {self.published.store(false,Ordering::Relaxed);} /// Whether the firmware may have seen any of this packet's commands. pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index 18b7775e0aa37..591389c5c9ea1 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -32,6 +32,43 @@ fn priority_fifo_and_partial_packet_yields() { assert!(!urgent_waiting([0,0].into_iter(),0)); } +#[test] +fn compute_batches_grow_only_under_a_budget() { + let budget=Some(4_000_000); + for joined in [false,true] { for explicit in [false,true] { + // One command always goes. + assert!(admits(false,joined,explicit,None,None,0)); + // A measured cost that fits the budget lets the batch grow. + assert!(admits(false,joined,explicit,Some(1_000_000),budget,3)); + assert!(admits(false,joined,explicit,Some(10_000),budget,1)); + }} + // Without an override: no budget or no measurement keeps one command per batch. + for joined in [false,true] { + assert!(!admits(false,joined,false,Some(10_000),None,1)); + assert!(!admits(false,joined,false,None,budget,1)); + assert!(!admits(false,joined,false,Some(1_000_000),budget,4)); + } + // An explicit override bounds the front packet's own commands alone, never joined ones. + assert!(admits(false,false,true,None,None,15)); + assert!(admits(false,false,true,Some(10_000_000),budget,15)); + assert!(!admits(false,true,true,None,None,1)); + assert!(!admits(false,true,true,Some(1_000_000),budget,4)); + // Render keeps its budget rule. + for joined in [false,true] { + assert!(admits(true,joined,false,None,budget,0)); + assert!(!admits(true,joined,false,None,budget,1)); + assert!(admits(true,joined,false,None,None,7)); + } +} + +#[test] +fn batched_commands_keep_their_deadline() { + assert_eq!(batch_deadline_secs(0),2); + assert_eq!(batch_deadline_secs(1),2); + assert_eq!(batch_deadline_secs(8),16); + assert_eq!(batch_deadline_secs(16),32); +} + #[test] fn initbm_vm_engine_and_notification_gates() { for bits in 0..32 {