diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index 27295660640ae..2558ea6ae6ab2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -48,8 +48,10 @@ pub(crate) fn urgent_waiting(urgencies: impl Iterator, active:u8) -> bo urgencies.into_iter().any(|p|p>active) } -pub(crate) const fn may_join(render:bool, next:usize, same_vm:bool, same_urgency:bool) -> bool { - render && next==0 && same_vm && same_urgency +/// A later pending packet may extend a batch of either engine from its first command, for the +/// batch's VM and at its urgency. The batch limit and GPU-time budget still bound the batch. +pub(crate) const fn may_join(next:usize, same_vm:bool, same_urgency:bool) -> bool { + next==0 && same_vm && same_urgency } pub(crate) const fn default_depth(t8122:bool) -> usize {if t8122 {2} else {1}} @@ -68,6 +70,23 @@ pub(crate) fn pipeline_room(depth: usize, batches: usize, passes: usize, Some(((last_base + last_count) % SLOTS, SLOTS - passes)) } +/// Whether the next command may extend the batch. Render passes always need the budget. A compute +/// batch always takes one command; further commands need a budget and the VM's measured compute +/// cost, so that without a budget every compute batch is one command, as before. Only an +/// explicit `m3_compute_batch_override` lets the front packet's own commands go unbudgeted. +pub(crate) fn admits(render:bool, joined:bool, explicit:bool, cost:Option, budget:Option, + passes:usize) -> bool { + if render || passes == 0 { return within_budget(cost, budget, passes); } + if explicit && !joined { return true; } + budget.is_some() && within_budget(cost, budget, passes) +} + +/// Seconds a published batch may run before it is failed: each command keeps the two seconds it +/// had when it was published alone, so batching never shortens a command's deadline. +pub(crate) const fn batch_deadline_secs(commands:usize) -> i64 { + 2 * (if commands == 0 { 1 } else { commands }) as i64 +} + /// Always admit one pass; subsequent passes must fit the measured budget. /// Unknown cost permits one pass until a measurement is available. pub(crate) fn within_budget(cost: Option, budget: Option, passes: usize) -> bool { diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index 85033c88e4192..628e927bc42a1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -208,8 +208,10 @@ pub(crate) fn render_internal_timestamps()->bool { /// `asahi.m3_render_batch_size`: bounded ordered render batch size (1..16). static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64); -/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). -static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). The default is the +/// compute storage's slot count. A compute batch grows past one command only under the GPU-time +/// budget (`m3_render_batch_budget_us`); without a budget each compute batch is one command. +static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(crate::m3::m3_compute_storage::SLOTS as u64); m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64); /// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies. static M3_EARLY_TILING: AtomicU64 = AtomicU64::new(0); @@ -263,6 +265,11 @@ static M3_COMPUTE_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, parse_compute_batch_override, 0o644, Some(get_atomic_param)); +/// Whether `m3_compute_batch_override` is set (it then bounds a packet's own commands alone). +pub(crate) fn compute_batch_overridden() -> bool { + M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed) != 0 +} + /// Snapshot once per packet, so a live write never changes an in-flight batch. pub(crate) fn compute_batch_size() -> usize { let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); @@ -483,10 +490,11 @@ pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize { } } -/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render batch takes in later packets -/// of its VM only while the VM's measured GPU time per pass, times the passes already in the -/// batch, stays below this budget (4000 µs on experimental T8122, disabled on Pro). -/// Coalescing a client's heavy frames +/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render or compute batch takes in +/// later packets of its VM only while the VM's measured GPU time per pass (or per compute +/// command), times the passes already in the batch, stays below this budget (4000 µs on +/// experimental T8122, disabled on Pro, where compute packets then never join). Each batched +/// command keeps its own completion deadline. Coalescing a client's heavy frames or dispatches /// otherwise makes every other client's work, the compositor's included, wait for all of them. /// 0 disables the budget; u64::MAX restores the SoC default. static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 8bd242a483432..81cdd4dbdac8b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -63,6 +63,9 @@ struct Inner { /// Whether a T8122 job verdict line was already emitted for the current job, so the /// execute-path fallback does not add a second one. T8122 only. t8122_verdict: bool, + /// Publish the front packet alone once: a batch that joined later packets failed before + /// anything reached the firmware, and the front packet must not take the blame for them. + solo_next: bool, /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. boot_step: crate::m3::t8122_start::BootStep, /// Packets that joined a batch started by an earlier packet (debug summary). @@ -72,6 +75,8 @@ struct Inner { /// Recent measured render GPU time per pass by VM. Smoothed over batches; /// replacing the least recently measured VM does not retain its mappings. pass_cost: policy::PassCosts, + /// Measured GPU time per compute command, per VM (the compute twin of `pass_cost`). + compute_cost: policy::PassCosts, /// Latched only after the isolated InitBM draw fully retired. render_initialized: bool, config: Config, @@ -221,6 +226,7 @@ impl Inner { self.packets.retain(|p| self.active.iter().any(|b|b.entries.iter().any(|(q,_)|Arc::ptr_eq(p,q)))); } fn fail_front(&mut self,error:Error) { + self.solo_next=false; if let Ok((packet,_))=self.pending.remove(0) { if !self.packet_active(&packet) && !self.fault_captured {packet.release_guard();} packet.finish(Err(error)); @@ -231,8 +237,8 @@ impl Inner { for (packet,_) in entries {packet.finish(result);} } /// Choose the next batch from the front of `pending`: the front packet's next commands of - /// one engine, and for render also the leading render commands of following packets of the - /// same VM, up to the ordered-batch limit. + /// one engine, and the leading commands of that engine of following packets of the same VM, + /// up to the ordered-batch limit and the batch's GPU-time budget. fn next_batch(&self,max:usize)->Result,usize)>> { let mut entries=KVec::new(); let Some((packet,next))=self.pending.first() else {return Ok(entries);}; @@ -256,17 +262,18 @@ impl Inner { .clamp(1,crate::m3::m3_compute_storage::SLOTS) }; let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..}); - let per_pass=self.pass_cost.get(packet.vm.id()); + let per_pass=if render {self.pass_cost.get(packet.vm.id())} else {self.compute_cost.get(packet.vm.id())}; let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc()); + let explicit=crate::m3::m3_params::compute_batch_overridden(); let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); for (position,(candidate,start)) in self.pending.iter().enumerate() { - // Later packets join only render batches, only from their first command, and only - // for the VM the batch is bound to, while the batch's expected GPU time is within - // the budget. - if position>0 && !policy::may_join(render,*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;} + // Later packets join only from their first command, only for the VM the batch is + // bound to and at its urgency, while the batch's expected GPU time is within the + // budget. Each queued submission otherwise pays a full firmware round trip. + if position>0 && (self.solo_next || !policy::may_join(*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency)) {break;} let mut index=*start; while index0,explicit,per_pass,budget,in_flight+entries.len()) {return Ok(entries);} entries.push((candidate.clone(),index),GFP_KERNEL)?; index+=1; } @@ -338,6 +345,7 @@ impl Inner { }; match self.publish(&entries,base,overlap) { Ok(batch)=>{ + self.solo_next=false; // Consume the published commands from `pending`. let Some((last,last_index))=entries.last().map(|(p,i)|(p.clone(),*i)) else {break;}; while let Some((packet,_))=self.pending.first() { @@ -372,6 +380,13 @@ impl Inner { // Not publishable behind the running batch (new VM binding or storage // growth): wait for it to retire. break; + } else if !self.solo_next && entries.iter().any(|(p,_)|!Arc::ptr_eq(p,&entries[0].0)) { + // Nothing reached the firmware, but later packets joined: they joined from + // their first command and were never published before, so they are + // unpublished again. Retry the front packet alone before failing it. + for (p,_) in entries.iter().filter(|(p,_)|!Arc::ptr_eq(p,&entries[0].0)) {p.clear_published();} + self.solo_next=true; + progress=true; } else { // Nothing reached the firmware: fail only the front packet, as a failed // execute() did before publication. @@ -732,7 +747,7 @@ impl Inner { return Ok(true); } Ok(false)=>{ - let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(2),b.previous_events)).ok_or(EIO)?; + let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(policy::batch_deadline_secs(b.entries.len())),b.previous_events)).ok_or(EIO)?; if expired { dev_err!(self.drm.as_ref(),"M3 pipelined completion events={} previous={}\n",self.config.completed_events,previous); self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); @@ -808,7 +823,7 @@ impl Inner { if evented && batch.events_ns<0 {batch.events_ns=at;} } if let Some(t)=poll_start {batch.polling_ns+=t.elapsed().as_nanos();batch.polls+=1;} - (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) + (evented,batch.start.elapsed()>=Delta::from_secs(policy::batch_deadline_secs(batch.entries.len())),batch.previous_events) }; if stamped && evented { let retire_mmio=crate::m3::m3_params::retire_mmio(self.device.soc()); @@ -906,6 +921,8 @@ impl Inner { let vm=batch.entries[0].0.vm.id(); self.pass_cost.record(vm,gpu_ns/batch_count as u64); self.render_initialized=true; + } else { + self.compute_cost.record(batch.entries[0].0.vm.id(),gpu_ns/batch_count as u64); } if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { let observed_tick=physical_counter(); @@ -1120,7 +1137,7 @@ impl Runtime { }; Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, window, drm, device, jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3::m3_pass_layout::SLOTS], - geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, + geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),compute_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false,solo_next:false, boot_step:crate::m3::t8122_start::BootStep::Publish })) }) } pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 155bda714e9d9..4c9dffdd7891a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -85,6 +85,9 @@ impl Packet { pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} /// Called before the first firmware-visible change of a batch that includes this packet. pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} + /// Undo `mark_published` for a packet that joined a batch from its first command when that + /// batch failed before any firmware-visible change. + pub(crate) fn clear_published(&self) {self.published.store(false,Ordering::Relaxed);} /// Whether the firmware may have seen any of this packet's commands. pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index d9a0a35c04dd0..591389c5c9ea1 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -21,11 +21,10 @@ fn priority_fifo_and_partial_packet_yields() { // An unfinished low-priority packet gets a high-priority predecessor next batch. let low_continuation=vec![0,0]; assert_eq!(pending_position(low_continuation.into_iter(),3),0); - assert!(!may_join(true,0,true,false)); - assert!(!may_join(true,1,true,true)); - assert!(!may_join(true,0,false,true)); - assert!(!may_join(false,0,true,true)); - assert!(may_join(true,0,true,true)); + assert!(!may_join(0,true,false)); + assert!(!may_join(1,true,true)); + assert!(!may_join(0,false,true)); + assert!(may_join(0,true,true)); // A sorted pending head cannot detect its own urgency advantage. Compare // with the active batch so same-VM urgent work also stops extending it. assert!(urgent_waiting([3,0].into_iter(),0)); @@ -33,6 +32,43 @@ fn priority_fifo_and_partial_packet_yields() { assert!(!urgent_waiting([0,0].into_iter(),0)); } +#[test] +fn compute_batches_grow_only_under_a_budget() { + let budget=Some(4_000_000); + for joined in [false,true] { for explicit in [false,true] { + // One command always goes. + assert!(admits(false,joined,explicit,None,None,0)); + // A measured cost that fits the budget lets the batch grow. + assert!(admits(false,joined,explicit,Some(1_000_000),budget,3)); + assert!(admits(false,joined,explicit,Some(10_000),budget,1)); + }} + // Without an override: no budget or no measurement keeps one command per batch. + for joined in [false,true] { + assert!(!admits(false,joined,false,Some(10_000),None,1)); + assert!(!admits(false,joined,false,None,budget,1)); + assert!(!admits(false,joined,false,Some(1_000_000),budget,4)); + } + // An explicit override bounds the front packet's own commands alone, never joined ones. + assert!(admits(false,false,true,None,None,15)); + assert!(admits(false,false,true,Some(10_000_000),budget,15)); + assert!(!admits(false,true,true,None,None,1)); + assert!(!admits(false,true,true,Some(1_000_000),budget,4)); + // Render keeps its budget rule. + for joined in [false,true] { + assert!(admits(true,joined,false,None,budget,0)); + assert!(!admits(true,joined,false,None,budget,1)); + assert!(admits(true,joined,false,None,None,7)); + } +} + +#[test] +fn batched_commands_keep_their_deadline() { + assert_eq!(batch_deadline_secs(0),2); + assert_eq!(batch_deadline_secs(1),2); + assert_eq!(batch_deadline_secs(8),16); + assert_eq!(batch_deadline_secs(16),32); +} + #[test] fn initbm_vm_engine_and_notification_gates() { for bits in 0..32 {