Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions drivers/gpu/drm/asahi/m3/m3_batch_policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,10 @@ pub(crate) fn urgent_waiting(urgencies: impl Iterator<Item=u8>, active:u8) -> bo
urgencies.into_iter().any(|p|p>active)
}

pub(crate) const fn may_join(render:bool, next:usize, same_vm:bool, same_urgency:bool) -> bool {
render && next==0 && same_vm && same_urgency
/// A later pending packet may extend a batch of either engine from its first command, for the
/// batch's VM and at its urgency. The batch limit and GPU-time budget still bound the batch.
pub(crate) const fn may_join(next:usize, same_vm:bool, same_urgency:bool) -> bool {
next==0 && same_vm && same_urgency
}

pub(crate) const fn default_depth(t8122:bool) -> usize {if t8122 {2} else {1}}
Expand All @@ -68,6 +70,23 @@ pub(crate) fn pipeline_room(depth: usize, batches: usize, passes: usize,
Some(((last_base + last_count) % SLOTS, SLOTS - passes))
}

/// Whether the next command may extend the batch. Render passes always need the budget. A compute
/// batch always takes one command; further commands need a budget and the VM's measured compute
/// cost, so that without a budget every compute batch is one command, as before. Only an
/// explicit `m3_compute_batch_override` lets the front packet's own commands go unbudgeted.
pub(crate) fn admits(render:bool, joined:bool, explicit:bool, cost:Option<u64>, budget:Option<u64>,
passes:usize) -> bool {
if render || passes == 0 { return within_budget(cost, budget, passes); }
if explicit && !joined { return true; }
budget.is_some() && within_budget(cost, budget, passes)
}

/// Seconds a published batch may run before it is failed: each command keeps the two seconds it
/// had when it was published alone, so batching never shortens a command's deadline.
pub(crate) const fn batch_deadline_secs(commands:usize) -> i64 {
2 * (if commands == 0 { 1 } else { commands }) as i64
}

/// Always admit one pass; subsequent passes must fit the measured budget.
/// Unknown cost permits one pass until a measurement is available.
pub(crate) fn within_budget(cost: Option<u64>, budget: Option<u64>, passes: usize) -> bool {
Expand Down
20 changes: 14 additions & 6 deletions drivers/gpu/drm/asahi/m3/m3_params.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,10 @@ pub(crate) fn render_internal_timestamps()->bool {
/// `asahi.m3_render_batch_size`: bounded ordered render batch size (1..16).
static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1);
m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64);
/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16).
static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(1);
/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). The default is the
/// compute storage's slot count. A compute batch grows past one command only under the GPU-time
/// budget (`m3_render_batch_budget_us`); without a budget each compute batch is one command.
static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(crate::m3::m3_compute_storage::SLOTS as u64);
m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64);
/// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies.
static M3_EARLY_TILING: AtomicU64 = AtomicU64::new(0);
Expand Down Expand Up @@ -263,6 +265,11 @@ static M3_COMPUTE_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0);
m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE,
parse_compute_batch_override, 0o644, Some(get_atomic_param));

/// Whether `m3_compute_batch_override` is set (it then bounds a packet's own commands alone).
pub(crate) fn compute_batch_overridden() -> bool {
M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed) != 0
}

/// Snapshot once per packet, so a live write never changes an in-flight batch.
pub(crate) fn compute_batch_size() -> usize {
let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed);
Expand Down Expand Up @@ -483,10 +490,11 @@ pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize {
}
}

/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render batch takes in later packets
/// of its VM only while the VM's measured GPU time per pass, times the passes already in the
/// batch, stays below this budget (4000 µs on experimental T8122, disabled on Pro).
/// Coalescing a client's heavy frames
/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render or compute batch takes in
/// later packets of its VM only while the VM's measured GPU time per pass (or per compute
/// command), times the passes already in the batch, stays below this budget (4000 µs on
/// experimental T8122, disabled on Pro, where compute packets then never join). Each batched
/// command keeps its own completion deadline. Coalescing a client's heavy frames or dispatches
/// otherwise makes every other client's work, the compositor's included, wait for all of them.
/// 0 disables the budget; u64::MAX restores the SoC default.
static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX);
Expand Down
39 changes: 28 additions & 11 deletions drivers/gpu/drm/asahi/m3/m3_runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ struct Inner {
/// Whether a T8122 job verdict line was already emitted for the current job, so the
/// execute-path fallback does not add a second one. T8122 only.
t8122_verdict: bool,
/// Publish the front packet alone once: a batch that joined later packets failed before
/// anything reached the firmware, and the front packet must not take the blame for them.
solo_next: bool,
/// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only.
boot_step: crate::m3::t8122_start::BootStep,
/// Packets that joined a batch started by an earlier packet (debug summary).
Expand All @@ -72,6 +75,8 @@ struct Inner {
/// Recent measured render GPU time per pass by VM. Smoothed over batches;
/// replacing the least recently measured VM does not retain its mappings.
pass_cost: policy::PassCosts,
/// Measured GPU time per compute command, per VM (the compute twin of `pass_cost`).
compute_cost: policy::PassCosts,
/// Latched only after the isolated InitBM draw fully retired.
render_initialized: bool,
config: Config,
Expand Down Expand Up @@ -221,6 +226,7 @@ impl Inner {
self.packets.retain(|p| self.active.iter().any(|b|b.entries.iter().any(|(q,_)|Arc::ptr_eq(p,q))));
}
fn fail_front(&mut self,error:Error) {
self.solo_next=false;
if let Ok((packet,_))=self.pending.remove(0) {
if !self.packet_active(&packet) && !self.fault_captured {packet.release_guard();}
packet.finish(Err(error));
Expand All @@ -231,8 +237,8 @@ impl Inner {
for (packet,_) in entries {packet.finish(result);}
}
/// Choose the next batch from the front of `pending`: the front packet's next commands of
/// one engine, and for render also the leading render commands of following packets of the
/// same VM, up to the ordered-batch limit.
/// one engine, and the leading commands of that engine of following packets of the same VM,
/// up to the ordered-batch limit and the batch's GPU-time budget.
fn next_batch(&self,max:usize)->Result<KVec<(Arc<crate::m3::m3_submit::Packet>,usize)>> {
let mut entries=KVec::new();
let Some((packet,next))=self.pending.first() else {return Ok(entries);};
Expand All @@ -256,17 +262,18 @@ impl Inner {
.clamp(1,crate::m3::m3_compute_storage::SLOTS)
};
let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..});
let per_pass=self.pass_cost.get(packet.vm.id());
let per_pass=if render {self.pass_cost.get(packet.vm.id())} else {self.compute_cost.get(packet.vm.id())};
let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc());
let explicit=crate::m3::m3_params::compute_batch_overridden();
let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum();
for (position,(candidate,start)) in self.pending.iter().enumerate() {
// Later packets join only render batches, only from their first command, and only
// for the VM the batch is bound to, while the batch's expected GPU time is within
// the budget.
if position>0 && !policy::may_join(render,*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;}
// Later packets join only from their first command, only for the VM the batch is
// bound to and at its urgency, while the batch's expected GPU time is within the
// budget. Each queued submission otherwise pays a full firmware round trip.
if position>0 && (self.solo_next || !policy::may_join(*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency)) {break;}
let mut index=*start;
while index<candidate.commands.len() && entries.len()<limit && same_engine(&candidate.commands[index]) {
if render && !policy::within_budget(per_pass,budget,in_flight+entries.len()) {return Ok(entries);}
if !policy::admits(render,position>0,explicit,per_pass,budget,in_flight+entries.len()) {return Ok(entries);}
entries.push((candidate.clone(),index),GFP_KERNEL)?;
index+=1;
}
Expand Down Expand Up @@ -338,6 +345,7 @@ impl Inner {
};
match self.publish(&entries,base,overlap) {
Ok(batch)=>{
self.solo_next=false;
// Consume the published commands from `pending`.
let Some((last,last_index))=entries.last().map(|(p,i)|(p.clone(),*i)) else {break;};
while let Some((packet,_))=self.pending.first() {
Expand Down Expand Up @@ -372,6 +380,13 @@ impl Inner {
// Not publishable behind the running batch (new VM binding or storage
// growth): wait for it to retire.
break;
} else if !self.solo_next && entries.iter().any(|(p,_)|!Arc::ptr_eq(p,&entries[0].0)) {
// Nothing reached the firmware, but later packets joined: they joined from
// their first command and were never published before, so they are
// unpublished again. Retry the front packet alone before failing it.
for (p,_) in entries.iter().filter(|(p,_)|!Arc::ptr_eq(p,&entries[0].0)) {p.clear_published();}
self.solo_next=true;
progress=true;
} else {
// Nothing reached the firmware: fail only the front packet, as a failed
// execute() did before publication.
Expand Down Expand Up @@ -732,7 +747,7 @@ impl Inner {
return Ok(true);
}
Ok(false)=>{
let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(2),b.previous_events)).ok_or(EIO)?;
let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(policy::batch_deadline_secs(b.entries.len())),b.previous_events)).ok_or(EIO)?;
if expired {
dev_err!(self.drm.as_ref(),"M3 pipelined completion events={} previous={}\n",self.config.completed_events,previous);
self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT);
Expand Down Expand Up @@ -808,7 +823,7 @@ impl Inner {
if evented && batch.events_ns<0 {batch.events_ns=at;}
}
if let Some(t)=poll_start {batch.polling_ns+=t.elapsed().as_nanos();batch.polls+=1;}
(evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events)
(evented,batch.start.elapsed()>=Delta::from_secs(policy::batch_deadline_secs(batch.entries.len())),batch.previous_events)
};
if stamped && evented {
let retire_mmio=crate::m3::m3_params::retire_mmio(self.device.soc());
Expand Down Expand Up @@ -906,6 +921,8 @@ impl Inner {
let vm=batch.entries[0].0.vm.id();
self.pass_cost.record(vm,gpu_ns/batch_count as u64);
self.render_initialized=true;
} else {
self.compute_cost.record(batch.entries[0].0.vm.id(),gpu_ns/batch_count as u64);
}
if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {
let observed_tick=physical_counter();
Expand Down Expand Up @@ -1120,7 +1137,7 @@ impl Runtime {
};
Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, window, drm, device,
jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3::m3_pass_layout::SLOTS],
geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false,
geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),compute_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false,solo_next:false,
boot_step:crate::m3::t8122_start::BootStep::Publish })) })
}
pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() }
Expand Down
3 changes: 3 additions & 0 deletions drivers/gpu/drm/asahi/m3/m3_submit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ impl Packet {
pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());}
/// Called before the first firmware-visible change of a batch that includes this packet.
pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);}
/// Undo `mark_published` for a packet that joined a batch from its first command when that
/// batch failed before any firmware-visible change.
pub(crate) fn clear_published(&self) {self.published.store(false,Ordering::Relaxed);}
/// Whether the firmware may have seen any of this packet's commands.
pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)}
pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));}
Expand Down
46 changes: 41 additions & 5 deletions tools/asahi/m3-batching-tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,54 @@ fn priority_fifo_and_partial_packet_yields() {
// An unfinished low-priority packet gets a high-priority predecessor next batch.
let low_continuation=vec![0,0];
assert_eq!(pending_position(low_continuation.into_iter(),3),0);
assert!(!may_join(true,0,true,false));
assert!(!may_join(true,1,true,true));
assert!(!may_join(true,0,false,true));
assert!(!may_join(false,0,true,true));
assert!(may_join(true,0,true,true));
assert!(!may_join(0,true,false));
assert!(!may_join(1,true,true));
assert!(!may_join(0,false,true));
assert!(may_join(0,true,true));
// A sorted pending head cannot detect its own urgency advantage. Compare
// with the active batch so same-VM urgent work also stops extending it.
assert!(urgent_waiting([3,0].into_iter(),0));
assert!(!urgent_waiting([3,0].into_iter(),3));
assert!(!urgent_waiting([0,0].into_iter(),0));
}

#[test]
fn compute_batches_grow_only_under_a_budget() {
let budget=Some(4_000_000);
for joined in [false,true] { for explicit in [false,true] {
// One command always goes.
assert!(admits(false,joined,explicit,None,None,0));
// A measured cost that fits the budget lets the batch grow.
assert!(admits(false,joined,explicit,Some(1_000_000),budget,3));
assert!(admits(false,joined,explicit,Some(10_000),budget,1));
}}
// Without an override: no budget or no measurement keeps one command per batch.
for joined in [false,true] {
assert!(!admits(false,joined,false,Some(10_000),None,1));
assert!(!admits(false,joined,false,None,budget,1));
assert!(!admits(false,joined,false,Some(1_000_000),budget,4));
}
// An explicit override bounds the front packet's own commands alone, never joined ones.
assert!(admits(false,false,true,None,None,15));
assert!(admits(false,false,true,Some(10_000_000),budget,15));
assert!(!admits(false,true,true,None,None,1));
assert!(!admits(false,true,true,Some(1_000_000),budget,4));
// Render keeps its budget rule.
for joined in [false,true] {
assert!(admits(true,joined,false,None,budget,0));
assert!(!admits(true,joined,false,None,budget,1));
assert!(admits(true,joined,false,None,None,7));
}
}

#[test]
fn batched_commands_keep_their_deadline() {
assert_eq!(batch_deadline_secs(0),2);
assert_eq!(batch_deadline_secs(1),2);
assert_eq!(batch_deadline_secs(8),16);
assert_eq!(batch_deadline_secs(16),32);
}

#[test]
fn initbm_vm_engine_and_notification_gates() {
for bits in 0..32 {
Expand Down