diff --git a/crates/hir-ty-codegen/backend.rs b/crates/hir-ty-codegen/backend.rs index 3288939b4f..3448ac896b 100644 --- a/crates/hir-ty-codegen/backend.rs +++ b/crates/hir-ty-codegen/backend.rs @@ -268,10 +268,14 @@ pub(super) enum Step { StoreRead, /// Encoding an entry and adding it to the store. StoreWrite, + TwinIndex, + Eligibility, + Canonicalize, + Remap, } impl Step { - const COUNT: usize = 7; + const COUNT: usize = 11; const ALL: [(Step, &str); Step::COUNT] = [ (Step::Infer, "infer"), (Step::Mir, "mir"), @@ -280,6 +284,10 @@ impl Step { (Step::Describe, "describe"), (Step::StoreRead, "store read"), (Step::StoreWrite, "store write"), + (Step::TwinIndex, "twin index"), + (Step::Eligibility, "eligibility"), + (Step::Canonicalize, "canonicalize"), + (Step::Remap, "remap"), ]; } diff --git a/crates/hir-ty-codegen/cached.rs b/crates/hir-ty-codegen/cached.rs index 77921d75b9..52c99e8806 100644 --- a/crates/hir-ty-codegen/cached.rs +++ b/crates/hir-ty-codegen/cached.rs @@ -10,7 +10,10 @@ //! program may link against a build of the crates without it, so a lookup checks that the //! symbols are there. -use std::sync::{Arc, Mutex, OnceLock}; +use std::{ + collections::VecDeque, + sync::{Arc, Mutex, OnceLock}, +}; use cranelift_codegen::{ binemit::{CodeOffset, Reloc}, @@ -482,6 +485,8 @@ impl Entry { /// The entries of the store that one program was asked about. pub(super) struct Entries { + pub(super) type_ids: bool, + pub(super) checkout: bool, store: &'static Store, /// Whether the entries asked for are the ones with what a debugger is told. debug: bool, @@ -495,12 +500,33 @@ pub(super) struct Entries { #[derive(Default)] struct Known { - entries: FxHashMap, Option>>, + entries: FxHashMap<[u8; KEY], Option>>, + /// Full entries are only a short-lived read cache. The usability summary below is the + /// durable result; entries can be read from the store again when stitching needs one. + entry_order: VecDeque<[u8; KEY]>, /// Whether the program can be built from an entry and everything it calls. - usable: FxHashMap, bool>, + usable: FxHashMap<[u8; KEY], bool>, types: FxHashMap<[u8; KEY], Option>, } +const ENTRY_CACHE_LIMIT: usize = 4096; + +impl Known { + fn remember(&mut self, key: [u8; KEY], entry: Option>) -> Option> { + if let Some(known) = self.entries.get(&key) { + return known.clone(); + } + if self.entries.len() == ENTRY_CACHE_LIMIT + && let Some(old) = self.entry_order.pop_front() + { + self.entries.remove(&old); + } + self.entries.insert(key, entry.clone()); + self.entry_order.push_back(key); + entry + } +} + #[derive(Serialize, Deserialize)] struct StoredTypeSet { types: Vec, @@ -523,7 +549,7 @@ impl KeptTypes { return *key; } let links = types.links.iter().map(|it| self.keep(store, it)).collect(); - let bytes = postcard::to_allocvec(&StoredTypeSet { types: types.types.clone(), links }) + let bytes = postcard::to_allocvec(&StoredTypeSet { types: types.types.to_vec(), links }) .unwrap_or_default(); let mut hasher = Sha256::new(); hasher.update(b"rustd types 2"); @@ -657,7 +683,7 @@ impl Drop for Writer { /// An entry on the way down from the one asked about. struct Visit { - name: Arc, + key: [u8; KEY], entry: Arc, /// How many of the functions the entry refers to have been looked at. seen: usize, @@ -676,7 +702,16 @@ impl Entries { cargo_home: Option, library: Option, ) -> Entries { - Entries { store, debug, program, cargo_home, library, known: Mutex::default() } + Entries { + type_ids: true, + checkout: true, + store, + debug, + program, + cargo_home, + library, + known: Mutex::default(), + } } /// Never use the entry again in this process: it did not fit the object it was read for. @@ -694,12 +729,19 @@ impl Entries { } pub(super) fn get(&self, name: &str) -> Option> { - if let Some(known) = self.known().entries.get(name) { + let key = Entry::key(name, self.debug, self.program.as_ref())?; + self.get_key(name, key) + } + + fn get_key(&self, name: &str, key: [u8; KEY]) -> Option> { + if !self.checkout && super::stable::StableNames::of_checkout(name) { + return None; + } + if let Some(known) = self.known().entries.get(&key) { return known.clone(); } let banned = Self::banned().lock().unwrap_or_else(|e| e.into_inner()).contains(name); let read = || { - let key = Entry::key(name, self.debug, self.program.as_ref())?; let mut entry = Entry::decode(&self.store.get(&key)?)?; if let Some(debug) = &mut entry.debug { let types = debug.stored_types.iter().map(|key| self.types(key)); @@ -712,8 +754,7 @@ impl Entries { .then(|| tally.step(super::backend::Step::StoreRead, read)) .flatten() .map(Arc::new); - self.known().entries.insert(Arc::from(name), entry.clone()); - entry + self.known().remember(key, entry) } /// The set of types kept under `key`, which every entry with a variable of the type @@ -747,35 +788,40 @@ impl Entries { .iter() .map(|key| self.types_at(key, depth + 1, visiting)) .collect::>()?; - Some(TypeSet::new(TypeList { types: stored.types, links })) + Some(TypeSet::new(TypeList { types: stored.types.into(), links })) }); visiting.remove(key); let Some(types) = types else { return None }; self.known().types.entry(*key).or_insert_with(|| Some(types.clone())).clone() } - fn settle(&self, name: Arc, usable: bool) { - self.known().usable.insert(name, usable); + fn settle(&self, key: [u8; KEY], usable: bool) { + self.known().usable.insert(key, usable); } /// Whether there is an entry for `name` and for everything it calls, and what they import /// is in the program, as `defines` tells. pub(super) fn usable(&self, name: &str, defines: &dyn Fn(&str) -> bool) -> bool { - let settled = |name: &str| self.known().usable.get(name).copied(); - if let Some(usable) = settled(name) { + let links = |entry: &Entry| { + entry.links(defines) + && (self.type_ids + || !entry.funcs.iter().any(|it| matches!(it.target, CachedTarget::TypeId(_)))) + }; + let Some(key) = Entry::key(name, self.debug, self.program.as_ref()) else { return false }; + let settled = |key: &[u8; KEY]| self.known().usable.get(key).copied(); + if let Some(usable) = settled(&key) { return usable; } - let name: Arc = Arc::from(name); - let Some(entry) = self.get(&name).filter(|it| it.links(defines)) else { - self.settle(name, false); + let Some(entry) = self.get_key(name, key).filter(|it| links(it)) else { + self.settle(key, false); return false; }; // Entries call each other in circles, so one is only known to be usable along with // the ones it leans on. - let mut unsettled = vec![name.clone()]; - let mut leaning = FxHashMap::, usize>::default(); - leaning.insert(name.clone(), 0); - let mut path = vec![Visit { name, entry, seen: 0, leans_on: 0, below: 0 }]; + let mut unsettled = vec![key]; + let mut leaning = FxHashMap::<[u8; KEY], usize>::default(); + leaning.insert(key, 0); + let mut path = vec![Visit { key, entry, seen: 0, leans_on: 0, below: 0 }]; while !path.is_empty() { let depth = path.len() - 1; let visit = &mut path[depth]; @@ -783,39 +829,44 @@ impl Entries { let Some(callee) = entry.named().nth(visit.seen) else { let Some(done) = path.pop() else { break }; if done.leans_on >= depth { - for name in unsettled.drain(done.below..) { - leaning.remove(&name); - self.settle(name, true); + for key in unsettled.drain(done.below..) { + leaning.remove(&key); + self.settle(key, true); } } else { - leaning.insert(done.name, done.leans_on); + leaning.insert(done.key, done.leans_on); let above = &mut path[depth - 1]; above.leans_on = above.leans_on.min(done.leans_on); } continue; }; visit.seen += 1; - if let Some(&on) = leaning.get(callee) { + let Some(key) = Entry::key(callee, self.debug, self.program.as_ref()) else { + for visit in path { + self.settle(visit.key, false); + } + return false; + }; + if let Some(&on) = leaning.get(&key) { visit.leans_on = visit.leans_on.min(on); continue; } - let found = match settled(callee) { + let found = match settled(&key) { Some(true) => continue, Some(false) => None, - None => self.get(callee).filter(|it| it.links(defines)), + None => self.get_key(callee, key).filter(|it| links(it)), }; - let callee: Arc = Arc::from(callee); let Some(entry) = found else { - self.settle(callee, false); + self.settle(key, false); for visit in path { - self.settle(visit.name, false); + self.settle(visit.key, false); } return false; }; - leaning.insert(callee.clone(), depth + 1); - unsettled.push(callee.clone()); + leaning.insert(key, depth + 1); + unsettled.push(key); path.push(Visit { - name: callee, + key, entry, seen: 0, leans_on: depth + 1, @@ -853,6 +904,139 @@ mod tests { signature } + fn with_entries(test: impl FnOnce(&Entries)) { + static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let at = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("rustd-entry-{}-{at}", std::process::id())); + let store = Box::leak(Box::new(Store::for_test(&dir))); + test(&Entries::new(store, true, None, None, None)); + std::fs::remove_dir_all(dir).unwrap(); + } + + fn keep_entry(entries: &Entries, name: &str, targets: Vec) { + let funcs = + targets.into_iter().map(|target| CachedFunc { target, sig: signature() }).collect(); + let entry = Entry { + label: name.to_owned(), + funcs, + datas: Vec::new(), + code: code(Vec::new()), + debug: None, + }; + let key = Entry::key(name, entries.debug, entries.program.as_ref()).unwrap(); + entries.store.insert(&key, &entry.encode().unwrap()); + entries.store.flush(); + } + + #[test] + fn missing_entries_keep_only_fixed_size_keys() { + with_entries(|entries| { + let name = "a long missing name::".repeat(4096); + assert!(entries.get(&name).is_none()); + assert!(!entries.usable(&name, &|_| true)); + assert!(!entries.usable(&name, &|_| true)); + let key = Entry::key(&name, true, None).unwrap(); + let known = entries.known(); + assert_eq!(known.entries.len(), 1); + assert_eq!(known.usable.len(), 1); + assert!(known.entries[&key].is_none()); + assert!(!known.usable[&key]); + }); + } + + #[test] + fn decoded_entries_are_bounded_and_repeated_reads_keep_one_slot() { + let mut known = Known::default(); + for index in 0..=ENTRY_CACHE_LIMIT { + let mut key = [0; KEY]; + key[..8].copy_from_slice(&(index as u64).to_le_bytes()); + known.remember(key, None); + known.remember(key, None); + } + assert_eq!(known.entries.len(), ENTRY_CACHE_LIMIT); + assert_eq!(known.entry_order.len(), ENTRY_CACHE_LIMIT); + assert!(!known.entries.contains_key(&[0; KEY])); + } + + #[test] + fn evicted_entries_can_be_read_again() { + with_entries(|entries| { + keep_entry(entries, "a", Vec::new()); + let first = entries.get("a").unwrap(); + let old = Arc::downgrade(&first); + drop(first); + for index in 0..ENTRY_CACHE_LIMIT { + entries + .known() + .remember(Entry::key(&format!("missing{index}"), true, None).unwrap(), None); + } + assert!(old.upgrade().is_none()); + assert!(entries.usable("a", &|_| true)); + assert_eq!(entries.get("a").unwrap().label, "a"); + }); + } + + #[test] + fn entry_cycles_are_usable_and_share_the_decoded_entry() { + with_entries(|entries| { + keep_entry(entries, "a", vec![CachedTarget::Named("b".to_owned())]); + keep_entry(entries, "b", vec![CachedTarget::Named("a".to_owned())]); + assert!(entries.usable("a", &|_| true)); + assert!(entries.usable("b", &|_| true)); + let first = entries.get("a").unwrap(); + assert!(Arc::ptr_eq(&first, &entries.get("a").unwrap())); + }); + } + + #[test] + fn type_ids_are_recompiled_through_cached_call_chains() { + with_entries(|entries| { + keep_entry(entries, "plain", Vec::new()); + keep_entry(entries, "caller", vec![CachedTarget::Named("id".to_owned())]); + keep_entry(entries, "id", vec![CachedTarget::TypeId(42)]); + assert!(entries.usable("caller", &|_| true)); + let mut restricted = Entries::new(entries.store, true, None, None, None); + restricted.type_ids = false; + assert!(restricted.usable("plain", &|_| true)); + assert!(!restricted.usable("id", &|_| true)); + assert!(!restricted.usable("caller", &|_| true)); + assert!(!restricted.usable("missing", &|_| true)); + }); + } + + #[test] + fn checkout_entries_are_kept_but_not_hydrated_for_patches() { + with_entries(|entries| { + let checkout = "fn ~123:body<>"; + keep_entry(entries, "plain", Vec::new()); + keep_entry(entries, checkout, Vec::new()); + keep_entry(entries, "caller", vec![CachedTarget::Named(checkout.to_owned())]); + assert!(entries.usable("caller", &|_| true)); + let mut restricted = Entries::new(entries.store, true, None, None, None); + restricted.checkout = false; + assert!(restricted.usable("plain", &|_| true)); + assert!(restricted.get(checkout).is_none()); + assert!(!restricted.usable(checkout, &|_| true)); + assert!(!restricted.usable("caller", &|_| true)); + assert!(!restricted.usable("missing", &|_| true)); + assert!(entries.get(checkout).is_some()); + }); + } + + #[test] + fn entries_with_missing_dependencies_or_imports_are_not_usable() { + with_entries(|entries| { + keep_entry(entries, "a", vec![CachedTarget::Named("missing".to_owned())]); + keep_entry( + entries, + "b", + vec![CachedTarget::Import { symbol: "missing".to_owned(), checked: true }], + ); + assert!(!entries.usable("a", &|_| false)); + assert!(!entries.usable("b", &|_| false)); + }); + } + fn reloc(offset: u32, namespace: u32, index: u32) -> CachedReloc { let name = ModuleRelocTarget::user(namespace, index); CachedReloc { offset, kind: Reloc::X86CallPCRel4, name, addend: -4 } @@ -901,7 +1085,7 @@ mod tests { debug.files.push("src/lib.rs".to_owned()); debug.declared = Some(Place { file: 0, line: 3, column: 1 }); debug.rows.push((4, Place { file: 0, line: 4, column: 5 })); - debug.types = vec![TypeSet::new(TypeList { types, links: Vec::new() })]; + debug.types = vec![TypeSet::new(TypeList { types: types.into(), links: Vec::new() })]; debug.scopes.push(Scope { parent: None, ranges: vec![(0, 8)] }); debug.frame_base = vec![cranelift_codegen::gimli::DW_OP_reg6.0]; debug.variables.push(VariableDebug { @@ -1066,7 +1250,7 @@ mod tests { } let links = types.links.iter().map(|it| keep(it, stored, keys)).collect(); let bytes = - postcard::to_allocvec(&StoredTypeSet { types: types.types.clone(), links }) + postcard::to_allocvec(&StoredTypeSet { types: types.types.to_vec(), links }) .unwrap(); let mut hasher = Sha256::new(); hasher.update(b"rustd types 2"); @@ -1094,7 +1278,7 @@ mod tests { .iter() .map(|key| read_types(key, stored, known, depth + 1)) .collect::>()?; - let types = TypeSet::new(TypeList { types, links }); + let types = TypeSet::new(TypeList { types: types.into(), links }); known.insert(*key, types.clone()); Some(types) } @@ -1170,17 +1354,19 @@ mod tests { .collect(); chain.variables[0].scope = Some(39_999); let mut linked = TypeSet::new(TypeList { - types: vec![DebugType::Base { name: "T".to_owned(), size: 4, encoding: 7 }], + types: vec![DebugType::Base { name: "T".to_owned(), size: 4, encoding: 7 }].into(), links: Vec::new(), }); for _ in 0..200 { - linked = - TypeSet::new(TypeList { types: vec![DebugType::Link(0)], links: vec![linked] }); + linked = TypeSet::new(TypeList { + types: vec![DebugType::Link(0)].into(), + links: vec![linked], + }); } let linked_entry = Entry { debug: Some(debug(vec![DebugType::Link(0)])), ..entry() }; let mut linked_entry = linked_entry; linked_entry.debug.as_mut().unwrap().types[0] = - TypeSet::new(TypeList { types: vec![DebugType::Link(0)], links: vec![linked] }); + TypeSet::new(TypeList { types: vec![DebugType::Link(0)].into(), links: vec![linked] }); assert!(read_back(&linked_entry).is_none()); let isa = Target::selected().and_then(|it| it.isa("backtracking", false)).unwrap(); // Only ELF and Mach-O have the tables. @@ -1195,11 +1381,14 @@ mod tests { #[test] fn linked_types_round_trip_and_share_a_die() { let shared = TypeSet::new(TypeList { - types: vec![DebugType::Base { name: "Shared".to_owned(), size: 4, encoding: 7 }], + types: vec![DebugType::Base { name: "Shared".to_owned(), size: 4, encoding: 7 }].into(), links: Vec::new(), }); let linked = || { - TypeSet::new(TypeList { types: vec![DebugType::Link(0)], links: vec![shared.clone()] }) + TypeSet::new(TypeList { + types: vec![DebugType::Link(0)].into(), + links: vec![shared.clone()], + }) }; let mut written = entry(); written.debug.as_mut().unwrap().types = vec![linked()]; diff --git a/crates/hir-ty-codegen/debuginfo.rs b/crates/hir-ty-codegen/debuginfo.rs index e595a516ed..3a15129f51 100644 --- a/crates/hir-ty-codegen/debuginfo.rs +++ b/crates/hir-ty-codegen/debuginfo.rs @@ -16,7 +16,7 @@ mod split; mod types; use codeview::CodeView; -pub(super) use types::Types; +pub(super) use types::{Descriptions, Types}; use cranelift_codegen::{ binemit::CodeOffset, @@ -107,7 +107,7 @@ pub(super) type TypeSet = std::sync::Arc; #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct TypeList { - pub(super) types: Vec, + pub(super) types: std::sync::Arc<[DebugType]>, pub(super) links: Vec, } @@ -490,11 +490,13 @@ const FUNCTIONS_OF_A_UNIT: usize = 256; pub(super) struct Dwarf { /// The unit functions are being added to. dwarf: DwarfUnit, - /// The units that have all their functions. - full: Vec, + /// The encoded sections of the units that have all their functions. + full: Sections, /// What the names of the files the entries of the units are read from start with, when /// they are kept apart from what is linked, and the units that are. - apart: Option<(String, Vec)>, + apart: Option<(String, split::Split)>, + units: usize, + error: Option, /// What tells the unit functions are being added to from another: what was added. id: FxHasher, encoding: Encoding, @@ -540,8 +542,19 @@ impl Dwarf { Encoding { format: Format::Dwarf32, version: 4, address_size: isa.pointer_bytes() }; let mut context = Dwarf { dwarf: DwarfUnit::new(encoding), - full: Vec::new(), - apart: apart.map(|name| (name, Vec::new())), + full: Sections::new(SectionWriter::default()), + apart: apart.map(|name| { + let split = split::Split::new( + encoding, + directory.as_bytes().to_vec(), + name.clone(), + isa.triple().binary_format, + isa.triple().architecture, + ); + (name, split) + }), + units: 0, + error: None, id: FxHasher::default(), addresses: Vec::new(), encoding, @@ -596,6 +609,12 @@ impl Dwarf { /// Ends the unit functions were added to, which is left out when it has none, and /// begins the next. fn end_unit(&mut self) { + if let Err(error) = self.write_unit() { + self.error.get_or_insert(error); + } + } + + fn write_unit(&mut self) -> Result<(), String> { let next = gimli::write::Unit::new(self.encoding, LineProgram::none()); let mut unit = std::mem::replace(&mut self.dwarf.unit, next); let ranges = std::mem::take(&mut self.ranges); @@ -606,22 +625,27 @@ impl Dwarf { self.namespaces.clear(); self.begin_unit(); if ranges.is_empty() { - return; + return Ok(()); } + let nth = self.units; + self.units += 1; let root = unit.root(); let addresses = std::mem::take(&mut self.addresses); if let Some((name, apart)) = &mut self.apart { let lines = std::mem::replace(&mut unit.line_program, LineProgram::none()); let root = unit.get_mut(root); - let name = split::file(name, apart.len()).into_bytes(); + let name = split::file(name, nth).into_bytes(); root.set(DW_AT_GNU_dwo_name, AttributeValue::String(name)); root.set(DW_AT_GNU_dwo_id, AttributeValue::Data8(id)); - apart.push(split::Apart { unit, lines, ranges, addresses, id }); - return; + return apart.add(split::Apart { unit, lines, ranges, addresses, id }); } let ranges = unit.ranges.add(RangeList(ranges)); unit.get_mut(root).set(DW_AT_ranges, AttributeValue::RangeListRef(ranges)); - self.full.push(unit); + let mut units = gimli::write::UnitTable::default(); + units.add(unit); + units + .write(&mut self.full, &mut self.dwarf.line_strings, &mut self.dwarf.strings) + .map_err(|e| e.to_string()) } fn file(&mut self, name: &str) -> FileId { @@ -1106,18 +1130,16 @@ impl Dwarf { /// described. fn sections(mut self) -> Result, String> { self.end_unit(); + if let Some(error) = self.error { + return Err(error); + } let mut sections = Vec::new(); - if self.full.is_empty() { + if self.units == 0 { return Ok(sections); } - let mut dwarf = gimli::write::Dwarf::new(); - (dwarf.strings, dwarf.line_strings) = (self.dwarf.strings, self.dwarf.line_strings); - for unit in self.full { - dwarf.units.add(unit); - } - let mut written = Sections::new(SectionWriter::default()); - dwarf.write(&mut written).map_err(|e| e.to_string())?; - written.for_each(|id, section: &SectionWriter| { + self.dwarf.line_strings.write(&mut self.full.debug_line_str).map_err(|e| e.to_string())?; + self.dwarf.strings.write(&mut self.full.debug_str).map_err(|e| e.to_string())?; + self.full.for_each(|id, section: &SectionWriter| { sections.push(Section { id, written: section.clone() }); Ok::<_, String>(()) })?; @@ -1128,18 +1150,17 @@ impl Dwarf { /// Answers the files of what is kept apart from it. fn emit(mut self, product: &mut ObjectProduct) -> Result, String> { self.end_unit(); + if let Some(error) = self.error.take() { + return Err(error); + } if !matches!(product.object.format(), BinaryFormat::Elf | BinaryFormat::MachO) { return Ok(Vec::new()); } - let Some((name, apart)) = self.apart.take() else { + let Some((_, split)) = self.apart.take() else { place(product, self.sections()?)?; return Ok(Vec::new()); }; - let mut split = split::Split::new(self.encoding, self.directory, name, &product.object); - for unit in apart { - split.add(unit)?; - } - let (sections, files) = split.finish(); + let (sections, files) = split.finish()?; place(product, sections)?; Ok(files) } @@ -1155,9 +1176,20 @@ struct Section { pub struct Sidecar { /// Where the unit left in the object says the file is. pub name: String, + /// A temporary file containing the unit, written while it was encoded. + /// It is renamed to `name` only after the object is ready. + pub staged: Option, pub bytes: Vec, } +impl Drop for Sidecar { + fn drop(&mut self) { + if let Some(path) = self.staged.take() { + let _ = std::fs::remove_file(path); + } + } +} + /// Writes the sections into the object, each with what the linker is to fill in. A Mach-O /// linker leaves the sections in the object, where a debugger reads them through the debug /// map: an offset into another section is written as it is, and an address is relocated @@ -1413,16 +1445,131 @@ impl Writer for SectionWriter { mod tests { use super::*; + fn split_isa() -> cranelift_codegen::isa::OwnedTargetIsa { + cranelift_codegen::isa::lookup_by_name("x86_64-unknown-linux-gnu") + .unwrap() + .finish(cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder())) + .unwrap() + } + + fn staging_directory() -> std::path::PathBuf { + let nonce = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let path = std::env::temp_dir().join(format!("rustd-dwo-{}-{nonce}", std::process::id())); + std::fs::create_dir(&path).unwrap(); + path + } + + #[test] + fn split_units_are_staged_and_cleaned_up() { + let directory = staging_directory(); + let object = directory.join("unit.o"); + let isa = split_isa(); + let mut dwarf = Dwarf::new(&*isa, "/", true, Some(object.to_str().unwrap().to_owned())); + let debug = FunctionDebug::new("f".to_owned(), "f".to_owned()); + dwarf.add_function(FuncId::from_u32(0), 8, &debug); + dwarf.end_unit(); + assert!(dwarf.error.is_none()); + let (_, split) = dwarf.apart.take().unwrap(); + let (_, files) = split.finish().unwrap(); + assert_eq!(files.len(), 1); + let staged = files[0].staged.as_ref().unwrap().clone(); + assert!(!std::fs::read(&staged).unwrap().is_empty()); + assert!(!std::path::Path::new(&files[0].name).exists()); + assert!(files[0].bytes.is_empty()); + drop(files); + assert!(!staged.exists()); + std::fs::remove_dir(directory).unwrap(); + } + + #[test] + fn unfinished_split_build_removes_staged_units() { + let directory = staging_directory(); + let object = directory.join("unit.o"); + let isa = split_isa(); + let mut dwarf = Dwarf::new(&*isa, "/", true, Some(object.to_str().unwrap().to_owned())); + let debug = FunctionDebug::new("f".to_owned(), "f".to_owned()); + dwarf.add_function(FuncId::from_u32(0), 8, &debug); + dwarf.end_unit(); + assert!(dwarf.error.is_none()); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), 1); + drop(dwarf); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), 0); + std::fs::remove_dir(directory).unwrap(); + } + + #[test] + fn split_write_errors_reach_emit() { + let directory = staging_directory(); + let object = directory.join("missing/unit.o"); + let isa = split_isa(); + let mut dwarf = Dwarf::new(&*isa, "/", true, Some(object.to_str().unwrap().to_owned())); + let debug = FunctionDebug::new("f".to_owned(), "f".to_owned()); + dwarf.add_function(FuncId::from_u32(0), 8, &debug); + dwarf.end_unit(); + let expected = dwarf.error.clone().unwrap(); + assert!(matches!(dwarf.sections(), Err(error) if error == expected)); + assert_eq!(std::fs::read_dir(&directory).unwrap().count(), 0); + std::fs::remove_dir(directory).unwrap(); + } + + #[test] + fn finished_units_are_encoded() { + let isa = + super::super::Target::selected().and_then(|it| it.isa("backtracking", false)).unwrap(); + let mut dwarf = Dwarf::new(&*isa, "/", true, None); + for at in 0..=FUNCTIONS_OF_A_UNIT { + let debug = FunctionDebug::new(format!("f{at}"), format!("f{at}")); + dwarf.add_function(FuncId::from_u32(at as u32), 8, &debug); + } + assert_eq!(dwarf.units, 1); + assert!(dwarf.full.debug_info.len() > 0); + assert!(dwarf.full.debug_abbrev.len() > 0); + assert_eq!(dwarf.full.debug_str.len(), 0); + assert_eq!(dwarf.ranges.len(), 1); + let sections = dwarf.sections().unwrap(); + assert!(sections.iter().any(|it| it.id == SectionId::DebugStr && it.written.len() > 0)); + } + + #[test] + fn empty_units_have_no_sections() { + let isa = + super::super::Target::selected().and_then(|it| it.isa("backtracking", false)).unwrap(); + for apart in [None, Some("empty".to_owned())] { + let dwarf = Dwarf::new(&*isa, "/", true, apart); + assert!(dwarf.sections().unwrap().is_empty()); + } + } + + #[test] + fn encoding_errors_reach_emit() { + let isa = + super::super::Target::selected().and_then(|it| it.isa("backtracking", false)).unwrap(); + let mut dwarf = Dwarf::new(&*isa, "/", true, None); + let encoding = Encoding { version: 1, ..dwarf.encoding }; + dwarf.dwarf.unit = gimli::write::Unit::new(encoding, LineProgram::none()); + dwarf.begin_unit(); + let debug = FunctionDebug::new("f".to_owned(), "f".to_owned()); + dwarf.add_function(FuncId::from_u32(0), 8, &debug); + dwarf.end_unit(); + let expected = gimli::write::Error::UnsupportedVersion(1).to_string(); + assert_eq!(dwarf.error.as_ref(), Some(&expected)); + assert!(matches!(dwarf.sections(), Err(error) if error == expected)); + } + #[test] fn linked_types_share_one_die() { let isa = super::super::Target::selected().and_then(|it| it.isa("backtracking", false)).unwrap(); let shared = TypeSet::new(TypeList { - types: vec![DebugType::Base { name: "Shared".to_owned(), size: 4, encoding: 7 }], + types: vec![DebugType::Base { name: "Shared".to_owned(), size: 4, encoding: 7 }].into(), links: Vec::new(), }); let linked = || { - TypeSet::new(TypeList { types: vec![DebugType::Link(0)], links: vec![shared.clone()] }) + TypeSet::new(TypeList { + types: vec![DebugType::Link(0)].into(), + links: vec![shared.clone()], + }) }; let mut dwarf = Dwarf::new(&*isa, "/", true, None); diff --git a/crates/hir-ty-codegen/debuginfo/split.rs b/crates/hir-ty-codegen/debuginfo/split.rs index f9411755c7..b5dc0cc5c3 100644 --- a/crates/hir-ty-codegen/debuginfo/split.rs +++ b/crates/hir-ty-codegen/debuginfo/split.rs @@ -17,11 +17,9 @@ use cranelift_codegen::gimli::{ Address, DebugLine, LineProgram, LineStringTable, Range, Sections, StringTable, Writer, }, }; -use cranelift_object::object::{ - self, BinaryFormat, SectionFlags, SectionKind, elf, - write::{Object, StandardSegment}, -}; +use cranelift_object::object::{self, SectionFlags, SectionKind, elf, write::StandardSegment}; use rustc_hash::FxHashMap; +use std::path::PathBuf; use super::{Reloc, Section, SectionWriter, Sidecar, Target}; @@ -85,10 +83,11 @@ pub(super) struct Apart { /// The sections of an object whose units are kept apart, as they are added to. pub(super) struct Split { encoding: Encoding, + format: object::BinaryFormat, + architecture: object::Architecture, directory: Vec, /// What the name of each file the entries are read from starts with. name: String, - object: (BinaryFormat, object::Architecture), info: SectionWriter, abbrev: SectionWriter, line: DebugLine, @@ -104,13 +103,30 @@ impl Split { encoding: Encoding, directory: Vec, name: String, - like: &Object<'_>, + format: target_lexicon::BinaryFormat, + architecture: target_lexicon::Architecture, ) -> Self { + let format = match format { + target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, + target_lexicon::BinaryFormat::Macho => object::BinaryFormat::MachO, + target_lexicon::BinaryFormat::Coff => object::BinaryFormat::Coff, + other => unreachable!("split DWARF on unsupported object format {other:?}"), + }; + let architecture = match architecture { + target_lexicon::Architecture::X86_32(_) => object::Architecture::I386, + target_lexicon::Architecture::X86_64 => object::Architecture::X86_64, + target_lexicon::Architecture::Arm(_) => object::Architecture::Arm, + target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64, + target_lexicon::Architecture::Riscv64(_) => object::Architecture::Riscv64, + target_lexicon::Architecture::S390x => object::Architecture::S390x, + other => unreachable!("split DWARF on unsupported architecture {other:?}"), + }; Split { encoding, + format, + architecture, directory, name, - object: (like.format(), like.architecture()), info: SectionWriter::default(), abbrev: SectionWriter::default(), line: DebugLine(SectionWriter::default()), @@ -183,22 +199,31 @@ impl Split { } let name = file(&self.name, self.files.len()); let lists = std::mem::take(&mut self.loc_apart); - let bytes = self.file([ - (SectionId::DebugInfo, &info), - (SectionId::DebugAbbrev, &by_index(&written.debug_abbrev.0)?), - (SectionId::DebugLoc, &lists), - ])?; + let sections = [ + (SectionId::DebugInfo, info), + (SectionId::DebugAbbrev, by_index(&written.debug_abbrev.0)?), + (SectionId::DebugLoc, lists), + ]; self.skeleton(name.as_bytes(), lines.0, id, of_unit, ranges_base, addr_base) .map_err(|e| e.to_string())?; - self.files.push(Sidecar { name, bytes }); + let bytes = Self::file(self.format, self.architecture, sections)?; + let mut temporary = PathBuf::from(&name); + temporary.as_mut_os_string().push(format!(".{}.tmp", std::process::id())); + let file = Sidecar { name, staged: Some(temporary.clone()), bytes: Vec::new() }; + std::fs::write(&temporary, bytes).map_err(|error| error.to_string())?; + self.files.push(file); Ok(()) } /// The file of one unit's entries: an object of nothing but these sections. - fn file(&self, sections: [(SectionId, &SectionWriter); 3]) -> Result, String> { - let (format, architecture) = self.object; + fn file( + format: object::BinaryFormat, + architecture: object::Architecture, + sections: [(SectionId, SectionWriter); 3], + ) -> Result, String> { // The sections are written with the low byte first, as every target of ours has it. - let mut object = Object::new(format, architecture, object::Endianness::Little); + let mut object = + object::write::Object::new(format, architecture, object::Endianness::Little); for (id, written) in sections { let bytes = written.bytes.slice(); let Some(name) = id.dwo_name().filter(|_| !bytes.is_empty()) else { continue }; @@ -332,7 +357,8 @@ impl Split { } /// The sections to link, and the files of what is kept apart. - pub(super) fn finish(self) -> (Vec
, Vec) { + pub(super) fn finish(self) -> Result<(Vec
, Vec), String> { + let files = self.files; let linked = [ (SectionId::DebugInfo, self.info), (SectionId::DebugAbbrev, self.abbrev), @@ -341,6 +367,6 @@ impl Split { (SectionId::DebugAddr, self.addr), ]; let linked = linked.into_iter().map(|(id, written)| Section { id, written }); - (linked.collect(), self.files) + Ok((linked.collect(), files)) } } diff --git a/crates/hir-ty-codegen/debuginfo/types.rs b/crates/hir-ty-codegen/debuginfo/types.rs index a9072a3fb2..1885466d72 100644 --- a/crates/hir-ty-codegen/debuginfo/types.rs +++ b/crates/hir-ty-codegen/debuginfo/types.rs @@ -20,6 +20,8 @@ use rustc_type_ir::{ UintTy, inherent::{BoundExistentialPredicates as _, GenericArgs as _, IntoKind}, }; +use sha2::{Digest, Sha256}; +use std::sync::{Arc, Mutex, Weak}; use super::super::vtable::{HEADER, VtableLayout}; use super::{Aggregate, DebugType, Field, Tag, TypeIndex, TypeList, TypeSet, Variant}; @@ -33,7 +35,6 @@ use crate::{ layout::CoroutineFrame, layout::struct_tail_erasing_lifetimes, layout::{Layout, RustcFieldIdx}, - mir::is_by_move_coroutine, mir::type_name::rustc_debug_name, next_solver::{ DbInterner, GenericArgs, Ty, TyKind, TypingMode, @@ -55,10 +56,31 @@ mod encoding { /// unless it is told of anyway: a list is as long as it likes. const POINTERS: u32 = 2; +#[derive(Default)] +pub(in super::super) struct Descriptions(Mutex>>); + +impl Descriptions { + fn intern(&self, types: Vec) -> Arc<[DebugType]> { + let Ok(bytes) = postcard::to_allocvec(&types) else { return types.into() }; + let key = Sha256::digest(bytes).into(); + let mut known = self.0.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(shared) = known.get(&key).and_then(Weak::upgrade) + && shared.as_ref() == types.as_slice() + { + return shared; + } + let shared: Arc<[DebugType]> = types.into(); + known.insert(key, Arc::downgrade(&shared)); + shared + } +} + /// The types of the variables of the functions of one environment. What a variable's type /// is told as depends on nothing but the type, so it is found once for every function with /// a variable of it: the layouts and the names are most of what telling of variables costs. pub(in super::super) struct Types<'db> { + sharing: super::super::Sharing, + descriptions: Arc, db: &'db dyn HirDatabase, env: ParamEnvAndCrate<'db>, interner: DbInterner<'db>, @@ -76,10 +98,17 @@ pub(in super::super) struct Types<'db> { } impl<'db> Types<'db> { - pub(in super::super) fn new(db: &'db dyn HirDatabase, env: ParamEnvAndCrate<'db>) -> Self { + pub(in super::super) fn new( + db: &'db dyn HirDatabase, + env: ParamEnvAndCrate<'db>, + descriptions: Arc, + sharing: super::super::Sharing, + ) -> Self { let interner = DbInterner::new_with(db, env.krate); let infcx = interner.infer_ctxt().build(TypingMode::PostAnalysis); Types { + sharing, + descriptions, db, env, interner, @@ -115,7 +144,7 @@ impl<'db> Types<'db> { self.roots.push(root); let described = self.ty(root.0, pointers).map(|_| { TypeSet::new(TypeList { - types: std::mem::take(&mut self.told), + types: self.descriptions.intern(std::mem::take(&mut self.told)), links: std::mem::take(&mut self.links), }) }); @@ -138,7 +167,13 @@ impl<'db> Types<'db> { let mut layout = None; let link = match ty.kind() { TyKind::Adt(..) => { - let found = self.db.layout_of_ty(ty.store(), self.env.store()).ok()?; + let found = super::super::remap::layout( + self.db, + ty.store(), + self.env.store(), + self.sharing.clone(), + ) + .ok()?; let link = !self.is_pointer(ty, found.size.bytes()); layout = Some(found); link @@ -165,7 +200,13 @@ impl<'db> Types<'db> { } let layout = match layout { Some(layout) => layout, - None => self.db.layout_of_ty(ty.store(), self.env.store()).ok()?, + None => super::super::remap::layout( + self.db, + ty.store(), + self.env.store(), + self.sharing.clone(), + ) + .ok()?, }; let name = self.name(ty)?; // A type that is made of itself finds its place here. What stands there until the @@ -494,7 +535,13 @@ impl<'db> Types<'db> { fields } - fn capture_names(&self, closure: InternedClosure<'db>) -> Option> { + fn capture_names(&self, mut closure: InternedClosure<'db>) -> Option> { + if let crate::InferBodyId::DefWithBodyId(hir_def::DefWithBodyId::FunctionId(func)) = + closure.owner + && let Some(source) = self.sharing.analysis_source(self.db, func) + { + closure.owner = source.into(); + } let store = ExpressionStore::of(self.db, closure.owner.expression_store_owner(self.db)); let infer = InferenceResult::of(self.db, closure.owner); let captured = infer.closures_data.get(&closure.expr)?.min_captures.values().flatten(); @@ -555,18 +602,18 @@ impl<'db> Types<'db> { let captured = self.captures(id.0.loc(self.db), args.tupled_upvars_ty(), offset_of, pointers); let parent_args = GenericArgs::new_from_slice(args.parent_args()); - let body = match is_by_move_coroutine(self.db, id.0, args) { - true => self.db.monomorphized_mir_body_for_coroutine_by_move( - id.0, - parent_args.store(), - self.env.store(), - ), - false => self.db.monomorphized_mir_body_for_coroutine( - id.0, - parent_args.store(), - self.env.store(), - ), - } + use salsa::plumbing::AsId; + let def = if self.sharing.coroutine_is_by_move(self.db, id.0, args) { + super::super::InstanceDef::CoroutineByMove(id.0.as_id()) + } else { + super::super::InstanceDef::Coroutine(id.0.as_id()) + }; + let body = super::super::program_instance_body( + self.db, + &super::super::Instance { def, args: parent_args.store(), checked: false }, + self.env.store(), + &self.sharing, + ) .ok()?; let held = body.locals_held_across_yields(&self.infcx, self.env.param_env); let closure = id.0.loc(self.db); @@ -640,3 +687,73 @@ impl Aggregate { Aggregate { name, size, fields: Vec::new(), parameters: Vec::new() } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn description(name: &str) -> Vec { + vec![DebugType::Base { name: name.to_owned(), size: 4, encoding: 7 }] + } + + #[test] + fn equal_descriptions_share_without_merging_sets() { + let descriptions = Descriptions::default(); + let first = TypeSet::new(TypeList { + types: descriptions.intern(description("Shared")), + links: Vec::new(), + }); + let mut types = description("Shared"); + types.reserve(100); + let second = TypeSet::new(TypeList { + types: descriptions.intern(types), + links: vec![first.clone()], + }); + assert!(Arc::ptr_eq(&first.types, &second.types)); + assert!(!Arc::ptr_eq(&first, &second)); + assert!(Arc::ptr_eq(&second.links[0], &first)); + } + + #[test] + fn empty_descriptions_share() { + let descriptions = Descriptions::default(); + let first = descriptions.intern(Vec::new()); + let second = descriptions.intern(Vec::new()); + assert!(first.is_empty()); + assert!(Arc::ptr_eq(&first, &second)); + } + + #[test] + fn descriptions_do_not_keep_payloads_alive() { + let descriptions = Descriptions::default(); + let first = descriptions.intern(description("Shared")); + let weak = Arc::downgrade(&first); + assert_eq!(Arc::strong_count(&first), 1); + drop(first); + assert!(weak.upgrade().is_none()); + let second = descriptions.intern(description("Shared")); + assert_eq!(second.as_ref(), description("Shared")); + assert_eq!(descriptions.0.lock().unwrap().len(), 1); + } + + #[test] + fn matching_hash_still_checks_content() { + let descriptions = Descriptions::default(); + let first = descriptions.intern(description("First")); + let other = description("Other"); + let key = Sha256::digest(postcard::to_allocvec(&other).unwrap()).into(); + descriptions.0.lock().unwrap().insert(key, Arc::downgrade(&first)); + let second = descriptions.intern(other.clone()); + assert!(!Arc::ptr_eq(&first, &second)); + assert_eq!(second.as_ref(), other); + } + + #[test] + fn descriptions_share_across_threads() { + let descriptions = Arc::new(Descriptions::default()); + let first = descriptions.intern(description("Shared")); + let second = + std::thread::spawn(move || descriptions.intern(description("Shared"))).join().unwrap(); + assert!(Arc::ptr_eq(&first, &second)); + } +} diff --git a/crates/hir-ty-codegen/fragment.rs b/crates/hir-ty-codegen/fragment.rs index 662e614c3c..4e7cf4bbf6 100644 --- a/crates/hir-ty-codegen/fragment.rs +++ b/crates/hir-ty-codegen/fragment.rs @@ -153,6 +153,7 @@ impl PartialEq for Code { /// leaves what was built from it alone. #[derive(PartialEq, Hash)] pub struct CompiledFunction { + pub(super) sharing: super::Sharing, pub(super) requests: Requests, pub(super) code: Code, /// The body the function was compiled from and the statements of it that @@ -253,6 +254,7 @@ pub(super) const INLINED: u32 = u32::MAX; /// Collects the requests of one function while its IR is built. pub(super) struct Recorder<'db> { + pub(super) sharing: super::Sharing, pub(super) db: &'db dyn HirDatabase, pub(super) isa: &'static dyn TargetIsa, /// What `isa` generates code for. @@ -300,6 +302,7 @@ impl<'db> Recorder<'db> { checked: bool, ) -> Self { Recorder { + sharing: super::Sharing::default(), db, isa, target, diff --git a/crates/hir-ty-codegen/home.rs b/crates/hir-ty-codegen/home.rs index d17241cd1e..7021917e40 100644 --- a/crates/hir-ty-codegen/home.rs +++ b/crates/hir-ty-codegen/home.rs @@ -122,6 +122,11 @@ pub fn body_files(db: &dyn HirDatabase, instance: &Instance) -> Option Option> { + let mentions = Mentions::of(db, instance); + (!mentions.opaque).then_some(mentions.bodies) +} + /// The file a body is written in, which for one a macro makes is where the macro is called. pub fn body_file(db: &dyn HirDatabase, body: DefWithBodyId) -> FileId { let file = match body { @@ -171,7 +176,12 @@ impl<'db> TypeVisitor> for Mentions<'db> { #[derive(Default)] pub(super) struct Homes { /// What each crate can see: itself and the crates whose impls it knows. - sees: FxHashMap>, + sees: FxHashMap>>, +} + +#[salsa::tracked(returns(clone))] +fn visibility(db: &dyn HirDatabase, krate: Crate) -> triomphe::Arc> { + triomphe::Arc::new(loaded_crates(db, krate).iter().copied().collect()) } impl Homes { @@ -192,10 +202,7 @@ impl Homes { return asked.clone(); } let home = mentions.crates.iter().copied().find(|&krate| { - let sees = self - .sees - .entry(krate) - .or_insert_with(|| loaded_crates(db, krate).iter().copied().collect()); + let sees = self.sees.entry(krate).or_insert_with(|| visibility(db, krate)); mentions.crates.iter().all(|it| sees.contains(it)) }); match home { diff --git a/crates/hir-ty-codegen/mod.rs b/crates/hir-ty-codegen/mod.rs index 23a61a3f3f..47555ccd9c 100644 --- a/crates/hir-ty-codegen/mod.rs +++ b/crates/hir-ty-codegen/mod.rs @@ -55,7 +55,6 @@ use syntax::{AstNode, AstToken, SyntaxElement, SyntaxKind, SyntaxNode, TextRange use triomphe::Arc; use unicode_width::UnicodeWidthChar; -use crate::mir::is_by_move_coroutine; use crate::{ CallableDefId, InferBodyId, InferenceResult, MemoryMap, ParamEnvAndCrate, consteval::try_const_usize, @@ -109,7 +108,26 @@ pub use home::body_file; mod llvm; mod patch; mod promote; +mod remap; mod settled; +mod sharing; +pub use sharing::Sharing; + +pub(crate) fn trace_analysis_query(db: &dyn HirDatabase, owner: InferBodyId<'_>, phase: &str) { + if std::env::var_os("RUSTD_CFG_ANALYSIS_TRACE").is_some() + && let InferBodyId::DefWithBodyId(DefWithBodyId::FunctionId(func)) = owner + { + let test = func + .krate(db) + .cfg_options(db) + .without_recording() + .check_atom(&cfg::CfgAtom::Flag(intern::sym::test)); + eprintln!( + "cfg analysis query: {phase} {}: test {test}", + FunctionSignature::of(db, func).name.as_str() + ); + } +} mod stable; pub use stable::{block_place, lower_declared}; mod store; @@ -129,7 +147,7 @@ mod vtable; pub mod wasm; mod wasm_object; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, salsa::SalsaValue)] pub enum CodegenError { /// The function uses something this backend can't compile yet; fall back to rustc. Unsupported(String), @@ -323,6 +341,72 @@ pub trait SymbolResolver { /// What an instance is called in every run of the daemon, if anything. type StableName = std::sync::Arc; +#[derive(Default)] +struct StableNameCache { + names: FxHashMap>, + bytes: usize, +} + +impl StableNameCache { + const BYTES: usize = 4 * 1024 * 1024; + const NAMES: usize = 4096; + + fn get(&self, instance: &Instance) -> Option<&Option> { + self.names.get(instance) + } + + fn insert(&mut self, instance: Instance, name: Option) { + let bytes = name.as_ref().map_or(0, |it| it.len()); + if bytes > Self::BYTES { + return; + } + if self.bytes + bytes > Self::BYTES || self.names.len() == Self::NAMES { + self.names.clear(); + self.bytes = 0; + } + self.bytes += bytes; + self.names.insert(instance, name); + } +} + +#[cfg(test)] +mod stable_name_cache_tests { + use super::*; + + fn instance(checked: bool) -> Instance { + Instance { def: InstanceDef::DropGlue, args: GenericArgs::empty().store(), checked } + } + + #[test] + fn names_are_shared_and_missing_names_are_memoized() { + let mut cache = StableNameCache::default(); + assert!(cache.get(&instance(true)).is_none()); + cache.insert(instance(true), None); + assert_eq!(cache.get(&instance(true)), Some(&None)); + let name = StableName::from("a name"); + cache.insert(instance(false), Some(name.clone())); + assert!(std::sync::Arc::ptr_eq( + cache.get(&instance(false)).unwrap().as_ref().unwrap(), + &name + )); + } + + #[test] + fn names_do_not_keep_more_than_the_byte_budget() { + let mut cache = StableNameCache::default(); + let name = StableName::from("x".repeat(StableNameCache::BYTES / 2 + 1)); + cache.insert(instance(true), Some(name.clone())); + cache.insert(instance(false), Some(name.clone())); + assert!(cache.get(&instance(true)).is_none()); + assert!(cache.get(&instance(false)).is_some()); + assert_eq!(cache.bytes, name.len()); + let oversized = StableName::from("x".repeat(StableNameCache::BYTES + 1)); + cache.insert(instance(true), Some(oversized)); + assert!(cache.get(&instance(true)).is_none()); + assert_eq!(cache.bytes, name.len()); + } +} + pub type ParallelWork<'a> = dyn Fn(&dyn HirDatabase, &dyn SymbolResolver, usize) + Sync + 'a; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -457,8 +541,9 @@ pub struct CodegenUnit { } /// The functions a build starts from. -#[derive(Clone, Copy)] +#[derive(Clone)] pub struct Roots<'a> { + pub sharing: Sharing, /// Compiled whether or not anything calls them. pub entries: &'a [FunctionId], /// Compiled once code in the object refers to them. A function not reached keeps rustc's @@ -486,13 +571,37 @@ impl CodegenUnit { denied: &FxHashSet, known: &mut Rejections, ) -> Result { + retry_sharing(!roots.sharing.is_empty(), known, |sharing, known| { + let mut roots = roots.clone(); + if !sharing { + roots.sharing = roots.sharing.disabled(); + } + Self::compile_inner(db, symbols, roots, denied, known) + }) + } + + fn compile_inner( + db: &dyn HirDatabase, + symbols: &dyn SymbolResolver, + roots: Roots<'_>, + denied: &FxHashSet, + known: &mut Rejections, + ) -> Result { + let mut sharing = roots.sharing.clone(); + sharing.prewarm(db); + let mut sharing_restarts = 0; let on_demand: FxHashSet = roots.on_demand.iter().copied().collect(); let all = || roots.entries.iter().chain(roots.on_demand).copied(); let mut declined = None; - // Code of the checkout comes from the store in the first build alone: what a patch is - // of has to be compiled here, to tell what an edit changes. - let hydrate = known.base.is_none(); - if let Some(base) = known.base.take().filter(|it| !it.outdated()) { + // A patch base needs the checkout's instance identities and dependency edges, + // which opaque store entries do not preserve. + let hydrate = !known.patching && known.base.is_none(); + if let Some(base) = known.base.take().filter(|it| { + if it.outdated() && std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("patch discard: cached checkout code invalidated by edit"); + } + !it.outdated() + }) { let own_statics = OwnStatics::new( db, all().map(|root| root.krate(db)).collect(), @@ -504,20 +613,38 @@ impl CodegenUnit { if stale.is_empty() { stale.push(base.stale(db, 0, 1)); } - let patch = - base.patch(db, symbols, roots, &on_demand, denied, known, own_statics, stale); + let patch = base.patch( + db, + symbols, + roots.clone(), + &on_demand, + denied, + known, + own_statics, + stale, + ); let patch = match patch { Ok(Ok(patch)) => Some(patch), Ok(Err(why)) => { + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("patch decline: {why}"); + } declined = Some(why); None } - Err(_) => None, + Err(error) => { + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("patch decline: {error}"); + } + None + } }; if let Some(patch) = patch { + known.reach.patch(patch.reach_changed); backend::Tally::global().report(); let compiled_now = known.compiled_since_last_build(); known.stats = BuildStats { + sharing: patch.sharing, bodies: base.bodies, reused: base.bodies.saturating_sub(compiled_now), compiled: compiled_now, @@ -539,6 +666,7 @@ impl CodegenUnit { return Ok(unit); } } + known.reach.clear(); let patching = known.patching; let used_by_rustc = known.used_by_rustc; // FIXME: a rejection outlives an edit that would lift it. Dropping them here costs one @@ -555,6 +683,8 @@ impl CodegenUnit { stale: _, frozen: _, used_by_rustc: _, + reach: _, + policy: _, } = known; let mut own_statics = OwnStatics::new( db, @@ -584,6 +714,13 @@ impl CodegenUnit { None, hydrate, )?; + cg.sharing = sharing.clone(); + cg.counting_sharing = roots.sharing.clone(); + if sharing.tentative() { + if let Some(entries) = cg.entries.as_mut() { + std::sync::Arc::get_mut(entries).expect("a new cache reader").type_ids = false; + } + } cg.patching = patching; let mut compiled = Vec::new(); let mut skipped = Vec::new(); @@ -616,9 +753,12 @@ impl CodegenUnit { let told = cg.debug.is_some().then_some(&cg.told); symbols.in_parallel(ours.len(), &|db, symbols, at| { if symbols.function(ours[at]).is_none() { - _ = db.mir_body(ours[at].into()); + let source = sharing + .analysis_source(db, ours[at]) + .unwrap_or_else(|| sharing.function(db, ours[at])); + _ = db.mir_body(source.into()); } else if let (Ok(compiled), Some(told)) = - (compiled_root(db, ours[at]).as_ref(), told) + (program_root(db, ours[at], &sharing).as_ref(), told) { told.ahead(db, &mut FxHashMap::default(), compiled); } @@ -662,6 +802,24 @@ impl CodegenUnit { break; } } + if cg.sharing_restart { + sharing = sharing.disabled(); + sharing_restarts += 1; + start_over("a shared type reaches TypeId"); + continue; + } + if !sharing.is_empty() { + if let Some((_, rejection)) = failed.first() { + return Err(CodegenError::Unsupported(rejection.why.clone())); + } + if let Some((_, error)) = report_errors.first().or_else(|| { + skipped.iter().find(|(root, error)| { + !denied.contains(root) && !left.contains(root) && !error.no_symbol() + }) + }) { + return Err(error.clone()); + } + } if cg.misfit { start_over("an entry of the store did not fit"); rejected.extend(failed); @@ -700,7 +858,7 @@ impl CodegenUnit { if failed.is_empty() && (!from_rustc.is_empty() || !generic.is_empty()) { let mut theirs = Some(FxHashSet::default()); for root in from_rustc.iter().chain(&generic) { - let blocks = async_blocks(db, *root).clone(); + let blocks = async_blocks(db, *root, sharing.clone()).clone(); theirs = theirs.zip(blocks).map(|(mut all, blocks)| { all.extend(blocks); all @@ -708,7 +866,7 @@ impl CodegenUnit { } let before = left.len(); left.extend(compiled.iter().map(|it| it.0).filter(|&root| { - match (async_blocks(db, root).clone(), &theirs) { + match (async_blocks(db, root, sharing.clone()).clone(), &theirs) { (Some(ours), Some(theirs)) => !ours.is_disjoint(theirs), (Some(ours), None) => !ours.is_empty(), (None, _) => true, @@ -721,30 +879,18 @@ impl CodegenUnit { } if failed.is_empty() { let instances = cg.instances.len(); - if let Some(store) = store::Store::global() { - let mut misses: Vec<_> = cg.names.misses.take().into_iter().collect(); - misses.sort_unstable_by_key(|it| std::cmp::Reverse(it.1)); - let (mut kept, writer) = (std::mem::take(&mut cg.kept), cg.writer.take()); - // The object and the link do not wait for the store to have what was - // kept: whoever reads the store next does. - store.settle_with(move || { - if let Some(mut writer) = writer { - kept.written(writer.finish()); - } - store.flush(); - let (found, added) = store.take_counts(); - if found + added != 0 { - eprintln!("store: {found} functions found, {added} compiled and added"); - } - kept.report(); - if !misses.is_empty() { - eprintln!("without a stable name: {misses:?}"); - } - }); - } + let mut misses: Vec<_> = cg.names.misses.take().into_iter().collect(); + misses.sort_unstable_by_key(|it| std::cmp::Reverse(it.1)); + let (mut kept, mut writer) = (std::mem::take(&mut cg.kept), cg.writer.take()); + let tentative = std::mem::take(&mut cg.tentative); + let checks = std::mem::take(&mut cg.tentative_checks); let total = COMPILED.load(std::sync::atomic::Ordering::Relaxed); let compiled_now = total - std::mem::replace(counted, total); *stats = BuildStats { + sharing: SharingStats { + type_id_restarts: sharing_restarts, + ..cg.sharing_stats + }, bodies: cg.bodies, reused: cg.bodies.saturating_sub(compiled_now), compiled: compiled_now, @@ -754,7 +900,17 @@ impl CodegenUnit { declined, }; if patching { - *base = patch::Base::record(db, &cg, roots.entries, &compiled, &skipped); + *base = patch::Base::record( + db, + &cg, + roots.entries, + roots.reports, + &compiled, + &skipped, + ); + if base.is_none() && std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("patch discard: could not record base"); + } } if let Some(path) = std::env::var_os("RUSTD_LAYOUTS") { let lines = cg @@ -773,6 +929,36 @@ impl CodegenUnit { let start = stdx::time::Instant::now(); let (object, apart) = cg.finish()?; stats.phases.object = start.elapsed(); + if let Some(store) = store::Store::global() { + for entry in checks { + entry.publish(store, symbols, &mut kept); + } + for entry in tentative { + writer + .get_or_insert_with(|| { + cached::Writer::new( + store, + symbols.cargo_home(), + symbols.library_sources(), + ) + }) + .keep(entry); + } + store.settle_with(move || { + if let Some(mut writer) = writer { + kept.written(writer.finish()); + } + store.flush(); + let (found, added) = store.take_counts(); + if found + added != 0 { + eprintln!("store: {found} functions found, {added} compiled and added"); + } + kept.report(); + if !misses.is_empty() { + eprintln!("without a stable name: {misses:?}"); + } + }); + } backend::Tally::global().report(); (*theirs, *unknown) = (own_statics.theirs, own_statics.unknown); let patch = if patching { Some(patch::empty_object()?) } else { None }; @@ -794,6 +980,61 @@ impl CodegenUnit { } } +fn retry_sharing( + sharing: bool, + known: &mut Rejections, + mut compile: impl FnMut(bool, &mut Rejections) -> Result, +) -> Result { + if !sharing { + return compile(false, known); + } + let before = known.clone(); + match compile(true, known) { + Ok(unit) => Ok(unit), + Err(error) => { + eprintln!("cfg sharing fallback: demoting all program items to unshared: {error}"); + *known = before; + let result = compile(false, known); + known.stats.sharing.fallback_restarts += 1; + result + } + } +} + +#[cfg(test)] +mod sharing_retry_tests { + use super::*; + + #[test] + fn retries_only_sharing_errors_and_restores_state() { + for (sharing, fails, attempts, succeeds) in [ + (false, 0, 1, true), + (false, 1, 1, false), + (true, 0, 1, true), + (true, 1, 2, true), + (true, 2, 2, false), + ] { + let mut known = Rejections::default(); + known.stats.bodies = 11; + let mut calls = 0; + let result = retry_sharing(sharing, &mut known, |enabled, known| { + calls += 1; + assert_eq!(enabled, sharing && calls == 1); + assert_eq!(known.stats.bodies, 11); + known.stats.bodies = 99; + if calls <= fails { + Err(CodegenError::Unsupported("refusal".to_owned())) + } else { + Ok(()) + } + }); + assert_eq!(result.is_ok(), succeeds); + assert_eq!(calls, attempts); + assert_eq!(known.stats.sharing.fallback_restarts, usize::from(attempts == 2)); + } + } +} + /// Which statics we define ourselves rather than use rustc's. A function left to rustc uses /// rustc's copy of a static, so the ones it refers to stay rustc's, along with whatever refers /// to them. @@ -894,6 +1135,8 @@ pub struct Rejections { base: Option, frozen: patch::Frozen, stale: Vec, + reach: reach::Cache, + policy: sharing::Cache, /// How many functions had been compiled when the last build finished. counted: usize, pub stats: BuildStats, @@ -917,9 +1160,14 @@ impl Rejections { self.base.is_some() } + pub fn impls_changed(&self) -> bool { + self.base.as_ref().is_none_or(|base| base.impls_changed()) + } + /// Makes the next build a full one. A patch looks at everything the last build compiled, /// and what an edit took away can only be looked at where a panic can be caught. pub fn forget_base(&mut self) { + self.reach.clear(); self.base = None; } @@ -932,6 +1180,8 @@ impl Rejections { /// Notes that a file changed: one of `crates`, or with `None` one we can't place. Every /// change since the last build has to be noted before [`Rejections::stale`] looks. pub fn touch(&mut self, db: &dyn HirDatabase, crates: Option<&[Crate]>) { + self.reach.clear(); + self.policy.clear(); match crates { Some(crates) => { self.frozen.edit(crates.iter().flat_map(|it| it.transitive_rev_deps(db))) @@ -956,14 +1206,57 @@ impl Rejections { /// Notes that a file changed in the bodies of functions alone, none of which the rest of /// the program sees into: not `const` or `async`, not returning `impl Trait`, declaring no /// items, and not handed to a macro. - pub fn touch_bodies(&mut self, db: &dyn HirDatabase, file: base_db::FileId) { + pub fn touch_bodies( + &mut self, + db: &dyn HirDatabase, + file: base_db::FileId, + functions: Option<&[FunctionId]>, + relocated: &[FunctionId], + ) { + self.reach.touch(file); + self.policy.touch(file, functions); self.frozen.edit_bodies(file); if let Some(base) = &mut self.base { - base.touch_bodies(db, file); + base.touch_bodies(db, file, functions, relocated); } self.forget(); } + pub fn reached( + &mut self, + db: &dyn HirDatabase, + sharing: Sharing, + edited: FxHashSet, + tests: &[FunctionId], + always: &[FunctionId], + kept: &[StaticId], + program: &str, + ) -> (Vec, Option, Vec) { + self.reach.reached(db, sharing, edited, tests, always, kept, program) + } + + pub fn sharing(&mut self, db: &dyn HirDatabase, sharing: Sharing, program: &str) -> Sharing { + let sharing = self.policy.sharing(db, sharing, program); + if self.base.as_ref().is_some_and(|base| !base.shares_with(&sharing)) { + eprintln!("patch discard: cfg sharing decisions changed"); + self.forget_base(); + } + sharing + } + + pub fn sharing_before_edit( + &mut self, + db: &dyn HirDatabase, + file: base_db::FileId, + functions: &[FunctionId], + ) { + self.policy.before_edit(db, file, functions); + } + + pub fn needs_sharing_facts(&self, files: &[base_db::FileId]) -> bool { + self.policy.needs_before_edit(files) + } + /// A function the last build left to rustc that is not what rustc compiled, and why it was /// left. rustc's copies are those of the build the daemon started from: the first call, to /// be made after that build, notes what they are copies of. @@ -986,11 +1279,14 @@ impl Rejections { self.stale.iter().map(|it| step(&it.steps)).max().unwrap_or_default() }; StaleSteps { + policy: longest(|it| it.policy), + rejected: longest(|it| it.rejected), roots: longest(|it| it.roots), instances: longest(|it| it.instances), statics: longest(|it| it.statics), left: longest(|it| it.left), roots_asked: self.stale.iter().map(|it| it.steps.roots_asked).sum(), + rejected_asked: self.stale.iter().map(|it| it.steps.rejected_asked).sum(), instances_asked: self.stale.iter().map(|it| it.steps.instances_asked).sum(), slowest: longest(|it| it.slowest), } @@ -1000,6 +1296,7 @@ impl Rejections { /// Measurements of the last build. #[derive(Default, Clone, Copy)] pub struct BuildStats { + pub sharing: SharingStats, /// Compiled functions in the object. pub bodies: usize, /// How many of them were compiled before this build, and how many during it. @@ -1015,6 +1312,15 @@ pub struct BuildStats { pub declined: Option<&'static str>, } +#[derive(Default, Clone, Copy)] +pub struct SharingStats { + pub t_compiled: usize, + pub redirected: usize, + pub analysis_shared_compiled: usize, + pub type_id_restarts: usize, + pub fallback_restarts: usize, +} + /// Time spent in each step of the last round. #[derive(Default, Clone, Copy)] pub struct Phases { @@ -1085,6 +1391,12 @@ enum Callee<'db> { /// Stitches compiled functions into one object: gives every function and data object they /// ask for an id in the module, and defines the ones that are ours. struct ObjectCodegen<'a, 'db> { + sharing: Sharing, + counting_sharing: Sharing, + sharing_restart: bool, + sharing_stats: SharingStats, + tentative: Vec, + tentative_checks: Vec, db: &'db dyn HirDatabase, symbols: &'a dyn SymbolResolver, module: ObjectModule, @@ -1148,7 +1460,7 @@ struct ObjectCodegen<'a, 'db> { names: stable::StableNames<'a, 'db>, /// Whether the code of the checkout's crates can come from the store. hydrate: bool, - stable_names: FxHashMap>, + stable_names: StableNameCache, kept: Kept, /// The entries of the store to build from in place of compiling, if any. entries: Option>, @@ -1168,6 +1480,48 @@ struct ObjectCodegen<'a, 'db> { /// Why an entry with a file of its own under what stands for cargo's directory is not kept. const OF_CARGO: &str = "a file under what stands for cargo's directory"; +struct CacheCheck { + key: [u8; store::KEY], + entry: Box, + name: String, + check: std::ffi::OsString, +} + +impl CacheCheck { + fn publish( + mut self, + store: &'static store::Store, + symbols: &dyn SymbolResolver, + kept: &mut Kept, + ) { + let stored = store.get(&self.key); + let mut alone = + cached::Writer::alone(store, symbols.cargo_home(), symbols.library_sources()); + let Some(bytes) = alone.encoded(&mut self.entry) else { + let _: Option<()> = kept.unkept(OF_CARGO); + return; + }; + match stored { + None => { + store.insert(&self.key, &bytes); + kept.added += 1; + } + Some(stored) if stored == bytes => kept.same += 1, + Some(stored) => { + kept.differing += 1; + let name = self.name; + eprintln!("entry of `{name}` is not what was compiled"); + if self.check != "1" { + let at = std::path::Path::new(&self.check).join(kept.differing.to_string()); + let was = cached::Entry::decode(&stored); + let entry = self.entry; + _ = std::fs::write(at, format!("{name}\n{was:#?}\n{entry:#?}")); + } + } + } + } +} + /// What became of the instances an object was built from, as entries of the store. #[derive(Default)] struct Kept { @@ -1375,6 +1729,26 @@ fn instance_serial(db: &dyn HirDatabase, instance: Instance, env: StoredParamEnv compiled_instance(db, instance, env).as_ref().map_or(0, print_of) } +#[salsa::tracked(returns(copy))] +fn program_root_serial(db: &dyn HirDatabase, func: FunctionId, sharing: Sharing) -> u128 { + program_root(db, func, &sharing).as_ref().map_or(0, print_of) +} + +#[salsa::tracked(returns(copy))] +fn report_serial(db: &dyn HirDatabase, func: FunctionId) -> u128 { + compiled_report(db, func).as_ref().map_or(0, print_of) +} + +#[salsa::tracked(returns(copy))] +fn program_instance_serial( + db: &dyn HirDatabase, + instance: Instance, + env: StoredParamEnvAndCrate, + sharing: Sharing, +) -> u128 { + program_instance(db, instance, env, &sharing).as_ref().map_or(0, print_of) +} + #[salsa::tracked(returns(copy))] fn static_serial(db: &dyn HirDatabase, id: StaticId) -> u128 { compiled_static(db, id).as_ref().map_or(0, print_of) @@ -1507,7 +1881,38 @@ fn rows_of(db: &dyn HirDatabase, compiled: &CompiledFunction) -> Vec { /// The code of a non-generic function. #[salsa::tracked(returns(ref))] fn compiled_root<'db>(db: &'db dyn HirDatabase, func: FunctionId) -> Result { - let (body, env) = root_body(db, func)?; + compile_root(db, func, Sharing::default()) +} + +fn program_root<'db>( + db: &'db dyn HirDatabase, + func: FunctionId, + sharing: &Sharing, +) -> &'db Result { + let canonical = sharing.function(db, func); + if canonical != func || !sharing.test_crate(func.krate(db)) { + compiled_root(db, canonical) + } else { + shared_root(db, func, sharing.clone()) + } +} + +#[salsa::tracked(returns(ref))] +fn shared_root<'db>( + db: &'db dyn HirDatabase, + func: FunctionId, + sharing: Sharing, +) -> Result { + compile_root(db, func, sharing) +} + +fn compile_root<'db>( + db: &'db dyn HirDatabase, + func: FunctionId, + sharing: Sharing, +) -> Result { + let (body, env) = program_body(db, func, &sharing)?; + let body = &*body; if std::env::var("RUSTD_DUMP_MIR") .is_ok_and(|want| FunctionSignature::of(db, func).name.as_str() == want) { @@ -1517,7 +1922,7 @@ fn compiled_root<'db>(db: &'db dyn HirDatabase, func: FunctionId) -> Result( db: &'db dyn HirDatabase, instance: Instance, env: StoredParamEnvAndCrate, +) -> Result { + compile_instance(db, instance, env, Sharing::default()) +} + +fn program_instance<'db>( + db: &'db dyn HirDatabase, + instance: Instance, + env: StoredParamEnvAndCrate, + sharing: &Sharing, +) -> &'db Result { + let canonical = sharing.instance(db, &instance); + let env = sharing.env(db, &canonical, &env); + if sharing.needs_policy(db, &canonical) { + shared_instance(db, canonical, env, sharing.clone()) + } else { + compiled_instance(db, canonical, env) + } +} + +#[salsa::tracked(returns(ref))] +fn shared_instance<'db>( + db: &'db dyn HirDatabase, + instance: Instance, + env: StoredParamEnvAndCrate, + sharing: Sharing, +) -> Result { + compile_instance(db, instance, env, sharing) +} + +fn compile_instance<'db>( + db: &'db dyn HirDatabase, + instance: Instance, + env: StoredParamEnvAndCrate, + sharing: Sharing, ) -> Result { let stored_env = env.clone(); let env = env.as_ref(db); let ty = || instance.args.as_ref().type_at(0); - let body = match instance.def { - InstanceDef::DropGlue => { - return compile(db, env, None, false, instance.checked, |fx| { - let ty = ty(); - fx.drop_glue(ty)?; - if std::env::var_os("RUSTD_DUMP_CLIF").is_some() { - eprintln!("drop glue for {ty:?}\n{}", fx.builder.func.display()); - } - Ok(()) - }); - } - InstanceDef::CallOnceShim => { - let args = instance.args.as_ref(); - return compile(db, env, None, false, instance.checked, |fx| match args.len() { - 1 => fx.call_once_shim(ty()), - _ => fx.call_once_by_pointer(args), - }); - } - InstanceDef::VtableShim(func) => { - let args = instance.args.as_ref(); - return compile(db, env, None, false, instance.checked, |fx| { - fx.vtable_shim(func, args) - }); - } - InstanceDef::FnPointerShim => { - return compile(db, env, None, false, instance.checked, |fx| fx.fn_pointer_shim(ty())); - } - InstanceDef::ReifyShim { receiver } => { - return compile(db, env, None, false, instance.checked, |fx| { - fx.reify_shim(ty(), receiver) - }); - } - InstanceDef::Fn(_) - | InstanceDef::Closure(_) - | InstanceDef::Coroutine(_) - | InstanceDef::CoroutineByMove(_) => instance_body(db, &instance, stored_env)?, - }; + let body = + match instance.def { + InstanceDef::DropGlue => { + return compile_shared(db, env, sharing, None, false, instance.checked, |fx| { + let ty = ty(); + fx.drop_glue(ty)?; + if std::env::var_os("RUSTD_DUMP_CLIF").is_some() { + eprintln!("drop glue for {ty:?}\n{}", fx.builder.func.display()); + } + Ok(()) + }); + } + InstanceDef::CallOnceShim => { + let args = instance.args.as_ref(); + return compile_shared(db, env, sharing, None, false, instance.checked, |fx| { + match args.len() { + 1 => fx.call_once_shim(ty()), + _ => fx.call_once_by_pointer(args), + } + }); + } + InstanceDef::VtableShim(func) => { + let args = instance.args.as_ref(); + return compile_shared(db, env, sharing, None, false, instance.checked, |fx| { + fx.vtable_shim(func, args) + }); + } + InstanceDef::FnPointerShim => { + return compile_shared(db, env, sharing, None, false, instance.checked, |fx| { + fx.fn_pointer_shim(ty()) + }); + } + InstanceDef::ReifyShim { receiver } => { + return compile_shared(db, env, sharing, None, false, instance.checked, |fx| { + fx.reify_shim(ty(), receiver) + }); + } + InstanceDef::Fn(_) + | InstanceDef::Closure(_) + | InstanceDef::Coroutine(_) + | InstanceDef::CoroutineByMove(_) => { + program_instance_body(db, &instance, stored_env, &sharing)? + } + }; let body = &*body; let dump = std::env::var("RUSTD_DUMP_MIR").ok(); let (tracked, abi, variadic) = match instance.def { @@ -1645,7 +2091,7 @@ fn compiled_instance<'db>( } _ => None, }; - let compiled = compile(db, env, Some(body), tracked, instance.checked, |fx| { + let compiled = compile_shared(db, env, sharing, Some(body), tracked, instance.checked, |fx| { fx.abi = abi; fx.variadic = variadic; fx.args = Some(args); @@ -1911,6 +2357,40 @@ fn instance_body<'db>( body.map_err(|e| CodegenError::Frontend(format!("mir: {e:?}"))) } +fn program_instance_body<'db>( + db: &'db dyn HirDatabase, + instance: &Instance, + env: StoredParamEnvAndCrate, + sharing: &Sharing, +) -> Result>> { + if sharing.is_empty() { + return instance_body(db, instance, env); + } + let analysis = sharing.analysis_instance(db, instance.def); + let def = analysis.map_or(instance.def, |(_, def)| def); + let raw = match def { + InstanceDef::Fn(func) => db.mir_body(func.into()), + InstanceDef::Closure(id) => db.mir_body_for_closure(InternedClosureId::from_id(id)), + InstanceDef::Coroutine(id) => db.mir_body_for_coroutine(InternedCoroutineId::from_id(id)), + InstanceDef::CoroutineByMove(id) => { + db.mir_body_for_coroutine_by_move(InternedCoroutineId::from_id(id)) + } + _ => unsupported!("analysis-sharing a shim"), + } + .map_err(|e| CodegenError::Frontend(format!("mir: {e:?}")))?; + let tally = backend::Tally::global(); + let mapped = tally.step(Step::Remap, || match analysis { + Some((source, _)) => remap::body(db, sharing, source, raw), + None => remap::opaques(db, sharing, raw), + })?; + tally + .step(Step::Monomorphize, || { + remap::monomorphized(db, sharing, &mapped, &instance.args, &env) + }) + .map(Cow::Owned) + .map_err(|e| CodegenError::Frontend(format!("mir: {e:?}"))) +} + /// The types a body takes and returns, which is all its callers have to know about it. Asked /// for on its own so that they do not depend on the rest of the body. #[derive(Clone, PartialEq, Eq)] @@ -1926,8 +2406,9 @@ fn body_types<'db>( db: &'db dyn HirDatabase, instance: Instance, env: StoredParamEnvAndCrate, + sharing: Sharing, ) -> Result { - let body = &*instance_body(db, &instance, env)?; + let body = &*program_instance_body(db, &instance, env, &sharing)?; Ok(BodyTypes { params: body.param_locals.iter().map(|&local| body.locals[local].ty.clone()).collect(), returned: body.locals[returned_local(body)].ty.clone(), @@ -1939,7 +2420,7 @@ fn body_types<'db>( /// another type. rustc lays such a future out differently, so the function that makes one has /// to come from the compiler that compiles what holds and polls it. #[salsa::tracked] -fn returns_async_block<'db>(db: &'db dyn HirDatabase, func: FunctionId) -> bool { +fn returns_async_block<'db>(db: &'db dyn HirDatabase, func: FunctionId, sharing: Sharing) -> bool { struct Finder { opaque: bool, coroutine: bool, @@ -1963,8 +2444,8 @@ fn returns_async_block<'db>(db: &'db dyn HirDatabase, func: FunctionId) -> bool if !finder.opaque && !FunctionSignature::of(db, func).is_async() { return false; } - let Ok((body, _)) = root_body(db, func) else { return false }; - let returned = body.locals[returned_local(body)].ty.as_ref(); + let Ok((body, _)) = program_body(db, func, &sharing) else { return false }; + let returned = body.locals[returned_local(&body)].ty.as_ref(); rustc_type_ir::TypeVisitable::visit_with(&returned, &mut finder); finder.coroutine } @@ -1986,10 +2467,11 @@ fn fn_trait_methods(lang_items: &LangItems) -> [[Option; 3]; 2] { fn coroutine_instance<'db>( db: &'db dyn HirDatabase, ty: Ty<'db>, + sharing: &Sharing, ) -> Option<(InstanceDef, GenericArgs<'db>)> { let TyKind::Coroutine(id, args) = ty.kind() else { return None }; let args = args.as_coroutine(); - let def = match is_by_move_coroutine(db, id.0, args) { + let def = match sharing.coroutine_is_by_move(db, id.0, args) { true => InstanceDef::CoroutineByMove(id.0.as_id()), false => InstanceDef::Coroutine(id.0.as_id()), }; @@ -2000,7 +2482,11 @@ fn coroutine_instance<'db>( /// can't be read. rustc lays a future out differently, so every function that touches one /// has to come from the same compiler. #[salsa::tracked(returns(ref))] -fn async_blocks<'db>(db: &'db dyn HirDatabase, func: FunctionId) -> Option> { +fn async_blocks<'db>( + db: &'db dyn HirDatabase, + func: FunctionId, + sharing: Sharing, +) -> Option> { struct Collector<'db>(Vec>); impl<'db> rustc_type_ir::TypeVisitor> for Collector<'db> { type Result = (); @@ -2014,7 +2500,7 @@ fn async_blocks<'db>(db: &'db dyn HirDatabase, func: FunctionId) -> Option(db: &'db dyn HirDatabase, func: FunctionId) -> Option db.monomorphized_mir_body_for_coroutine_by_move( - id.0, - parent, - env.store(), - ), - false => db.monomorphized_mir_body_for_coroutine(id.0, parent, env.store()), - } + let def = if sharing.coroutine_is_by_move(db, id.0, args) { + InstanceDef::CoroutineByMove(id.0.as_id()) + } else { + InstanceDef::Coroutine(id.0.as_id()) + }; + program_instance_body( + db, + &Instance { def, args: parent, checked: false }, + env.store(), + &sharing, + ) } TyKind::CoroutineClosure(id, args) => { let parent = GenericArgs::new_from_slice(args.as_coroutine_closure().parent_args()); let closure = id.0.as_closure(db); - db.monomorphized_mir_body_for_closure(closure, parent.store(), env.store()) + program_instance_body( + db, + &Instance { + def: InstanceDef::Closure(closure.as_id()), + args: parent.store(), + checked: false, + }, + env.store(), + &sharing, + ) } TyKind::Closure(id, args) => { let parent = GenericArgs::new_from_slice(args.as_closure().parent_args()); - db.monomorphized_mir_body_for_closure(id.0, parent.store(), env.store()) + program_instance_body( + db, + &Instance { + def: InstanceDef::Closure(id.0.as_id()), + args: parent.store(), + checked: false, + }, + env.store(), + &sharing, + ) } _ => continue, }; @@ -2234,12 +2741,15 @@ impl Ready<'_> { struct ToldTypes( #[allow(clippy::type_complexity)] std::sync::Arc< - std::sync::Mutex>>, + std::sync::Mutex< + FxHashMap<(StoredParamEnvAndCrate, StoredTy, Sharing), Option>, + >, >, + std::sync::Arc, ); /// The types a thread tells of itself, by the environment they are in. -type OwnTypes<'db> = FxHashMap>; +type OwnTypes<'db> = FxHashMap<(StoredParamEnvAndCrate, Sharing), debuginfo::Types<'db>>; impl ToldTypes { fn told<'db>( @@ -2248,13 +2758,15 @@ impl ToldTypes { own: &mut OwnTypes<'db>, env: &StoredParamEnvAndCrate, ty: &StoredTy, + sharing: &Sharing, ) -> Option { - let key = (env.clone(), ty.clone()); + let key = (env.clone(), ty.clone(), sharing.clone()); if let Some(told) = self.0.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { return told.clone(); } - let types = - own.entry(env.clone()).or_insert_with(|| debuginfo::Types::new(db, env.as_ref(db))); + let types = own.entry((env.clone(), sharing.clone())).or_insert_with(|| { + debuginfo::Types::new(db, env.as_ref(db), self.1.clone(), sharing.clone()) + }); let told = types.describe(ty.as_ref()); // Another thread may have told of it meanwhile, and the functions share one set. self.0.lock().unwrap_or_else(|e| e.into_inner()).entry(key).or_insert(told).clone() @@ -2268,7 +2780,7 @@ impl ToldTypes { compiled: &CompiledFunction, ) { if let (Some(_), Some((env, variables))) = (&compiled.source, &compiled.variables) { - variables.iter().for_each(|it| _ = self.told(db, own, env, &it.ty)); + variables.iter().for_each(|it| _ = self.told(db, own, env, &it.ty, &compiled.sharing)); } } } @@ -2279,7 +2791,7 @@ impl ToldTypes { fn describe_variables<'db>( db: &'db dyn HirDatabase, owner: DefWithBodyId, - (told, own, env): (&ToldTypes, &mut OwnTypes<'db>, &StoredParamEnvAndCrate), + (told, own, env, sharing): (&ToldTypes, &mut OwnTypes<'db>, &StoredParamEnvAndCrate, &Sharing), variables: &[Variable], places: &SlotPlaces, scopes: &FxHashMap, @@ -2303,7 +2815,9 @@ fn describe_variables<'db>( let ty = match firsts.get(&ty) { Some(&first) => first, None => { - let Some(described) = told.told(db, own, env, &variable.ty) else { continue }; + let Some(described) = told.told(db, own, env, &variable.ty, sharing) else { + continue; + }; let first = debug.types.len() as u32; debug.types.push(described); firsts.insert(ty, first); @@ -2882,9 +3396,22 @@ fn compile<'db>( tracked: bool, checked: bool, fill: impl for<'cx> FnOnce(&mut FunctionCx<'cx, 'db>) -> Result<()>, +) -> Result { + compile_shared(db, env, Sharing::default(), body, tracked, checked, fill) +} + +fn compile_shared<'db>( + db: &'db dyn HirDatabase, + env: ParamEnvAndCrate<'db>, + sharing: Sharing, + body: Option<&'db MirBody<'db>>, + tracked: bool, + checked: bool, + fill: impl for<'cx> FnOnce(&mut FunctionCx<'cx, 'db>) -> Result<()>, ) -> Result { let isa = &**target()?; let mut cg = Recorder::new(db, isa, selected_target()?, checked); + cg.sharing = sharing; let tally = backend::Tally::global(); /// `RUSTD_PROMOTE=0` leaves every local in its slot of the frame. static PROMOTE: LazyLock = @@ -2941,15 +3468,16 @@ fn compile<'db>( .map(|Named { binding, param, ty, slot, pointer }| Variable { binding, param, - ty, + ty: cg.sharing.fold(db, env.krate, ty.as_ref()).store(), slot: slot.as_u32(), pointer, }) .collect::>(); let variables = Some((env.store(), variables)).filter(|it| !it.1.is_empty()); + let sharing = cg.sharing.clone(); let (requests, layouts) = cg.finish(); COMPILED.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - Ok(CompiledFunction { requests, code, source, closure: None, variables, layouts }) + Ok(CompiledFunction { sharing, requests, code, source, closure: None, variables, layouts }) } fn build<'db>( @@ -3036,6 +3564,7 @@ fn inlinable( db: &dyn HirDatabase, instance: Instance, env: StoredParamEnvAndCrate, + sharing: Sharing, ) -> Option { /// The most blocks and statements the body of a function that is inlined has, and /// the most instructions it is built to with what it calls inlined into it. @@ -3043,7 +3572,8 @@ fn inlinable( const STATEMENTS: usize = 12; const INSTRUCTIONS: usize = 64; let InstanceDef::Fn(func) = instance.def else { return None }; - let generic = db.mir_body(func.into()).ok()?; + let source = sharing.analysis_source(db, func).unwrap_or(func); + let generic = db.mir_body(source.into()).ok()?; let statements: usize = generic.basic_blocks.iter().map(|it| it.1.statements.len()).sum(); if generic.basic_blocks.len() > BLOCKS || statements > STATEMENTS @@ -3052,8 +3582,9 @@ fn inlinable( { return None; } - let body = instance_body(db, &instance, env.clone()).ok()?; + let body = program_instance_body(db, &instance, env.clone(), &sharing).ok()?; let mut cg = Recorder::new(db, &**target().ok()?, selected_target().ok()?, instance.checked); + cg.sharing = sharing; cg.unlocated = true; let args = instance.args.clone(); let mut ctx = build(&mut cg, env.as_ref(db), Some(&*body), false, |fx| { @@ -3076,6 +3607,7 @@ fn inlinable_cycle( _: salsa::Id, _: Instance, _: StoredParamEnvAndCrate, + _: Sharing, ) -> Option { None } @@ -3240,6 +3772,41 @@ fn emit_wasm(func: &cranelift_codegen::ir::Function, calls: Calls) -> Result( + db: &'db dyn HirDatabase, + func: FunctionId, + sharing: &Sharing, +) -> Result<(Cow<'db, MirBody<'db>>, ParamEnvAndCrate<'db>)> { + if let Some(source) = sharing.analysis_source(db, func) { + let raw = db + .mir_body(source.into()) + .map_err(|e| CodegenError::Frontend(format!("mir: {e:?}")))?; + let mapped = remap::body(db, sharing, source, raw)?; + let env = ParamEnvAndCrate { + param_env: crate::next_solver::ParamEnv::empty(), + krate: func.krate(db), + }; + let args = GenericArgs::new_from_slice(&[]).store(); + let body = remap::monomorphized(db, sharing, &mapped, &args, &env.store()) + .map_err(|e| CodegenError::Frontend(format!("mir: {e:?}")))?; + Ok((Cow::Owned(body), env)) + } else if !sharing.is_empty() { + let raw = + db.mir_body(func.into()).map_err(|e| CodegenError::Frontend(format!("mir: {e:?}")))?; + let mapped = remap::opaques(db, sharing, raw)?; + let env = ParamEnvAndCrate { + param_env: crate::next_solver::ParamEnv::empty(), + krate: func.krate(db), + }; + let args = GenericArgs::new_from_slice(&[]).store(); + let body = remap::monomorphized(db, sharing, &mapped, &args, &env.store()) + .map_err(|e| CodegenError::Frontend(format!("mir: {e:?}")))?; + Ok((Cow::Owned(body), env)) + } else { + root_body(db, func).map(|(body, env)| (Cow::Borrowed(body), env)) + } +} + fn root_body<'db>( db: &'db dyn HirDatabase, func: FunctionId, @@ -3299,6 +3866,12 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { store.settled(); } Ok(ObjectCodegen { + sharing: Sharing::default(), + counting_sharing: Sharing::default(), + sharing_restart: false, + sharing_stats: SharingStats::default(), + tentative: Vec::new(), + tentative_checks: Vec::new(), db, symbols, module, @@ -3343,22 +3916,19 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { source_paths: FxHashMap::default(), builder_ctx: FunctionBuilderContext::new(), location_counter: 0, - names: stable::StableNames::new(db, symbols, own_statics.crates.clone(), hydrate), + names: stable::StableNames::new(db, symbols, own_statics.crates.clone(), true), hydrate, - stable_names: FxHashMap::default(), + stable_names: StableNameCache::default(), kept: Kept::default(), // A patch gets these from its base, and a check compiles them to compare. entries: store .filter(|_| base.is_none() && std::env::var_os("RUSTD_CACHE_CHECK").is_none()) .map(|store| { let (home, library) = (symbols.cargo_home(), symbols.library_sources()); - std::sync::Arc::new(cached::Entries::new( - store, - debugged, - program.flatten(), - home, - library, - )) + let mut entries = + cached::Entries::new(store, debugged, program.flatten(), home, library); + entries.checkout = hydrate; + std::sync::Arc::new(entries) }), named: FxHashMap::default(), found: Vec::new(), @@ -3430,7 +4000,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { /// itself is left to rustc. `false` when the body can't be compiled to find them. fn want_reachable_from(&mut self, func: FunctionId) -> bool { let db = self.db; - let Ok(compiled) = compiled_root(db, func).as_ref() else { return false }; + let Ok(compiled) = program_root(db, func, &self.sharing).as_ref() else { return false }; self.want_requested(&compiled.requests); true } @@ -3448,7 +4018,8 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { InstanceDef::Fn(func) if instance.args.as_ref().is_empty() => self.want(func), _ if !seen.insert(instance) => {} _ => { - let compiled = compiled_instance(db, instance.clone(), env.clone()); + let compiled = + program_instance(db, instance.clone(), env.clone(), &self.sharing); stack.extend(compiled.as_ref().ok().map(|it| &it.requests)); } } @@ -3467,7 +4038,6 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { /// Puts a non-generic function in the object, exported under its rustc symbol name. fn define_function(&mut self, func: FunctionId) -> Result { let db = self.db; - db.mir_body(func.into()).map_err(|e| CodegenError::Frontend(format!("mir: {e:?}")))?; let Some(symbol) = self.symbols.function(func) else { // FIXME: with a `main` of ours only what it reaches is compiled, and a function // left to rustc can still call one private to its object. @@ -3475,7 +4045,8 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { return Err(if private { CodegenError::Private } else { CodegenError::NoSymbol }); }; let start = stdx::time::Instant::now(); - let compiled = compiled_root(db, func); + let compiled = program_root(db, func, &self.sharing); + self.count_sharing(func, self.sharing.function(db, func) != func); let took = start.elapsed(); self.phases.compile += took; let compiled = compiled.as_ref().map_err(Clone::clone)?; @@ -3486,11 +4057,36 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { let name = super::type_name::rustc_debug_function(db, func, &[]) .unwrap_or_else(|| FunctionSignature::of(db, func).name.as_str().to_owned()); let declared = self.declaration_at(func); - let rows = self.debug.as_ref().map(|_| root_rows(db, func)); - self.describe(id, compiled, rows, FunctionDebug::new(name, symbol.clone()), declared); + let rows = self.debug.as_ref().map(|_| rows_of(db, compiled)); + self.describe( + id, + compiled, + rows.as_ref(), + FunctionDebug::new(name, symbol.clone()), + declared, + ); Ok(symbol) } + fn count_sharing(&mut self, func: FunctionId, redirected: bool) { + let db = self.db; + if self.counting_sharing.test_crate(func.krate(db)) { + self.sharing_stats.redirected += usize::from(redirected); + self.sharing_stats.t_compiled += usize::from(!redirected); + let analysis = !redirected && self.sharing.analysis_source(db, func).is_some(); + self.sharing_stats.analysis_shared_compiled += usize::from(analysis); + if std::env::var("RUSTD_CFG_TWINS").is_ok_and(|value| value == "items") { + eprintln!( + "cfg codegen body: {}: T compiled {}, redirected {}, analysis-shared {}", + FunctionSignature::of(db, func).name.as_str(), + usize::from(!redirected), + usize::from(redirected), + usize::from(analysis) + ); + } + } + } + /// Puts the `main` the C runtime calls in the object. fn define_start(&mut self, main: FunctionId) -> Result<()> { let compiled = compiled_start(self.db, main).as_ref().map_err(Clone::clone)?; @@ -3536,10 +4132,11 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { let told = self.debug.is_some().then_some(&self.told); self.symbols.in_parallel(wave.len(), &|db, _, at| { let (instance, env, _) = &wave[at]; - let compiled = compiled_instance(db, instance.clone(), env.clone()); + let compiled = + program_instance(db, instance.clone(), env.clone(), &self.sharing); if let (Ok(compiled), Some(told)) = (compiled.as_ref(), told) { told.ahead(db, &mut FxHashMap::default(), compiled); - instance_rows(db, instance.clone(), env.clone()); + rows_of(db, compiled); } }); self.phases.compile += start.elapsed(); @@ -3616,6 +4213,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { let (rejected, hydrate) = (self.rejected, self.hydrate); let threads = std::thread::available_parallelism().map_or(1, |it| it.get()); let (entries, owned) = (self.entries.clone(), &self.own_statics.crates.clone()); + let sharing = self.sharing.clone(); let entries = entries.as_deref(); let told = self.debug.is_some().then(|| self.told.clone()); let told = told.as_ref(); @@ -3656,18 +4254,16 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { let (mut found, mut called) = (Vec::new(), Vec::new()); let compiled = match &job { // One without a symbol is left to rustc, and so is what only it calls. - either::Either::Left(root) => symbols.function(*root).and_then(|_| { - if told.is_some() { - root_rows(db, *root); - } - compiled_root(db, *root).as_ref().ok() - }), + either::Either::Left(root) => symbols + .function(*root) + .and_then(|_| program_root(db, *root, &sharing).as_ref().ok()), either::Either::Right((instance, env)) => { - let compiled = compiled_instance(db, instance.clone(), env.clone()); + let compiled = + program_instance(db, instance.clone(), env.clone(), &sharing); let compiled = compiled.as_ref().ok(); if let (Some(compiled), Some(told)) = (compiled, told) { told.ahead(db, &mut types, compiled); - instance_rows(db, instance.clone(), env.clone()); + rows_of(db, compiled); } compiled } @@ -3728,7 +4324,10 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { id: FuncId, ) -> Result<()> { let start = stdx::time::Instant::now(); - let compiled = compiled_instance(self.db, instance.clone(), env.clone()); + let compiled = program_instance(self.db, instance.clone(), env.clone(), &self.sharing); + if let Some(owner) = home::written_in(self.db, instance) { + self.count_sharing(owner, self.sharing.instance(self.db, instance).def != instance.def); + } let took = start.elapsed(); self.phases.compile += took; let compiled = compiled.as_ref().map_err(Clone::clone)?; @@ -3769,9 +4368,9 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { _ => None, }; let known = known.unwrap_or_else(|| name.clone()); - let rows = - self.debug.as_ref().map(|_| instance_rows(self.db, instance.clone(), env.clone())); - let debug = self.describe(id, compiled, rows, FunctionDebug::new(known, name), declared); + let rows = self.debug.as_ref().map(|_| rows_of(self.db, compiled)); + let debug = + self.describe(id, compiled, rows.as_ref(), FunctionDebug::new(known, name), declared); let tally = backend::Tally::global(); tally.step(Step::StoreWrite, || self.keep(instance, compiled, &ids, debug)); Ok(()) @@ -3868,7 +4467,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { }; rows.sort_by_key(|row| row.0); let scopes = describe_scopes(db, *owner, variables, &rows, len, &mut debug); - let types = (&self.told, &mut self.types, env); + let types = (&self.told, &mut self.types, env, &compiled.sharing); describe_variables(db, *owner, types, variables, places, &scopes, &mut debug); } let len = compiled.code.bytes.len() as u32; @@ -3878,10 +4477,18 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { } fn stable_name(&mut self, instance: &Instance) -> Option { + let canonical = self.sharing.instance(self.db, instance); + let instance = &canonical; if let Some(known) = self.stable_names.get(instance) { return known.clone(); } - let name = self.names.instance(instance).map(StableName::from); + let name = self.names.instance(instance).map(|name| { + if let Some((known, _)) = self.named.get_key_value(name.as_str()) { + known.clone() + } else { + StableName::from(name) + } + }); self.stable_names.insert(instance.clone(), name.clone()); name } @@ -3909,39 +4516,21 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { let sets = || debug.iter().flat_map(|it| &it.types).cloned().collect::>(); if check.is_none() && store.has(&key) { let kept = cached::Kept::Types(sets()); - return self.writer(store).keep(kept); + return self.keep_tentative(store, kept); } let Some(entry) = self.entry(instance, compiled, ids, debug) else { self.kept.unnamed_requests += 1; return; }; - let mut entry = Box::new(entry); + let entry = Box::new(entry); let Some(check) = check else { - return self.writer(store).keep(cached::Kept::Entry { key, entry }); + return self.keep_tentative(store, cached::Kept::Entry { key, entry }); }; - // The entry as it would be written is set against the one that is there. - let stored = store.get(&key); - let mut alone = - cached::Writer::alone(store, self.symbols.cargo_home(), self.symbols.library_sources()); - let Some(bytes) = alone.encoded(&mut entry) else { - let _: Option<()> = self.kept.unkept(OF_CARGO); - return; - }; - match stored { - None => { - store.insert(&key, &bytes); - self.kept.added += 1; - } - Some(stored) if stored == bytes => self.kept.same += 1, - Some(stored) => { - self.kept.differing += 1; - eprintln!("entry of `{name}` is not what was compiled"); - if check != "1" { - let at = std::path::Path::new(&check).join(self.kept.differing.to_string()); - let was = cached::Entry::decode(&stored); - _ = std::fs::write(at, format!("{name}\n{was:#?}\n{entry:#?}")); - } - } + let entry = CacheCheck { key, entry, name: name.to_string(), check }; + if self.sharing.tentative() { + self.tentative_checks.push(entry); + } else { + entry.publish(store, self.symbols, &mut self.kept); } } @@ -3952,6 +4541,14 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { }) } + fn keep_tentative(&mut self, store: &'static store::Store, kept: cached::Kept) { + if self.sharing.tentative() { + self.tentative.push(kept); + } else { + self.writer(store).keep(kept); + } + } + /// Whether the instance is compiled with the flags of the crate its function is written /// in, as the copy under the symbol of the function is. A crate that uses an `#[inline]` /// function of another compiles a copy with its own. @@ -4065,6 +4662,13 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { for (request, &id) in compiled.requests.funcs.iter().zip(&ids.funcs) { let decl = self.module.declarations().get_function_decl(id); let target = if let Some(&hash) = self.type_ids.get(&id) { + if let FuncTarget::Instance(callee, _) = &request.target + && let InstanceDef::Fn(func) = callee.def + && let Some(of) = self.type_id_argument(func, callee.args.as_ref()) + && self.sharing.link_type(self.db, of) != of + { + return self.kept.unkept("program-specific TypeId"); + } CachedTarget::TypeId(hash) } else if decl.linkage == Linkage::Import { // rustc's copy of a function of the checkout is of the checkout as rustc @@ -4194,7 +4798,12 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { /// The function of an entry and everything it calls, declared here and to be defined from /// the store. `None` when the store does not have all of it, or `body` is given and is /// not the signature the entry was compiled with. - fn found_callee(&mut self, name: &str, body: Option<&Signature>) -> Result> { + fn found_callee( + &mut self, + name: &str, + body: Option<&Signature>, + shared: Option<&StableName>, + ) -> Result> { if let Some(&id) = self.named.get(name) { return Ok(Some(id)); } @@ -4220,7 +4829,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { .module .declare_function(&symbol, linkage, entry.code.signature()) .map_err(|e| CodegenError::Backend(e.to_string()))?; - let name = StableName::from(name); + let name = shared.cloned().unwrap_or_else(|| StableName::from(name)); self.named.insert(name.clone(), id); self.found.push((name.clone(), id)); self.found_pending.push((name, id)); @@ -4250,7 +4859,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { self.import(&symbol, &func.sig)? } CachedTarget::Named(callee) => { - let Some(id) = self.found_callee(callee, None)? else { + let Some(id) = self.found_callee(callee, None, None)? else { unsupported!("an entry calls `{callee}`, which the store lacks"); }; id @@ -4469,7 +5078,10 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { }; self.import(&symbol, &request.sig) } - FuncTarget::Instance(instance, env) => self.callee(instance, env, &request.sig), + FuncTarget::Instance(instance, env) => { + let instance = self.sharing.link_instance(self.db, instance); + self.callee(&instance, env, &request.sig) + } FuncTarget::Asm(text) => self.assembly(text, &request.sig), } } @@ -4503,7 +5115,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { self.vtables.insert(key, id); Ok(id) } - DataRequest::Static(id) => self.static_data(*id), + DataRequest::Static(id) => self.static_data(self.sharing.link_static(self.db, *id)), DataRequest::Location(at) => self.location(*at), DataRequest::Declaration(func) => { let found = self.declaration_at(*func); @@ -4726,6 +5338,13 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { ) -> Result { let instance = &self.settled(instance)?; let (def, args) = (instance.def, instance.args.as_ref()); + if let InstanceDef::Fn(func) = def + && let Some(of) = self.type_id_argument(func, args) + && self.sharing.contains_unified(self.db, of) + { + self.sharing_restart = true; + unsupported!("a shared type reaches TypeId"); + } if let InstanceDef::Fn(func) = def && args.iter().all(|arg| arg.region().is_some()) && let Some(symbol) = self.symbols.function(func) @@ -4733,7 +5352,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { && let Some(symbol) = self.rustc_copy(symbol, instance.checked) // rustc's copy of a function of another crate would hand out rustc's future. && (self.own_statics.crates.contains(&func.krate(self.db)) - || !returns_async_block(self.db, func)) + || !returns_async_block(self.db, func, self.sharing.clone())) { self.want(func); return self.import(&symbol, sig); @@ -4784,10 +5403,15 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { // A `TypeId` is a hash only rustc can compute, and the two compilers have to agree on // it for `downcast` to work between their code. So take rustc's. if let InstanceDef::Fn(func) = def - && let Some(of) = self.type_id_of(func, args) + && let Some(of) = self.type_id_argument(func, args) { - let mut hasher = - type_id::TypeIdHasher::new(self.db, self.symbols, &self.own_statics.crates); + let of = self.sharing.link_type(self.db, of); + let mut hasher = type_id::TypeIdHasher::new( + self.db, + self.symbols, + &self.own_statics.crates, + self.sharing.clone(), + ); if let Some(hash) = hasher.type_id(of) { return self.constant_type_id(instance.clone(), sig, hash); } @@ -4863,7 +5487,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { return self.import(symbol, sig); } let body = signature.is_some().then_some(sig); - if let Some(id) = self.found_callee(name, body)? { + if let Some(id) = self.found_callee(name, body, Some(name))? { self.instances.insert(instance.clone(), id); return Ok(id); } @@ -4957,7 +5581,8 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { } let mut callers: FxHashMap<&Instance, Vec> = FxHashMap::default(); for (at, (instance, env)) in self.envs.iter().enumerate() { - let Ok(compiled) = compiled_instance(self.db, instance.clone(), env.clone()).as_ref() + let Ok(compiled) = + program_instance(self.db, instance.clone(), env.clone(), &self.sharing).as_ref() else { continue; }; @@ -4990,7 +5615,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { fn ours_alone(&self, id: StaticId) -> bool { self.symbols.private_static_owner(id).is_some_and(|func| { !self.own_statics.crates.contains(&func.krate(self.db)) - && *returns_async_block(self.db, func) + && *returns_async_block(self.db, func, self.sharing.clone()) }) } @@ -4999,7 +5624,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { let InstanceDef::Fn(func) = instance.def else { return None }; // rustc's copy of a function of another crate would hand out rustc's future. if !self.own_statics.crates.contains(&func.krate(self.db)) - && *returns_async_block(self.db, func) + && *returns_async_block(self.db, func, self.sharing.clone()) { return None; } @@ -5113,10 +5738,12 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { /// The `TypeId::of` that returns `hash`. fn type_id_shim(&mut self, sig: &Signature, hash: u128) -> Result { - if let Some(&id) = self.type_id_shims.get(&hash) { + if let Some(&id) = self.type_id_shims.get(&hash) + && self.module.declarations().get_function_decl(id).signature == *sig + { return Ok(id); } - let name = format!("rustd::type_id::h{}", self.type_id_shims.len()); + let name = format!("rustd::type_id::h{}", self.type_ids.len()); let id = self .module .declare_function(&name, Linkage::Local, sig) @@ -5131,7 +5758,7 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { let words = [hash as u64, (hash >> 64) as u64].map(|it| it as i64); let returned = match *sig.returns.as_slice() { [] => { - let &[out] = builder.block_params(block) else { + let Some(&out) = builder.block_params(block).first() else { unsupported!("a `TypeId` returned as `{sig}`"); }; for (word, offset) in words.into_iter().zip([0, 8]) { @@ -5159,6 +5786,25 @@ impl<'a, 'db> ObjectCodegen<'a, 'db> { Ok(id) } + fn type_id_argument(&self, func: FunctionId, args: GenericArgs<'db>) -> Option> { + if let Some(ty) = self.type_id_of(func, args) { + return Some(ty); + } + if FunctionSignature::of(self.db, func).name.as_str() != "type_id" { + return None; + } + let ItemContainerId::ImplId(id) = func.lookup(self.db).container else { + return (self.symbols.item(func.into()).as_deref() + == Some("core::intrinsics::type_id")) + .then(|| args.as_slice().first().and_then(|arg| arg.ty())) + .flatten(); + }; + let trait_ref = self.db.impl_trait(id)?.skip_binder(); + (self.symbols.item(trait_ref.def_id.0.into()).as_deref() == Some("core::any::Any")) + .then(|| args.as_slice().first().and_then(|arg| arg.ty())) + .flatten() + } + /// The `T` of `TypeId::of::`, when `func` is that function. fn type_id_of(&self, func: FunctionId, args: GenericArgs<'db>) -> Option> { let db = self.db; @@ -6014,13 +6660,11 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { } fn layout(&self, ty: Ty<'db>) -> Result> { + let env = self.env; if let Some(layout) = self.layouts.borrow().get(&ty) { return Ok(layout.clone()); } - let layout = self - .cg - .db - .layout_of_ty(ty.store(), self.env.store()) + let layout = remap::layout(self.cg.db, ty.store(), env.store(), self.cg.sharing.clone()) .map_err(|e| CodegenError::Frontend(format!("layout of {ty:?}: {e:?}")))?; if let Some(log) = &self.cg.layouts { log.borrow_mut().entry(ty).or_insert((layout.size.bytes(), layout.align.abi.bytes())); @@ -6939,14 +7583,30 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { if layout.size.bytes() == 0 || self.mentions_location(body, expr) { return None; } - let constant = monomorphized_promoted_mir_body_query( - db, - body.owner, - expr, - self.args.clone()?, - self.env.store(), - ) - .ok()?; + let constant = if let InferBodyId::DefWithBodyId(DefWithBodyId::FunctionId(source)) = + body.owner + && self.cg.sharing.analysis_item(db, source, source.into()) != source.into() + { + remap::promoted( + db, + source, + expr, + self.args.clone()?, + self.env.store(), + self.cg.sharing.clone(), + ) + .as_ref() + .ok()? + } else { + monomorphized_promoted_mir_body_query( + db, + body.owner, + expr, + self.args.clone()?, + self.env.store(), + ) + .ok()? + }; let allocation = interpret_mir(db, constant, false, Some(self.env)).ok()?.0.ok()?; let ty = allocation.ty; let blob = self.allocation_blob(allocation, false).ok()?; @@ -7090,7 +7750,9 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { fn body_types(&self, def: InstanceDef, args: GenericArgs<'db>) -> Result<&'db BodyTypes> { // The types of a body are the same however it is compiled. let instance = Instance { def, args: args.store(), checked: false }; - body_types(self.cg.db, instance, self.env.store()).as_ref().map_err(Clone::clone) + body_types(self.cg.db, instance, self.env.store(), self.cg.sharing.clone()) + .as_ref() + .map_err(Clone::clone) } /// The signature the body of an instance is compiled with. @@ -9736,7 +10398,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { if is_dyn_method(self.interner, self.env.param_env, func, generic_args).is_some() { return self.reified(fn_ty, false); } - let (callee, callee_args) = db.lookup_impl_method(self.env, func, generic_args); + let (callee, callee_args) = self.lookup_impl_method(func, generic_args); let either::Either::Left(callee) = callee else { unsupported!("fn pointer to a builtin derive method"); }; @@ -10784,7 +11446,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { TyKind::CoroutineClosure(id, generic_args) => { let closure = generic_args.as_coroutine_closure(); if let TyKind::Coroutine(future, future_args) = dst_ty.kind() - && is_by_move_coroutine( + && self.cg.sharing.coroutine_is_by_move( self.cg.db, future.0, future_args.as_coroutine(), @@ -10899,8 +11561,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { fn implements_fn_itself(&self, method: FunctionId, ty: Ty<'db>, tuple_ty: Ty<'db>) -> bool { let db = self.cg.db; let args = GenericArgs::new_from_slice(&[ty.into(), tuple_ty.into()]); - let (either::Either::Left(callee), _) = db.lookup_impl_method(self.env, method, args) - else { + let (either::Either::Left(callee), _) = self.lookup_impl_method(method, args) else { return false; }; let ItemContainerId::ImplId(impl_) = callee.lookup(db).container else { @@ -11208,7 +11869,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { return self.cg.callee(InstanceDef::CallOnceShim, trait_ref.args, self.env, &sig); } if Some(func) == lang_items.FuturePoll - && let Some((def, parent_args)) = coroutine_instance(db, self_ty) + && let Some((def, parent_args)) = coroutine_instance(db, self_ty, &self.cg.sharing) { let sig = self.body_signature(self.body_types(def, parent_args)?)?; return self.cg.callee(def, parent_args, self.env, &sig); @@ -11216,7 +11877,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { let Some(args) = self.vtable_method_args(func, trait_ref) else { unsupported!("generic method in a vtable"); }; - let (callee, callee_args) = db.lookup_impl_method(self.env, func, args); + let (callee, callee_args) = self.lookup_impl_method(func, args); let either::Either::Left(callee) = callee else { unsupported!("builtin derive method in a vtable"); }; @@ -11362,7 +12023,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { unsupported!("shim for an intrinsic"); } let db = self.cg.db; - let (callee, _) = db.lookup_impl_method(self.env, func, generic_args); + let (callee, _) = self.lookup_impl_method(func, generic_args); if let either::Either::Left(callee) = callee && track_caller(db, callee) { @@ -11558,7 +12219,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { let Some(func) = self.interner.lang_items().FnOnce_call_once else { unsupported!("no `FnOnce::call_once`"); }; - let (callee, callee_args) = self.cg.db.lookup_impl_method(self.env, func, args); + let (callee, callee_args) = self.lookup_impl_method(func, args); let either::Either::Left(callee) = callee else { unsupported!("builtin derive method in a vtable"); }; @@ -11683,6 +12344,20 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { sig } + fn lookup_impl_method( + &self, + func: FunctionId, + args: GenericArgs<'db>, + ) -> ( + either::Either< + FunctionId, + (hir_def::BuiltinDeriveImplId, hir_def::builtin_derive::BuiltinDeriveImplMethod), + >, + GenericArgs<'db>, + ) { + self.cg.db.lookup_impl_method(self.env, func, args) + } + /// Calls `func` with arguments that live at the given addresses. fn call_direct( &mut self, @@ -11705,7 +12380,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { let dyn_ty = generic_args.type_at(0); return self.call_virtual(func, dyn_ty, receiver, receiver_ty, rest, dst, dst_ty); } - let (callee, callee_args) = db.lookup_impl_method(self.env, func, generic_args); + let (callee, callee_args) = self.lookup_impl_method(func, generic_args); let either::Either::Left(callee) = callee else { unsupported!("call to builtin derive method"); }; @@ -11859,7 +12534,7 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { _ => Region::new_erased(interner), }); let instance = Instance { def, args: args.store(), checked: self.cg.checked }; - let body = inlinable(db, instance, self.env.store()).as_ref()?; + let body = inlinable(db, instance, self.env.store(), self.cg.sharing.clone()).as_ref()?; if body.func.signature != *sig { return None; } @@ -13736,13 +14411,13 @@ impl<'cx, 'db> FunctionCx<'cx, 'db> { /// Calls the `poll` of the `async` block of type `ty`, which is what `Future::poll` is for /// it. `args` are the pinned reference and the context. fn coroutine_callee(&mut self, ty: Ty<'db>) -> Option> { - let (def, parent_args) = coroutine_instance(self.cg.db, ty)?; + let (def, parent_args) = coroutine_instance(self.cg.db, ty, &self.cg.sharing)?; Some(Callee::Instance(def, parent_args)) } /// Drops the `async` block at `addr`, by polling it with no context. fn drop_coroutine(&mut self, addr: Value, ty: Ty<'db>) -> Result<()> { - let Some((def, parent_args)) = coroutine_instance(self.cg.db, ty) else { + let Some((def, parent_args)) = coroutine_instance(self.cg.db, ty, &self.cg.sharing) else { unsupported!("drop glue for {ty:?}"); }; let body = self.body_types(def, parent_args)?; diff --git a/crates/hir-ty-codegen/patch.rs b/crates/hir-ty-codegen/patch.rs index 1065748eca..b39ecbccf9 100644 --- a/crates/hir-ty-codegen/patch.rs +++ b/crates/hir-ty-codegen/patch.rs @@ -13,7 +13,7 @@ use std::{ use base_db::{Crate, FileId}; use cranelift_module::{Linkage, Module}; use cranelift_object::{ObjectBuilder, ObjectModule}; -use hir_def::{FunctionId, HasModule, StaticId}; +use hir_def::{DefWithBodyId, FunctionId, HasModule, StaticId}; use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet}; type FxIndexMap = indexmap::IndexMap; @@ -26,8 +26,9 @@ use super::{ compiled_static, declaration_location, fragment::{CompiledFunction, DataRequest, FuncTarget, Requests}, home::{body_file, body_files}, - instance_serial, is_thread_local, left_to_rustc, root_serial, row_location, source_location, - static_serial, target, + instance_serial, is_thread_local, left_to_rustc, program_instance, program_instance_serial, + program_root, program_root_serial, root_serial, row_location, source_location, static_serial, + target, }; /// What a compiled function or static was when the base took it in. @@ -232,7 +233,7 @@ impl Left { let code = compiled_root(db, root).as_ref().ok(); Left { code: code.map(|it| Mark::new(db, root_serial(db, root), &it.requests)), - async_blocks: async_blocks(db, root).clone(), + async_blocks: async_blocks(db, root, super::Sharing::default()).clone(), } } } @@ -301,14 +302,16 @@ impl Frozen { /// The last object built in full, as far as patching it goes. #[derive(Clone)] pub(super) struct Base { + sharing: super::Sharing, + decisions: Vec, /// Whether the object says which lines its code is of, as a patch of it then does. debug: bool, entries: Vec, pub(super) compiled: Vec<(FunctionId, String)>, marks: Vec, + reports: FxHashMap, /// The crate of each of `compiled`, and the file it is written in. crates: Vec, - files: Vec, touched: Touched, pub(super) skipped: Vec<(FunctionId, CodegenError)>, /// For each of `skipped` that rustc has a copy of. @@ -340,15 +343,21 @@ struct Homes { root: FxHashMap, roots_of_crate: FxHashMap>, roots_of_file: FxHashMap>, + roots_of_function: FxHashMap>, + roots_of_other_body: FxHashMap>, instances_of_crate: FxHashMap>, instances_of_file: FxHashMap>, + instances_of_function: FxHashMap>, + instances_of_other_body: FxHashMap>, /// The instances that the bodies of any file may be part of. instances_of_any_file: Vec, } impl Homes { fn new( + db: &dyn HirDatabase, compiled: &[(FunctionId, String)], + bodies: &[Vec], crates: &[Crate], files: &[FileId], instances: &FxIndexMap, @@ -358,8 +367,20 @@ impl Homes { homes.root.insert(root, i as u32); homes.roots_of_crate.entry(crates[i]).or_default().push(i as u32); homes.roots_of_file.entry(files[i]).or_default().push(i as u32); + for &body in &bodies[i] { + match body { + DefWithBodyId::FunctionId(func) => { + homes.roots_of_function.entry(func).or_default().push(i as u32) + } + _ => homes + .roots_of_other_body + .entry(body_file(db, body)) + .or_default() + .push(i as u32), + } + } } - for (i, was) in instances.values().enumerate() { + for (i, (instance, was)) in instances.iter().enumerate() { homes.instances_of_crate.entry(was.env.krate).or_default().push(i as u32); match &was.files { Some(files) => { @@ -369,6 +390,23 @@ impl Homes { } None => homes.instances_of_any_file.push(i as u32), } + match super::home::body_owners(db, instance) { + Some(bodies) => { + for body in bodies { + match body { + DefWithBodyId::FunctionId(func) => { + homes.instances_of_function.entry(func).or_default().push(i as u32) + } + _ => homes + .instances_of_other_body + .entry(body_file(db, body)) + .or_default() + .push(i as u32), + } + } + } + None => homes.instances_of_any_file.push(i as u32), + } } homes } @@ -392,6 +430,8 @@ struct Touched { crates: FxHashSet, /// The files that changed in the bodies of functions alone. bodies: FxHashSet, + wide_bodies: FxHashSet, + functions: FxHashSet, /// A file changed that we can't tell the crates of. any: bool, /// What the last look found changed. @@ -441,6 +481,7 @@ impl Touched { pub struct Stale { roots: Vec, instances: Vec, + reach_changed: FxHashSet, /// The statics with another value, which a patch defines again under their name. statics: Vec, /// What a patch can't replace, and why. That only matters for what the program still @@ -456,6 +497,8 @@ pub struct Stale { /// How long looking at each kind of thing in the base took, in the order they are looked at. #[derive(Default, Clone, Copy)] pub struct StaleSteps { + pub policy: std::time::Duration, + pub rejected: std::time::Duration, pub roots: std::time::Duration, pub instances: std::time::Duration, pub statics: std::time::Duration, @@ -463,6 +506,7 @@ pub struct StaleSteps { pub left: std::time::Duration, /// How many roots and instances an edit could have changed, and so were asked about. pub roots_asked: usize, + pub rejected_asked: usize, pub instances_asked: usize, /// The longest any one of them took to answer. pub slowest: std::time::Duration, @@ -487,6 +531,8 @@ pub(super) struct Patch { pub(super) object: Vec, pub(super) bodies: usize, pub(super) phases: Phases, + pub(super) sharing: super::SharingStats, + pub(super) reach_changed: Option>, } /// An object that defines nothing. @@ -510,20 +556,43 @@ impl Base { db: &dyn HirDatabase, cg: &ObjectCodegen<'_, '_>, entries: &[FunctionId], + reports: &[FunctionId], compiled: &[(FunctionId, String)], skipped: &[(FunctionId, CodegenError)], ) -> Option { + let sharing = &cg.sharing; let decls = cg.module.declarations(); let debug = cg.debug.is_some(); let root_mark = |db: &dyn HirDatabase, root| { - let compiled = compiled_root(db, root).as_ref().ok()?; - Some(Mark::of_function(db, root_serial(db, root), compiled, debug)) + let compiled = program_root(db, root, sharing).as_ref().ok()?; + let mut bodies = vec![root.into()]; + if let Some((owner, _)) = &compiled.source { + bodies.push(*owner); + } + for data in &compiled.requests.datas { + match data { + DataRequest::Location(Some((owner, _))) => bodies.push(*owner), + DataRequest::Declaration(func) => bodies.push((*func).into()), + _ => {} + } + } + Some(( + Mark::of_function( + db, + program_root_serial(db, root, sharing.clone()), + compiled, + debug, + ), + bodies, + )) }; let instance_mark = |db: &dyn HirDatabase, (instance, env): &(Instance, StoredParamEnvAndCrate)| { - let compiled = compiled_instance(db, instance.clone(), env.clone()).as_ref().ok(); + let compiled = + program_instance(db, instance.clone(), env.clone(), sharing).as_ref().ok(); let mark = compiled.map(|it| { - let serial = instance_serial(db, instance.clone(), env.clone()); + let serial = + program_instance_serial(db, instance.clone(), env.clone(), sharing.clone()); Mark::of_function(db, serial, it, debug) }); (mark, body_files(db, instance)) @@ -542,6 +611,7 @@ impl Base { .zip(root_marks) .map(|(&(root, _), mark)| mark.into_inner().unwrap_or_else(|| root_mark(db, root))) .collect::>>()?; + let (marks, root_bodies): (Vec<_>, Vec<_>) = marks.into_iter().unzip(); let mut instances = envs .iter() .zip(instance_marks) @@ -593,10 +663,11 @@ impl Base { } }; for &(root, _) in compiled { - note(Node::Root(root), &compiled_root(db, root).as_ref().ok()?.requests); + note(Node::Root(root), &program_root(db, root, sharing).as_ref().ok()?.requests); } for (instance, known) in &instances { - if let Ok(compiled) = compiled_instance(db, instance.clone(), known.env.clone()) { + if let Ok(compiled) = program_instance(db, instance.clone(), known.env.clone(), sharing) + { note(Node::Instance(instance.clone()), &compiled.requests); } } @@ -622,7 +693,7 @@ impl Base { .rejected .iter() .filter(|(instance, it)| { - compiled_instance(db, (*instance).clone(), it.env.clone()).is_err() + program_instance(db, (*instance).clone(), it.env.clone(), sharing).is_err() }) .map(|(instance, it)| (instance.clone(), it.env.clone())) .collect(); @@ -630,14 +701,22 @@ impl Base { let files: Vec = compiled.iter().map(|&(root, _)| body_file(db, root.into())).collect(); Some(Base { + sharing: sharing.clone(), + decisions: sharing.decisions(db), debug, rejected, entries: entries.to_vec(), compiled: compiled.to_vec(), marks, - homes: Homes::new(compiled, &crates, &files, &instances), + reports: reports + .iter() + .map(|&test| { + let compiled = super::compiled_report(db, test).as_ref().ok()?; + Some((test, Mark::new(db, super::report_serial(db, test), &compiled.requests))) + }) + .collect::>()?, + homes: Homes::new(db, compiled, &root_bodies, &crates, &files, &instances), crates, - files, touched: Touched::default(), skipped: skipped.to_vec(), left, @@ -668,6 +747,18 @@ impl Base { self.outdated } + pub(super) fn impls_changed(&self) -> bool { + self.touched.any || !self.touched.crates.is_empty() + } + + fn decisions_changed(&self, db: &dyn HirDatabase) -> bool { + self.decisions != self.sharing.decisions(db) + } + + pub(super) fn shares_with(&self, sharing: &super::Sharing) -> bool { + (self.sharing.is_empty() && sharing.is_empty()) || &self.sharing == sharing + } + /// Notes that a file changed: one of `crates`, or with `None` one we can't place. pub(super) fn touch(&mut self, db: &dyn HirDatabase, crates: Option<&[Crate]>) { let Some(crates) = crates else { @@ -686,8 +777,21 @@ impl Base { /// Notes that a file changed in the bodies of functions alone, none of which the rest of /// the program sees into. - pub(super) fn touch_bodies(&mut self, db: &dyn HirDatabase, file: FileId) { + pub(super) fn touch_bodies( + &mut self, + db: &dyn HirDatabase, + file: FileId, + functions: Option<&[FunctionId]>, + relocated: &[FunctionId], + ) { self.touched.bodies.insert(file); + match functions { + Some(functions) => self.touched.functions.extend(functions), + None => { + self.touched.wide_bodies.insert(file); + } + } + self.touched.functions.extend(relocated); self.outdated |= self.hydrated && base_db::relevant_crates(db, file).iter().any(|it| self.crates.contains(it)); } @@ -736,8 +840,14 @@ impl Base { } let homes = &self.homes; let of_crates = touched.crates.iter().filter_map(|it| homes.roots_of_crate.get(it)); - let of_files = touched.bodies.iter().filter_map(|it| homes.roots_of_file.get(it)); + let of_files = touched.wide_bodies.iter().filter_map(|it| homes.roots_of_file.get(it)); let mut asked: Vec = of_crates.chain(of_files).flatten().copied().collect(); + asked.extend( + touched.functions.iter().filter_map(|it| homes.roots_of_function.get(it)).flatten(), + ); + asked.extend( + touched.bodies.iter().filter_map(|it| homes.roots_of_other_body.get(it)).flatten(), + ); asked.extend(touched.roots.iter().filter_map(|it| homes.root.get(it))); Box::new(part_of(asked, part, parts)) } @@ -755,8 +865,14 @@ impl Base { } let homes = &self.homes; let of_crates = touched.crates.iter().filter_map(|it| homes.instances_of_crate.get(it)); - let of_files = touched.bodies.iter().filter_map(|it| homes.instances_of_file.get(it)); + let of_files = touched.wide_bodies.iter().filter_map(|it| homes.instances_of_file.get(it)); let mut asked: Vec = of_crates.chain(of_files).flatten().copied().collect(); + asked.extend( + touched.functions.iter().filter_map(|it| homes.instances_of_function.get(it)).flatten(), + ); + asked.extend( + touched.bodies.iter().filter_map(|it| homes.instances_of_other_body.get(it)).flatten(), + ); if !touched.bodies.is_empty() { asked.extend(&homes.instances_of_any_file); } @@ -775,57 +891,105 @@ impl Base { pub(super) fn stale(&self, db: &dyn HirDatabase, part: usize, parts: usize) -> Stale { let start = stdx::time::Instant::now(); let mut stale = Stale::default(); + let changed = self.decisions_changed(db); + stale.steps.policy = start.elapsed(); + if changed { + return stale.declined("cfg sharing decisions changed"); + } let every = |len: usize| (part..len).step_by(parts); // A check of the skipping below: looks at what no edit can have changed too, and says // so if one did. - let check_all = std::env::var_os("RUSTD_CHECK_ALL").is_some(); + let verify = std::env::var_os("RUSTD_VERIFY_FASTPATH").is_some_and(|it| it == "1"); + let check_all = std::env::var_os("RUSTD_CHECK_ALL").is_some() || verify; for i in every(self.rejected.len()) { + stale.steps.rejected_asked += 1; let (instance, env) = &self.rejected[i]; - let compiles = probe(|| compiled_instance(db, instance.clone(), env.clone()).is_ok()); + let compiles = probe(|| { + program_instance(db, instance.clone(), env.clone(), &self.sharing).is_ok() + }); if compiles != Some(false) { return stale.declined("an instance that did not compile is another now"); } } + stale.steps.rejected = start.elapsed() - stale.steps.policy; + let start = stdx::time::Instant::now(); + let selected: Vec<_> = + if check_all { self.roots_to_ask(false, 0, 1).collect() } else { Vec::new() }; + if part == 0 && std::env::var_os("RUSTD_STATS").is_some() { + eprintln!( + "patch root candidates: {} of {}", + self.roots_to_ask(false, 0, 1).count(), + self.compiled.len() + ); + eprintln!( + "patch instance candidates: {} of {}; opaque={}", + self.instances_to_ask(false, 0, 1).count(), + self.instances.len(), + self.homes.instances_of_any_file.iter().collect::>().len() + ); + } for i in self.roots_to_ask(check_all, part, parts) { let root = self.compiled[i].0; - let touched = self.touched.reaches(self.crates[i]) - || self.touched.reaches_bodies(Some(&[self.files[i]])) - || self.touched.roots.contains(&root); + let touched = !check_all || selected.binary_search(&i).is_ok(); if !touched && !check_all { continue; } let asked = stdx::time::Instant::now(); let mark = probe(|| { - let compiled = compiled_root(db, root).as_ref().ok()?; - Some(Mark::of_function(db, root_serial(db, root), compiled, self.debug)) + let compiled = program_root(db, root, &self.sharing).as_ref().ok()?; + Some(Mark::of_function( + db, + program_root_serial(db, root, self.sharing.clone()), + compiled, + self.debug, + )) }); stale.steps.roots_asked += 1; stale.steps.slowest = stale.steps.slowest.max(asked.elapsed()); match mark { Some(Some(mark)) if mark == self.marks[i] => {} _ if !touched => { - eprintln!("check-all: MISSED root {root:?} of {:?}", self.crates[i]); + eprintln!( + "check-all: MISSED root {root:?} of {:?}: symbol={} file={:?} before={:?} after={:?}", + self.crates[i], + self.compiled[i].1, + body_file(db, root.into()), + (self.marks[i].serial, self.marks[i].locations, self.marks[i].lines), + mark.map(|it| it.map(|mark| (mark.serial, mark.locations, mark.lines))) + ); + assert!(!verify, "root invalidation fast path missed {root:?}"); continue; } - Some(Some(_)) => stale.roots.push(root), + Some(Some(mark)) => { + stale.roots.push(root); + if mark.seen() != self.marks[i].seen() { + stale.reach_changed.insert(Node::Root(root)); + } + } Some(None) => return stale.declined("a root of the base no longer compiles"), None => return stale.declined("a root of the base is gone"), } } stale.steps.roots = start.elapsed(); + let selected: Vec<_> = + if check_all { self.instances_to_ask(false, 0, 1).collect() } else { Vec::new() }; for i in self.instances_to_ask(check_all, part, parts) { let Some((instance, was)) = self.instances.get_index(i) else { continue }; - let touched = self.touched.reaches(was.env.krate) - || self.touched.reaches_bodies(was.files.as_deref()) - || self.touched.instances.contains(instance); + let touched = !check_all || selected.binary_search(&i).is_ok(); if !touched && !check_all { continue; } let asked = stdx::time::Instant::now(); let mark = probe(|| { - let compiled = compiled_instance(db, instance.clone(), was.env.clone()); + let compiled = + program_instance(db, instance.clone(), was.env.clone(), &self.sharing); compiled.as_ref().ok().map(|it| { - let serial = instance_serial(db, instance.clone(), was.env.clone()); + let serial = program_instance_serial( + db, + instance.clone(), + was.env.clone(), + self.sharing.clone(), + ); Mark::of_function(db, serial, it, self.debug) }) }); @@ -838,9 +1002,15 @@ impl Base { Some(mark) if mark == was.mark => {} _ if !touched => { eprintln!("check-all: MISSED instance {instance:?} of {:?}", was.env.krate); + assert!(!verify, "instance invalidation fast path missed {instance:?}"); continue; } - Some(Some(_)) if !was.forwarded => stale.instances.push(instance.clone()), + Some(Some(mark)) if !was.forwarded => { + stale.instances.push(instance.clone()); + if Some(mark.seen()) != was.mark.map(|it| it.seen()) { + stale.reach_changed.insert(Node::Instance(instance.clone())); + } + } // One that had no code of its own might have now. _ => stale.broken.push(( Node::Instance(instance.clone()), @@ -865,6 +1035,7 @@ impl Base { } if !touched { eprintln!("check-all: MISSED static {id:?} of {:?}", was.krate); + assert!(!verify, "static invalidation fast path missed {id:?}"); continue; } stale.their_walk_changed |= was.theirs; @@ -872,6 +1043,7 @@ impl Base { let writable = probe(|| compiled_static(db, id).as_ref().ok().map(|it| it.writable)); if writable == Some(Some(was.writable)) && !was.theirs && !is_thread_local(db, id) { stale.statics.push(id); + stale.reach_changed.insert(Node::Static(id)); } else { stale.broken.push((Node::Static(id), "a static of the base changed")); } @@ -960,9 +1132,12 @@ impl Base { return true; } let requests = match parent { - Node::Root(root) => compiled_root(db, *root).as_ref().ok().map(|it| &it.requests), + Node::Root(root) => { + program_root(db, *root, &self.sharing).as_ref().ok().map(|it| &it.requests) + } Node::Instance(instance) => self.instances.get(instance).and_then(|known| { - let compiled = compiled_instance(db, instance.clone(), known.env.clone()); + let compiled = + program_instance(db, instance.clone(), known.env.clone(), &self.sharing); compiled.as_ref().ok().map(|it| &it.requests) }), Node::Static(id) => compiled_static(db, *id).as_ref().ok().map(|it| &it.requests), @@ -999,6 +1174,9 @@ impl Base { stale: Vec, ) -> Result> { let start = stdx::time::Instant::now(); + if roots.sharing != self.sharing || self.decisions_changed(db) { + return Ok(Err("cfg sharing decisions changed")); + } if roots.entries != self.entries { return Ok(Err("the roots compiled whether called or not changed")); } @@ -1014,6 +1192,7 @@ impl Base { let mut changed_instances = Vec::new(); let mut changed_statics = Vec::new(); let mut broken = Vec::new(); + let mut reach_changed = Some(FxHashSet::default()); for part in stale { if let Some(why) = part.declined { return Ok(Err(why)); @@ -1022,6 +1201,15 @@ impl Base { changed_instances.extend(part.instances); changed_statics.extend(part.statics); broken.extend(part.broken); + if part.their_walk_changed { + reach_changed = None; + } + if let Some(changed) = &mut reach_changed { + changed.extend(part.reach_changed); + } + } + if !broken.is_empty() { + reach_changed = None; } // What the edit took away is still in the base, where nothing refers to it any more. let unchanged = Unchanged { @@ -1049,16 +1237,33 @@ impl Base { Some(self), false, )?; + cg.sharing = self.sharing.clone(); + cg.counting_sharing = roots.sharing.clone(); + if cg.sharing.tentative() { + if let Some(entries) = cg.entries.as_mut() { + std::sync::Arc::get_mut(entries).expect("a new cache reader").type_ids = false; + } + } cg.phases.compile = verify; // What nothing reached any more can be reached again by the code compiled now. cg.replaced = unchanged; + let mut reports = 0; for &test in roots.reports { - cg.define_report(test)?; + let compiled = super::compiled_report(db, test).as_ref().map_err(Clone::clone)?; + let mark = Mark::new(db, super::report_serial(db, test), &compiled.requests); + if self.reports.get(&test) != Some(&mark) { + cg.define_report(test)?; + reports += 1; + } + } + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("patch reports: {reports} of {}", roots.reports.len()); } // rustc's copies are the ones the base was built against: what they call is in it. for instance in &changed_instances { let was = &self.instances[instance]; - let Ok(compiled) = compiled_instance(db, instance.clone(), was.env.clone()).as_ref() + let Ok(compiled) = + program_instance(db, instance.clone(), was.env.clone(), &self.sharing).as_ref() else { return Ok(Err("an instance of the base no longer compiles")); }; @@ -1075,7 +1280,8 @@ impl Base { return Ok(Err("a static of the base no longer has a value")); } } - let mut queued: FxHashSet = known_roots().collect(); + let base_roots: FxHashSet = known_roots().collect(); + let mut queued = base_roots.clone(); let mut queue = changed_roots; queue.extend(cg.wanted.drain(..).filter(|&func| queued.insert(func))); let mut next = 0; @@ -1101,7 +1307,8 @@ impl Base { // A future is laid out by one compiler: a function that now handles one of rustc's // has to be left to rustc as well. if let Some(theirs) = self.their_async_blocks() { - let shares = |root| match (async_blocks(db, root).clone(), &theirs) { + let shares = |root| match (async_blocks(db, root, cg.sharing.clone()).clone(), &theirs) + { (Some(ours), Some(theirs)) => !ours.is_disjoint(theirs), (Some(ours), None) => !ours.is_empty(), (None, _) => true, @@ -1110,11 +1317,32 @@ impl Base { return Ok(Err("a function now shares an `async` block with one left to rustc")); } } - let (bodies, mut phases) = (cg.bodies, cg.phases); + if let Some(changed) = &mut reach_changed { + changed.extend( + queue.iter().filter(|root| !base_roots.contains(root)).copied().map(Node::Root), + ); + changed.extend( + cg.envs + .iter() + .filter(|(instance, _)| !self.instances.contains_key(instance)) + .map(|(instance, _)| Node::Instance(instance.clone())), + ); + changed.extend( + cg.own_static_order + .iter() + .filter(|id| !self.statics.contains_key(id)) + .copied() + .map(Node::Static), + ); + } + let (bodies, mut phases, sharing) = (cg.bodies, cg.phases, cg.sharing_stats); + if cg.sharing_restart { + return Ok(Err("a shared type reaches TypeId")); + } let start = stdx::time::Instant::now(); let (object, _) = cg.finish()?; phases.object = start.elapsed(); - Ok(Ok(Patch { object, bodies, phases })) + Ok(Ok(Patch { object, bodies, phases, sharing, reach_changed })) } } diff --git a/crates/hir-ty-codegen/reach.rs b/crates/hir-ty-codegen/reach.rs index 09b8ef4bc5..32bfdecf1b 100644 --- a/crates/hir-ty-codegen/reach.rs +++ b/crates/hir-ty-codegen/reach.rs @@ -9,20 +9,24 @@ use hir_def::{ }; use rustc_hash::{FxHashMap, FxHashSet}; use rustc_type_ir::inherent::IntoKind as _; +use std::sync::Arc; use crate::next_solver::TyKind; use crate::{db::HirDatabase, traits::StoredParamEnvAndCrate}; use super::{ - CodegenError, Instance, InstanceDef, compiled_instance, compiled_root, compiled_static, + CodegenError, Instance, InstanceDef, compiled_static, fragment::{DataRequest, FuncTarget, Requests}, - instance_serial, is_thread_local, + is_thread_local, patch::Mark, - print_of, root_serial, static_serial, + print_of, static_serial, }; -#[derive(Clone, PartialEq, Eq, Hash)] +#[path = "reach_dynamic.rs"] +mod dynamic; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] enum Node { Root(FunctionId), Instance(Instance, StoredParamEnvAndCrate), @@ -51,20 +55,187 @@ impl Node { } } -#[derive(Default)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] struct Graph { + sharing: super::Sharing, ids: FxHashMap, - /// The print of each node's own code and of the source locations it reports. - own: Vec>, + nodes: Vec, edges: Vec>, /// The statics among the nodes that a running program can change. written: Vec, /// The crates an edit can change the code of. - edited: FxHashSet, + edited: Arc<[Crate]>, /// The first nodes that did not compile, and why. unknown: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct Input { + sharing: super::Sharing, + edited: Arc<[Crate]>, + tests: Vec, + always: Vec, + kept: Vec, +} + +#[salsa::interned(debug, unsafe(no_lifetime), revisions = usize::MAX)] +struct ReachInput { + #[returns(ref)] + data: Arc, +} + +#[salsa::interned(debug, unsafe(no_lifetime), revisions = usize::MAX)] +struct Component { + #[returns(ref)] + members: Vec, + #[returns(ref)] + below: Vec, +} + +/// A deduplicated reach set, chunked independently of call edges. +#[salsa::interned(debug, unsafe(no_lifetime), revisions = usize::MAX)] +struct PrintTree { + #[returns(ref)] + members: Vec, + #[returns(ref)] + children: Vec, +} + +fn print_tree(db: &dyn HirDatabase, roots: &[Component]) -> PrintTree { + let mut seen = FxHashSet::default(); + let mut pending = roots.to_vec(); + while let Some(component) = pending.pop() { + if seen.insert(component) { + pending.extend(component.below(db)); + } + } + let mut members: Vec<_> = seen.into_iter().map(|it| (print_of(&it), it)).collect(); + members.sort_by_key(|it| it.0); + print_subtree(db, &members, 128) +} + +fn print_subtree(db: &dyn HirDatabase, members: &[(u128, Component)], bits: u32) -> PrintTree { + if members.len() <= 32 || bits == 0 { + return PrintTree::new(db, members.iter().map(|it| it.1).collect::>(), Vec::new()); + } + let bits = bits.saturating_sub(5); + // Fixed hash prefixes let overlapping reach sets share their unchanged subtrees. + let mut children = Vec::new(); + let mut rest = members; + while let Some(&(key, _)) = rest.first() { + let end = rest.partition_point(|it| it.0 >> bits == key >> bits); + children.push(print_subtree(db, &rest[..end], bits)); + rest = &rest[end..]; + } + PrintTree::new(db, Vec::new(), children) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Edges { + targets: Vec, + written: bool, + unknown: Option, +} + +fn settled(db: &dyn HirDatabase, node: &Node, edited: &[Crate]) -> bool { + let krate = match node { + Node::Root(func) => func.krate(db), + Node::Instance(Instance { def: InstanceDef::Fn(func), args, .. }, _) + if args.as_ref().iter().all(|it| it.region().is_some()) => + { + func.krate(db) + } + Node::Instance(..) => return false, + Node::Static(id) => id.krate(db), + }; + !edited.contains(&krate) +} + +fn compiled<'db>( + db: &'db dyn HirDatabase, + node: &Node, + sharing: &super::Sharing, +) -> Result<(u128, &'db Requests), &'db CodegenError> { + match node { + Node::Root(func) => super::program_root(db, *func, sharing) + .as_ref() + .map(|it| (super::program_root_serial(db, *func, sharing.clone()), &it.requests)), + Node::Instance(instance, env) => { + super::program_instance(db, instance.clone(), env.clone(), sharing).as_ref().map(|it| { + ( + super::program_instance_serial( + db, + instance.clone(), + env.clone(), + sharing.clone(), + ), + &it.requests, + ) + }) + } + Node::Static(id) => { + compiled_static(db, *id).as_ref().map(|it| (static_serial(db, *id), &it.requests)) + } + } +} + +#[salsa::tracked(returns(ref))] +fn reach_edges( + db: &dyn HirDatabase, + node: Node, + sharing: super::Sharing, + edited: Arc<[Crate]>, +) -> Edges { + let mut result = Edges { targets: Vec::new(), written: false, unknown: None }; + if settled(db, &node, &edited) || foreign(db, &node) { + return result; + } + if let Node::Static(id) = node { + result.written = is_thread_local(db, id) + || compiled_static(db, id).as_ref().map_or(true, |it| it.writable); + } + let (_, requests) = match compiled(db, &node, &sharing) { + Ok(it) => it, + Err(why) => { + if !matches!(&node, Node::Instance(it, _) if names_type_id(db, it)) { + result.unknown = Some(format!("{}: {why:?}", node.name(db))); + } + return result; + } + }; + let mut homes = super::home::Homes::default(); + result.targets.extend(requests.funcs.iter().filter_map(|it| match &it.target { + FuncTarget::Instance(instance, env) => Some(match instance.def { + InstanceDef::Fn(func) if instance.args.as_ref().is_empty() => Node::Root(func), + _ => Node::Instance(instance.clone(), homes.home(db, instance, env)), + }), + FuncTarget::Symbol(_) | FuncTarget::Path(_) | FuncTarget::Asm(_) => None, + })); + result.targets.extend(requests.datas.iter().filter_map(|it| match it { + DataRequest::Static(id) => Some(Node::Static(*id)), + _ => None, + })); + result.targets.sort_by_key(print_of); + result.targets.dedup(); + result +} + +#[salsa::tracked(returns(copy))] +fn reach_own( + db: &dyn HirDatabase, + node: Node, + sharing: super::Sharing, + edited: Arc<[Crate]>, +) -> Option { + if settled(db, &node, &edited) || foreign(db, &node) { + return Some(0); + } + match compiled(db, &node, &sharing) { + Ok((serial, requests)) => Some(print_of(&Mark::new(db, serial, requests).seen())), + Err(_) => matches!(&node, Node::Instance(it, _) if names_type_id(db, it)).then_some(0), + } +} + /// Whether the instance is `core::any::TypeId::of::`. fn names_type_id(db: &dyn HirDatabase, instance: &Instance) -> bool { let InstanceDef::Fn(func) = instance.def else { return false }; @@ -92,7 +263,7 @@ impl Graph { if let Some(&id) = self.ids.get(&node) { return id; } - let first = self.own.len(); + let first = self.nodes.len(); let mut stack = vec![(first, self.add(db, node).into_iter())]; while let Some((id, targets)) = stack.last_mut() { let id = *id; @@ -102,7 +273,7 @@ impl Graph { }; let known = self.ids.get(&target).copied(); let target = known.unwrap_or_else(|| { - let new = self.own.len(); + let new = self.nodes.len(); stack.push((new, self.add(db, target).into_iter())); new }); @@ -113,80 +284,26 @@ impl Graph { /// Takes the node in and says what it refers to. fn add(&mut self, db: &dyn HirDatabase, node: Node) -> Vec { - let id = self.own.len(); + let id = self.nodes.len(); self.ids.insert(node.clone(), id); + self.nodes.push(node.clone()); self.edges.push(Vec::new()); - if self.settled(db, &node) || foreign(db, &node) { - self.own.push(Some(0)); - return Vec::new(); + let edges = reach_edges(db, node, self.sharing.clone(), self.edited.clone()); + if edges.written { + self.written.push(id); } - let compiled: Result<(u128, &Requests), &CodegenError> = match &node { - Node::Root(func) => { - compiled_root(db, *func).as_ref().map(|it| (root_serial(db, *func), &it.requests)) - } - Node::Instance(instance, env) => compiled_instance(db, instance.clone(), env.clone()) - .as_ref() - .map(|it| (instance_serial(db, instance.clone(), env.clone()), &it.requests)), - Node::Static(id) => { - compiled_static(db, *id).as_ref().map(|it| (static_serial(db, *id), &it.requests)) - } - }; - if let Node::Static(it) = node - && (is_thread_local(db, it) - || compiled_static(db, it).as_ref().map_or(true, |it| it.writable)) + if let Some(unknown) = &edges.unknown + && self.unknown.len() < 8 { - self.written.push(id); + self.unknown.push(unknown.clone()); } - let (serial, requests) = match compiled { - Ok(compiled) => compiled, - Err(why) => { - // `TypeId::of` is a number the build takes from rustc, of the type alone. - let known = matches!(&node, Node::Instance(it, _) if names_type_id(db, it)); - if !known && self.unknown.len() < 8 { - self.unknown.push(format!("{}: {why:?}", node.name(db))); - } - self.own.push(known.then_some(0)); - return Vec::new(); - } - }; - self.own.push(Some(print_of(&Mark::new(db, serial, requests).seen()))); - let funcs = requests.funcs.iter().filter_map(|it| match &it.target { - FuncTarget::Instance(instance, env) => Some(match instance.def { - InstanceDef::Fn(func) if instance.args.as_ref().is_empty() => Node::Root(func), - _ => Node::Instance(instance.clone(), env.clone()), - }), - // Code rustc compiled, which is the same until cargo builds again, or a name the - // linker resolves, which the caller hands in as reached by everything. - FuncTarget::Symbol(_) | FuncTarget::Path(_) | FuncTarget::Asm(_) => None, - }); - let statics = requests.datas.iter().filter_map(|it| match it { - DataRequest::Static(id) => Some(Node::Static(*id)), - _ => None, - }); - funcs.chain(statics).collect() - } - - /// Whether the node is of a crate no edit reaches, and of nothing else: a function - /// without type arguments, or a static. What it calls is then of such crates too, - /// unless it was handed it when the program ran, by code that is a node of its own. - fn settled(&self, db: &dyn HirDatabase, node: &Node) -> bool { - let krate = match node { - Node::Root(func) => func.krate(db), - Node::Instance(Instance { def: InstanceDef::Fn(func), args, .. }, _) - if args.as_ref().iter().all(|it| it.region().is_some()) => - { - func.krate(db) - } - Node::Instance(..) => return false, - Node::Static(id) => id.krate(db), - }; - !self.edited.contains(&krate) + edges.targets.clone() } /// The strongly connected components, each after the ones it refers to, and the /// component of each node. fn components(&self) -> (Vec>, Vec) { const NONE: usize = usize::MAX; - let count = self.own.len(); + let count = self.edges.len(); let (mut index, mut low) = (vec![NONE; count], vec![0; count]); let mut of = vec![NONE; count]; let mut components: Vec> = Vec::new(); @@ -228,10 +345,13 @@ impl Graph { } /// A set of small numbers. -#[derive(Clone, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] struct Bits(Vec); impl Bits { + fn contains(&self, at: usize) -> bool { + self.0.get(at / 64).is_some_and(|word| word >> (at % 64) & 1 == 1) + } fn insert(&mut self, at: usize) { if self.0.len() <= at / 64 { self.0.resize(at / 64 + 1, 0); @@ -258,28 +378,32 @@ impl Bits { /// others. `always` and `kept` are what runs or is found without any code naming it: every /// test is taken to reach them. With it, the print of what all of them reach, and the first /// things reached that did not compile. -pub fn reached( - db: &dyn HirDatabase, - edited: FxHashSet, - tests: &[FunctionId], - always: &[FunctionId], - kept: &[StaticId], -) -> (Vec, Option, Vec) { - let mut graph = Graph { edited, ..Graph::default() }; - let tests: Vec = tests.iter().map(|&it| graph.node(db, Node::Root(it))).collect(); - let mut ambient: Vec = always.iter().map(|&it| graph.node(db, Node::Root(it))).collect(); - ambient.extend(kept.iter().map(|&it| graph.node(db, Node::Static(it)))); +#[derive(Debug, Clone, PartialEq, Eq)] +struct Topology { + graph: Graph, + components: Vec, + of: Vec, + tests: Vec, + ambient: Vec, + groups: Vec, +} + +#[salsa::tracked(returns(ref))] +fn reach_topology(db: &dyn HirDatabase, input: ReachInput) -> Topology { + let input = input.data(db); + let mut graph = + Graph { sharing: input.sharing.clone(), edited: input.edited.clone(), ..Graph::default() }; + let tests: Vec = input.tests.iter().map(|&it| graph.node(db, Node::Root(it))).collect(); + let mut ambient: Vec = + input.always.iter().map(|&it| graph.node(db, Node::Root(it))).collect(); + ambient.extend(input.kept.iter().map(|&it| graph.node(db, Node::Static(it)))); let (components, of) = graph.components(); let written: FxHashMap = graph.written.iter().enumerate().map(|(at, &node)| (node, at)).collect(); - // Per component: the print of all it reaches, and the written statics among that. - let mut prints: Vec> = Vec::with_capacity(components.len()); + // Per component: its dependencies and the written statics among them. + let mut reachable = Vec::with_capacity(components.len()); let mut shared: Vec = Vec::with_capacity(components.len()); for (at, members) in components.iter().enumerate() { - let mut own: Option> = members.iter().map(|&it| graph.own[it]).collect(); - if let Some(own) = &mut own { - own.sort_unstable(); - } let mut below: Vec = members.iter().flat_map(|&it| &graph.edges[it]).map(|&it| of[it]).collect(); below.sort_unstable(); @@ -287,20 +411,16 @@ pub fn reached( below.retain(|&it| it != at); let mut statics = Bits::default(); members.iter().filter_map(|it| written.get(it)).for_each(|&it| statics.insert(it)); - let mut reached: Option> = below.iter().map(|&it| prints[it]).collect(); - if let Some(reached) = &mut reached { - reached.sort_unstable(); - } for &it in &below { statics.union(&shared[it]); } - prints.push(own.zip(reached).map(|it| print_of(&it))); + let mut members: Vec<_> = members.iter().map(|&it| graph.nodes[it].clone()).collect(); + members.sort_by_key(print_of); + let mut below: Vec<_> = below.into_iter().map(|id| reachable[id]).collect(); + below.sort_by_key(print_of); + reachable.push(Component::new(db, members, below)); shared.push(statics); } - let mut everywhere: Vec> = ambient.iter().map(|&it| prints[of[it]]).collect(); - everywhere.sort_unstable(); - everywhere.dedup(); - let everywhere: Option> = everywhere.into_iter().collect(); let mut ambient_shared = Bits::default(); ambient.iter().for_each(|&it| ambient_shared.union(&shared[of[it]])); // The tests that reach a static in common are one group. @@ -320,15 +440,693 @@ pub fn reached( group[a.max(b)] = a.min(b); } } - let alone: Vec> = tests + let group: Vec = (0..tests.len()).map(|at| find(&mut group, at)).collect(); + Topology { graph, components: reachable, of, tests, ambient, groups: group } +} + +#[salsa::tracked(returns(copy))] +fn reach_component_print( + db: &dyn HirDatabase, + component: Component, + sharing: super::Sharing, + edited: Arc<[Crate]>, +) -> Option { + let mut own: Option> = component + .members(db) .iter() - .map(|&it| prints[of[it]].zip(everywhere.as_ref()).map(|it| print_of(&it))) + .map(|node| reach_own(db, node.clone(), sharing.clone(), edited.clone())) .collect(); - let group: Vec = (0..tests.len()).map(|at| find(&mut group, at)).collect(); + if let Some(own) = &mut own { + own.sort_unstable(); + } + own.map(|it| print_of(&it)) +} + +#[salsa::tracked(returns(copy))] +fn reach_tree_print( + db: &dyn HirDatabase, + tree: PrintTree, + sharing: super::Sharing, + edited: Arc<[Crate]>, +) -> Option { + let mut sum = 0u128; + for &component in tree.members(db) { + sum = sum.wrapping_add(reach_component_print( + db, + component, + sharing.clone(), + edited.clone(), + )?); + } + for &child in tree.children(db) { + sum = sum.wrapping_add(reach_tree_print(db, child, sharing.clone(), edited.clone())?); + } + Some(sum) +} + +#[salsa::tracked(returns(ref))] +fn reach_ambient_components(db: &dyn HirDatabase, input: ReachInput) -> Vec { + let topology = reach_topology(db, input); + topology.ambient.iter().map(|&id| topology.components[topology.of[id]]).collect() +} + +#[salsa::tracked(returns(copy))] +fn reach_ambient_tree(db: &dyn HirDatabase, input: ReachInput) -> PrintTree { + print_tree(db, reach_ambient_components(db, input)) +} + +#[salsa::tracked(returns(ref))] +fn reach_ambient(db: &dyn HirDatabase, input: ReachInput) -> Option> { + reach_tree_print( + db, + reach_ambient_tree(db, input), + input.data(db).sharing.clone(), + input.data(db).edited.clone(), + ) + .map(|print| vec![print]) +} + +#[salsa::tracked(returns(copy))] +fn reach_root_component(db: &dyn HirDatabase, input: ReachInput, test: usize) -> Component { + let topology = reach_topology(db, input); + topology.components[topology.of[topology.tests[test]]] +} + +#[salsa::tracked(returns(copy))] +fn reach_root_tree(db: &dyn HirDatabase, input: ReachInput, test: usize) -> PrintTree { + print_tree(db, &[reach_root_component(db, input, test)]) +} + +#[salsa::tracked(returns(copy))] +fn reach_root_code(db: &dyn HirDatabase, input: ReachInput, test: usize) -> Option { + reach_tree_print( + db, + reach_root_tree(db, input, test), + input.data(db).sharing.clone(), + input.data(db).edited.clone(), + ) +} + +#[salsa::tracked(returns(copy))] +fn reach_root_print(db: &dyn HirDatabase, input: ReachInput, test: usize) -> Option { + reach_root_code(db, input, test).zip(reach_ambient(db, input).as_ref()).map(|it| print_of(&it)) +} + +pub fn reached( + db: &dyn HirDatabase, + sharing: super::Sharing, + mut edited: FxHashSet, + tests: &[FunctionId], + always: &[FunctionId], + kept: &[StaticId], +) -> (Vec, Option, Vec) { + sharing.edited_crates(&mut edited); + let mut edited: Vec<_> = edited.into_iter().collect(); + edited.sort_by_key(print_of); + let input = ReachInput::new( + db, + Arc::new(Input { + sharing, + edited: edited.into(), + tests: tests.to_vec(), + always: always.to_vec(), + kept: kept.to_vec(), + }), + ); + reach_program(db, input).clone() +} + +type Answer = (Vec, Option, Vec); + +#[derive(Default, Clone)] +pub(super) struct Cache { + state: Option>, + changed: FxHashSet, + patch: Option>, + previous: FxHashSet, +} + +#[derive(Clone)] +struct Snapshot { + input: ReachInput, + topology: Arc, + nodes: Arc>>, + bodies: Arc>>, + edges: Arc>, + own: Vec>, + components: Vec>, + roots: Arc>, + ambient: Arc>, + code: Vec>, + ambient_code: Option, + alone: Vec>, + groups: Arc>>, + answer: Answer, + dynamic: Option, + edges_changed: bool, +} + +#[derive(Clone)] +struct DynamicSnapshot { + graph: Graph, + engine: dynamic::Dynamic, + edges: Vec, + roots: Vec<(usize, usize)>, + groups: Vec, + unknown: FxHashSet, +} + +impl Cache { + pub(super) fn clear(&mut self) { + *self = Self::default(); + } + + pub(super) fn touch(&mut self, file: base_db::FileId) { + self.changed.insert(file); + self.patch = None; + } + + pub(super) fn patch(&mut self, changed: Option>) { + let Some(changed) = changed else { + self.clear(); + return; + }; + let mut candidates = std::mem::replace(&mut self.previous, changed); + candidates.extend(self.previous.iter().cloned()); + self.patch = Some(candidates); + } + + pub(super) fn reached( + &mut self, + db: &dyn HirDatabase, + sharing: super::Sharing, + mut edited: FxHashSet, + tests: &[FunctionId], + always: &[FunctionId], + kept: &[StaticId], + program: &str, + ) -> Answer { + sharing.edited_crates(&mut edited); + let mut edited: Vec<_> = edited.into_iter().collect(); + edited.sort_by_key(print_of); + let input = ReachInput::new( + db, + Arc::new(Input { + sharing, + edited: edited.into(), + tests: tests.to_vec(), + always: always.to_vec(), + kept: kept.to_vec(), + }), + ); + let changed = std::mem::take(&mut self.changed); + let patch = self.patch.take(); + let update = match &mut self.state { + None => Err("no retained snapshot"), + Some(state) if state.input != input => { + let before = state.input.data(db); + let after = input.data(db); + Err(if before.sharing != after.sharing { + "sharing policy changed" + } else if before.edited != after.edited { + "edited crates changed" + } else if before.tests != after.tests { + "test roots changed" + } else if before.always != after.always { + "always-reached roots changed" + } else { + "kept statics changed" + }) + } + Some(state) => Arc::make_mut(state).update(db, &changed, patch.as_ref(), program), + }; + if update.is_ok() { + let state = self.state.as_ref().unwrap(); + if std::env::var_os("RUSTD_VERIFY_FASTPATH").is_some_and(|it| it == "1") { + assert_eq!(&state.answer, reach_program(db, input), "reach fast path"); + } + if std::env::var_os("RUSTD_STATS").is_some() { + let path = if state.edges_changed { "updated edges" } else { "unchanged edges" }; + eprintln!("reach fast path: {path}; program={program}"); + } + return state.answer.clone(); + } + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("reach full path: {}; program={program}", update.unwrap_err()); + } + let answer = reach_program(db, input).clone(); + self.state = Some(Arc::new(Snapshot::new(db, input, answer.clone()))); + answer + } +} + +impl Snapshot { + fn new(db: &dyn HirDatabase, input: ReachInput, answer: Answer) -> Self { + let data = input.data(db); + let topology = Arc::new(reach_topology(db, input).clone()); + let mut nodes: FxHashMap<_, Vec> = FxHashMap::default(); + let mut functions: FxHashSet = FxHashSet::default(); + let mut edges = Vec::new(); + let mut own = Vec::new(); + for (id, node) in topology.graph.nodes.iter().enumerate() { + if let Some(func) = match node { + Node::Root(func) => Some(*func), + Node::Instance(instance, _) => super::home::written_in(db, instance), + Node::Static(_) => None, + } { + functions.insert(func); + } + let key = match node { + Node::Root(id) => super::patch::Node::Root(*id), + Node::Static(id) => super::patch::Node::Static(*id), + Node::Instance(instance, _) => super::patch::Node::Instance(instance.clone()), + }; + nodes.entry(key).or_default().push(id); + edges.push( + reach_edges(db, node.clone(), data.sharing.clone(), data.edited.clone()).clone(), + ); + own.push(reach_own(db, node.clone(), data.sharing.clone(), data.edited.clone())); + } + let ids: FxHashMap<_, _> = + topology.components.iter().copied().enumerate().map(|(i, c)| (c, i)).collect(); + let mut roots = vec![Bits::default(); topology.components.len()]; + for (root, &node) in topology.tests.iter().enumerate() { + roots[topology.of[node]].insert(root); + } + let mut ambient: FxHashSet<_> = + topology.ambient.iter().map(|&node| topology.of[node]).collect(); + for (id, component) in topology.components.iter().enumerate().rev() { + let here = roots[id].clone(); + for below in component.below(db) { + let below = ids[below]; + roots[below].union(&here); + if ambient.contains(&id) { + ambient.insert(below); + } + } + } + let components = topology + .components + .iter() + .map(|&c| reach_component_print(db, c, data.sharing.clone(), data.edited.clone())) + .collect(); + let code = (0..data.tests.len()).map(|id| reach_root_code(db, input, id)).collect(); + let ambient_code = reach_ambient(db, input).as_ref().map(|it| it[0]); + let alone = (0..data.tests.len()).map(|id| reach_root_print(db, input, id)).collect(); + let mut groups: FxHashMap<_, Vec<_>> = FxHashMap::default(); + for (root, &group) in topology.groups.iter().enumerate() { + groups.entry(group).or_default().push(root); + } + let mut bodies: FxHashMap<_, Vec<_>> = FxHashMap::default(); + for func in functions { + if hir_def::signatures::FunctionSignature::of(db, func).has_body() { + bodies + .entry(super::home::body_file(db, func.into())) + .or_default() + .push((func, crate::sharing::body_item_inventory(db, func).clone())); + } + } + Self { + input, + topology, + nodes: Arc::new(nodes), + bodies: Arc::new(bodies), + edges: Arc::new(edges), + own, + components, + roots: Arc::new(roots), + ambient: Arc::new(ambient), + code, + ambient_code, + alone, + groups: Arc::new(groups), + answer, + dynamic: None, + edges_changed: false, + } + } + + fn update_dynamic( + &mut self, + db: &dyn HirDatabase, + candidates: &FxHashSet, + program: &str, + ) { + let data = self.input.data(db); + let count = data.tests.len(); + let state = self.dynamic.get_or_insert_with(|| { + let graph = self.topology.graph.clone(); + let mut roots: Vec<_> = self + .topology + .tests + .iter() + .copied() + .enumerate() + .map(|(root, node)| (node, root)) + .collect(); + roots.extend(self.topology.ambient.iter().map(|&node| (node, count))); + let engine = + dynamic::Dynamic::new(graph.edges.clone(), self.own.clone(), &roots, count + 1); + DynamicSnapshot { + graph, + engine, + edges: self.edges.as_ref().clone(), + roots, + groups: self.topology.groups.clone(), + unknown: self + .edges + .iter() + .enumerate() + .filter_map(|(id, edges)| edges.unknown.is_some().then_some(id)) + .collect(), + } + }); + state.engine.affected = Bits::default(); + let mut pending: Vec<_> = candidates.iter().copied().collect(); + let mut pulled = FxHashSet::default(); + let mut changes = Vec::new(); + let mut added = state.graph.nodes.len(); + while let Some(id) = pending.pop() { + if !pulled.insert(id) { + continue; + } + let node = state.graph.nodes[id].clone(); + let edges = + reach_edges(db, node.clone(), data.sharing.clone(), data.edited.clone()).clone(); + let own = reach_own(db, node, data.sharing.clone(), data.edited.clone()); + let targets: Vec<_> = + edges.targets.iter().map(|node| state.graph.node(db, node.clone())).collect(); + while added < state.graph.nodes.len() { + let node = state.graph.nodes[added].clone(); + let edges = + reach_edges(db, node.clone(), data.sharing.clone(), data.edited.clone()) + .clone(); + let own = reach_own(db, node.clone(), data.sharing.clone(), data.edited.clone()); + assert_eq!(state.engine.add_node(own), added); + if edges.unknown.is_some() { + state.unknown.insert(added); + } + state.edges.push(edges); + let (key, func) = match node { + Node::Root(id) => (super::patch::Node::Root(id), Some(id)), + Node::Instance(instance, _) => { + let func = super::home::written_in(db, &instance); + (super::patch::Node::Instance(instance), func) + } + Node::Static(id) => (super::patch::Node::Static(id), None), + }; + Arc::make_mut(&mut self.nodes).entry(key).or_default().push(added); + if let Some(func) = func.filter(|&func| FunctionSignature::of(db, func).has_body()) + { + let bodies = Arc::make_mut(&mut self.bodies) + .entry(super::home::body_file(db, func.into())) + .or_default(); + if !bodies.iter().any(|&(id, _)| id == func) { + bodies.push((func, crate::sharing::body_item_inventory(db, func).clone())); + } + } + pending.push(added); + added += 1; + } + if state.engine.edges[id] != targets + || state.edges[id].written != edges.written + || state.edges[id].unknown != edges.unknown + { + self.edges_changed = true; + } + if edges.unknown.is_some() { + state.unknown.insert(id); + } else { + state.unknown.remove(&id); + } + for &target in &targets { + if state.engine.masks[state.engine.owner[target]].iter().next().is_none() { + pending.push(target); + } + } + if edges.written != state.edges[id].written { + state.graph.written.retain(|&node| node != id); + if edges.written { + state.graph.written.push(id); + } + } + state.graph.edges[id] = targets.clone(); + state.edges[id] = edges; + state.engine.set_own(id, own); + changes.push((id, targets)); + } + let mut seeds = Vec::new(); + for (id, targets) in changes { + state.engine.change_edges(id, targets, &state.roots, &mut seeds); + } + state.engine.settle(seeds); + let mut changed_groups = FxHashSet::default(); + if self.edges_changed { + let new_groups = dynamic_groups(&state.graph.written, &state.engine, count); + for (root, (&old, &new)) in state.groups.iter().zip(&new_groups).enumerate() { + if old != new || state.engine.affected.contains(root) { + changed_groups.insert(old); + changed_groups.insert(new); + } + } + state.groups = new_groups; + let mut groups: FxHashMap<_, Vec<_>> = FxHashMap::default(); + for (root, &group) in state.groups.iter().enumerate() { + groups.entry(group).or_default().push(root); + } + self.groups = Arc::new(groups); + self.answer + .0 + .iter_mut() + .zip(&state.groups) + .for_each(|(answer, &group)| answer.group = group); + if !self.answer.2.is_empty() || !state.unknown.is_empty() { + let mut seen = FxHashSet::default(); + let mut pending: Vec<_> = self + .topology + .tests + .iter() + .chain(&self.topology.ambient) + .copied() + .rev() + .collect(); + self.answer.2.clear(); + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + if let Some(unknown) = &state.edges[id].unknown { + self.answer.2.push(unknown.clone()); + if self.answer.2.len() == 8 { + break; + } + } + pending.extend(state.graph.edges[id].iter().rev().copied()); + } + } + } + let ambient = state.engine.code(count).map(|code| vec![code]); + let ambient_changed = self.answer.1 != ambient.as_ref().map(print_of); + let affected: Vec<_> = if ambient_changed { + (0..count).collect() + } else { + state.engine.affected.iter().filter(|&root| root < count).collect() + }; + let mut groups: FxHashSet<_> = + changed_groups.into_iter().filter(|group| self.groups.contains_key(group)).collect(); + for &root in &affected { + self.alone[root] = + state.engine.code(root).zip(ambient.as_ref()).map(|it| print_of(&it)); + groups.insert(state.groups[root]); + } + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!( + "reach incremental graph: pulled={} roots={} groups={}; program={program}", + pulled.len(), + affected.len(), + groups.len() + ); + } + refresh_groups(&self.groups, &self.alone, groups, &mut self.answer.0); + self.answer.1 = ambient.as_ref().map(print_of); + } + + fn update( + &mut self, + db: &dyn HirDatabase, + files: &FxHashSet, + patch: Option<&FxHashSet>, + program: &str, + ) -> Result<(), &'static str> { + self.edges_changed = false; + let data = self.input.data(db); + if !files.is_empty() && patch.is_none() { + return Err("patch candidates unavailable"); + } + for file in files { + if self + .bodies + .get(file) + .into_iter() + .flatten() + .any(|(func, items)| crate::sharing::body_item_inventory(db, *func) != items) + { + return Err("body item inventory changed"); + } + } + let candidates: FxHashSet<_> = patch + .into_iter() + .flatten() + .flat_map(|node| self.nodes.get(node).into_iter().flatten().copied()) + .collect(); + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("reach patch candidates: {}; program={program}", candidates.len()); + } + if self.dynamic.is_some() + || candidates.iter().any(|&id| { + reach_edges( + db, + self.topology.graph.nodes[id].clone(), + data.sharing.clone(), + data.edited.clone(), + ) != &self.edges[id] + }) + { + self.update_dynamic(db, &candidates, program); + return Ok(()); + } + let mut changed = FxHashSet::default(); + for id in candidates { + let node = &self.topology.graph.nodes[id]; + let own = reach_own(db, node.clone(), data.sharing.clone(), data.edited.clone()); + if own != self.own[id] { + self.own[id] = own; + changed.insert(self.topology.of[id]); + } + } + if changed.is_empty() { + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("reach retained answer: unchanged component prints; program={program}"); + } + return Ok(()); + } + let mut affected = Bits::default(); + let before_ambient = self.ambient_code; + for id in changed { + let mut own: Option> = self.topology.components[id] + .members(db) + .iter() + .map(|node| self.own[self.topology.graph.ids[node]]) + .collect(); + if let Some(own) = &mut own { + own.sort_unstable(); + } + let new = own.map(|own| print_of(&own)); + let old = std::mem::replace(&mut self.components[id], new); + if old == new { + continue; + } + let (Some(old), Some(new)) = (old, new) else { + return Err("component code became known or unknown"); + }; + for root in self.roots[id].iter() { + self.code[root] = + self.code[root].map(|code| code.wrapping_sub(old).wrapping_add(new)); + affected.insert(root); + } + if self.ambient.contains(&id) { + self.ambient_code = + self.ambient_code.map(|code| code.wrapping_sub(old).wrapping_add(new)); + } + } + if before_ambient != self.ambient_code { + for root in 0..self.code.len() { + affected.insert(root); + } + } + let ambient = self.ambient_code.map(|code| vec![code]); + let mut groups = FxHashSet::default(); + for root in affected.iter() { + let alone = self.code[root].zip(ambient.as_ref()).map(|it| print_of(&it)); + if self.alone[root] != alone { + self.alone[root] = alone; + groups.insert(self.topology.groups[root]); + } + } + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!( + "reach updated roots: {}, groups: {}; program={program}", + affected.iter().count(), + groups.len() + ); + } + refresh_groups(&self.groups, &self.alone, groups, &mut self.answer.0); + self.answer.1 = ambient.as_ref().map(print_of); + Ok(()) + } +} + +fn dynamic_groups(written: &[usize], graph: &dynamic::Dynamic, count: usize) -> Vec { + let mut groups: Vec<_> = (0..count).collect(); + fn find(groups: &mut [usize], mut at: usize) -> usize { + while groups[at] != at { + groups[at] = groups[groups[at]]; + at = groups[at]; + } + at + } + for &node in written { + let mask = &graph.masks[graph.owner[node]]; + let roots: Vec<_> = if mask.contains(count) { + (0..count).collect() + } else { + mask.iter().filter(|&root| root < count).collect() + }; + if let Some(&first) = roots.first() { + for &root in &roots[1..] { + let (a, b) = (find(&mut groups, first), find(&mut groups, root)); + groups[a.max(b)] = a.min(b); + } + } + } + (0..count).map(|root| find(&mut groups, root)).collect() +} + +fn refresh_groups( + members: &FxHashMap>, + alone: &[Option], + changed: FxHashSet, + reached: &mut [Reached], +) { + for group in changed { + let members = &members[&group]; + let mut together: Option> = members.iter().map(|&root| alone[root]).collect(); + if let Some(prints) = &mut together { + prints.sort_unstable(); + } + for &root in members { + reached[root].print = alone[root].zip(together.as_ref()).map(|it| print_of(&it)); + } + } +} + +#[salsa::tracked(returns(ref))] +fn reach_program( + db: &dyn HirDatabase, + input: ReachInput, +) -> (Vec, Option, Vec) { + let topology = reach_topology(db, input); + let alone: Vec<_> = + (0..input.data(db).tests.len()).map(|id| reach_root_print(db, input, id)).collect(); + finish(topology, alone, reach_ambient(db, input).as_ref().map(print_of)) +} + +fn finish(topology: &Topology, alone: Vec>, ambient: Option) -> Answer { + let group = &topology.groups; // What one test of a group leaves in a static is what another finds: each has the // result it has for the code of all of them. let mut together: FxHashMap>> = FxHashMap::default(); - for (&group, &print) in std::iter::zip(&group, &alone) { + for (&group, &print) in std::iter::zip(group, &alone) { let prints = together.entry(group).or_insert_with(|| Some(Vec::new())); match (prints.as_mut(), print) { (Some(prints), Some(print)) => prints.push(print), @@ -336,11 +1134,70 @@ pub fn reached( } } together.values_mut().flatten().for_each(|it| it.sort_unstable()); - let reached = std::iter::zip(group, alone) + let reached = std::iter::zip(group.iter().copied(), alone) .map(|(group, alone)| { let all = together[&group].as_ref(); Reached { print: alone.zip(all).map(|it| print_of(&it)), group } }) .collect(); - (reached, everywhere.as_ref().map(print_of), graph.unknown) + (reached, ambient, topology.graph.unknown.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn edge_updates_split_static_groups_and_include_ambient_writes() { + let roots = [(0, 0), (1, 1), (3, 2)]; + let mut graph = dynamic::Dynamic::new( + vec![vec![2], vec![2], vec![], vec![]], + vec![Some(1); 4], + &roots, + 3, + ); + assert_eq!(dynamic_groups(&[2], &graph, 2), vec![0, 0]); + for (node, edges, expected) in [ + (0, vec![], vec![0, 1]), + (3, vec![2], vec![0, 0]), + (3, vec![], vec![0, 1]), + (0, vec![2], vec![0, 0]), + ] { + let mut seeds = Vec::new(); + graph.change_edges(node, edges, &roots, &mut seeds); + graph.settle(seeds); + assert_eq!(dynamic_groups(&[2], &graph, 2), expected); + } + assert_eq!(dynamic_groups(&[], &graph, 2), vec![0, 1]); + } + + #[test] + fn refreshing_groups_matches_full_aggregation() { + for mut alone in [Vec::new(), vec![Some(3), Some(5), Some(7)], vec![None, Some(5), Some(7)]] + { + let groups: Vec<_> = + (0..alone.len()).map(|root| if root < 2 { 0 } else { 2 }).collect(); + let topology = Topology { + graph: Graph::default(), + components: Vec::new(), + of: Vec::new(), + tests: Vec::new(), + ambient: Vec::new(), + groups, + }; + let mut members: FxHashMap<_, Vec<_>> = FxHashMap::default(); + for (root, &group) in topology.groups.iter().enumerate() { + members.entry(group).or_default().push(root); + } + let mut reached = finish(&topology, alone.clone(), None).0; + refresh_groups(&members, &alone, FxHashSet::default(), &mut reached); + assert_eq!(reached, finish(&topology, alone.clone(), None).0); + for value in [Some(11), None, Some(3)] { + let Some(first) = alone.first_mut() else { continue }; + *first = value; + refresh_groups(&members, &alone, [0].into_iter().collect(), &mut reached); + assert_eq!(reached, finish(&topology, alone.clone(), None).0); + } + } + } } diff --git a/crates/hir-ty-codegen/reach_dynamic.rs b/crates/hir-ty-codegen/reach_dynamic.rs new file mode 100644 index 0000000000..e689983559 --- /dev/null +++ b/crates/hir-ty-codegen/reach_dynamic.rs @@ -0,0 +1,398 @@ +//! Push-based SCC and root-mask maintenance, adapted from PR #23's reach-incr prototype. + +use std::collections::VecDeque; + +use super::{Bits, Graph, print_of}; +use rustc_hash::FxHashSet; + +#[derive(Clone)] +pub(super) struct Dynamic { + pub(super) edges: Vec>, + pub(super) owner: Vec, + members: Vec>, + below: Vec>, + above: Vec>, + prints: Vec>, + roots_at: Vec, + pub(super) masks: Vec, + own: Vec>, + sums: Vec, + missing: Vec, + free: Vec, + pub(super) affected: Bits, +} + +impl Dynamic { + pub(super) fn new( + edges: Vec>, + own: Vec>, + roots: &[(usize, usize)], + count: usize, + ) -> Self { + let (members, owner) = Graph { edges: edges.clone(), ..Graph::default() }.components(); + let n = members.len(); + let mut this = Self { + edges, + owner, + members, + below: vec![Vec::new(); n], + above: vec![Vec::new(); n], + prints: vec![None; n], + roots_at: vec![Bits::default(); n], + masks: vec![Bits::default(); n], + own, + sums: vec![0; count], + missing: vec![0; count], + free: Vec::new(), + affected: Bits::default(), + }; + for id in 0..n { + this.prints[id] = this.print(id); + } + for &(node, root) in roots { + this.roots_at[this.owner[node]].insert(root); + } + let mut seeds = Vec::new(); + this.repair(&(0..n).collect::>(), &mut seeds); + this.propagate((0..n).rev().collect()); + this + } + + fn print(&self, id: usize) -> Option { + let mut own: Vec<_> = + self.members[id].iter().map(|&node| self.own[node]).collect::>()?; + own.sort_unstable(); + Some(print_of(&own)) + } + + pub(super) fn code(&self, root: usize) -> Option { + (self.missing[root] == 0).then_some(self.sums[root]) + } + + fn contribution(&mut self, root: usize, print: Option, add: bool) { + match print { + Some(print) => { + self.sums[root] = if add { + self.sums[root].wrapping_add(print) + } else { + self.sums[root].wrapping_sub(print) + } + } + None if add => self.missing[root] += 1, + None => self.missing[root] -= 1, + } + self.affected.insert(root); + } + + fn replace_print(&mut self, id: usize, new: Option) { + let old = std::mem::replace(&mut self.prints[id], new); + if old != new { + for root in self.masks[id].clone().iter() { + self.contribution(root, old, false); + self.contribution(root, new, true); + } + } + } + + pub(super) fn set_own(&mut self, node: usize, own: Option) { + if self.own[node] != own { + self.own[node] = own; + let id = self.owner[node]; + self.replace_print(id, self.print(id)); + } + } + + fn add_component(&mut self, members: Vec) -> usize { + let id = self.free.pop().unwrap_or(self.members.len()); + for &node in &members { + self.owner[node] = id; + } + if id == self.members.len() { + self.members.push(members); + self.prints.push(None); + self.below.push(Vec::new()); + self.above.push(Vec::new()); + self.roots_at.push(Bits::default()); + self.masks.push(Bits::default()); + } else { + self.members[id] = members; + } + self.prints[id] = self.print(id); + id + } + + pub(super) fn add_node(&mut self, own: Option) -> usize { + let node = self.edges.len(); + self.edges.push(Vec::new()); + self.owner.push(usize::MAX); + self.own.push(own); + self.add_component(vec![node]); + node + } + + fn repair(&mut self, sources: &[usize], seeds: &mut Vec) { + for &id in sources { + let old = std::mem::take(&mut self.below[id]); + let mut below: Vec<_> = self.members[id] + .iter() + .flat_map(|&node| &self.edges[node]) + .map(|&to| self.owner[to]) + .filter(|&to| to != id) + .collect(); + below.sort_unstable(); + below.dedup(); + for &to in &old { + self.above[to].retain(|&from| from != id); + } + for &to in &below { + self.above[to].push(id); + } + seeds.push(id); + seeds.extend(old); + seeds.extend(&below); + self.below[id] = below; + } + } + + fn repartition( + &mut self, + old: &[usize], + groups: Vec>, + root_nodes: &[(usize, usize)], + seeds: &mut Vec, + ) { + let mut sources = old.to_vec(); + let mut roots = Bits::default(); + for &id in old { + sources.extend(&self.above[id]); + roots.union(&std::mem::take(&mut self.roots_at[id])); + for root in std::mem::take(&mut self.masks[id]).iter() { + self.contribution(root, self.prints[id], false); + } + self.members[id].clear(); + self.prints[id] = Some(print_of(&Vec::::new())); + } + for members in groups { + sources.push(self.add_component(members)); + } + for &(node, root) in root_nodes { + if roots.contains(root) { + self.roots_at[self.owner[node]].insert(root); + } + } + sources.sort_unstable(); + sources.dedup(); + self.repair(&sources, seeds); + self.free.extend(old); + } + + fn closure(start: usize, edges: &[Vec]) -> FxHashSet { + let mut seen = FxHashSet::default(); + let mut pending = vec![start]; + while let Some(node) = pending.pop() { + if seen.insert(node) { + pending.extend(&edges[node]); + } + } + seen + } + + fn split(&mut self, id: usize, roots: &[(usize, usize)], seeds: &mut Vec) { + let members = &self.members[id]; + let local: rustc_hash::FxHashMap<_, _> = + members.iter().enumerate().map(|(i, &node)| (node, i)).collect(); + let edges: Vec> = members + .iter() + .map(|&node| self.edges[node].iter().filter_map(|to| local.get(to).copied()).collect()) + .collect(); + let (groups, _) = Graph { edges, ..Graph::default() }.components(); + if groups.len() > 1 { + let groups = groups + .into_iter() + .map(|group| group.into_iter().map(|i| members[i]).collect()) + .collect(); + self.repartition(&[id], groups, roots, seeds); + } + } + + fn propagate(&mut self, seeds: Vec) { + let mut queued = FxHashSet::default(); + let mut pending = VecDeque::new(); + for id in seeds { + if queued.insert(id) { + pending.push_back(id); + } + } + while let Some(id) = pending.pop_front() { + queued.remove(&id); + let mut mask = self.roots_at[id].clone(); + for &from in &self.above[id] { + mask.union(&self.masks[from]); + } + let old = std::mem::replace(&mut self.masks[id], mask); + let new = &self.masks[id]; + if old == *new { + continue; + } + let removed: Vec<_> = old.iter().filter(|&root| !new.contains(root)).collect(); + let added: Vec<_> = new.iter().filter(|&root| !old.contains(root)).collect(); + for root in removed { + self.contribution(root, self.prints[id], false); + } + for root in added { + self.contribution(root, self.prints[id], true); + } + for &to in &self.below[id] { + if queued.insert(to) { + pending.push_back(to); + } + } + } + } + + pub(super) fn change_edges( + &mut self, + node: usize, + edges: Vec, + roots: &[(usize, usize)], + seeds: &mut Vec, + ) { + let old = self.edges[node].clone(); + for to in old.iter().copied().filter(|to| !edges.contains(to)) { + let from = self.owner[node]; + let target = self.owner[to]; + self.edges[node].retain(|&edge| edge != to); + if from == target { + self.split(from, roots, seeds); + } else { + self.repair(&[from], seeds); + } + } + for to in edges.iter().copied().filter(|to| !old.contains(to)) { + self.edges[node].push(to); + let from = self.owner[node]; + let target = self.owner[to]; + if from == target { + continue; + } + let forward = Self::closure(target, &self.below); + if forward.contains(&from) { + let backward = Self::closure(from, &self.above); + let merged: Vec<_> = forward.intersection(&backward).copied().collect(); + let members = + merged.iter().flat_map(|&id| self.members[id].iter().copied()).collect(); + self.repartition(&merged, vec![members], roots, seeds); + } else { + self.repair(&[from], seeds); + } + } + self.edges[node] = edges; + } + + pub(super) fn settle(&mut self, seeds: Vec) { + self.propagate(seeds); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check(graph: &Dynamic, roots: &[(usize, usize)], count: usize) { + let (components, of) = + Graph { edges: graph.edges.clone(), ..Graph::default() }.components(); + let prints: Vec<_> = components + .iter() + .map(|members| { + let mut own: Vec<_> = + members.iter().map(|&node| graph.own[node]).collect::>()?; + own.sort_unstable(); + Some(print_of(&own)) + }) + .collect(); + for root in 0..count { + let mut seen = FxHashSet::default(); + let mut pending: Vec<_> = + roots.iter().filter_map(|&(node, id)| (id == root).then_some(node)).collect(); + while let Some(node) = pending.pop() { + if seen.insert(node) { + pending.extend(&graph.edges[node]); + } + } + let reached: FxHashSet<_> = seen.iter().map(|&node| of[node]).collect(); + let expected = reached + .iter() + .try_fold(0u128, |sum, &id| prints[id].map(|print| sum.wrapping_add(print))); + assert_eq!(graph.code(root), expected, "root {root}"); + for node in 0..graph.edges.len() { + assert_eq!( + graph.masks[graph.owner[node]].contains(root), + seen.contains(&node), + "node {node} root {root}" + ); + for other in 0..graph.edges.len() { + assert_eq!(graph.owner[node] == graph.owner[other], of[node] == of[other]); + } + } + } + } + + #[test] + fn add_remove_and_merge_split_match_full_traversal() { + let roots = [(0, 0), (3, 1), (2, 2), (3, 2)]; + let mut graph = Dynamic::new( + vec![vec![1], vec![2], vec![], vec![]], + vec![Some(1), Some(2), Some(3), None], + &roots, + 3, + ); + check(&graph, &roots, 3); + for (node, edges) in [(0, vec![1, 3]), (0, vec![1]), (2, vec![0]), (2, vec![])] { + let mut seeds = Vec::new(); + graph.change_edges(node, edges, &roots, &mut seeds); + graph.settle(seeds); + check(&graph, &roots, 3); + } + graph.set_own(3, Some(4)); + check(&graph, &roots, 3); + let node = graph.add_node(Some(5)); + let mut seeds = Vec::new(); + graph.change_edges(0, vec![1, node], &roots, &mut seeds); + graph.change_edges(node, vec![0], &roots, &mut seeds); + graph.settle(seeds); + check(&graph, &roots, 3); + } + + #[test] + fn mixed_edge_batches_and_code_changes_match_full_traversal() { + let n = 24; + let roots = [(0, 0), (1, 1), (2, 2), (3, 2)]; + let mut graph = + Dynamic::new(vec![Vec::new(); n], (0..n).map(|i| Some(i as u128)).collect(), &roots, 3); + let mut random = 7u64; + for edit in 0..600 { + let mut seeds = Vec::new(); + for _ in 0..3 { + random = random.wrapping_mul(6364136223846793005).wrapping_add(1); + let node = random as usize % n; + let to = (random >> 32) as usize % n; + let mut edges = graph.edges[node].clone(); + if edges.contains(&to) { + edges.retain(|&id| id != to); + } else { + edges.push(to); + } + graph.change_edges(node, edges, &roots, &mut seeds); + graph.set_own(node, (edit % 13 != 0).then_some(random as u128)); + } + graph.settle(seeds); + check(&graph, &roots, 3); + } + } + + #[test] + fn empty_graph_has_empty_ambient_print() { + let graph = Dynamic::new(Vec::new(), Vec::new(), &[], 1); + assert_eq!(graph.code(0), Some(0)); + } +} diff --git a/crates/hir-ty-codegen/remap.rs b/crates/hir-ty-codegen/remap.rs new file mode 100644 index 0000000000..3f25b37cf8 --- /dev/null +++ b/crates/hir-ty-codegen/remap.rs @@ -0,0 +1,320 @@ +use hir_def::{AdtId, AttrDefId, FunctionId, HasModule, VariantId}; +use rustc_type_ir::Interner; +use triomphe::Arc; + +use super::{ + AggregateKind, CodegenError, MirBody, Operand, OperandKind, ProjectionElem, Result, Rvalue, + Sharing, StatementKind, TerminatorKind, +}; +use crate::{ + db::HirDatabase, + layout::{Layout, LayoutError, LayoutProvider}, + next_solver::{Allocation, AllocationData, StoredGenericArgs, StoredTy}, + traits::StoredParamEnvAndCrate, +}; + +pub(super) fn monomorphized<'db>( + db: &'db dyn HirDatabase, + sharing: &Sharing, + body: &MirBody<'db>, + args: &StoredGenericArgs, + env: &StoredParamEnvAndCrate, +) -> std::result::Result, super::super::MirLowerError<'db>> { + let interner = crate::next_solver::DbInterner::new_with(db, env.krate); + crate::mir::monomorphization::monomorphized_with(db, body, args, env, &|id, args| { + if let crate::ImplTraitId::ReturnTypeImplTrait(func, index) = id.loc(db) + && let Some(source) = sharing.body_source(db, func) + { + let source_id = crate::db::InternedOpaqueTyId::new( + db, + crate::ImplTraitId::ReturnTypeImplTrait(source, index), + ); + let hidden = interner.type_of(source_id.into()).instantiate_identity().skip_norm_wip(); + let hidden = sharing.analysis_fold(db, source, hidden); + return crate::next_solver::EarlyBinder::bind(hidden) + .instantiate(interner, args) + .skip_norm_wip(); + } + interner.type_of(id.into()).instantiate(interner, args).skip_norm_wip() + }) +} + +struct ProgramLayout<'a, 'db> { + db: &'db dyn HirDatabase, + sharing: &'a Sharing, +} + +impl<'db> LayoutProvider<'db> for ProgramLayout<'_, 'db> { + fn coroutine_is_by_move( + &self, + id: crate::db::InternedCoroutineId<'db>, + args: rustc_type_ir::CoroutineArgs>, + ) -> bool { + self.sharing.coroutine_is_by_move(self.db, id, args) + } + fn db(&self) -> &'db dyn HirDatabase { + self.db + } + fn layout( + &self, + ty: StoredTy, + env: StoredParamEnvAndCrate, + ) -> std::result::Result, LayoutError> { + layout(self.db, ty, env, self.sharing.clone()) + } + fn adt( + &self, + def: AdtId, + args: StoredGenericArgs, + env: StoredParamEnvAndCrate, + ) -> std::result::Result, LayoutError> { + crate::layout::adt::layout_of_adt_with(self.db, def, args, env, self) + } + fn coroutine( + &self, + id: crate::db::InternedCoroutineId<'db>, + args: StoredGenericArgs, + env: StoredParamEnvAndCrate, + by_move: bool, + ) -> std::result::Result>, LayoutError> { + use salsa::plumbing::AsId; + let def = if by_move { + super::InstanceDef::CoroutineByMove(id.as_id()) + } else { + super::InstanceDef::Coroutine(id.as_id()) + }; + super::program_instance_body( + self.db, + &super::Instance { def, args, checked: false }, + env, + self.sharing, + ) + .map_err(|_| LayoutError::Unknown) + } +} + +#[salsa::tracked(returns(clone), cycle_result = layout_cycle)] +pub(super) fn layout( + db: &dyn HirDatabase, + ty: StoredTy, + env: StoredParamEnvAndCrate, + sharing: Sharing, +) -> std::result::Result, LayoutError> { + if sharing.is_empty() { + return db.layout_of_ty(ty, env); + } + crate::layout::layout_of_ty_with(db, ty, env, &ProgramLayout { db, sharing: &sharing }) +} + +fn layout_cycle( + _: &dyn HirDatabase, + _: salsa::Id, + _: StoredTy, + _: StoredParamEnvAndCrate, + _: Sharing, +) -> std::result::Result, LayoutError> { + Err(LayoutError::RecursiveTypeWithoutIndirection) +} + +pub(super) fn body<'db>( + db: &'db dyn HirDatabase, + sharing: &Sharing, + source: FunctionId, + original: &MirBody<'db>, +) -> Result> { + mapped_body(db, sharing, Some(source), original) +} + +pub(super) fn opaques<'db>( + db: &'db dyn HirDatabase, + sharing: &Sharing, + original: &MirBody<'db>, +) -> Result> { + mapped_body(db, sharing, None, original) +} + +#[salsa::tracked(returns(ref))] +pub(super) fn promoted<'db>( + db: &'db dyn HirDatabase, + source: FunctionId, + expr: hir_def::hir::ExprId, + args: StoredGenericArgs, + env: StoredParamEnvAndCrate, + sharing: Sharing, +) -> Result> { + let raw = super::super::promoted_mir_body_query(db, source.into(), expr) + .map_err(|e| CodegenError::Frontend(format!("promoted mir: {e:?}")))?; + let mapped = body(db, &sharing, source, &raw)?; + monomorphized(db, &sharing, &mapped, &args, &env) + .map_err(|e| CodegenError::Frontend(format!("promoted mir: {e:?}"))) +} + +fn mapped_body<'db>( + db: &'db dyn HirDatabase, + sharing: &Sharing, + source: Option, + original: &MirBody<'db>, +) -> Result> { + let mut body = original.clone(); + let item = |item| source.map_or(item, |source| sharing.analysis_item(db, source, item)); + let krate = original.owner.module(db).krate(db); + let variant = |variant| match variant { + VariantId::EnumVariantId(id) => match item(AttrDefId::EnumVariantId(id)) { + AttrDefId::EnumVariantId(id) => VariantId::EnumVariantId(id), + _ => variant, + }, + VariantId::StructId(id) => match item(AdtId::StructId(id).into()) { + AttrDefId::AdtId(AdtId::StructId(id)) => VariantId::StructId(id), + _ => variant, + }, + VariantId::UnionId(id) => match item(AdtId::UnionId(id).into()) { + AttrDefId::AdtId(AdtId::UnionId(id)) => VariantId::UnionId(id), + _ => variant, + }, + }; + macro_rules! fold { + ($value:expr) => { + *$value = match source { + Some(source) => sharing.analysis_fold(db, source, $value.as_ref()), + None => sharing.reuse_opaques(db, krate, $value.as_ref()), + } + .store() + }; + } + let operand = |op: &mut Operand| -> Result<()> { + match &mut op.kind { + OperandKind::Constant { konst, ty } => { + fold!(konst); + fold!(ty); + } + OperandKind::Allocation { allocation } => { + let value = allocation.as_ref(); + if source.is_some() && matches!(value.memory_map, crate::MemoryMap::Complex(_)) { + return Err(CodegenError::Unsupported( + "analysis-sharing a complex allocation".into(), + )); + } + *allocation = Allocation::new(AllocationData { + ty: match source { + Some(source) => sharing.analysis_fold(db, source, value.ty), + None => sharing.reuse_opaques(db, krate, value.ty), + }, + memory: value.memory.clone(), + memory_map: value.memory_map.clone(), + }) + .store(); + } + OperandKind::Static(id) => { + if let AttrDefId::StaticId(mapped) = item((*id).into()) { + *id = mapped; + } + } + OperandKind::Copy(_) | OperandKind::Move(_) => {} + } + Ok(()) + }; + for (_, local) in body.locals.iter_mut() { + fold!(&mut local.ty); + } + body.walk_places(|place| { + place.projection = super::super::Projection::new_from_iter( + place.projection.as_slice().iter().map(|&elem| match elem { + ProjectionElem::Downcast(id) => ProjectionElem::Downcast(variant(id)), + elem => elem, + }), + ) + .store(); + }); + for (_, block) in body.basic_blocks.iter_mut() { + for statement in &mut block.statements { + match &mut statement.kind { + StatementKind::Assign(_, value) => match value { + Rvalue::Aggregate(kind, operands) => { + for op in &mut **operands { + operand(op)?; + } + match kind { + AggregateKind::Array(ty) + | AggregateKind::Tuple(ty) + | AggregateKind::Closure(ty) => { + fold!(ty); + } + AggregateKind::Adt(id, args) => { + *id = variant(*id); + fold!(args); + } + AggregateKind::Union(id, field) => { + if let VariantId::UnionId(mapped) = variant(VariantId::UnionId(*id)) + { + *id = mapped; + field.parent = VariantId::UnionId(mapped); + } + } + } + } + Rvalue::Use(op) | Rvalue::UnaryOp(_, op) => operand(op)?, + Rvalue::Repeat(op, len) => { + operand(op)?; + fold!(len); + } + Rvalue::Cast(_, op, ty) => { + operand(op)?; + fold!(ty); + } + Rvalue::CheckedBinaryOp(_, lhs, rhs) => { + operand(lhs)?; + operand(rhs)?; + } + Rvalue::Ref(..) + | Rvalue::Len(_) + | Rvalue::Discriminant(_) + | Rvalue::CopyForDeref(_) => {} + Rvalue::ThreadLocalRef(n) + | Rvalue::AddressOf(n) + | Rvalue::BinaryOp(n) + | Rvalue::NullaryOp(n) => match *n {}, + }, + StatementKind::InlineAsm(asm) => { + for op in &mut asm.operands { + match op { + super::super::AsmOperand::Reg { input: Some(op), .. } + | super::super::AsmOperand::Const(op) + | super::super::AsmOperand::Sym(op) => operand(op)?, + super::super::AsmOperand::Reg { input: None, .. } + | super::super::AsmOperand::Label => {} + } + } + } + StatementKind::Deinit(_) + | StatementKind::FakeRead(_) + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Nop => {} + } + } + if let Some(terminator) = &mut block.terminator { + match &mut terminator.kind { + TerminatorKind::Call { func, args, .. } => { + operand(func)?; + for op in &mut **args { + operand(op)?; + } + } + TerminatorKind::SwitchInt { discr, .. } => operand(discr)?, + TerminatorKind::DropAndReplace { value, .. } + | TerminatorKind::Yield { value, .. } => operand(value)?, + TerminatorKind::Assert { cond, .. } => operand(cond)?, + TerminatorKind::Goto { .. } + | TerminatorKind::UnwindResume + | TerminatorKind::Abort + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::Drop { .. } + | TerminatorKind::CoroutineDrop + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } => {} + } + } + } + Ok(body) +} diff --git a/crates/hir-ty-codegen/sharing.rs b/crates/hir-ty-codegen/sharing.rs new file mode 100644 index 0000000000..4a283e3008 --- /dev/null +++ b/crates/hir-ty-codegen/sharing.rs @@ -0,0 +1,1123 @@ +use base_db::Crate; +use hir_def::{AttrDefId, CallableDefId, DefWithBodyId, FunctionId, HasModule, Lookup}; +use rustc_hash::{FxHashMap, FxHashSet}; +use rustc_type_ir::{ + TypeFoldable, TypeFolder, TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitor, + inherent::{IntoKind, Ty as _}, +}; +use std::{ + hash::{Hash, Hasher}, + sync::Arc, + time::Instant, +}; + +use crate::{ + InferBodyId, + db::{HirDatabase, InternedClosureId, InternedCoroutineClosureId, InternedCoroutineId}, + next_solver::{DbInterner, Ty, TyKind}, + traits::{ParamEnvAndCrate, StoredParamEnvAndCrate}, +}; + +use super::{Instance, InstanceDef}; + +#[salsa::tracked(returns(ref))] +fn reverse_twins( + db: &dyn HirDatabase, + normal: Crate, + test: Crate, +) -> rustc_hash::FxHashMap { + hir_def::twins::twin_index(db, normal, test).twins.iter().map(|(&t, &n)| (n, t)).collect() +} + +#[salsa::tracked(returns(copy))] +fn decision_serial(db: &dyn HirDatabase, normal: Crate, test: Crate, both: bool) -> u128 { + let twins = reverse_twins(db, normal, test); + let eligibility = crate::sharing::pair_eligibility(db, normal, test, both); + policy_serial(twins, eligibility) +} + +fn policy_serial( + twins: &FxHashMap, + eligibility: &crate::sharing::Eligibility, +) -> u128 { + let mut twins: Vec<_> = twins.iter().map(|(&n, &t)| super::print_of(&(&t, &n))).collect(); + let mut full: Vec<_> = eligibility.candidates.iter().map(super::print_of).collect(); + let mut analysis: Vec<_> = eligibility.analysis_shared.iter().map(super::print_of).collect(); + twins.sort_unstable(); + full.sort_unstable(); + analysis.sort_unstable(); + super::print_of(&(twins, full, analysis, &eligibility.pair_veto)) +} + +/// The crate copies present in one program. An empty policy compiles without sharing. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct Sharing { + pairs: Vec<(Crate, Crate, bool)>, + enabled: bool, + snapshot: Option>, +} + +#[derive(Debug, Clone)] +struct PairPolicy { + key: (Crate, Crate, bool, u128), + twins: hir_def::twins::TwinIndex, + reverse: FxHashMap, + eligibility: crate::sharing::Eligibility, +} + +#[derive(Debug, Clone)] +struct Policy(Vec); + +impl PartialEq for Policy { + fn eq(&self, other: &Self) -> bool { + self.0.iter().map(|it| it.key).eq(other.0.iter().map(|it| it.key)) + } +} +impl Eq for Policy {} +impl Hash for Policy { + fn hash(&self, state: &mut H) { + self.0.len().hash(state); + for pair in &self.0 { + pair.key.hash(state); + } + } +} + +#[derive(Default, Clone)] +pub(super) struct Cache { + input: Option, + shared: Option, + files: FxHashMap, + changed: FxHashMap>>, + eligibility: Vec<((Crate, Crate, bool), crate::sharing::EligibilityState)>, +} + +#[derive(Default, Clone)] +struct FileFacts { + trees: Vec<(Crate, hir_expand::HirFileId, Option>)>, + bodies: Vec<(FunctionId, bool, crate::sharing::BodySharingFacts)>, + by_function: FxHashMap, +} + +impl FileFacts { + fn push(&mut self, func: FunctionId, normal: bool, facts: crate::sharing::BodySharingFacts) { + self.by_function.insert(func, self.bodies.len()); + self.bodies.push((func, normal, facts)); + } +} + +impl Cache { + pub(super) fn clear(&mut self) { + *self = Self::default(); + } + + pub(super) fn touch(&mut self, file: base_db::FileId, functions: Option<&[FunctionId]>) { + let functions = functions.map(|it| it.iter().copied().collect::>()); + self.changed + .entry(file) + .and_modify(|was| match (was.as_mut(), functions.as_ref()) { + (Some(was), Some(now)) => was.extend(now), + _ => *was = None, + }) + .or_insert(functions); + } + + pub(super) fn needs_before_edit(&self, files: &[base_db::FileId]) -> bool { + self.shared.as_ref().is_some_and(|sharing| !sharing.is_empty()) + && files.iter().any(|file| !self.files.contains_key(file)) + } + + pub(super) fn before_edit( + &mut self, + db: &dyn HirDatabase, + file: base_db::FileId, + functions: &[FunctionId], + ) { + if self.shared.as_ref().is_none_or(Sharing::is_empty) + || self.files.contains_key(&file) + || functions.is_empty() + { + return; + } + let mut facts = FileFacts::default(); + for &func in functions { + let source = func.lookup(db).id.file_id; + let krate = func.krate(db); + if func.module(db).block(db).is_none() + && !facts.trees.iter().any(|&(k, f, _)| k == krate && f == source) + { + facts.trees.push(( + krate, + source, + hir_def::item_tree::file_item_tree_snapshot(db, source, krate), + )); + } + if hir_def::signatures::FunctionSignature::of(db, func).has_body() { + facts.push(func, true, crate::sharing::body_sharing_facts(db, func, true).clone()); + } + } + self.files.insert(file, facts); + } + + pub(super) fn sharing( + &mut self, + db: &dyn HirDatabase, + input: Sharing, + program: &str, + ) -> Sharing { + let start = Instant::now(); + let changed = std::mem::take(&mut self.changed); + let mut reason = "policy input changed"; + let mut updates = Vec::new(); + let mut checked = 0; + let mut bodies = FxHashSet::default(); + let stable = self.input.as_ref() == Some(&input) + && changed.iter().all(|(file, functions)| { + let Some(facts) = self.files.get(file) else { + reason = "changed file has no retained facts"; + return false; + }; + let indexes: Vec<_> = match functions { + Some(functions) => functions.iter().filter_map(|func| facts.by_function.get(func).copied()).collect(), + None => (0..facts.bodies.len()).collect(), + }; + facts.trees.iter().all(|(krate, file, tree)| { + reason = "item tree changed"; + &hir_def::item_tree::file_item_tree_snapshot(db, *file, *krate) == tree + }) && indexes.into_iter().all(|index| { + let (func, normal, facts) = &facts.bodies[index]; + checked += 1; + reason = "body facts changed"; + let new = crate::sharing::body_sharing_facts(db, *func, *normal); + if new == facts { + return true; + } + if std::env::var_os("RUSTD_STATS").is_some() { + for (field, old, new) in facts.changed_fields(new) { + eprintln!( + "sharing fact mismatch: owner={func:?} name={} normal={normal} field={field} old={old} new={new}; program={program}", + hir_def::signatures::FunctionSignature::of(db, *func).name.as_str(), + ); + } + } + if !facts.same_structure(new) { + return false; + } + let krate = func.krate(db); + let crates: Vec<_> = input + .pairs + .iter() + .filter(|&&(normal, _, _)| normal == krate) + .flat_map(|&(normal, test, _)| [normal, test]) + .collect(); + if facts.policy_projection(db, &crates) != new.policy_projection(db, &crates) { + bodies.insert(*func); + } + updates.push((*file, index, new.clone())); + true + }) + }); + let validated = start.elapsed(); + if !input.is_empty() && std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("sharing candidates: bodies={checked}; program={program}"); + } + let rejected_before: usize = self + .shared + .as_ref() + .into_iter() + .flat_map(|it| it.snapshot.iter()) + .flat_map(|it| &it.0) + .map(|it| it.eligibility.rejected.len()) + .sum(); + let retained = if stable { self.shared.clone() } else { None }; + let mut affected = 0; + let retained = retained.and_then(|mut shared| { + if bodies.is_empty() { + return Some(shared); + } + let mut policy = shared.snapshot.as_deref()?.clone(); + let mut handled = FxHashSet::default(); + for ((normal, test, both), state) in &mut self.eligibility { + let changed: Vec<_> = + bodies.iter().copied().filter(|func| func.krate(db) == *normal).collect(); + if changed.is_empty() { + continue; + } + affected += state.update(db, *normal, *test, *both, &changed)?; + handled.extend(changed); + let pair = policy + .0 + .iter_mut() + .find(|pair| (pair.key.0, pair.key.1, pair.key.2) == (*normal, *test, *both))?; + if &pair.eligibility != state.result() { + pair.eligibility = state.result().clone(); + pair.key.3 = policy_serial(&pair.reverse, &pair.eligibility); + } + } + if handled != bodies { + reason = "changed body is outside retained eligibility graph"; + return None; + } + shared.snapshot = Some(Arc::new(policy)); + Some(shared) + }); + if let Some(shared) = retained { + if std::env::var_os("RUSTD_VERIFY_FASTPATH").is_some_and(|it| it == "1") { + let full = input.clone().capture(db); + for (cached, full) in shared + .snapshot + .iter() + .flat_map(|it| &it.0) + .zip(full.snapshot.iter().flat_map(|it| &it.0)) + { + assert_eq!(cached.twins, full.twins, "sharing twins fast path"); + assert_eq!( + cached.eligibility, full.eligibility, + "sharing eligibility fast path" + ); + } + assert_eq!( + shared.decisions(db), + full.decisions(db), + "sharing fingerprint fast path" + ); + } + if !input.is_empty() && std::env::var_os("RUSTD_STATS").is_some() { + if bodies.is_empty() { + eprintln!("sharing fast path: unchanged facts; program={program}"); + } else { + let rejected: usize = shared + .snapshot + .iter() + .flat_map(|it| &it.0) + .map(|it| it.eligibility.rejected.len()) + .sum(); + eprintln!( + "sharing incremental path: bodies={} affected={affected} rejected={rejected_before}->{rejected}; program={program}", + bodies.len() + ); + } + eprintln!("sharing validate-us: {}; program={program}", validated.as_micros()); + } + for (file, index, facts) in updates { + self.files.get_mut(&file).unwrap().bodies[index].2 = facts; + } + self.shared = Some(shared.clone()); + return shared; + } + if !input.is_empty() && std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("sharing full path: {reason}; program={program}"); + } + let shared = input.clone().capture(db); + let captured = start.elapsed(); + self.eligibility.clear(); + self.files.clear(); + if !input.is_empty() { + self.eligibility = input + .pairs + .iter() + .map(|&(normal, test, both)| { + let state = crate::sharing::EligibilityState::new(db, normal, test, both); + assert_eq!( + state.result(), + crate::sharing::pair_eligibility(db, normal, test, both), + "sharing retained evaluator initialization" + ); + ((normal, test, both), state) + }) + .collect(); + let mut crates = FxHashMap::default(); + for &(normal, test, _) in &input.pairs { + crates.insert(normal, true); + crates.entry(test).or_insert(false); + } + for (krate, normal) in crates { + for &item in &hir_def::twins::crate_items(db, krate).items { + let AttrDefId::FunctionId(func) = item else { continue }; + if !hir_def::signatures::FunctionSignature::of(db, func).has_body() { + continue; + } + let source = func.lookup(db).id.file_id; + let file = super::home::body_file(db, func.into()); + let facts = self.files.entry(file).or_default(); + if func.module(db).block(db).is_none() + && !facts.trees.iter().any(|&(k, f, _)| k == krate && f == source) + { + facts.trees.push(( + krate, + source, + hir_def::item_tree::file_item_tree_snapshot(db, source, krate), + )); + } + facts.push( + func, + normal, + crate::sharing::body_sharing_facts(db, func, normal).clone(), + ); + } + } + } + self.input = Some(input); + self.shared = Some(shared.clone()); + if !shared.is_empty() && std::env::var_os("RUSTD_STATS").is_some() { + eprintln!( + "sharing phases-us: validate={} capture={} inventory={}; program={program}", + validated.as_micros(), + (captured - validated).as_micros(), + (start.elapsed() - captured).as_micros(), + ); + } + shared + } +} + +impl Sharing { + fn capture(mut self, db: &dyn HirDatabase) -> Self { + if !self.is_empty() { + self.snapshot = Some(Arc::new(Policy( + self.pairs + .iter() + .map(|&(n, t, both)| PairPolicy { + key: (n, t, both, decision_serial(db, n, t, both)), + twins: hir_def::twins::twin_index(db, n, t).clone(), + reverse: reverse_twins(db, n, t).clone(), + eligibility: crate::sharing::pair_eligibility(db, n, t, both).clone(), + }) + .collect(), + ))); + } + self + } + + fn policy(&self, normal: Crate, test: Crate) -> Option<&PairPolicy> { + self.snapshot.as_ref()?.0.iter().find(|it| (it.key.0, it.key.1) == (normal, test)) + } + + fn twins<'a>( + &'a self, + db: &'a dyn HirDatabase, + normal: Crate, + test: Crate, + ) -> &'a hir_def::twins::TwinIndex { + self.policy(normal, test) + .map(|it| &it.twins) + .unwrap_or_else(|| hir_def::twins::twin_index(db, normal, test)) + } + + fn reverse<'a>( + &'a self, + db: &'a dyn HirDatabase, + normal: Crate, + test: Crate, + ) -> &'a FxHashMap { + self.policy(normal, test) + .map(|it| &it.reverse) + .unwrap_or_else(|| reverse_twins(db, normal, test)) + } + + fn eligibility<'a>( + &'a self, + db: &'a dyn HirDatabase, + normal: Crate, + test: Crate, + both: bool, + ) -> &'a crate::sharing::Eligibility { + self.policy(normal, test) + .map(|it| &it.eligibility) + .unwrap_or_else(|| crate::sharing::pair_eligibility(db, normal, test, both)) + } + + pub(super) fn prewarm(&self, db: &dyn HirDatabase) { + if self.is_empty() || self.snapshot.is_some() { + return; + } + let tally = super::backend::Tally::global(); + for &(normal, test, both) in &self.pairs { + tally.step(super::Step::TwinIndex, || hir_def::twins::twin_index(db, normal, test)); + tally.step(super::Step::Eligibility, || { + crate::sharing::pair_eligibility(db, normal, test, both) + }); + } + } + + pub(super) fn decisions(&self, db: &dyn HirDatabase) -> Vec { + if self.is_empty() { + return Vec::new(); + } + if let Some(snapshot) = &self.snapshot { + return snapshot.0.iter().map(|it| it.key.3).collect(); + } + self.pairs.iter().map(|&(n, t, both)| decision_serial(db, n, t, both)).collect() + } + + pub fn new(pairs: Vec<(Crate, Crate, bool)>) -> Self { + Self { pairs, enabled: true, snapshot: None } + } + + pub fn disabled(mut self) -> Self { + self.enabled = false; + self + } + + pub(super) fn is_empty(&self) -> bool { + !self.enabled || self.pairs.is_empty() + } + + pub(super) fn tentative(&self) -> bool { + self.enabled && self.pairs.iter().any(|it| it.2) + } + + pub(super) fn test_crate(&self, krate: Crate) -> bool { + self.pairs.iter().any(|it| it.1 == krate) + } + + pub(super) fn edited_crates(&self, edited: &mut rustc_hash::FxHashSet) { + if self.is_empty() { + return; + } + for &(normal, test, _) in &self.pairs { + if edited.contains(&test) { + edited.insert(normal); + } + } + } + + pub(super) fn item(&self, db: &dyn HirDatabase, item: AttrDefId) -> AttrDefId { + if self.is_empty() { + return item; + } + for &(normal, test, both) in &self.pairs { + if item.module(db).krate(db) != test { + continue; + } + let eligibility = self.eligibility(db, normal, test, both); + if eligibility.pair_veto.is_empty() && eligibility.candidates.contains(&item) { + return self.twins(db, normal, test).twins.get(&item).copied().unwrap_or(item); + } + } + item + } + + pub(super) fn analysis_shared(&self, db: &dyn HirDatabase, func: FunctionId) -> bool { + self.pairs.iter().any(|&(n, t, both)| { + func.krate(db) == t + && self.eligibility(db, n, t, both).analysis_shared.contains(&func.into()) + }) + } + + pub(super) fn analysis_source( + &self, + db: &dyn HirDatabase, + func: FunctionId, + ) -> Option { + if self.is_empty() || !self.analysis_shared(db, func) { + return None; + } + for &(normal, test, both) in &self.pairs { + if func.krate(db) == test + && self.eligibility(db, normal, test, both).pair_veto.is_empty() + && let Some(&AttrDefId::FunctionId(source)) = + self.twins(db, normal, test).twins.get(&func.into()) + { + return Some(source); + } + } + None + } + + pub(super) fn analysis_fold<'db, T: TypeFoldable>>( + &self, + db: &'db dyn HirDatabase, + source: FunctionId, + value: T, + ) -> T { + value.fold_with(&mut Canonical { + db, + sharing: self, + interner: DbInterner::new_with(db, source.krate(db)), + reverse: false, + analysis: Some(source.krate(db)), + opaque_only: false, + }) + } + + pub(super) fn body_source(&self, db: &dyn HirDatabase, func: FunctionId) -> Option { + self.analysis_source(db, func).or_else(|| { + let canonical = self.function(db, func); + (canonical != func).then_some(canonical) + }) + } + + pub(super) fn reuse_opaques<'db, T: TypeFoldable>>( + &self, + db: &'db dyn HirDatabase, + krate: Crate, + value: T, + ) -> T { + value.fold_with(&mut Canonical { + db, + sharing: self, + interner: DbInterner::new_with(db, krate), + reverse: false, + analysis: None, + opaque_only: true, + }) + } + + pub(super) fn analysis_item( + &self, + db: &dyn HirDatabase, + source: FunctionId, + item: AttrDefId, + ) -> AttrDefId { + let normal = source.krate(db); + for &(n, t, _) in &self.pairs { + if n == normal && item.module(db).krate(db) == n { + return self.reverse(db, n, t).get(&item).copied().unwrap_or(item); + } + } + item + } + + pub(super) fn analysis_instance( + &self, + db: &dyn HirDatabase, + def: InstanceDef, + ) -> Option<(FunctionId, InstanceDef)> { + use salsa::plumbing::{AsId, FromId}; + let owner = match def { + InstanceDef::Fn(f) => { + return self.body_source(db, f).map(|source| (source, InstanceDef::Fn(source))); + } + InstanceDef::Closure(id) => InternedClosureId::from_id(id).loc(db).owner, + InstanceDef::Coroutine(id) | InstanceDef::CoroutineByMove(id) => { + InternedCoroutineId::from_id(id).loc(db).owner + } + _ => return None, + }; + let InferBodyId::DefWithBodyId(DefWithBodyId::FunctionId(func)) = owner else { + return None; + }; + let source = self.body_source(db, func)?; + let def = match def { + InstanceDef::Closure(id) => { + let mut loc = InternedClosureId::from_id(id).loc(db); + loc.owner = source.into(); + InstanceDef::Closure(InternedClosureId::new(db, loc).as_id()) + } + InstanceDef::Coroutine(id) | InstanceDef::CoroutineByMove(id) => { + let mut loc = InternedCoroutineId::from_id(id).loc(db); + loc.owner = source.into(); + let id = InternedCoroutineId::new(db, loc).as_id(); + if matches!(def, InstanceDef::CoroutineByMove(_)) { + InstanceDef::CoroutineByMove(id) + } else { + InstanceDef::Coroutine(id) + } + } + _ => return None, + }; + Some((source, def)) + } + + pub(super) fn function(&self, db: &dyn HirDatabase, func: FunctionId) -> FunctionId { + match self.item(db, func.into()) { + AttrDefId::FunctionId(it) => it, + _ => func, + } + } + + fn owner<'db>(&self, db: &'db dyn HirDatabase, owner: InferBodyId<'db>) -> InferBodyId<'db> { + match owner { + InferBodyId::DefWithBodyId(DefWithBodyId::FunctionId(f)) => self.function(db, f).into(), + _ => owner, + } + } + + fn def(&self, db: &dyn HirDatabase, def: InstanceDef) -> InstanceDef { + use salsa::plumbing::{AsId, FromId}; + match def { + InstanceDef::Fn(f) => InstanceDef::Fn(self.function(db, f)), + InstanceDef::VtableShim(f) => InstanceDef::VtableShim(self.function(db, f)), + InstanceDef::Closure(id) => { + let mut loc = InternedClosureId::from_id(id).loc(db); + loc.owner = self.owner(db, loc.owner); + InstanceDef::Closure(InternedClosureId::new(db, loc).as_id()) + } + InstanceDef::Coroutine(id) | InstanceDef::CoroutineByMove(id) => { + let mut loc = InternedCoroutineId::from_id(id).loc(db); + loc.owner = self.owner(db, loc.owner); + let id = InternedCoroutineId::new(db, loc).as_id(); + if matches!(def, InstanceDef::Coroutine(_)) { + InstanceDef::Coroutine(id) + } else { + InstanceDef::CoroutineByMove(id) + } + } + _ => def, + } + } + + pub(super) fn fold<'db, T: TypeFoldable>>( + &self, + db: &'db dyn HirDatabase, + krate: Crate, + value: T, + ) -> T { + if self.is_empty() { + return value; + } + value.fold_with(&mut Canonical { + db, + sharing: self, + interner: DbInterner::new_with(db, krate), + reverse: false, + analysis: None, + opaque_only: false, + }) + } + + pub(super) fn link_type<'db>(&self, db: &'db dyn HirDatabase, ty: Ty<'db>) -> Ty<'db> { + if self.is_empty() { + return ty; + } + ty.fold_with(&mut Canonical { + db, + sharing: self, + interner: DbInterner::new_no_crate(db), + reverse: true, + analysis: None, + opaque_only: false, + }) + } + + pub(super) fn coroutine_is_by_move<'db>( + &self, + db: &'db dyn HirDatabase, + id: InternedCoroutineId<'db>, + args: rustc_type_ir::CoroutineArgs>, + ) -> bool { + use salsa::plumbing::{AsId, FromId}; + let id = match self.analysis_instance(db, InstanceDef::Coroutine(id.as_id())) { + Some((_, InstanceDef::Coroutine(source))) => InternedCoroutineId::from_id(source), + _ => id, + }; + crate::mir::is_by_move_coroutine(db, id, args) + } + + fn link_item(&self, db: &dyn HirDatabase, item: AttrDefId) -> AttrDefId { + for &(normal, test, both) in &self.pairs { + if !both && item.module(db).krate(db) == normal { + if let Some(&t) = self.reverse(db, normal, test).get(&item) { + return t; + } + } + } + item + } + + pub(super) fn instance(&self, db: &dyn HirDatabase, instance: &Instance) -> Instance { + if self.is_empty() { + return instance.clone(); + } + super::backend::Tally::global() + .step(super::Step::Canonicalize, || self.canonical_instance(db, instance)) + } + + fn canonical_instance(&self, db: &dyn HirDatabase, instance: &Instance) -> Instance { + let Some(krate) = self.pairs.first().map(|it| it.0) else { return instance.clone() }; + let args = self.fold(db, krate, instance.args.as_ref()); + // A test-only argument can select an impl absent from N's trait namespace. + // Keep that instance's semantic identity in T until instance-level eligibility proves it. + let mut remaining = TestTypes { db, sharing: self, found: false }; + args.visit_with(&mut remaining); + let def = if remaining.found { instance.def } else { self.def(db, instance.def) }; + let canonical = Instance { + def, + args: if remaining.found { instance.args.clone() } else { args.store() }, + checked: instance.checked, + }; + if self.test_body(db, &canonical) { instance.clone() } else { canonical } + } + + pub(super) fn env( + &self, + db: &dyn HirDatabase, + instance: &Instance, + env: &StoredParamEnvAndCrate, + ) -> StoredParamEnvAndCrate { + if self.is_empty() || self.needs_policy(db, instance) { + return env.clone(); + } + let home = super::home::Homes::default().home(db, instance, env); + if home.krate == env.krate { + return env.clone(); + } + let env = env.as_ref(db); + let krate = home.krate; + ParamEnvAndCrate { krate, param_env: self.fold(db, krate, env.param_env) }.store() + } + + pub(super) fn needs_policy(&self, db: &dyn HirDatabase, instance: &Instance) -> bool { + let mut types = TestTypes { db, sharing: self, found: false }; + instance.args.as_ref().visit_with(&mut types); + types.found || self.test_body(db, instance) + } + + fn test_body(&self, db: &dyn HirDatabase, instance: &Instance) -> bool { + let owner = match instance.def { + InstanceDef::VtableShim(func) => Some(func), + _ => super::home::written_in(db, instance), + }; + owner.is_some_and(|f| self.test_crate(f.krate(db))) + } + + pub(super) fn link_instance(&self, db: &dyn HirDatabase, instance: &Instance) -> Instance { + let mut instance = instance.clone(); + if let InstanceDef::Fn(func) = instance.def { + instance.def = InstanceDef::Fn(self.link_function(db, func)); + } + instance + } + + pub(super) fn link_static( + &self, + db: &dyn HirDatabase, + id: hir_def::StaticId, + ) -> hir_def::StaticId { + if self.is_empty() { + return id; + } + for &(normal, test, both) in &self.pairs { + if !both && id.krate(db) == normal { + if let Some((&AttrDefId::StaticId(t), _)) = + self.twins(db, normal, test).twins.iter().find(|(_, n)| **n == id.into()) + { + return t; + } + } + } + id + } + + pub(super) fn contains_unified(&self, db: &dyn HirDatabase, ty: Ty<'_>) -> bool { + if !self.tentative() { + return false; + } + let mut visitor = Unified { db, sharing: self, found: false }; + ty.visit_with(&mut visitor); + visitor.found + } + + pub(super) fn link_function(&self, db: &dyn HirDatabase, func: FunctionId) -> FunctionId { + if self.is_empty() { + return func; + } + for &(normal, test, both) in &self.pairs { + if !both && func.krate(db) == normal { + if let Some(&AttrDefId::FunctionId(t)) = + self.reverse(db, normal, test).get(&func.into()) + { + return t; + } + } + } + func + } +} + +struct Canonical<'a, 'db> { + db: &'db dyn HirDatabase, + sharing: &'a Sharing, + interner: DbInterner<'db>, + reverse: bool, + analysis: Option, + opaque_only: bool, +} + +impl<'db> Canonical<'_, 'db> { + fn item(&self, item: AttrDefId) -> AttrDefId { + if self.opaque_only { + return item; + } + if let Some(normal) = self.analysis { + for &(n, t, _) in &self.sharing.pairs { + if n == normal && item.module(self.db).krate(self.db) == n { + return self.sharing.reverse(self.db, n, t).get(&item).copied().unwrap_or(item); + } + } + return item; + } + if self.reverse { + self.sharing.link_item(self.db, item) + } else { + self.sharing.item(self.db, item) + } + } + + fn function(&self, id: FunctionId) -> FunctionId { + match self.item(id.into()) { + AttrDefId::FunctionId(id) => id, + _ => id, + } + } + + fn owner(&self, id: InferBodyId<'db>) -> InferBodyId<'db> { + match id { + InferBodyId::DefWithBodyId(DefWithBodyId::FunctionId(f)) => self.function(f).into(), + _ => id, + } + } + + fn solver_def( + &self, + id: crate::next_solver::SolverDefId<'db>, + ) -> crate::next_solver::SolverDefId<'db> { + use crate::{ImplTraitId, db::InternedOpaqueTyId, next_solver::SolverDefId}; + match id { + SolverDefId::ConstId(id) => match self.item(id.into()) { + AttrDefId::ConstId(id) => id.into(), + _ => id.into(), + }, + SolverDefId::TypeAliasId(id) => match self.item(id.into()) { + AttrDefId::TypeAliasId(id) => id.into(), + _ => id.into(), + }, + SolverDefId::InternedOpaqueTyId(id) => { + let loc = match id.loc(self.db) { + ImplTraitId::ReturnTypeImplTrait(func, index) => { + ImplTraitId::ReturnTypeImplTrait(self.function(func), index) + } + ImplTraitId::TypeAliasImplTrait(alias, index) => { + match self.item(alias.into()) { + AttrDefId::TypeAliasId(alias) => { + ImplTraitId::TypeAliasImplTrait(alias, index) + } + _ => id.loc(self.db), + } + } + }; + InternedOpaqueTyId::new(self.db, loc).into() + } + _ => id, + } + } +} + +impl<'db> TypeFolder> for Canonical<'_, 'db> { + fn cx(&self) -> DbInterner<'db> { + self.interner + } + + fn fold_const( + &mut self, + value: crate::next_solver::Const<'db>, + ) -> crate::next_solver::Const<'db> { + use crate::{ + db::GeneralConstId, + next_solver::{Const, ConstKind}, + }; + let value = value.super_fold_with(self); + if let ConstKind::Unevaluated(mut unevaluated) = value.kind() + && let GeneralConstId::ConstId(id) = unevaluated.def.0 + && let AttrDefId::ConstId(id) = self.item(id.into()) + { + unevaluated.def = GeneralConstId::ConstId(id).into(); + return Const::new(self.interner, ConstKind::Unevaluated(unevaluated)); + } + value + } + + fn fold_predicate( + &mut self, + value: crate::next_solver::Predicate<'db>, + ) -> crate::next_solver::Predicate<'db> { + use crate::next_solver::{ClauseKind, Predicate, PredicateKind}; + let value = value.super_fold_with(self).kind().map_bound(|mut kind| { + if let PredicateKind::Clause(ClauseKind::Trait(tr)) = &mut kind + && let AttrDefId::TraitId(id) = self.item(tr.trait_ref.def_id.0.into()) + { + tr.trait_ref.def_id = id.into(); + } + if let PredicateKind::Clause(ClauseKind::Projection(projection)) = &mut kind { + let term = projection.projection_term; + projection.projection_term = rustc_type_ir::AliasTerm::new_from_def_id( + self.interner, + self.solver_def(term.def_id()), + term.args, + ); + } + kind + }); + Predicate::new(self.interner, value) + } + + fn fold_ty(&mut self, ty: Ty<'db>) -> Ty<'db> { + match ty.kind() { + TyKind::Adt(adt, args) => { + if let AttrDefId::AdtId(id) = self.item(adt.def_id().into()) { + return Ty::new_adt(self.interner, id, args.fold_with(self)); + } + } + TyKind::FnDef(def, args) => { + let def = match def.0 { + CallableDefId::FunctionId(f) => self.function(f).into(), + CallableDefId::StructId(s) => { + match self.item(hir_def::AdtId::StructId(s).into()) { + AttrDefId::AdtId(hir_def::AdtId::StructId(s)) => s.into(), + _ => def.0, + } + } + CallableDefId::EnumVariantId(v) => match self.item(v.into()) { + AttrDefId::EnumVariantId(v) => v.into(), + _ => def.0, + }, + }; + return Ty::new_fn_def(self.interner, def.into(), args.fold_with(self)); + } + TyKind::Closure(id, args) => { + let mut loc = id.0.loc(self.db); + loc.owner = self.owner(loc.owner); + return Ty::new_closure( + self.interner, + InternedClosureId::new(self.db, loc).into(), + args.fold_with(self), + ); + } + TyKind::Coroutine(id, args) => { + let mut loc = id.0.loc(self.db); + loc.owner = self.owner(loc.owner); + return Ty::new_coroutine( + self.interner, + InternedCoroutineId::new(self.db, loc).into(), + args.fold_with(self), + ); + } + TyKind::CoroutineClosure(id, args) => { + let mut loc = id.0.loc(self.db); + loc.owner = self.owner(loc.owner); + return Ty::new_coroutine_closure( + self.interner, + InternedCoroutineClosureId::new(self.db, loc).into(), + args.fold_with(self), + ); + } + TyKind::Dynamic(bounds, region) => { + use crate::next_solver::{BoundExistentialPredicates, ExistentialPredicate}; + let interner = self.interner; + let predicates = bounds.iter().map(|bound| { + bound.map_bound(|pred| { + let mut pred = pred.fold_with(self); + match &mut pred { + ExistentialPredicate::Trait(tr) => { + if let AttrDefId::TraitId(id) = self.item(tr.def_id.0.into()) { + tr.def_id = id.into(); + } + } + ExistentialPredicate::AutoTrait(id) => { + if let AttrDefId::TraitId(mapped) = self.item(id.0.into()) { + *id = mapped.into(); + } + } + ExistentialPredicate::Projection(projection) => { + projection.def_id = self + .solver_def(projection.def_id.into()) + .try_into() + .unwrap_or(projection.def_id); + } + } + pred + }) + }); + return Ty::new_dynamic( + interner, + BoundExistentialPredicates::new_from_iter(predicates), + region, + ); + } + TyKind::Alias(mut alias) => { + alias.kind = rustc_type_ir::AliasTyKind::new_from_def_id( + self.interner, + self.solver_def(alias.kind.def_id()), + ); + alias.args = alias.args.fold_with(self); + return Ty::new(self.interner, TyKind::Alias(alias)); + } + _ => {} + } + ty.super_fold_with(self) + } +} + +struct TestTypes<'a, 'db> { + db: &'db dyn HirDatabase, + sharing: &'a Sharing, + found: bool, +} +impl<'db> TypeVisitor> for TestTypes<'_, 'db> { + type Result = (); + fn visit_ty(&mut self, ty: Ty<'db>) { + if let Some(item) = type_item(self.db, ty) { + self.found |= self.sharing.test_crate(item.module(self.db).krate(self.db)); + } + ty.super_visit_with(self); + } +} + +struct Unified<'a, 'db> { + db: &'db dyn HirDatabase, + sharing: &'a Sharing, + found: bool, +} +impl<'db> TypeVisitor> for Unified<'_, 'db> { + type Result = (); + fn visit_ty(&mut self, ty: Ty<'db>) { + if let Some(item) = type_item(self.db, ty) { + for &(normal, test, both) in &self.sharing.pairs { + if !both { + continue; + } + let twins = self.sharing.twins(self.db, normal, test); + self.found |= twins + .twins + .iter() + .any(|(&t, &n)| (item == t || item == n) && self.sharing.item(self.db, t) == n); + } + } + ty.super_visit_with(self); + } +} + +fn type_item(db: &dyn HirDatabase, ty: Ty<'_>) -> Option { + let owner = match ty.kind() { + TyKind::Adt(adt, _) => return Some(adt.def_id().into()), + TyKind::FnDef(def, _) => { + return Some(match def.0 { + CallableDefId::FunctionId(id) => id.into(), + CallableDefId::StructId(id) => hir_def::AdtId::StructId(id).into(), + CallableDefId::EnumVariantId(id) => id.into(), + }); + } + TyKind::Dynamic(bounds, _) => { + use rustc_type_ir::inherent::BoundExistentialPredicates as _; + return bounds.principal_def_id().map(|id| id.0.into()); + } + TyKind::Alias(alias) => match alias.kind.def_id() { + crate::next_solver::SolverDefId::TypeAliasId(id) => return Some(id.into()), + crate::next_solver::SolverDefId::InternedOpaqueTyId(id) => { + return Some(match id.loc(db) { + crate::ImplTraitId::ReturnTypeImplTrait(id, _) => id.into(), + crate::ImplTraitId::TypeAliasImplTrait(id, _) => id.into(), + }); + } + _ => return None, + }, + TyKind::Closure(id, _) => id.0.loc(db).owner, + TyKind::Coroutine(id, _) => id.0.loc(db).owner, + _ => return None, + }; + match owner { + InferBodyId::DefWithBodyId(DefWithBodyId::FunctionId(id)) => Some(id.into()), + _ => None, + } +} diff --git a/crates/hir-ty-codegen/store.rs b/crates/hir-ty-codegen/store.rs index 133cf53548..9be4bcc00d 100644 --- a/crates/hir-ty-codegen/store.rs +++ b/crates/hir-ty-codegen/store.rs @@ -974,6 +974,11 @@ pub fn is_off(value: &std::ffi::OsStr) -> bool { } impl Store { + #[cfg(test)] + pub(super) fn for_test(dir: &Path) -> Store { + Self::open(dir, Identity::new(b"test", "test"), Tuning { seal_at: 1, limit: None }).unwrap() + } + /// The store of this process: the directory `RUSTD_CACHE` names, when it names one. A /// directory that cannot be used is said once, and the process compiles without one. pub(super) fn global() -> Option<&'static Store> { diff --git a/crates/hir-ty-codegen/type_id.rs b/crates/hir-ty-codegen/type_id.rs index a575fbd114..3042d64201 100644 --- a/crates/hir-ty-codegen/type_id.rs +++ b/crates/hir-ty-codegen/type_id.rs @@ -107,6 +107,7 @@ fn hash_str(hasher: &mut StableSipHasher128, text: &str) { } pub(super) struct TypeIdHasher<'a, 'db> { + sharing: super::Sharing, db: &'db dyn HirDatabase, symbols: &'a dyn SymbolResolver, /// The binders around what is being hashed, outermost first: for each, its bound @@ -133,8 +134,10 @@ impl<'a, 'db> TypeIdHasher<'a, 'db> { db: &'db dyn HirDatabase, symbols: &'a dyn SymbolResolver, owned: &'a FxHashSet, + sharing: super::Sharing, ) -> Self { Self { + sharing, db, symbols, binders: Vec::new(), @@ -573,7 +576,7 @@ impl<'a, 'db> TypeIdHasher<'a, 'db> { self.closure(hasher, id.0.loc(self.db))?; // The future of an `async` closure is another type when it owns what the // closure captured. - let by_move = crate::mir::is_by_move_coroutine(self.db, id.0, args.as_coroutine()); + let by_move = self.sharing.coroutine_is_by_move(self.db, id.0, args.as_coroutine()); hasher.write_u8(by_move.into()); let parent = GenericArgs::new_from_slice(args.as_coroutine().parent_args()); self.args(parent)?.hash(hasher); diff --git a/crates/rustd-wasm/src/lib.rs b/crates/rustd-wasm/src/lib.rs index ec0c289c04..56787b68b7 100644 --- a/crates/rustd-wasm/src/lib.rs +++ b/crates/rustd-wasm/src/lib.rs @@ -387,14 +387,14 @@ impl Compiler { // those are shows in what a build leaves undefined. known.patching = true; let unit = loop { - let roots = - Roots { - entries: &entries, - on_demand: &on_demand, - start: None, - reports: &[], - kept: &[], - }; + let roots = Roots { + sharing: Default::default(), + entries: &entries, + on_demand: &on_demand, + start: None, + reports: &[], + kept: &[], + }; let unit = hir::attach_db(db, || { CodegenUnit::compile(db, resolver, roots, &Default::default(), known) }) diff --git a/crates/rustd/src/cargo/daemon.rs b/crates/rustd/src/cargo/daemon.rs index d9a80f9c2b..16b2c3b307 100644 --- a/crates/rustd/src/cargo/daemon.rs +++ b/crates/rustd/src/cargo/daemon.rs @@ -135,7 +135,6 @@ pub(crate) struct WorkerConfig { pub(crate) repository: PathBuf, /// The directory of the package being built, from the root of the checkout. pub(crate) project: PathBuf, - pub(crate) cfg_test: bool, /// Which build the worker stands in for, as the daemon's own plan of it has it. pub(crate) selection: crate::plan::Selection, /// The variables the worker runs with that decide what it compiles and with what. @@ -155,9 +154,6 @@ impl WorkerConfig { fn args(&self, root: &Path, crates: &[PathBuf]) -> Vec { let mut args: Vec = vec![root.join(&self.project).into(), "--own-target".into()]; args.extend(self.selection.flags().into_iter().map(Into::into)); - if self.cfg_test { - args.push("--cfg-test".into()); - } if self.selection.dev_deps { args.push("--dev-deps".into()); } @@ -190,6 +186,7 @@ pub(crate) struct Checkout { /// When cargo started, as a stamp: source newer than this is not what rustc compiled. pub(crate) built_at: u128, pub(crate) config: WorkerConfig, + pub(crate) unit_test: bool, /// The root of the checkout. pub(crate) root: PathBuf, pub(crate) programs: Vec, @@ -219,13 +216,14 @@ impl Checkout { // Each with a comma after it: the program is of these crates and no others. let crates: String = plan.crates.iter().map(|it| named(it) + ",").collect(); let mut line = format!( - "{verb}\t{}\t{}\t{}\t{}\t{}\t{}", + "{verb}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", self.project().display(), crates, plan.object.display(), plan.patch.display(), plan.link_cmd, - plan.binary.display() + plan.binary.display(), + self.unit_test ); for file in &plan.symbol_files { line.push('\t'); @@ -286,6 +284,9 @@ impl Worker { let _ = std::fs::remove_file(&socket); let config = &checkout.config; let mut args = config.args(&checkout.root, &first.crates); + if checkout.unit_test { + args.push("--cfg-test".into()); + } let named: [(&str, &Path); 5] = [ ("--out", &first.object), ("--patch", &first.patch), diff --git a/crates/rustd/src/cargo/mod.rs b/crates/rustd/src/cargo/mod.rs index c6e3b17489..f67767fb04 100644 --- a/crates/rustd/src/cargo/mod.rs +++ b/crates/rustd/src/cargo/mod.rs @@ -206,13 +206,35 @@ enum Left { /// them. Rustc sees none of it: the daemon builds what rust-analyzer passes, which has no /// borrow checker and few of rustc's lints. fn by_daemon(mut state: State) -> Result { + let start = std::time::Instant::now(); let changed = state.refusal.clone().or_else(|| state.broken_guard()); - if let Some(reason) = changed.or_else(|| state.stale_cargos()) { + let changed = changed.or_else(|| state.stale_cargos()); + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("client fingerprints: {:.1?}", start.elapsed()); + } + if let Some(reason) = changed { return Err(Left::Again(Box::new(state), reason)); } - let answers = - state.checkouts.iter().map(built).collect::, _>>().map_err(Left::Failed)?; + let start = std::time::Instant::now(); + let answers = std::thread::scope(|scope| { + let requests: Vec<_> = + state.checkouts.iter().map(|checkout| scope.spawn(|| built(checkout))).collect(); + requests + .into_iter() + .map(|request| { + request.join().unwrap_or_else(|_| Err("a daemon request panicked".to_owned())) + }) + .collect::, _>>() + }) + .map_err(Left::Failed)?; + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("client daemon: {:.1?} ({} checkouts)", start.elapsed(), state.checkouts.len()); + } + let start = std::time::Instant::now(); let cargos = state.settle(&answers).map_err(Left::Failed)?; + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!("client settle: {:.1?}", start.elapsed()); + } if let Some((_, why)) = cargos.first() { return Err(Left::Failed(why.clone())); } diff --git a/crates/rustd/src/cargo/state.rs b/crates/rustd/src/cargo/state.rs index 77fb0f03e9..a35567ee28 100644 --- a/crates/rustd/src/cargo/state.rs +++ b/crates/rustd/src/cargo/state.rs @@ -332,7 +332,6 @@ impl Staged { let config = WorkerConfig { repository: repository(&root), project: project.strip_prefix(&root).unwrap_or(Path::new("")).to_owned(), - cfg_test: false, selection: asked.selection(), env: worker_env(), }; @@ -639,10 +638,9 @@ impl State { BTreeMap::new() } }; - let config = |unit_test: bool| WorkerConfig { + let config = WorkerConfig { repository: repository(&root), project: project.strip_prefix(&root).unwrap_or(Path::new("")).to_owned(), - cfg_test: unit_test, selection: asked.selection(), env: worker_env(), }; @@ -651,7 +649,8 @@ impl State { key: format!("{key}{}", if unit_test { "-unit" } else { "" }), generation: 0, built_at: built.started, - config: config(unit_test), + config: config.clone(), + unit_test, root: root.clone(), programs: Vec::new(), sources: Vec::new(), diff --git a/crates/rustd/src/main.rs b/crates/rustd/src/main.rs index d0bd0896e4..249359c0bc 100644 --- a/crates/rustd/src/main.rs +++ b/crates/rustd/src/main.rs @@ -47,8 +47,8 @@ use std::{ use anyhow::{Context, bail}; use hir::{AsAssocItem, AssocItemContainer, Crate, Module, ModuleDef, db::HirDatabase}; use hir_def::{ - AssocItemId, DefWithBodyId, FunctionId, ItemContainerId, Lookup, ModuleDefId, StaticId, - expr_store::Body, item_scope::ItemScope, + AssocItemId, DefWithBodyId, FunctionId, HasModule, ItemContainerId, Lookup, ModuleDefId, + StaticId, expr_store::Body, item_scope::ItemScope, }; use hir_ty::db::InternedClosureId; use hir_ty::mir::codegen::{ @@ -109,8 +109,8 @@ struct Args { deny: Option, /// Directory that gets one object per compiled function, for testing each in isolation. isolate: Option, - /// Build the workspace's crates as `cargo test` does, with `cfg(test)`. - cfg_test: bool, + /// This program is the `cfg(test)` build of its crate. + unit_test: bool, /// Run build scripts and build proc macros in a directory of rustd's own under the /// target directory, so that nothing here waits for a cargo that is building there. own_target: bool, @@ -131,6 +131,8 @@ struct Args { base: Option, /// The crates of `checkout`. One that is the same in another checkout is of both. members: Arc>, + /// Root files of member libs/bins, whose tested copies a normal program leaves out. + unit_roots: Arc>, } impl Args { @@ -151,6 +153,12 @@ impl Args { /// `cfg(test)` as the program of the tests, and without for whatever links it. fn as_built(&self, db: &RootDatabase, krate: Crate, named: &str) -> bool { let krate = ide_db::base_db::Crate::from(krate); + if !self.unit_test { + return !self.selection.dev_deps + || !is_tested(db, krate) + || crate_root_file(db, krate.into()) + .is_none_or(|root| !self.unit_roots.contains(&root)); + } let member = matches!(krate.data(db).origin, CrateOrigin::Local { .. }); let own = named == self.krate; // What the tests of a library use may use the library: the program then has both. @@ -261,7 +269,7 @@ impl Args { report, deny, isolate, - cfg_test, + unit_test: cfg_test, own_target, selection: plan::Selection { profile: profile.unwrap_or_else(|| "dev".to_owned()), @@ -279,6 +287,7 @@ impl Args { checkout: PathBuf::new(), base: None, members: Arc::default(), + unit_roots: Arc::default(), }) } } @@ -1867,7 +1876,7 @@ fn crates_called(db: &RootDatabase, krate: &str, args: &Args) -> Vec { }), false => it.display_name(db).is_some_and(|name| name.crate_name().as_str() == krate), }) - .filter(|it| !args.cfg_test || args.as_built(db, *it, krate)) + .filter(|it| args.as_built(db, *it, krate)) .filter(|it| !doctest::is_documented(db, (*it).into())) .collect() } @@ -1981,6 +1990,17 @@ impl Checkouts { let at = self.roots.iter().position(|(_, it)| *it == root).expect("it has a root"); let base = self.splits[at].as_ref().map(|it| it.base.clone()); let members = Arc::new(self.loaded.crates(at).iter().copied().collect()); + let mut unit_roots = FxHashSet::default(); + if let ProjectWorkspaceKind::Cargo { cargo, .. } = &self.loaded.workspaces()[at].kind { + for package in cargo.packages().filter(|&it| cargo[it].is_member) { + for &target in &cargo[package].targets { + if matches!(cargo[target].kind, TargetKind::Lib { .. } | TargetKind::Bin) { + unit_roots.insert(PathBuf::from(cargo[target].root.clone())); + } + } + } + } + let unit_roots = Arc::new(unit_roots); // An override for measuring one against the other. let level = match std::env::var("RUSTD_DEBUGINFO").ok().as_deref() { Some("0") => plan::Level::Nothing, @@ -1989,7 +2009,7 @@ impl Checkouts { None => self.debug[at].debug.level, }; let debug = plan::Described { level, ..self.debug[at].debug }; - Ok(Parts { root, base, members, debug }) + Ok(Parts { root, base, members, unit_roots, debug }) } /// Whether the program of the checkout `project` is of is one rustc links from bitcode. @@ -2380,6 +2400,7 @@ struct Parts { root: PathBuf, base: Option, members: Arc>, + unit_roots: Arc>, /// What the build has a debugger told. debug: plan::Described, } @@ -2632,9 +2653,146 @@ fn resolve_crates(db: &RootDatabase, args: &Args) -> anyhow::Result { } } resolve_all(db, &crates); + hir::attach_db(db, || dump_twins(db, args)); Ok(crates.len()) } +fn normal_twin( + db: &RootDatabase, + args: &Args, + test: ide_db::base_db::Crate, +) -> Option { + let mut normal = args.members.iter().copied().filter(|&krate| { + !is_tested(db, krate) + && !doctest::is_documented(db, krate) + && krate.root_file_id(db) == test.root_file_id(db) + }); + normal.next().filter(|_| normal.next().is_none()) +} + +fn program_sharing(db: &RootDatabase, args: &Args) -> hir_ty::mir::codegen::Sharing { + let roots: Vec<_> = args + .crates() + .flat_map(|name| crates_named(db, name, args).unwrap_or_default()) + .map(ide_db::base_db::Crate::from) + .collect(); + let present: FxHashSet<_> = roots + .iter() + .copied() + .chain(roots.iter().flat_map(|&krate| krate.transitive_deps(db))) + .collect(); + let mut pairs = Vec::new(); + for &test in args.members.iter() { + if !is_tested(db, test) || !present.contains(&test) { + continue; + } + if let Some(normal) = normal_twin(db, args, test) { + pairs.push((normal, test, present.contains(&normal))); + } + } + let sharing = hir_ty::mir::codegen::Sharing::new(pairs); + if std::env::var_os("RUSTD_CFG_SHARING").is_some_and(|it| it == "0") { + sharing.disabled() + } else { + sharing + } +} + +fn dump_twins(db: &RootDatabase, args: &Args) { + if std::env::var_os("RUSTD_CFG_TWINS").is_none() { + return; + } + let program_roots: Vec<_> = args + .crates() + .flat_map(|name| crates_named(db, name, args).unwrap_or_default()) + .map(ide_db::base_db::Crate::from) + .collect(); + let present: FxHashSet<_> = program_roots + .iter() + .copied() + .chain(program_roots.iter().flat_map(|&krate| krate.transitive_deps(db))) + .collect(); + let roots: FxHashSet<_> = program_roots.iter().map(|krate| krate.root_file_id(db)).collect(); + for &test in args + .members + .iter() + .filter(|&&krate| is_tested(db, krate) && roots.contains(&krate.root_file_id(db))) + { + let Some(normal) = normal_twin(db, args, test) else { continue }; + let index = hir_def::twins::twin_index(db, normal, test); + let name = hir::Crate::from(test) + .display_name(db) + .map(|name| name.to_string()) + .unwrap_or_default(); + eprintln!( + "cfg twins: {name}: twins {}, cfg-differing {}, test-only {}, unmatched {}, normal-only {}, unknown {}; function bodies: twins {}, cfg-differing {}, test-only {}, unmatched {}, unknown {}", + index.twins.len(), + index.cfg_differing, + index.test_only.len(), + index.unmatched.len(), + index.normal_only, + index.unknown, + index.function_bodies.twins, + index.function_bodies.cfg_differing, + index.function_bodies.test_only, + index.function_bodies.unmatched, + index.function_bodies.unknown + ); + let both = present.contains(&normal) && present.contains(&test); + let eligibility = hir_ty::sharing::pair_eligibility(db, normal, test, both); + let bodies = |items: &FxHashSet| { + items.intersection(&index.function_items).count() + }; + let rejected = |reason| { + eligibility + .rejected + .iter() + .filter(|(id, why)| **why == reason && index.function_items.contains(id)) + .count() + }; + let veto = if eligibility.pair_veto.is_empty() { + "none".to_owned() + } else { + eligibility.pair_veto.join(",") + }; + let enabled = eligibility.pair_veto.is_empty(); + eprintln!( + "cfg eligibility: {name}: fully-shared bodies {}; analysis-shared bodies {}; rejected bodies: cfg-differing {}, transitive {}, generic-unresolved {}, unknown {}; statics {}, test-only statics {}; pair veto: {veto}; both copies {both}; TypeId: deferred to codegen", + if enabled { bodies(&eligibility.candidates) } else { 0 }, + if enabled { bodies(&eligibility.analysis_shared) } else { 0 }, + rejected(hir_ty::sharing::Rejection::CfgDiffering), + rejected(hir_ty::sharing::Rejection::Transitive), + rejected(hir_ty::sharing::Rejection::GenericUnresolved), + rejected(hir_ty::sharing::Rejection::Unknown), + eligibility.statics, + eligibility.test_only_statics, + ); + if std::env::var("RUSTD_CFG_TWINS").is_ok_and(|value| value == "items") { + for &item in &index.function_items { + if !index.twins.contains_key(&item) { + continue; + } + let hir_def::AttrDefId::FunctionId(id) = item else { continue }; + let shared = if eligibility.analysis_shared.contains(&item) { + "analysis-shared" + } else { + "fully-shared" + }; + let outcome = eligibility.rejected.get(&item).map_or(shared, |why| match why { + hir_ty::sharing::Rejection::CfgDiffering => "cfg-differing", + hir_ty::sharing::Rejection::Transitive => "transitive", + hir_ty::sharing::Rejection::GenericUnresolved => "generic-unresolved", + hir_ty::sharing::Rejection::Unknown => "unknown", + }); + eprintln!( + "cfg body: {name}::{}: {outcome}", + hir::Function::from(id).name(db).as_str() + ); + } + } + } +} + /// Resolves the names of `crates` again after an edit to many files, on every core: whoever /// asks first otherwise reads every file that changed on one. fn resolve_names(db: &RootDatabase, crates: &[ide_db::base_db::Crate]) { @@ -2898,6 +3056,21 @@ fn declared_in(db: &RootDatabase, krate: Crate) -> Declared { fn functions_of_modules(db: &RootDatabase, krate: Crate) -> Vec { let mut declared = Declared::default(); declared.in_modules(db, krate); + let mut modules = vec![krate.root_module(db)]; + while let Some(module) = modules.pop() { + modules.extend(module.children(db)); + for def in module.declarations(db) { + if let hir::ModuleDef::Trait(trait_) = def { + let trait_: hir_def::TraitId = trait_.into(); + declared.functions.extend(trait_.trait_items(db).items.iter().filter_map( + |&(_, item)| match item { + AssocItemId::FunctionId(func) => Some(func), + _ => None, + }, + )); + } + } + } declared.functions } @@ -2977,12 +3150,30 @@ fn lower(db: &RootDatabase, bodies: Vec) -> Vec { bodies } +type SkippedReport = Arc<[(String, CodegenError)]>; + +#[derive(Clone, Default)] +struct BuildText { + skipped: Option, + harness: Option>, +} + +impl BuildText { + fn skipped(&mut self, verify: bool, fresh: impl Fn() -> SkippedReport) -> SkippedReport { + if verify && let Some(cached) = &self.skipped { + assert_eq!(cached, &fresh(), "skipped diagnostics fast path"); + } + self.skipped.get_or_insert_with(fresh).clone() + } +} + struct Build { compiled: usize, - skipped: Vec<(String, CodegenError)>, + skipped: SkippedReport, instances: usize, codegen: Duration, emit: Duration, + emitted: [Duration; 4], /// How long finding the roots, checking what is cached and compiling took, in that order. steps: [Duration; 3], /// Within the checking, the longest a thread took over each kind of cached thing. @@ -3004,8 +3195,10 @@ fn build( args: &Args, known: &mut Rejections, listed: &Listed, + text: &mut BuildText, ) -> anyhow::Result { let start = Instant::now(); + hir::attach_db(db, || dump_twins(db, args)); let contents = &content::Contents::default(); let (root, base) = (args.checkout.clone(), args.base.clone()); let debug = args.debug; @@ -3043,7 +3236,7 @@ fn build( let listed = listed.of(db, krate); // With `cfg(test)` the program starts in the harness rustc wrote, which calls // every test: the crate's own `main` is one function among the others. - if at == 0 && !args.cfg_test { + if at == 0 && !args.unit_test { main = main.or(listed.main); } let listed_here = listed.functions.iter().copied().zip(listed.exported.iter().copied()); @@ -3080,7 +3273,11 @@ fn build( }; // The program's `main` is called by one rustc wrote, when rustc built the crate. let own = |main: &FunctionId| resolver.own_crate_id(hir_def::HasModule::krate(main, db)); + let sharing = hir::attach_db(db, || { + known.sharing(db, program_sharing(db, args), &args.out.display().to_string()) + }); let roots = Roots { + sharing: sharing.clone(), entries: &entries, on_demand: &on_demand, start: main.filter(|it| own(it).is_some()), @@ -3106,19 +3303,28 @@ fn build( // What every question about a crate's code comes down to first, asked one crate to a // thread: left to the questions, whichever gets there has the others wait for all of it. let next = std::sync::atomic::AtomicUsize::new(0); + let impls = std::sync::atomic::AtomicU64::new(0); + let prewarm = known.impls_changed(); + let impl_crates = if prewarm { local.as_slice() } else { &[] }; let stale: Option> = std::thread::scope(|scope| { let parts: Vec<_> = (0..threads) .map(|part| { - let (db, known, local, next) = (db.clone(), &*known, &local, &next); + let (db, known, local, next, impls) = + (db.clone(), &*known, impl_crates, &next, &impls); scope.spawn(move || { PROBING.set(true); hir::attach_db(&db, || { + let start = Instant::now(); while let Some(&krate) = local.get(next.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) { hir_ty::method_resolution::TraitImpls::for_crate(&db, krate.into()); hir_ty::method_resolution::InherentImpls::for_crate(&db, krate.into()); } + impls.fetch_max( + start.elapsed().as_micros() as u64, + std::sync::atomic::Ordering::Relaxed, + ); known.stale(&db, part, threads) }) }) @@ -3127,12 +3333,27 @@ fn build( parts.into_iter().map(|it| it.join().ok().flatten()).collect() }); hir::attach_db(db, || known.set_stale(hir_db, stale.unwrap_or_default())); + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!( + "check impls prewarm: {}; program={}", + if prewarm { "enabled" } else { "skipped" }, + args.out.display() + ); + eprintln!( + "check impls longest-thread-us: {}; program={}", + impls.load(std::sync::atomic::Ordering::Relaxed), + args.out.display() + ); + } let looked = known.stale_steps(); let revalidate = start.elapsed(); // Salsa attaches the database for the duration of the closure; the solver needs it. let mut unit = hir::attach_db(db, || CodegenUnit::compile(hir_db, &resolver, roots, &denied, known)) .map_err(anyhow::Error::msg)?; + if known.stats.patched.is_none() { + *text = BuildText::default(); + } let name = |func| resolver.path(func).unwrap_or_else(|| "".to_owned()); let codegen = start.elapsed(); if std::env::var_os("RUSTD_STATS").is_some() { @@ -3150,45 +3371,62 @@ fn build( weaken::definitions(object, |name| !cargo::plan::is_rust_symbol(name)); // The files a debugger is sent to are there before the object that sends it. for file in &unit.apart { - replace(Path::new(&file.name), &file.bytes)?; + if let Some(staged) = &file.staged { + std::fs::rename(staged, &file.name)?; + } else { + replace(Path::new(&file.name), &file.bytes)?; + } } replace(&args.out, object)?; } if let (Some(path), Some(patch)) = (&args.patch, &unit.patch) { replace(path, patch)?; } + let written = start.elapsed(); + let mut reached = Duration::ZERO; let mut harness_unfit = None; + let retained_harness = text.harness.is_some(); if let Some(harness) = harness { // What each test can run, for the client to run only the ones an edit can change. - Reached::of(db, listed, args).write(&args.out)?; - let text = hir::attach_db(db, || match programs_own.first() { - Some(&krate) if doctest => { - let runners: Vec<(String, Option)> = listed - .of(db, krate) - .functions - .iter() - .filter(|&&it| doctest::is_runner(db, it.into())) - .map(|&it| { - let name = hir::Function::from(it).name(db).as_str().to_owned(); - (name, resolver.function_named(it)) - }) - .collect(); - doctest::harness(db, krate.into(), &runners) - } - _ => harness_of(db, &resolver, &programs_own), - }); - match text { + Reached::of(db, listed, args, known, sharing).write(&args.out)?; + reached = start.elapsed() - written; + let make_harness = || { + hir::attach_db(db, || match programs_own.first() { + Some(&krate) if doctest => { + let runners: Vec<(String, Option)> = listed + .of(db, krate) + .functions + .iter() + .filter(|&&it| doctest::is_runner(db, it.into())) + .map(|&it| { + let name = hir::Function::from(it).name(db).as_str().to_owned(); + (name, resolver.function_named(it)) + }) + .collect(); + doctest::harness(db, krate.into(), &runners) + } + _ => harness_of(db, &resolver, &programs_own), + }) + }; + if std::env::var_os("RUSTD_VERIFY_FASTPATH").is_some_and(|it| it == "1") + && let Some(cached) = &text.harness + { + assert_eq!(cached, &make_harness(), "harness text fast path"); + } + let harness_text = text.harness.get_or_insert_with(make_harness); + match harness_text { Ok(text) => { // A splice adds the edit to the executable and leaves the harness as it was linked. - if harness.write(&text)? + if harness.write(text)? && let Some(binary) = &args.splice { lock(&LINKED).remove(binary); } } - Err(reason) => harness_unfit = Some(reason), + Err(reason) => harness_unfit = Some(reason.clone()), } } + let harnessed = start.elapsed() - written - reached; // A function of a crate rustc built nothing of has no copy to stand in for it. let missing = unit.skipped.iter().find(|(func, error)| { !error.no_symbol() && !resolver.rustc_built(hir_def::HasModule::krate(func, db)) @@ -3201,16 +3439,32 @@ fn build( .report_errors .first() .map(|(func, error)| format!("the report shim for test `{}` failed: {error}", name(*func))); + let make_skipped = || { + unit.skipped + .iter() + .map(|(func, error)| (name(*func), with_failed_macro(db, *func, error.clone()))) + .collect::() + }; + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!( + "write text: retained skipped={} harness={}; program={}", + text.skipped.is_some(), + retained_harness, + args.out.display() + ); + } + let skipped = text.skipped( + std::env::var_os("RUSTD_VERIFY_FASTPATH").is_some_and(|it| it == "1"), + make_skipped, + ); + let diagnosed = start.elapsed() - written - reached - harnessed; let build = Build { compiled: unit.compiled.len(), - skipped: unit - .skipped - .into_iter() - .map(|(func, e)| (name(func), with_failed_macro(db, func, e))) - .collect(), + skipped, instances: unit.instances, codegen, emit: start.elapsed(), + emitted: [written, reached, harnessed, diagnosed], steps: [setup, revalidate - setup, codegen - revalidate], looked, unfit: harness_unfit.or(report_unfit).or(missing).or_else(|| { @@ -3228,6 +3482,7 @@ fn build( for (i, &(func, _)) in unit.compiled.iter().enumerate() { let single = hir::attach_db(db, || { let roots = Roots { + sharing: program_sharing(db, args), entries: &[func], on_demand: &[], start: None, @@ -3258,7 +3513,7 @@ impl Build { for name in compiled { writeln!(out, "compiled\t{name}\t")?; } - for (name, error) in &self.skipped { + for (name, error) in self.skipped.iter() { let error = error.to_string().replace(['\t', '\n'], " "); writeln!(out, "skipped\t{name}\t{error}")?; } @@ -3313,14 +3568,14 @@ fn report(build: &Build) { } } if build.skipped.len() <= 8 { - for (name, error) in &build.skipped { + for (name, error) in build.skipped.iter() { eprintln!(" left to rustc: {name}: {error}"); } return; } // Bucket by the innermost reason, without the payload. let mut reasons: FxHashMap = FxHashMap::default(); - for (name, error) in &build.skipped { + for (name, error) in build.skipped.iter() { let error = error.to_string(); let reason = error.rsplit("`: ").next().unwrap_or(&error); let reason = reason.split(['(', '{']).next().unwrap_or(reason).trim(); @@ -3642,7 +3897,7 @@ fn selected(args: &Args) -> Vec { // Dev-dependencies are for tests: cargo runs the build scripts and the proc macros of // those only for a build that has them. Not `--all-targets`, which fails on a bench or an // example a published package lists and left out the file of, as `cargo test` does not. - if args.cfg_test || selection.dev_deps { + if selection.dev_deps { selected.push("--tests".to_owned()); } selected @@ -3953,7 +4208,7 @@ fn build_batch(db: &RootDatabase, args: Args) -> anyhow::Result<()> { report: Some(report.join(format!("{krate}.tsv"))), deny: None, isolate: None, - cfg_test: args.cfg_test, + unit_test: args.unit_test, own_target: args.own_target, selection: args.selection.clone(), also: Vec::new(), @@ -3961,9 +4216,17 @@ fn build_batch(db: &RootDatabase, args: Args) -> anyhow::Result<()> { checkout: args.checkout.clone(), base: args.base.clone(), members: args.members.clone(), + unit_roots: args.unit_roots.clone(), }; let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - build(db, table, &args, &mut Rejections::default(), &mut Listed::default()) + build( + db, + table, + &args, + &mut Rejections::default(), + &mut Listed::default(), + &mut BuildText::default(), + ) })); let error = match built { Ok(Ok(_)) => continue, @@ -4019,6 +4282,7 @@ struct Target { /// The checkout it is a program of. `None` is the one the daemon was started in. project: Option, krate: String, + unit_test: bool, /// The other crates of the workspace that are in it, when the client says: the names /// after the first in `--crate a,b,c`. Otherwise the ones the daemon was started with. batch: Option>, @@ -4034,14 +4298,16 @@ struct Target { impl Target { fn is(&self, args: &Args) -> bool { - self.krate == args.krate && self.out == args.out + self.krate == args.krate && self.out == args.out && self.unit_test == args.unit_test } /// The daemon's arguments, were it started for this program. fn args(self, daemon: &Args, checkouts: &Checkouts) -> anyhow::Result { - let Target { project, krate, batch, out, patch, link_cmd, splice, symbol_files } = self; + let Target { project, krate, unit_test, batch, out, patch, link_cmd, splice, symbol_files } = + self; let project = project.unwrap_or_else(|| daemon.project.clone()); - let Parts { root: checkout, base, members, debug } = checkouts.parts(&project)?; + let Parts { root: checkout, base, members, unit_roots, debug } = + checkouts.parts(&project)?; let symbol_files = if symbol_files.is_empty() { daemon.symbol_files.clone() } else { symbol_files }; let batch = batch.unwrap_or_else(|| { @@ -4053,8 +4319,10 @@ impl Target { checkout, base, members, + unit_roots, debug, krate, + unit_test, batch, widened: Vec::new(), out, @@ -4071,7 +4339,12 @@ impl Target { let (mut krate, mut out, mut patch, mut link_cmd) = (None, None, None, None); let (mut project, mut splice) = (None, None); let mut symbol_files = Vec::new(); + let mut unit_test = false; while let Some(flag) = flags.next() { + if flag == "--cfg-test" { + unit_test = true; + continue; + } let value = flags.next().with_context(|| format!("{flag} needs a value"))?; match flag.as_str() { "--crate" => krate = Some(value), @@ -4090,6 +4363,7 @@ impl Target { Ok(Some(Target { project, krate, + unit_test, batch, out, patch, @@ -4103,7 +4377,8 @@ impl Target { && link_cmd.is_none() && splice.is_none() && symbol_files.is_empty() - && project.is_none() => + && project.is_none() + && !unit_test => { Ok(None) } @@ -4113,7 +4388,9 @@ impl Target { /// The fields after the request's name on its line, each after a tab. Empty is absent. fn parse(fields: &[&str]) -> Option { - let &[project, krate, out, patch, link_cmd, splice, ref symbol_files @ ..] = fields else { + let &[project, krate, out, patch, link_cmd, splice, unit_test, ref symbol_files @ ..] = + fields + else { return None; }; let given = |it: &str| (!it.is_empty()).then(|| it.to_owned()); @@ -4121,6 +4398,7 @@ impl Target { Some(Target { project: given(project).map(PathBuf::from), krate, + unit_test: unit_test.parse().ok()?, batch, out: PathBuf::from(out), patch: given(patch).map(PathBuf::from), @@ -4148,13 +4426,14 @@ impl Target { None => self.krate.clone(), }; let mut fields = format!( - "\t{}\t{}\t{}\t{}\t{}\t{}", + "\t{}\t{}\t{}\t{}\t{}\t{}\t{}", self.project.as_ref().map(|it| it.display().to_string()).unwrap_or_default(), crates, self.out.display(), self.patch.as_ref().map(|it| it.display().to_string()).unwrap_or_default(), self.link_cmd.as_deref().unwrap_or_default(), - self.splice.as_ref().map(|it| it.display().to_string()).unwrap_or_default() + self.splice.as_ref().map(|it| it.display().to_string()).unwrap_or_default(), + self.unit_test ); for file in &self.symbol_files { fields.push('\t'); @@ -4170,6 +4449,7 @@ struct Program { args: Args, table: Arc, known: Rejections, + text: BuildText, unbuilt: Unbuilt, /// The object the last build was asked for. wrote: Option, @@ -4692,6 +4972,7 @@ fn lower_until( })); } } + functions.retain(|f| !is_tested(db, f.krate(db))); let count = functions.len(); let unnamed = hir_ty::mir::codegen::Unnamed::new(functions); let threads = std::thread::available_parallelism().map_or(1, |it| it.get()); @@ -4747,7 +5028,7 @@ fn cargo_config(args: &Args) -> CargoConfig { extra_env_for_commands_only: true, sysroot: Some(sysroot), target: args.selection.target.clone(), - set_test: args.cfg_test, + set_test: args.selection.dev_deps, features: match args.selection.all_features { true => CargoFeatures::All, false => CargoFeatures::Selected { @@ -4842,6 +5123,7 @@ impl Held { } self.checkouts.cargo_config = cargo_config.clone(); self.checkouts.selection = selection.clone(); + self.checkouts.inputs = inputs::Inputs::default(); if let Err(why) = self.checkouts.add(projects, &mut self.db, &mut self.vfs) { eprintln!("spare: not used: {why:#}"); return None; @@ -5136,7 +5418,7 @@ fn follow_cargo( } seen.1 }; - let Parts { root, base, members, debug } = checkouts.parts(&args.project)?; + let Parts { root, base, members, unit_roots, debug } = checkouts.parts(&args.project)?; let mut names = args.crates().filter(|it| ready.contains(&root.join(it.as_str()))).cloned(); if let Some(krate) = names.next() { let of_ready = Args { @@ -5146,6 +5428,7 @@ fn follow_cargo( checkout: root, base, members, + unit_roots, debug, ..args.clone() }; @@ -5303,8 +5586,9 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul None => eprintln!("store: the toolchain of the library cannot be told from another"), } let built = follow_cargo(&mut db, &mut vfs, &mut checkouts, &args, &listed)?; - let Parts { root, base, members, debug } = checkouts.parts(&args.project)?; + let Parts { root, base, members, unit_roots, debug } = checkouts.parts(&args.project)?; (args.checkout, args.base, args.members, args.debug) = (root, base, members, debug); + args.unit_roots = unit_roots; // The names are resolved before cargo says which program it built: those of the crates // it built are, and not of every crate of the workspace. if let Some(built) = built { @@ -5347,8 +5631,8 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul // What the other checkouts don't share with this one, before a request waits for it. for also in &args.also { let start = Instant::now(); - let Parts { root: checkout, base, members, debug } = checkouts.parts(also)?; - let of_it = Args { checkout, base, members, debug, ..args.clone() }; + let Parts { root: checkout, base, members, unit_roots, debug } = checkouts.parts(also)?; + let of_it = Args { checkout, base, members, unit_roots, debug, ..args.clone() }; let resolved = resolve_crates(&db, &of_it)?; eprintln!("resolve {}: {resolved} crates in {:.1?}", also.display(), start.elapsed()); } @@ -5385,6 +5669,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul checkout: args.checkout, base: args.base, members: args.members, + unit_roots: args.unit_roots, debug: args.debug, selection: args.selection, ..whole @@ -5424,11 +5709,12 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul known.touch(&db, None); listed.touch(&db, None); } - let mut cold = build(&db, &table, &args, &mut known, &listed)?; + let mut text = BuildText::default(); + let mut cold = build(&db, &table, &args, &mut known, &listed, &mut text)?; // An `include!` in a body is first seen when the body is compiled. if WatchedFile::include_sources(&mut files, &mut tree, &mut db) { known.touch(&db, None); - cold = build(&db, &table, &args, &mut known, &listed)?; + cold = build(&db, &table, &args, &mut known, &listed, &mut text)?; } if let Some(unfit) = &cold.unfit { report(&cold); @@ -5492,7 +5778,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul } memory::report(&mut db, "warm-up")?; eprintln!("watching {} files", files.len()); - ide_db::executions::take(&db); + trace_queries(&db); // Requests wait here while a build is under way, and those that have come by the time it // is done are built together. let (sender, events) = std::sync::mpsc::channel(); @@ -5522,6 +5808,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul args, table: Arc::new(table), known, + text, unbuilt: Unbuilt::default(), wrote, whole, @@ -5562,7 +5849,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul if WatchedFile::watch_included(&mut files, &mut db) | WatchedFile::include_sources(&mut files, &mut tree, &mut db) { - programs.iter_mut().for_each(|it| it.known.touch(&db, None)); + programs.iter_mut().for_each(|it| it.touch(&db, None)); } // The program each client asks for. let mut clients: Vec<(usize, Asked)> = Vec::new(); @@ -5642,9 +5929,10 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul .filter(|it| it.is(&program.args)) .filter(|it| !it.symbol_files.is_empty()); let same = parts.members == program.args.members + && parts.unit_roots == program.args.unit_roots && parts.debug == program.args.debug; if same && (linked.is_none() || new.is_empty()) { - program.known.touch(&db, (!included).then_some(&new)); + program.touch(&db, (!included).then_some(&new)); program.unbuilt.stale |= !new.is_empty(); continue; } @@ -5673,7 +5961,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul let reading = process::turn(&mut db, stream.env.clone()); if !reading.is_empty() { for program in &mut programs { - program.known.touch(&db, Some(&reading)); + program.touch(&db, Some(&reading)); program.unbuilt.stale = true; } listed.touch(&db, Some(&reading)); @@ -5744,7 +6032,9 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul } // The checkouts read as one share its programs. (Some(target), Some(view)) if shared => programs.iter().position(|it| { - it.args.krate == target.krate && it.args.checkout == view.base + it.args.krate == target.krate + && it.args.unit_test == target.unit_test + && it.args.checkout == view.base }), (Some(_), _) => None, (None, _) => Some(0), @@ -5839,6 +6129,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul args, table, known, + text: BuildText::default(), unbuilt, wrote: None, whole: None, @@ -5877,11 +6168,13 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul checkouts.share(&mut db, tree.vfs(), differences)?; watch_loaded(&mut db, &mut tree, &mut files); for program in &mut programs { - let Parts { base, members, .. } = checkouts.parts(&program.args.project)?; + let Parts { base, members, unit_roots, .. } = + checkouts.parts(&program.args.project)?; program.moved |= program.args.members != members; (program.args.base, program.args.members) = (base, members); + program.args.unit_roots = unit_roots; program.unbuilt.stale = true; - program.known.touch(&db, None); + program.touch(&db, None); } listed.touch(&db, None); eprintln!("split again: {:.1?}", start.elapsed()); @@ -6014,7 +6307,20 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul Watched::Source(id) => Some(id), Watched::Included(_) => None, }); - outlines.before_edit(&db, &listed, &editing.collect::>()); + let editing: Vec<_> = editing.collect(); + outlines.before_edit(&db, &listed, &editing); + if programs.iter().any(|program| program.known.needs_sharing_facts(&editing)) { + let functions = listed.in_files(&db, &editing); + hir::attach_db(&db, || { + for (&file, functions) in editing.iter().zip(functions) { + if let Some(functions) = functions { + for program in &mut programs { + program.known.sharing_before_edit(&db, file, &functions); + } + } + } + }); + } for &index in &gone { if let Watched::Source(id) = files[index].file { change.change_file(id, None); @@ -6040,6 +6346,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul if changed != 0 || !background { db.apply_change(change); } + let changed_db = start.elapsed(); if !removed.is_empty() { tree.remove(&mut db, &removed); let mut index = 0; @@ -6050,7 +6357,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul } // Asked after the change: a file can have joined a crate with it. if !included.is_empty() { - programs.iter_mut().for_each(|it| it.known.touch(&db, None)); + programs.iter_mut().for_each(|it| it.touch(&db, None)); listed.touch(&db, None); } for path in included { @@ -6064,6 +6371,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul unbuilt.edited.sort_unstable(); unbuilt.edited.dedup(); } + let noted = start.elapsed(); let paths: FxHashMap<_, _> = files .iter() .filter_map(|it| match it.file { @@ -6073,36 +6381,59 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul .collect(); let paths = Arc::new(Sources { paths }); let sources = outlines.after_edit(&db, &listed, sources); - for (file, bodies_only) in sources { + let outlined = start.elapsed(); + for (file, bodies_only, functions) in sources { if bodies_only { if std::env::var_os("RUSTD_STATS").is_some() { - eprintln!("bodies only: {file:?}"); + eprintln!( + "bodies only: {file:?}; changed functions={:?}; relocated functions={:?}", + functions.as_ref().map(|it| it.code.len()), + functions.as_ref().map(|it| it.locations.len()) + ); } - programs.iter_mut().for_each(|it| it.known.touch_bodies(&db, file)); + programs.iter_mut().for_each(|it| { + it.known.touch_bodies( + &db, + file, + functions.as_ref().map(|it| it.code.as_slice()), + functions.as_ref().map_or(&[], |it| it.locations.as_slice()), + ) + }); continue; } let crates = ide_db::base_db::relevant_crates(&db, file); let crates = (!crates.is_empty()).then_some(crates); - programs.iter_mut().for_each(|it| it.known.touch(&db, crates)); + programs.iter_mut().for_each(|it| it.touch(&db, crates)); listed.touch(&db, crates); } if !moved.is_empty() { programs.iter_mut().for_each(|it| { - it.known.touch(&db, Some(&moved)); + it.touch(&db, Some(&moved)); it.unbuilt.stale = true; }); listed.touch(&db, Some(&moved)); } + let invalidated = start.elapsed(); // The examples of the libraries are written as their documentation says now. let crates = write_examples(&mut db, emits.triple(), |db| builds.settle(db, &mut programs))?; if !crates.is_empty() { programs.iter_mut().for_each(|it| { - it.known.touch(&db, Some(&crates)); + it.touch(&db, Some(&crates)); it.unbuilt.stale = true; }); listed.touch(&db, Some(&crates)); } + if std::env::var_os("RUSTD_STATS").is_some() { + eprintln!( + "apply-us: db={} bookkeeping={} outlines={} invalidate={} examples={}", + changed_db.as_micros(), + (noted - changed_db).as_micros(), + (outlined - noted).as_micros(), + (invalidated - outlined).as_micros(), + (start.elapsed() - invalidated).as_micros(), + ); + } let edit = Edit { start, age, settled, read, apply: start.elapsed(), changed }; if background { let mut asked: FxHashMap> = FxHashMap::default(); @@ -6126,6 +6457,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul args: program.args.clone(), table: program.table.clone(), known: Rejections::default(), + text: BuildText::default(), unbuilt: Unbuilt::default(), wrote: None, whole: None, @@ -6196,7 +6528,7 @@ fn run_holding(mut args: Args, held: Option, ahead: bool) -> anyhow::Resul break answers; } for program in &mut programs { - program.known.touch(&db, Some(&late)); + program.touch(&db, Some(&late)); program.unbuilt.stale = true; } listed.touch(&db, Some(&late)); @@ -6388,6 +6720,10 @@ struct Edit { } impl Program { + fn touch(&mut self, db: &RootDatabase, crates: Option<&[ide_db::base_db::Crate]>) { + self.text = BuildText::default(); + self.known.touch(db, crates); + } /// Notes the crates rustc did not compile from the files this program is of, when it joins /// linked with what rustc built: of the first checkout, whose files as the daemon read them /// are `as_compiled`, or of its own checkout as it was when rustc built it. @@ -6438,9 +6774,10 @@ impl Program { new: &[ide_db::base_db::Crate], files: &[WatchedFile], ) -> anyhow::Result<()> { - let Parts { root, base, members, debug } = parts; + let Parts { root, base, members, unit_roots, debug } = parts; (self.args.checkout, self.args.base) = (root, base); (self.args.members, self.args.debug) = (members, debug); + self.args.unit_roots = unit_roots; let patching = self.known.patching; self.known = Rejections::default(); self.known.patching = patching; @@ -6551,7 +6888,7 @@ impl Program { && self.known.patches() && self.known.patching == other.known.patching && Arc::ptr_eq(&self.table, &other.table) - && (ours.cfg_test, &ours.selection) == (theirs.cfg_test, &theirs.selection) + && (ours.unit_test, &ours.selection) == (theirs.unit_test, &theirs.selection) && crates(ours).is_some_and(|it| Some(it) == crates(theirs)) } @@ -6566,11 +6903,13 @@ impl Program { wanted: Option<&Outputs>, link_it: bool, ) -> anyhow::Result> { + let queued = edit.start.elapsed().saturating_sub(edit.apply); + let preparing = Instant::now(); // The crates left to rustc are as edited when a client has named what it built since. let named_anew = std::mem::take(&mut self.unbuilt.named_anew); let unnamed = if named_anew { None } else { self.widen(db) }; self.read_copies_again(db)?; - let Program { args, table, known, unbuilt, wrote, whole, tests, .. } = self; + let Program { args, table, known, text, unbuilt, wrote, whole, tests, .. } = self; let Outputs { out, patch, link_cmd, splice } = wanted.cloned().unwrap_or_else(|| Outputs::of(args)); let args = &Args { out, patch, link_cmd, splice, ..args.clone() }; @@ -6591,13 +6930,20 @@ impl Program { // Neither is what no crate that is built reads: an example, or a test. edited.retain(|&file| listed.sees(db, file)); // The check of the edited files does not wait for the build: it reads other things. - let (warm, broken) = std::thread::scope(|scope| { + let prepared = preparing.elapsed(); + let (warm, broken, diagnostics, diagnostics_wait) = std::thread::scope(|scope| { let check = { let (db, edited) = (db.clone(), &edited); - scope.spawn(move || broken_in(&db, edited, paths)) + scope.spawn(move || { + let start = Instant::now(); + let broken = broken_in(&db, edited, paths); + (broken, start.elapsed()) + }) }; - let warm = build(db, table, args, known, listed); - (warm, check.join().ok().flatten()) + let warm = build(db, table, args, known, listed, text); + let joining = Instant::now(); + let (broken, diagnostics) = check.join().unwrap_or_default(); + (warm, broken, diagnostics, joining.elapsed()) }); let warm = warm?; if known.stats.patched.is_none() { @@ -6666,25 +7012,43 @@ impl Program { ), ), ("write", warm.emit), - ("build-rest", built.saturating_sub(apply + warm.codegen + warm.emit)), + ("write-object", warm.emitted[0]), + ("write-reach", warm.emitted[1]), + ("write-harness", warm.emitted[2]), + ("write-diagnostics", warm.emitted[3]), + ("schedule", queued), + ("sync-prepare", prepared), + ("diagnostics", diagnostics), + ("diagnostics-wait", diagnostics_wait), + ( + "build-rest", + built.saturating_sub( + apply + queued + prepared + warm.codegen + warm.emit + diagnostics_wait, + ), + ), ("report", reported - built), ("link", total - reported), ]; let steps: Vec = steps.iter().map(|(name, took)| format!("{name}={}", took.as_micros())).collect(); - eprintln!("timeline-us: {}", steps.join(" ")); + eprintln!("timeline-us: {}; program={}", steps.join(" "), args.out.display()); eprintln!( - "asked: {} roots, {} instances, slowest {:.1?}; longest on a thread: roots {:.1?}, \ - instances {:.1?}, statics {:.1?}, left {:.1?}", + "asked: {} roots, {} instances, slowest {:.1?}; longest on a thread: policy {:.1?}, \ + rejected {:.1?}, roots {:.1?}, \ + instances {:.1?}, statics {:.1?}, left {:.1?}; {} rejected instances", warm.looked.roots_asked, warm.looked.instances_asked, warm.looked.slowest, + warm.looked.policy, + warm.looked.rejected, warm.looked.roots, warm.looked.instances, warm.looked.statics, warm.looked.left, + warm.looked.rejected_asked, ); } + trace_queries(db); eprintln!("edit-to-binary: {total:.1?}"); if let Some(why) = failed { return Ok(Err(why)); @@ -6717,6 +7081,14 @@ fn report_stats(stats: &hir_ty::mir::codegen::BuildStats) { "stats: {} of {} functions compiled, {} reused", stats.compiled, stats.bodies, stats.reused ); + eprintln!( + "cfg codegen: T bodies compiled {}, redirected to N {}, analysis-shared compiled {}, TypeId restarts {}, sharing fallback restarts {}", + stats.sharing.t_compiled, + stats.sharing.redirected, + stats.sharing.analysis_shared_compiled, + stats.sharing.type_id_restarts, + stats.sharing.fallback_restarts + ); if let Some(patched) = stats.patched { eprintln!("patch: {patched} functions"); } @@ -6734,6 +7106,63 @@ fn report_stats(stats: &hir_ty::mir::codegen::BuildStats) { mod tests { use super::*; + #[test] + fn skipped_text_is_retained_and_verified() { + for report in [ + Vec::new(), + vec![("module::function".to_owned(), CodegenError::Frontend("failed".to_owned()))], + ] { + let calls = std::cell::Cell::new(0); + let fresh = || { + calls.set(calls.get() + 1); + SkippedReport::from(report.clone()) + }; + let mut text = BuildText::default(); + let first = text.skipped(false, fresh); + assert_eq!(&*first, &*report); + assert!(Arc::ptr_eq(&first, &text.skipped(false, fresh))); + assert_eq!(calls.get(), 1); + assert!(Arc::ptr_eq(&first, &text.skipped(true, fresh))); + assert_eq!(calls.get(), 2); + text = BuildText::default(); + assert_eq!(text.skipped(false, fresh), first); + assert_eq!(calls.get(), 3); + } + } + + #[test] + #[should_panic(expected = "skipped diagnostics fast path")] + fn skipped_text_verification_detects_changed_errors() { + let mut text = BuildText::default(); + text.skipped(false, || Arc::from([])); + text.skipped(true, || { + Arc::from([("changed".to_owned(), CodegenError::Frontend("error".to_owned()))]) + }); + } + + #[test] + fn target_requests_keep_the_programs_test_configuration() { + assert!(Target::from_flags(["--cfg-test".to_owned()].into_iter()).is_err()); + assert!(Target::from_flags(std::iter::empty()).unwrap().is_none()); + for unit_test in [false, true] { + let mut flags = vec!["--crate", "lib,dep", "--out", "lib.o", "--symbols", "dep.rlib"]; + if unit_test { + flags.push("--cfg-test"); + } + let target = Target::from_flags(flags.into_iter().map(str::to_owned)).unwrap().unwrap(); + assert_eq!(target.unit_test, unit_test); + let fields = target.fields(); + let fields: Vec<_> = fields[1..].split('\t').collect(); + let parsed = Target::parse(&fields).unwrap(); + assert_eq!(parsed.unit_test, unit_test); + assert_eq!(parsed.krate, "lib"); + assert_eq!(parsed.batch, Some(vec!["dep".to_owned()])); + assert_eq!(parsed.symbol_files, vec![PathBuf::from("dep.rlib")]); + } + assert!(Target::parse(&["", "lib", "lib.o", "", "", "", "invalid"]).is_none()); + assert!(Target::parse(&[]).is_none()); + } + #[test] fn library_sources_are_named_as_rustc_names_them() { let windows = r"C:\t\1.99.0\lib\rustlib\src\rust\library\core\src\iter\traits\accum.rs"; @@ -6794,7 +7223,13 @@ impl Tests { /// What each test of the program can run, as lines for the client that decides which /// of them to run: `test `. A print of `-` is of code that is /// not all known. - fn reached(db: &RootDatabase, listed: &Listed, args: &Args) -> String { + fn reached( + db: &RootDatabase, + listed: &Listed, + args: &Args, + known: &mut Rejections, + sharing: hir_ty::mir::codegen::Sharing, + ) -> String { use std::fmt::Write as _; let (mut always, mut kept) = (Vec::new(), Vec::new()); let mut edited = FxHashSet::default(); @@ -6809,8 +7244,9 @@ impl Tests { } let tests = Tests::named(db, listed, args); let roots: Vec = tests.iter().map(|it| it.0).collect(); + let program = args.out.display().to_string(); let (reached, _, unknown) = - hir_ty::mir::codegen::reached(db, edited, &roots, &always, &kept); + known.reached(db, sharing, edited, &roots, &always, &kept, &program); if !unknown.is_empty() { eprintln!("reached: not known: {}", unknown.join("; ")); } @@ -6849,12 +7285,27 @@ fn lint_of( Ok(lint::linted(db, &crates, &paths)) } +fn trace_queries(db: &RootDatabase) { + let ran = ide_db::executions::take(db); + if std::env::var_os("RUSTD_TRACE").is_some() { + let ran: Vec<_> = + ran.into_iter().map(|(query, count)| format!("{query}={count}")).collect(); + eprintln!("queries: {}", ran.join(" ")); + } +} + /// What each test of a program can run, as [`Tests::reached`] writes it. struct Reached(String); impl Reached { - fn of(db: &RootDatabase, listed: &Listed, args: &Args) -> Reached { - Reached(hir::attach_db(db, || Tests::reached(db, listed, args))) + fn of( + db: &RootDatabase, + listed: &Listed, + args: &Args, + known: &mut Rejections, + sharing: hir_ty::mir::codegen::Sharing, + ) -> Reached { + Reached(hir::attach_db(db, || Tests::reached(db, listed, args, known, sharing))) } fn path(object: &Path, suffix: &str) -> PathBuf { diff --git a/crates/rustd/src/outline.rs b/crates/rustd/src/outline.rs index ff6de7e7f5..6c418ba95a 100644 --- a/crates/rustd/src/outline.rs +++ b/crates/rustd/src/outline.rs @@ -7,7 +7,10 @@ //! nor one a macro is handed, nor one that declares items other than statics and constants. //! Such bodies count as part of what the rest of the program sees. -use std::hash::{Hash, Hasher}; +use std::{ + hash::{Hash, Hasher}, + time::Instant, +}; use hir_def::{FunctionId, Lookup, ModuleDefId, expr_store::Body}; use ide_db::{ @@ -42,7 +45,39 @@ const INERT: &[&str] = &[ /// Each file as it was when last looked at. #[derive(Default)] pub(crate) struct Outlines { - files: FxHashMap>, + files: FxHashMap>, +} + +#[derive(Clone)] +struct Outline { + outside: u64, + functions: Option>, +} + +pub(crate) struct ChangedFunctions { + pub(crate) code: Vec, + pub(crate) locations: Vec, +} + +fn function_changes( + was: &FxHashMap, + now: &FxHashMap, +) -> Option<(Vec, Vec)> { + if now.len() != was.len() || now.keys().any(|id| !was.contains_key(id)) { + return None; + } + let mut code = Vec::new(); + let mut locations = Vec::new(); + for (id, source) in now { + let old = was[id]; + if old.0 != source.0 { + code.push(*id); + } + if (old.1, old.2) != (source.1, source.2) { + locations.push(*id); + } + } + Some((code, locations)) } impl Outlines { @@ -60,20 +95,34 @@ impl Outlines { db: &RootDatabase, listed: &Listed, files: Vec, - ) -> Vec<(FileId, bool)> { + ) -> Vec<(FileId, bool, Option)> { let off = std::env::var_os("RUSTD_NO_BODIES").is_some(); let outlines = outlines(db, listed, &files); let files: Vec<_> = files .into_iter() .zip(outlines) .map(|(file, now)| { - let was = self.files.insert(file, now); - (file, !off && now.is_some() && was == Some(now)) + let was = self.files.remove(&file).flatten(); + let bodies_only = !off + && now + .as_ref() + .zip(was.as_ref()) + .is_some_and(|(now, was)| now.outside == was.outside); + let functions = bodies_only + .then(|| { + let now = now.as_ref()?.functions.as_ref()?; + let was = was.as_ref()?.functions.as_ref()?; + let (code, locations) = function_changes(was, now)?; + Some(ChangedFunctions { code, locations }) + }) + .flatten(); + self.files.insert(file, now); + (file, bodies_only, functions) }) .collect(); // An edit to more than bodies can change which functions there are, in any file, and // what is kept here was found by asking the ones listed before it. - if files.iter().any(|&(_, bodies_only)| !bodies_only) { + if files.iter().any(|(_, bodies_only, _)| !bodies_only) { self.files.clear(); } files @@ -82,7 +131,8 @@ impl Outlines { /// The [`outline`] of each file. A wide edit is of thousands of files: their bodies are /// lowered together and the files are hashed on all threads. -fn outlines(db: &RootDatabase, listed: &Listed, files: &[FileId]) -> Vec> { +fn outlines(db: &RootDatabase, listed: &Listed, files: &[FileId]) -> Vec> { + let start = Instant::now(); let stats = std::env::var_os("RUSTD_STATS").is_some(); if files.len() > 1 { let mut crates: Vec<_> = @@ -93,7 +143,9 @@ fn outlines(db: &RootDatabase, listed: &Listed, files: &[FileId]) -> Vec Vec Option { +fn outline(db: &RootDatabase, file: FileId, functions: &[FunctionId]) -> Option { let Some(seen) = Written::of(db, functions) else { if std::env::var_os("RUSTD_STATS").is_some() { eprintln!("outline: {file:?} is included in another file"); @@ -144,7 +205,43 @@ fn outline(db: &RootDatabase, file: FileId, functions: &[FunctionId]) -> Option< let tree = EditionedFileId::new(db, file, edition).parse(db).syntax_node(); let (hash, found) = hash_outside_bodies(&tree, &seen.declaring); // Each of them has to be one of the bodies that were kept in. - (found == seen.declaring.len()).then_some(hash) + if found != seen.declaring.len() { + return None; + } + let text = tree.text().to_string(); + let lines = ide_db::line_index(db, file); + let functions = functions + .iter() + .map(|&func| { + let at = func.lookup(db).id; + if at.file_id.original_file(db).file_id(db) != file { + return None; + } + let mut hash = FxHasher::default(); + let range = if at.file_id.is_macro() { + at.to_node(db).syntax().text().to_string().hash(&mut hash); + let call = at.file_id.original_call_node(db)?; + if call.file_id.file_id(db) != file { + return None; + } + call.value + .ancestors() + .find_map(ast::Item::cast)? + .syntax() + .text() + .to_string() + .hash(&mut hash); + call.value.text_range() + } else { + let range = at.to_range(db); + text.get(std::ops::Range::::from(range))?.hash(&mut hash); + range + }; + let start = lines.try_line_col(range.start())?; + Some((func, (hash.finish(), start.line, start.col))) + }) + .collect(); + Some(Outline { outside: hash, functions }) } /// What looking at the functions written in a file tells that how they are written does not. @@ -375,6 +472,25 @@ mod tests { use super::*; use syntax::SourceFile; + #[test] + fn changed_function_sources() { + let was = FxHashMap::from_iter([(1, (10, 0, 0)), (2, (20, 1, 0))]); + let now = FxHashMap::from_iter([(1, (11, 0, 0)), (2, (20, 2, 0))]); + assert_eq!(function_changes(&was, &now), Some((vec![1], vec![2]))); + assert_eq!(function_changes(&was, &was), Some((vec![], vec![]))); + assert_eq!( + function_changes::(&FxHashMap::default(), &FxHashMap::default()), + Some((vec![], vec![])) + ); + assert_eq!(function_changes(&was, &FxHashMap::from_iter([(1, (10, 0, 0))])), None); + assert_eq!( + function_changes(&was, &FxHashMap::from_iter([(1, (10, 0, 0)), (3, (20, 1, 0))])), + None + ); + let now = FxHashMap::from_iter([(1, (10, 0, 1)), (2, (20, 1, 0))]); + assert_eq!(function_changes(&was, &now), Some((vec![], vec![1]))); + } + fn hash(text: &str) -> u64 { let tree = SourceFile::parse(text, Edition::CURRENT).syntax_node(); hash_outside_bodies(&tree, &FxHashSet::default()).0 diff --git a/crates/rustd/src/symbols.rs b/crates/rustd/src/symbols.rs index 07cad49b39..af78295a18 100644 --- a/crates/rustd/src/symbols.rs +++ b/crates/rustd/src/symbols.rs @@ -3,6 +3,7 @@ use std::{ hash::{DefaultHasher, Hash, Hasher}, path::{Path, PathBuf}, + sync::{Arc, LazyLock, Mutex, Weak}, time::SystemTime, }; @@ -39,7 +40,7 @@ pub(crate) struct SymbolTable { /// When the files started to be read. read: Option, /// The copies crates made of one instance, by the symbol without the crate that made it. - copies: FxHashMap>, + copies: SharedMap>, /// The crates that made a copy of an instance of a function, by the function's name /// alone: what there is to go by for an instance whose types cannot be spelled. makers: FxHashMap>, @@ -51,12 +52,65 @@ pub(crate) struct Paths { /// More than one symbol for the items that print as one path: of two `const _` blocks of /// a module, or of one name in one function. Linking one to another's symbol is a /// miscompile. - by_path: FxHashMap>, + by_path: SharedMap>, /// Symbols of items nested in closures and anonymous constants, by their path without the /// `{closure#0}`-like segments, with the numbers of the closures among those. - nested: FxHashMap>, + nested: SharedMap>, } +#[derive(Default)] +struct SharedMap { + entries: FxHashMap, Arc>, +} + +type SharedEntries = FxHashMap, Weak)>>; + +impl SharedMap { + fn or_default(&mut self, key: String) -> &mut V { + Arc::make_mut(self.entries.entry(key.into()).or_default()) + } + + fn get(&self, key: &str) -> Option<&V> { + self.entries.get(key).map(|value| &**value) + } + + fn len(&self) -> usize { + self.entries.len() + } + + fn share(&mut self, cache: &Mutex>) { + use std::hash::BuildHasher; + let mut cache = cache.lock().unwrap(); + cache.retain(|_, entries| { + entries.retain(|(key, value)| key.strong_count() != 0 && value.strong_count() != 0); + !entries.is_empty() + }); + self.entries = self + .entries + .drain() + .map(|(key, value)| { + let hash = rustc_hash::FxBuildHasher.hash_one((&*key, &*value)); + let candidates = cache.entry(hash).or_default(); + for (known_key, known_value) in &*candidates { + if let (Some(known_key), Some(known_value)) = + (known_key.upgrade(), known_value.upgrade()) + && known_key == key + && known_value == value + { + return (known_key, known_value); + } + } + candidates.push((Arc::downgrade(&key), Arc::downgrade(&value))); + (key, value) + }) + .collect(); + } +} + +static PLACED: LazyLock>>> = LazyLock::new(Mutex::default); +static NESTED: LazyLock>>> = LazyLock::new(Mutex::default); +static COPIES: LazyLock>>> = LazyLock::new(Mutex::default); + /// Who can refer to a symbol an object defines. #[derive(Clone, Copy, PartialEq, Eq)] enum Reach { @@ -68,6 +122,7 @@ enum Reach { PrivateStatic, } +#[derive(Clone, PartialEq, Eq, Hash)] struct NestedSymbol { name: String, closures: Vec, @@ -75,6 +130,7 @@ struct NestedSymbol { around: Vec, } +#[derive(Clone, PartialEq, Eq, Hash)] struct Placed { name: String, /// The place of the item among those of its name in what it is written in, which rustc @@ -292,6 +348,7 @@ impl SymbolTable { } table.place(unplaced); table.index(symbols); + table.share(); table.files = files.to_vec(); Ok(table) } @@ -305,9 +362,18 @@ impl SymbolTable { table.add_name(Named::new(name, Reach::Linkable, 0), &mut symbols); } table.index(symbols); + table.share(); table } + fn share(&mut self) { + self.linkable.by_path.share(&PLACED); + self.linkable.nested.share(&NESTED); + self.private_statics.by_path.share(&PLACED); + self.private_statics.nested.share(&NESTED); + self.copies.share(&COPIES); + } + /// Files the symbols under their paths, once every crate named in one is known. fn index(&mut self, symbols: Vec) { let table = self; @@ -347,7 +413,7 @@ impl SymbolTable { && self.names.insert(Self::hashed(&name)) && instance.len() != name.len() { - self.copies.entry(instance.to_owned()).or_default().push(name.clone()); + self.copies.or_default(instance.to_owned()).push(name.clone()); } let Some((with_ids, path)) = demangled else { return }; for (krate, id) in crate_ids(&with_ids) { @@ -528,13 +594,13 @@ impl Paths { .split("::") .filter_map(|it| it.strip_prefix("{closure#")?.strip_suffix('}')?.parse().ok()) .collect(); - let symbols = self.nested.entry(named.join("::")).or_default(); + let symbols = self.nested.or_default(named.join("::")); if symbols.iter().all(|it| it.name != name) { let around = around.clone(); symbols.push(NestedSymbol { name: name.clone(), closures, place, around }); } } - let known = self.by_path.entry(path).or_default(); + let known = self.by_path.or_default(path); let same = |it: &Placed| { without_instantiating_crate(&it.name) == without_instantiating_crate(&name) }; @@ -1046,11 +1112,83 @@ impl V0<'_> { #[cfg(test)] mod tests { - use std::path::PathBuf; + use std::{ + hash::BuildHasher, + path::PathBuf, + sync::{Arc, Mutex}, + }; use rustc_hash::FxHashMap; - use super::{Paths, SymbolTable, V0, function_name}; + use super::{Paths, SharedEntries, SharedMap, SymbolTable, V0, function_name}; + + #[test] + fn tables_share_normalized_symbols_and_copies() { + let of_lib = "_RINvCslFwujm09uXn_4kdep3sumhECsjPNOOTmEH7l_4klib"; + let of_app = "_RINvCslFwujm09uXn_4kdep3sumhECsjSK0TBt3L2V_4kapp"; + let first = SymbolTable::of_names([of_lib, of_app]); + let second = SymbolTable::of_names([of_lib, of_app, "_RNvCslFwujm09uXn_4kdep5small"]); + for (key, value) in &first.linkable.by_path.entries { + let (other_key, other_value) = + second.linkable.by_path.entries.get_key_value(&**key).unwrap(); + assert!(Arc::ptr_eq(key, other_key)); + assert!(Arc::ptr_eq(value, other_value)); + } + for (key, value) in &first.copies.entries { + let (other_key, other_value) = second.copies.entries.get_key_value(&**key).unwrap(); + assert!(Arc::ptr_eq(key, other_key)); + assert!(Arc::ptr_eq(value, other_value)); + } + assert!(second.len() > first.len()); + } + + #[test] + fn shared_paths_keep_ambiguity_and_mutate_independently() { + let cache = Mutex::new(SharedEntries::default()); + let mut first = Paths::default(); + let mut second = Paths::default(); + for paths in [&mut first, &mut second] { + paths.insert("dep::f".to_owned(), "first".to_owned(), Some(0), vec![]); + paths.by_path.share(&cache); + } + second.insert("dep::f".to_owned(), "second".to_owned(), Some(0), vec![]); + second.by_path.share(&cache); + assert_eq!(first.get("dep::f"), Some("first")); + assert_eq!(second.get("dep::f"), None); + assert_eq!(first.get("dep::missing"), None); + let mut empty = Paths::default(); + empty.by_path.share(&cache); + assert_eq!(empty.get("dep::f"), None); + } + + #[test] + fn shared_entries_check_hash_collisions_and_release_dead_data() { + let cache = Mutex::new(SharedEntries::default()); + let mut first = SharedMap::>::default(); + first.or_default("path".to_owned()).push("first".to_owned()); + first.share(&cache); + let (key, value) = first.entries.get_key_value("path").unwrap(); + let weak_key = Arc::downgrade(key); + let weak_value = Arc::downgrade(value); + let hash = rustc_hash::FxBuildHasher.hash_one(("path", &vec!["second".to_owned()])); + cache.lock().unwrap().insert(hash, vec![(weak_key.clone(), weak_value.clone())]); + let mut second = SharedMap::>::default(); + second.or_default("path".to_owned()).push("second".to_owned()); + second.share(&cache); + assert_eq!(second.get("path").unwrap(), &["second"]); + drop(first); + assert!(weak_key.upgrade().is_none()); + assert!(weak_value.upgrade().is_none()); + drop(second); + SharedMap::>::default().share(&cache); + assert!(cache.lock().unwrap().is_empty()); + } + + #[test] + fn missing_symbol_file_is_not_a_cached_table() { + let missing = PathBuf::from("/no-such-rustd-symbol-file"); + assert!(SymbolTable::load(&[missing], &PathBuf::from(".")).is_err()); + } fn placed(crates: &[(&str, &[(u64, usize)])]) -> Vec<(String, u64)> { let mut table = SymbolTable::default(); diff --git a/harness/bench.py b/harness/bench.py index 522bb51edf..db820cd80f 100755 --- a/harness/bench.py +++ b/harness/bench.py @@ -172,7 +172,7 @@ def daemon_steps(text): """The steps the daemon timed in `text`, in milliseconds, summed over its builds.""" steps = {} for line in re.findall(r"^timeline-us: (.*)$", text, re.M): - for name, micros in (pair.split("=") for pair in line.split()): + for name, micros in (pair.split("=") for pair in line.split(";", 1)[0].split()): # How long ago the file was saved is not the daemon's time. if name != "notice": steps[name] = round(steps.get(name, 0) + int(micros) / 1000, 2) diff --git a/harness/cargo_rustd.py b/harness/cargo_rustd.py index c63b350b84..c9af892238 100644 --- a/harness/cargo_rustd.py +++ b/harness/cargo_rustd.py @@ -963,7 +963,7 @@ def body_statics(work, check): const A: u32 = 1; #[cfg(test)] mod tests { - #[test] fn ta() { assert_eq!(super::a(), 1); } + #[test] fn ta() { assert_eq!(shared::identity(super::a()), 1); } #[test] fn tb() { assert_eq!(super::b(), 2); } #[test] fn h1() { assert!(super::hit() >= 1); } #[test] fn h2() { assert!(super::hit() >= 1); } @@ -980,7 +980,10 @@ def incremental(work): ws = os.path.realpath(os.path.join(work, "incremental")) lib = os.path.join(ws, "src/lib.rs") os.makedirs(os.path.dirname(lib), exist_ok=True) - open(os.path.join(ws, "Cargo.toml"), "w").write('[package]\nname = "lib"\nversion = "0.1.0"\nedition = "2021"\n') + open(os.path.join(ws, "Cargo.toml"), "w").write('[package]\nname = "lib"\nversion = "0.1.0"\nedition = "2021"\n\n[dependencies]\nshared = { path = "shared" }\n') + os.makedirs(os.path.join(ws, "shared/src"), exist_ok=True) + open(os.path.join(ws, "shared/Cargo.toml"), "w").write('[package]\nname = "shared"\nversion = "0.1.0"\nedition = "2021"\n') + open(os.path.join(ws, "shared/src/lib.rs"), "w").write('pub fn identity(value: T) -> T { value }\n') open(lib, "w").write(INCREMENTAL_LIB) side = Side(ws, base_env(os.path.join(work, "home-incremental")), True) checks = [] @@ -1028,6 +1031,859 @@ def check(name, cached, *args, env=None, listed=7, failed=()): return checks +def test_graph(work): + ws = os.path.realpath(os.path.join(work, "test-graph")) + files = { + "Cargo.toml": '[package]\nname = "test_graph"\nversion = "0.1.0"\nedition = "2021"\n\n[features]\ndefault = ["test-support"]\ntest-support = []\n\n[workspace]\nmembers = ["probe"]\n\n[dev-dependencies]\nprobe = { path = "probe" }\n', + "probe/Cargo.toml": '[package]\nname = "probe"\nversion = "0.1.0"\nedition = "2021"\n\n[dependencies]\ntest_graph = { path = ".." }\n', + "probe/src/lib.rs": 'pub fn tested() -> bool { test_graph::tested() }\n', + "src/lib.rs": '''pub fn tested() -> bool { cfg!(test) } +pub fn value() -> u32 { 40 + 2 } +#[cfg(not(test))] const CONFIG: u32 = 7; +#[cfg(test)] const CONFIG: u32 = 8; +pub fn configured() -> u32 { CONFIG } +#[cfg_attr(test, inline)] pub fn attributed() {} +pub fn gated() -> u32 { let mut n = 0; #[cfg(test)] { n = 1; } n } +pub struct Shape; +#[derive(Debug, Clone)] +pub struct Derived; +pub fn formatted() -> String { format!("{:?}", Derived.clone()) } +pub enum Choice { One } +pub type Alias = Shape; +pub const SHARED: u32 = 1; +pub static STATE: u32 = 2; +pub trait Value { fn get(&self) -> u32; } +impl Value for Shape { fn get(&self) -> u32 { SHARED } } +impl Shape { pub fn own(&self) -> u32 { STATE } } +pub fn nested() { fn local() {} local(); } +macro_rules! generate { () => { pub fn generated() {} } } +generate!(); +#[cfg(test)] +mod tests { + #[test] + fn unit() { + assert!(super::tested()); + assert!(!probe::tested()); + assert_eq!(super::value(), 42); + assert_eq!(super::configured(), 8); + assert_eq!(super::gated(), 1); + assert_eq!(super::formatted(), "Derived"); + } +} +''', + "tests/integration.rs": '''#[test] +fn integration() { + assert!(cfg!(test)); + assert!(!test_graph::tested()); + assert_eq!(test_graph::value(), 42); +} +''', + } + for file, text in files.items(): + os.makedirs(os.path.dirname(os.path.join(ws, file)), exist_ok=True) + open(os.path.join(ws, file), "w").write(text) + side = Side(ws, dict(base_env(os.path.join(work, "home-test-graph")), RUSTD_CFG_TWINS="items"), True) + checks = [] + + def check(name, held, detail): + checks.append((f"test graph: {name}", held, detail)) + print(f" {'ok ' if held else 'NOT'} test graph: {name} {detail}", flush=True) + + try: + _, cold = side.cargo("test") + check("cold unit and integration tests", cold.returncode == 0, cold.stderr[-200:]) + _, states = wait_warm(side, ws) + _, status = side.cargo("status") + workers = re.findall(r"worker: pid (\d+)", status.stdout) + check("both checkouts share one worker", len(workers) == 1 and states == ["ready", "ready"], f"workers {workers}, states {states}") + for manifest in pathlib.Path(side.env["RUSTD_HOME"]).glob("spares/*/package/Cargo.toml"): + rewrite(str(manifest), 'version = "0.0.0"', 'version = "0.0.1"') + for label in ("warm", "edited"): + if label == "edited": + rewrite(os.path.join(ws, "src/lib.rs"), "40 + 2", "41 + 2") + for file in ("src/lib.rs", "tests/integration.rs"): + rewrite(os.path.join(ws, file), ", 42)", ", 43)") + _, ran = side.cargo("test") + check(f"{label} unit and integration tests are daemon-built", ran.returncode == 0 and how_built(ran.stderr) == "daemon" and "tests::unit" in ran.stdout and "test integration" in ran.stdout, f"exit {ran.returncode}, {how_built(ran.stderr)}; {ran.stderr[-200:]}") + logs = pathlib.Path(side.env["RUSTD_HOME"]) + loads = sum(path.read_text(errors="replace").count("load workspace:") for path in logs.glob("**/log")) + check("the graph was loaded once", loads == 1, f"loads {loads}") + twin_counts = [tuple(map(int, match)) for path in logs.glob("**/log") for match in re.findall( + r"cfg twins: test_graph: twins (\d+), cfg-differing (\d+), test-only (\d+), unmatched (\d+)", path.read_text(errors="replace"))] + check("cfg source and expansion twins before and after the edit", len(twin_counts) >= 2 and all(count == (26, 3, 3, 0) for count in twin_counts), str(twin_counts)) + body_counts = [tuple(map(int, match)) for path in logs.glob("**/log") for match in re.findall( + r"cfg twins: test_graph: [^\n]+; function bodies: twins (\d+), cfg-differing (\d+), test-only (\d+), unmatched (\d+), unknown (\d+)", path.read_text(errors="replace"))] + check("generated and format bodies have known twins", len(body_counts) >= 2 and all(count == (13, 3, 1, 0, 0) for count in body_counts), str(body_counts)) + text = "\n".join(path.read_text(errors="replace") for path in logs.glob("**/log")) + check("unrelated body is fully shared", "cfg body: test_graph::value: fully-shared" in text, "value") + check("paired cfg constants reject their caller transitively", "cfg body: test_graph::configured: transitive" in text, "configured") + check("static reverse closure is analysis shared", "cfg body: test_graph::own: analysis-shared" in text, "Shape::own") + eligibility = re.findall(r"cfg eligibility: test_graph: [^\n]+pair veto: ([^;]+);", text) + check("the main pair has no safety veto", eligibility and all(reason == "none" for reason in eligibility), str(eligibility)) + check("TypeId is deferred to codegen", "both copies true; TypeId: deferred to codegen" in text and "would veto program" not in text, "deferred") + + source = os.path.join(ws, "src/lib.rs") + original = pathlib.Path(source).read_text() + integration = pathlib.Path(ws, "tests/integration.rs") + integration_source = integration.read_text() + before = {path: len(path.read_text(errors="replace")) for path in logs.glob("**/log")} + positive = original.split("#[cfg(test)]\nmod tests")[0] + '''#[cfg(test)] +mod tests { + #[test] fn unit() { super::generated(); let _ = probe::tested(); } +} +''' + pathlib.Path(source).write_text(positive) + integration.write_text("#[test] fn integration() { test_graph::generated(); }\n") + _, ran = side.cargo("test") + fresh = "\n".join(path.read_text(errors="replace")[before.get(path, 0):] for path in logs.glob("**/log")) + check("positive program defers TypeId", ran.returncode == 0 and "both copies true; TypeId: deferred to codegen" in fresh, f"exit {ran.returncode}, expected deferred TypeId with both copies") + integration.write_text(integration_source) + variants = [ + ("predicate result", """ +#[cfg(any(test, feature = "test-support"))] pub fn predicate_gate() {} +pub fn predicate_caller() { predicate_gate(); } +""", "cfg body: test_graph::predicate_caller: fully-shared"), + ("rejected function remapping", """ +pub fn different_function() -> u32 { if cfg!(test) { 9 } else { 7 } } +pub fn call_different_function() -> u32 { different_function() } +""", "cfg body: test_graph::call_different_function: analysis-shared"), + ("baked const length", """ +pub const DIFFERENT_LENGTH: usize = if cfg!(test) { 9 } else { 7 }; +pub fn baked_length() -> usize { let value = [0u8; DIFFERENT_LENGTH]; value.len() } +""", "cfg body: test_graph::baked_length: transitive"), + ("baked rejection stops at call boundary", """ +pub const DIFFERENT_LENGTH: usize = if cfg!(test) { 9 } else { 7 }; +pub fn baked_length() -> usize { let value = [0u8; DIFFERENT_LENGTH]; value.len() } +pub fn call_baked_length() -> usize { baked_length() } +""", "cfg body: test_graph::call_baked_length: analysis-shared"), + ("baked const generic", """ +pub const DIFFERENT_LENGTH: usize = if cfg!(test) { 9 } else { 7 }; +fn length() -> usize { N } +pub fn baked_generic() -> usize { length::() } +""", "cfg body: test_graph::baked_generic: transitive"), + ("baked const annotation", """ +pub const DIFFERENT_LENGTH: usize = if cfg!(test) { 9 } else { 7 }; +pub fn baked_annotation() -> usize { let value: [u8; DIFFERENT_LENGTH] = Default::default(); value.len() } +""", "cfg body: test_graph::baked_annotation: transitive"), + ("concrete impl precision", """ +#[derive(Debug, Clone)] +pub struct Changed { #[cfg(test)] extra: u32 } +pub fn unrelated_debug() -> String { format!("{:?}", Derived.clone()) } +""", "cfg body: test_graph::unrelated_debug: fully-shared"), + ("default derive helper", """ +#[derive(Default)] +pub enum DefaultChoice { #[default] First, Second } +pub fn default_choice() -> DefaultChoice { DefaultChoice::default() } +""", "cfg body: test_graph::default_choice: fully-shared"), + ("per-copy initializer", """ +pub static PER_COPY: u32 = if cfg!(test) { 9 } else { 7 }; +pub fn per_copy_value() -> u32 { PER_COPY } +""", "cfg body: test_graph::per_copy_value: analysis-shared"), + ("static absent counterpart", """ +#[cfg(not(test))] pub static ABSENT: u32 = 7; +#[cfg(test)] pub static ABSENT: u32 = 9; +pub fn absent_static() -> u32 { ABSENT } +""", "cfg body: test_graph::absent_static: transitive"), + ("per-copy type changes", """ +pub struct StaticShape { #[cfg(test)] extra: u32 } +pub static STATIC_SHAPE: StaticShape = StaticShape { #[cfg(test)] extra: 1 }; +pub fn changed_static_type() -> &'static StaticShape { &STATIC_SHAPE } +""", "cfg body: test_graph::changed_static_type: transitive"), + ("generic dispatch", "\npub fn unresolved(value: &T) -> u32 { value.get() }\n", "cfg body: test_graph::unresolved: fully-shared"), + ("dyn dispatch", "\npub fn dynamic(value: &dyn Value) -> u32 { value.get() }\n", "cfg body: test_graph::dynamic: fully-shared"), + ("generic impl edges", "\npub struct Other;\nimpl Value for Other { fn get(&self) -> u32 { STATE } }\npub fn dispatch_all(value: &T) -> u32 { value.get() }\n", "cfg body: test_graph::dispatch_all: analysis-shared"), + ("dyn impl edges", "\npub struct Other;\nimpl Value for Other { fn get(&self) -> u32 { STATE } }\npub fn dynamic_all(value: &dyn Value) -> u32 { value.get() }\n", "cfg body: test_graph::dynamic_all: analysis-shared"), + ("projection edges", "\npub trait Project { type Output; }\nimpl Project for Shape { type Output = u32; }\npub fn projected(value: T::Output) -> T::Output { value }\n", "cfg body: test_graph::projected: fully-shared"), + ("foreign local argument", "\npub fn foreign_local() -> Option { Some(Shape) }\n", "cfg body: test_graph::foreign_local: fully-shared"), + ("inherent impl type identity", "\nimpl Shape { pub fn changed(&self) -> bool { cfg!(test) } }\npub fn inherent_shape() -> Option { Some(Shape) }\n", "cfg body: test_graph::inherent_shape: analysis-shared"), + ("blanket impl type identity", "\npub trait Blanket { fn changed(&self) -> bool; }\nimpl Blanket for T { fn changed(&self) -> bool { cfg!(test) } }\npub fn blanket_shape() -> Option { Some(Shape) }\n", "cfg body: test_graph::blanket_shape: analysis-shared"), + ("blanket predicate type identity", "\npub trait Bound {}\npub trait Blanket { fn changed(&self) -> bool; }\nimpl> Blanket for Vec { fn changed(&self) -> bool { cfg!(test) } }\npub fn blanket_bound_shape() -> Option { Some(Shape) }\n", "cfg body: test_graph::blanket_bound_shape: analysis-shared"), + ("trait argument type identity", "\npub trait Accept { fn changed(&self) -> bool; }\nimpl Accept for u32 { fn changed(&self) -> bool { cfg!(test) } }\npub fn argument_shape() -> Option { Some(Shape) }\n", "cfg body: test_graph::argument_shape: analysis-shared"), + ("Self variant", "\nimpl Choice { pub fn one() -> Self { Self::One } }\n", "cfg body: test_graph::one: fully-shared"), + ("indirect unknown", "\npub fn indirect(f: fn() -> u32) -> u32 { f() }\n", "cfg body: test_graph::indirect: unknown"), + ("projected indirect unknown", "\npub struct Pointer { call: fn() -> u32 }\npub fn indirect_field(value: Pointer) -> u32 { (value.call)() }\n", "cfg body: test_graph::indirect_field: unknown"), + ("TLS", '''\nthread_local! { static TLS: core::cell::Cell = const { core::cell::Cell::new(3) }; } +pub fn tls_value() -> u32 { TLS.with(|value| value.get()) } +#[test] fn tls_probe() { assert_eq!(tls_value(), 3); } +''', "cfg body: test_graph::tls_value: analysis-shared"), + ("TypeId", "\n#[test] fn identity_probe() { let _ = std::any::TypeId::of::(); }\n", "TypeId: deferred to codegen"), + ("common trait impl", "\n#[cfg(test)] impl core::fmt::Display for Shape { fn fmt(&self, _: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { Ok(()) } }\n", "pair veto: test-only-trait-impl"), + ("test-only trait exception", "\n#[cfg(test)] trait Only {}\n#[cfg(test)] impl Only for Shape {}\n", "pair veto: none"), + ("blanket impl", "\n#[cfg(test)] trait Only {}\n#[cfg(test)] impl Only for T {}\n", "pair veto: test-only-blanket-impl"), + ("prelude shadowing", "\n#[cfg(test)] pub struct Option;\n", "pair veto: namespace-shadowing"), + ("block scope shadowing", "\npub fn scoped() { #[cfg(test)] struct Option; }\n", "pair veto: namespace-shadowing"), + ("value namespace shadowing", "\n#[cfg(test)] fn Some() {}\n", "pair veto: namespace-shadowing"), + ("macro namespace shadowing", "\n#[cfg(test)] use std::stringify as vec;\n", "pair veto: namespace-shadowing"), + ("inherent method collision", "\n#[cfg(not(test))] impl Shape { fn collision() {} }\n#[cfg(test)] impl Shape { fn collision() {} }\n", "pair veto: colliding-inherent-method"), + ("exported test macro", "\n#[cfg(test)] #[macro_export] macro_rules! only_test { () => { () } }\n", "pair veto: cfg-test-macro-rules"), + ("isolated test macro", "\n#[cfg(test)] mod isolated { macro_rules! only_test { () => { () } } fn use_it() { only_test!(); } }\n", "pair veto: none"), + ("shared foreign test macro", "\n#[cfg(test)] use core::assert as foreign_assert;\n", "pair veto: none"), + ("unused tail test macro", "\n#[cfg(test)] macro_rules! only_test { () => { () } }\n", "pair veto: none"), + ("textual test macro", "\n#[cfg(test)] macro_rules! only_test { () => { () } }pub fn after_macro() {}\n", "pair veto: cfg-test-macro-rules"), + ("macro_use test macro", "\n#[cfg(test)] #[macro_use] mod macros { macro_rules! only_test { () => { () } } }\npub fn after_macro() {}\n", "pair veto: cfg-test-macro-rules"), + ("re-exported test macro", "\n#[cfg(test)] mod macros { macro_rules! only_test { () => { () } } pub(crate) use only_test; }\n#[cfg(test)] use macros::only_test;\n", "pair veto: cfg-test-macro-rules"), + ("drop closure", "\npub struct Dropped;\nimpl Drop for Dropped { fn drop(&mut self) { let _ = STATE; } }\npub fn dropped() { let _ = Dropped; }\n", "cfg body: test_graph::dropped: analysis-shared"), + ] + for label, addition, expected in variants: + before = {path: len(path.read_text(errors="replace")) for path in logs.glob("**/log")} + pathlib.Path(source).write_text(original + addition) + _, ran = side.cargo("test") + fresh = "\n".join(path.read_text(errors="replace")[before.get(path, 0):] for path in logs.glob("**/log")) + check(f"{label} diagnostics", ran.returncode == 0 and expected in fresh, f"exit {ran.returncode}, expected {expected}") + finally: + side.cargo("stop") + return checks + test_graph_sharing(work) + + +def test_graph_sharing(work): + ws = os.path.realpath(os.path.join(work, "graph-sharing")) + files = { + "Cargo.toml": '[package]\nname = "graph_sharing"\nversion = "0.1.0"\nedition = "2021"\n[workspace]\nmembers = ["probe", "format_helper"]\n[dev-dependencies]\nprobe = { path = "probe" }\nformat_helper = { path = "format_helper" }\n', + "format_helper/Cargo.toml": '[package]\nname = "format_helper"\nversion = "0.1.0"\nedition = "2021"\n', + "format_helper/src/lib.rs": '''pub struct Compare(pub T); +impl std::fmt::Display for Compare { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self.0) } +} +''', + "probe/Cargo.toml": '[package]\nname = "probe"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\ngraph_sharing = { path = ".." }\n', + "probe/src/lib.rs": '''pub fn normal() { assert_eq!(graph_sharing::lease(), 900); assert!(!graph_sharing::tested()); assert_eq!(graph_sharing::caller(), 7); assert_eq!(graph_sharing::const_operand(), 17); assert_eq!(graph_sharing::analysis_closure(), 7); assert_eq!(graph_sharing::analysis_result(true), Ok(7)); assert_eq!(graph_sharing::analysis_result(false), Err(())); } +pub fn shared() -> u32 { graph_sharing::shared() } +pub fn pointer() -> usize { graph_sharing::shared as usize } +pub fn bump() -> usize { graph_sharing::bump() } +pub fn read() -> usize { graph_sharing::read() } +pub fn tls_bump() -> usize { graph_sharing::tls_bump() } +pub fn tls_read() -> usize { graph_sharing::tls_read() } +pub fn default_value() -> u32 { graph_sharing::analysis_default() } +pub use graph_sharing::Defaulted as NormalDefault; +pub trait ExternalValue { type Inner; } +impl ExternalValue for u8 { type Inner = u8; } +pub struct External { pub value: T::Inner } +pub fn async_closure_value() -> u32 { + let mut future = std::pin::pin!(graph_sharing::analysis_async_closure()); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + match std::future::Future::poll(future.as_mut(), &mut context) { + std::task::Poll::Ready(value) => value, + std::task::Poll::Pending => panic!("ready future"), + } +} +pub fn hidden_size() -> usize { graph_sharing::hidden_size() } +''', + "src/main.rs": 'fn main() { println!("{}", graph_sharing::shared()); }\n', + "src/lib.rs": '''use std::sync::atomic::{AtomicUsize, Ordering}; +#[inline(never)] pub fn shared() -> u32 { 40 + 2 } +pub fn share_switch() -> u32 { 17 } +pub async fn shared_async() -> u32 { 13 } +#[cfg(test)] const LEASE: u32 = 1; +#[cfg(not(test))] const LEASE: u32 = 900; +pub fn lease() -> u32 { LEASE } +pub fn tested() -> bool { cfg!(test) } +pub fn different() -> u32 { if cfg!(test) { 9 } else { 7 } } +pub fn caller() -> u32 { different() } +const PER_COPY: u32 = if cfg!(test) { 19 } else { 17 }; +pub fn const_operand() -> u32 { PER_COPY } +pub fn analysis_closure() -> u32 { let call = || different(); call() } +pub async fn analysis_async() -> usize { + let iter = changed(); + std::future::ready(()).await; + iter.count() +} +pub fn analysis_result(ok: bool) -> Result { + if ok { Ok(different()) } else { Err(()) } +} +pub struct Defaulted(pub u32); +impl Default for Defaulted { + fn default() -> Self { Self(if cfg!(test) { 19 } else { 17 }) } +} +pub fn analysis_default() -> u32 { Some(()).map(|_| Defaulted::default().0).unwrap() } +pub async fn analysis_async_closure() -> u32 { let call = async || different(); call().await } +pub trait AsyncRunner { async fn run(&self) -> u32; } +pub struct AsyncWorker; +impl AsyncRunner for AsyncWorker { async fn run(&self) -> u32 { different() } } +pub async fn analysis_trait_async() -> u32 { AsyncWorker.run().await } +pub trait AsyncDefault { async fn default_value() -> u32; } +impl AsyncDefault for Defaulted { + async fn default_value() -> u32 { Self::default().0 } +} +pub async fn analysis_trait_generic() -> u32 { + let _ = different(); + T::default_value().await +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TaggedSegment { Normal(String), Tagged(T) } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlanSegment { Normal(String), Tagged(u32) } +fn map_segment(segment: TaggedSegment) -> PlanSegment { + match segment { + TaggedSegment::Normal(text) => PlanSegment::Normal(text), + TaggedSegment::Tagged(value) => PlanSegment::Tagged(value), + } +} +pub fn analysis_collect_segments() -> Vec { + let _ = different(); + vec![TaggedSegment::Normal(String::from("plan")), TaggedSegment::Tagged(23)] + .into_iter().map(map_segment).collect() +} +static STATE: AtomicUsize = AtomicUsize::new(0); +pub fn bump() -> usize { STATE.fetch_add(1, Ordering::SeqCst) } +pub fn read() -> usize { STATE.load(Ordering::SeqCst) } +std::thread_local! { static TLS: std::cell::Cell = const { std::cell::Cell::new(0) }; } +pub fn tls_bump() -> usize { TLS.with(|value| value.replace(value.get() + 1)) } +pub fn tls_read() -> usize { TLS.with(|value| value.get()) } +pub fn static_generic(value: T) -> String { + let _ = STATE.load(Ordering::SeqCst); + format!("{:?}", value) +} +pub struct Batch(pub [u8; 1]); +impl IntoIterator for Batch { + type Item = u8; + type IntoIter = std::array::IntoIter; + fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } +} +pub struct Shape; +impl std::fmt::Debug for Shape { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("shape") } +} +pub trait Calc { fn calc(&self) -> u32; } +impl Calc for Shape { fn calc(&self) -> u32 { 3 } } +pub fn generic(value: T) -> u32 { value.calc() } +pub fn closure() -> u32 { let value = || shared(); value() } +pub fn changed() -> impl Iterator { + #[cfg(test)] { [0u8; 128].into_iter() } + #[cfg(not(test))] { [0u8; 1].into_iter() } +} +pub fn count() -> usize { changed().count() } +pub fn hidden_size() -> usize { std::mem::size_of_val(&changed()) } +#[cfg(test)] mod tests { + struct Only; + impl super::Calc for Only { fn calc(&self) -> u32 { 11 } } + #[test] fn s3c_identity() { + let local: super::Defaulted = Default::default(); + let normal: probe::NormalDefault = Default::default(); + assert_eq!(local.0, 19); + assert_eq!(normal.0, 17); + } + #[test] fn dev_dependency_layout() { + let value = probe::External:: { value: 3 }; + assert_eq!(value.value, 3); + } + #[test] fn both_copies() { + assert_eq!(super::lease(), 1); + assert!(super::tested()); + assert_eq!(super::caller(), 9); + assert_eq!(super::share_switch(), 17); + assert_eq!(super::const_operand(), 19); + assert_eq!(super::analysis_closure(), 9); + assert_eq!(super::analysis_result(true), Ok(9)); + assert_eq!(super::analysis_result(false), Err(())); + assert_eq!(super::analysis_default(), 19); + assert_eq!(probe::default_value(), 17); + let mut closure = std::pin::pin!(super::analysis_async_closure()); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert_eq!(std::future::Future::poll(closure.as_mut(), &mut context), std::task::Poll::Ready(9)); + assert_eq!(probe::async_closure_value(), 7); + let mut trait_future = std::pin::pin!(super::analysis_trait_async()); + assert_eq!(std::future::Future::poll(trait_future.as_mut(), &mut context), std::task::Poll::Ready(9)); + let mut shared_future = std::pin::pin!(super::shared_async()); + assert_eq!(std::future::Future::poll(shared_future.as_mut(), &mut context), std::task::Poll::Ready(13)); + let mut local_generic = std::pin::pin!(super::analysis_trait_generic::()); + assert_eq!(std::future::Future::poll(local_generic.as_mut(), &mut context), std::task::Poll::Ready(19)); + let mut future = std::pin::pin!(super::analysis_async()); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert_eq!(std::future::Future::poll(future.as_mut(), &mut context), std::task::Poll::Ready(128)); + probe::normal(); + assert_eq!(super::shared(), 42); + assert_eq!(probe::shared(), 42); + assert_eq!(super::generic(super::Shape), 3); + assert_eq!(super::generic(Only), 11); + let object: &dyn super::Calc = &super::Shape; + assert_eq!(object.calc(), 3); + let debug: &dyn std::fmt::Debug = &super::Shape; + assert_eq!(format!("{:?}", debug), "shape"); + assert_eq!(format!("{}", format_helper::Compare(super::Shape)), "shape"); + assert_eq!(super::analysis_collect_segments(), vec![ + super::PlanSegment::Normal(String::from("plan")), super::PlanSegment::Tagged(23) + ]); + assert!(Vec::>::new().into_iter().map(super::map_segment) + .collect::>().is_empty()); + assert_eq!(super::static_generic(super::Shape), "shape"); + let mut flattened = vec![super::Batch([3])].into_iter().flatten(); + assert_eq!(flattened.next(), Some(3)); + assert_eq!(flattened.next(), None); + assert_eq!(super::closure(), 42); + assert_ne!(super::shared as usize, probe::pointer()); + assert_eq!(super::bump(), 0); + assert_eq!(probe::read(), 0); + assert_eq!(probe::bump(), 0); + assert_eq!(super::read(), 1); + assert_eq!(super::tls_bump(), 0); + assert_eq!(probe::tls_read(), 0); + assert_eq!(probe::tls_bump(), 0); + assert_eq!(super::tls_read(), 1); + assert_eq!(super::count(), 128); + assert_ne!(super::hidden_size(), probe::hidden_size()); + } +} +''', + "tests/integration.rs": '''#[test] fn normal_copy() { + assert_eq!(graph_sharing::shared(), 42); + assert_eq!(graph_sharing::lease(), 900); + assert!(!graph_sharing::tested()); + assert_eq!(graph_sharing::caller(), 7); + assert_eq!(graph_sharing::count(), 1); +} +''', + } + for file, text in files.items(): + path = pathlib.Path(ws, file) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + side = Side(ws, dict(base_env(os.path.join(work, "home-graph-sharing")), RUSTD_CFG_TWINS="items", RUSTD_CACHE_CHECK="1", RUSTD_CFG_ANALYSIS_TRACE="1"), True) + checks = [] + + def check(name, held, detail): + checks.append((f"cfg sharing: {name}", bool(held), detail)) + print(f" {'ok ' if held else 'NOT'} cfg sharing: {name} {detail}", flush=True) + + logs = pathlib.Path(side.env["RUSTD_HOME"]) + def snapshot(): + return {path: path.read_text(errors="replace") for path in logs.glob("**/log")} + + def log(before=None): + return "\n".join(text[len((before or {}).get(path, "")):] for path, text in snapshot().items()) + + try: + _, cold = side.cargo("test") + check("cold runtime", cold.returncode == 0, cold.stderr[-300:]) + wait_warm(side, ws) + _, warm = side.cargo("test") + check("both copies, statics and opaque layout", warm.returncode == 0 and how_built(warm.stderr) == "daemon" and "tests::both_copies" in warm.stdout and "test normal_copy" in warm.stdout, warm.stderr[-300:]) + text = log() + shared = re.findall(r"cfg codegen body: shared: T compiled (\d+), redirected (\d+)", text) + check("shared function has no T compilation", bool(shared) and all(t == "0" and n == "1" for t, n in shared), str(shared)) + check("opaque callers stay analysis-shared", "cfg body: graph_sharing::count: analysis-shared" in text and "cfg body: graph_sharing::hidden_size: analysis-shared" in text, "count, hidden_size") + queries = re.findall(r"cfg analysis query: (infer|mir) shared_async: test (true|false)", text) + check("shared async has no T analysis", bool(queries) and all(copy == "false" for _, copy in queries), str(queries)) + for body in ("caller", "count", "hidden_size", "bump", "read", "tls_bump", "tls_read", "static_generic", "const_operand", "analysis_closure", "analysis_async", "analysis_result", "analysis_default", "analysis_async_closure", "analysis_trait_async", "analysis_trait_generic", "analysis_collect_segments"): + queries = re.findall(rf"cfg analysis query: (infer|mir) {body}: test (true|false)", text) + check(f"{body} reuses N analysis", bool(queries) and all(copy == "false" for _, copy in queries), str(queries)) + compiled = re.findall(rf"cfg codegen body: {body}: T compiled 1, redirected 0, analysis-shared (\d+)", text) + check(f"{body} compiled from N MIR", bool(compiled) and all(count == "1" for count in compiled), str(compiled)) + _, built = side.cargo("run", "--bin", "graph_sharing") + check("CLI before edit", built.returncode == 0 and built.stdout.strip() == "42", built.stderr[-200:]) + rewrite(os.path.join(ws, "src/lib.rs"), "40 + 2", "41 + 2") + for file in ("src/lib.rs", "tests/integration.rs"): + path = pathlib.Path(ws, file) + path.write_text(path.read_text().replace(", 42)", ", 43)")) + before = snapshot() + _, edited = side.cargo("run", "--bin", "graph_sharing") + fresh = log(before) + check("CLI shared edit splices one function", edited.returncode == 0 and edited.stdout.strip() == "43" and "spliced: 1 functions redirected" in fresh, edited.stderr[-200:] + fresh[-600:]) + before = snapshot() + _, tests = side.cargo("test") + check("both programs observe shared edit", tests.returncode == 0 and how_built(tests.stderr) == "daemon" and "tests::both_copies" in tests.stdout and "test normal_copy" in tests.stdout, tests.stderr[-300:]) + fresh = log(before) + check("test shared edit patches with sharing", fresh.count("patch:") >= 2 and "spliced:" in fresh and "the whole object was written again" not in fresh, fresh[-900:]) + source = pathlib.Path(ws, "src/lib.rs") + source.write_text(source.read_text().replace("share_switch() -> u32 { 17 }", "share_switch() -> u32 { if cfg!(test) { 19 } else { 17 } }").replace("super::share_switch(), 17", "super::share_switch(), 19")) + before = snapshot() + _, switched = side.cargo("test") + fresh = log(before) + check("changed sharing decisions rebuild", switched.returncode == 0 and "cfg sharing decisions changed" in fresh, switched.stderr[-300:] + fresh[-900:]) + probe = pathlib.Path(ws, "probe/src/lib.rs") + probe.write_text(probe.read_text() + '\npub fn boxed() -> Box { Box::new(graph_sharing::Shape) }\n') + source.write_text(source.read_text() + ''' +#[cfg(test)] mod identity { + #[test] fn cross_copy_downcast() { + assert!(probe::boxed().downcast::().is_err()); + let value: Box = Box::new(super::Shape); + assert!(value.downcast::().is_ok()); + } +} +''') + before = snapshot() + _, identity = side.cargo("test") + fresh = log(before) + check("cross-copy TypeId restarts unshared", identity.returncode == 0 and "identity::cross_copy_downcast" in identity.stdout and "TypeId restarts 1" in fresh, identity.stderr[-400:] + fresh[-600:]) + finally: + side.cargo("stop") + disabled = Side(ws, dict(base_env(os.path.join(work, "home-sharing-disabled")), RUSTD_CFG_TWINS="items", RUSTD_CFG_SHARING="0"), True) + try: + disabled.cargo("test") + wait_warm(disabled, ws) + _, result = disabled.cargo("test") + text = "\n".join(path.read_text(errors="replace") for path in pathlib.Path(disabled.env["RUSTD_HOME"]).glob("**/log")) + check("sharing switch keeps T compilation", result.returncode == 0 and how_built(result.stderr) == "daemon" and "cfg codegen body: shared: T compiled 1, redirected 0" in text and "TypeId restarts 1" not in text, result.stderr[-300:]) + finally: + disabled.cargo("stop") + + single_ws = os.path.join(work, "sharing-single") + os.makedirs(os.path.join(single_ws, "src")) + pathlib.Path(single_ws, "Cargo.toml").write_text('[package]\nname="sharing_single"\nversion="0.1.0"\nedition="2021"\n') + pathlib.Path(single_ws, "src/lib.rs").write_text('''use std::sync::atomic::{AtomicUsize, Ordering}; +pub struct Token; +pub struct Batch(Vec); +impl IntoIterator for Batch { + type Item = u8; + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } +} +static STATE: AtomicUsize = AtomicUsize::new(0); +pub fn bump() -> usize { STATE.fetch_add(1, Ordering::SeqCst) } +pub fn boxed() -> Box { Box::new(Token) } +pub fn changed() -> u8 { if cfg!(test) { 3 } else { 4 } } +pub fn flatten() -> Option { + let _ = STATE.load(Ordering::SeqCst); + vec![Batch(vec![changed()])].into_iter().flatten().next() +} +#[test] fn one_copy() { + let mut flattened = vec![Batch(vec![3])].into_iter().flatten(); + assert_eq!(flattened.next(), Some(3)); + assert_eq!(flattened.next(), None); + assert_eq!(flatten(), Some(3)); + assert_eq!(bump(), 0); + assert_eq!(STATE.load(Ordering::SeqCst), 1); + let value: Box = Box::new(Token); + assert!(value.is::()); + assert!(value.downcast::().is_ok()); + assert!(boxed().downcast::().is_ok()); +} +''') + single = Side(single_ws, dict(base_env(os.path.join(work, "home-sharing-single")), RUSTD_CFG_TWINS="items", RUSTD_CFG_ANALYSIS_TRACE="1"), True) + try: + single.cargo("test") + wait_warm(single, single_ws) + _, result = single.cargo("test") + text = "\n".join(path.read_text(errors="replace") for path in pathlib.Path(single.env["RUSTD_HOME"]).glob("**/log")) + check("one copy retains static and TypeId identity", result.returncode == 0 and how_built(result.stderr) == "daemon" and "cfg codegen body: bump: T compiled 0, redirected 1" in text and "TypeId restarts 1" not in text, result.stderr[-300:]) + check("one-copy analysis Flatten uses N impl environment", result.returncode == 0 and "cfg codegen body: flatten: T compiled 1, redirected 0, analysis-shared 1" in text and "cfg analysis query: mir flatten: test true" not in text and "cfg analysis query: infer flatten: test true" not in text, result.stderr[-300:]) + finally: + single.cargo("stop") + return checks + test_graph_sharing_incremental(work) + + +def test_graph_sharing_incremental(work): + checks, counts = [], {} + for mode in ("on", "off"): + ws = os.path.realpath(os.path.join(work, "sharing-incremental-" + mode)) + functions = "\n".join(f"pub fn f{i}(x: T) -> (T, u64) {{ (x, if enabled(1) {{ {i} }} else {{ 0 }}) }}" for i in range(400)) + total = "+".join(f"f{i}(1u64).1" for i in range(400)) + feature = "pub fn enabled(x: u64) -> bool { x != 0 }\nfn identity(x: u64) -> u64 { x }\n#[derive(Clone)] pub struct Flag;\nmacro_rules! generated { () => { pub fn generated_value() -> u64 { 7 } } }\ngenerated!();\nimpl Flag { pub fn check(x: u64) -> bool { enabled(x) } }\npub fn dependency_value() -> u64 { named_dependency::value() }\npub fn schema_value() -> u64 { schema_probe::value() }\npub fn policy_value() -> u64 { policy_cycle(0) }\nfn policy_cycle(x: u8) -> u64 { if x == 0 { 7 } else { policy_value() } }\n" + functions + "\n" + feature += "named_dependency::macro_value!(0);\n" + reports = "\n".join(f'#[test] fn report{i}() -> Result<(), &\'static str> {{ Ok(()) }}' for i in range(20)) + files = { + "Cargo.toml": '[package]\nname="sharing_incremental"\nversion="0.1.0"\nedition="2021"\n[workspace]\nmembers=["probe", "named_dependency", "schema", "unused_dependency"]\n[dependencies]\nnamed_dependency={path="named_dependency"}\nschema_probe={path="schema"}\nunused_dependency={path="unused_dependency"}\n[dev-dependencies]\nprobe={path="probe"}\n', + "schema/Cargo.toml": '[package]\nname="schema_probe"\nversion="0.1.0"\nedition="2021"\n[dev-dependencies]\nsharing_incremental={path=".."}\n', + "schema/src/lib.rs": 'pub fn value() -> u64 { 7 }\n#[cfg(test)] mod tests { #[test] fn dependency() { assert_eq!(sharing_incremental::schema_value(), 7); } }\n', + "named_dependency/Cargo.toml": '[package]\nname="named_dependency"\nversion="0.1.0"\nedition="2021"\n', + "named_dependency/src/lib.rs": 'pub fn value() -> u64 { 7 }\npub static VALUE: u64 = 7;\n#[macro_export] macro_rules! macro_value { (0) => { pub fn macro_dependency_value() -> u64 { 7 } }; (1) => { pub fn macro_dependency_value() -> u64 { unused_dependency::value() } }; }\n', + "unused_dependency/Cargo.toml": '[package]\nname="unused_dependency"\nversion="0.1.0"\nedition="2021"\n', + "unused_dependency/src/lib.rs": 'pub fn value() -> u64 { 7 }\n', + "probe/Cargo.toml": '[package]\nname="probe"\nversion="0.1.0"\nedition="2021"\n[dependencies]\nsharing_incremental={path=".."}\n', + "probe/src/lib.rs": 'pub fn total() -> u64 { sharing_incremental::total() }\npub fn different() -> u64 { sharing_incremental::different() }\npub fn policy_value() -> u64 { sharing_incremental::policy_value() }\n', + "src/features.rs": feature, + "src/lib.rs": f'''mod features; +pub use features::*; +thread_local! {{ static SAID: std::cell::RefCell> = None.into(); }} +pub fn macro_local() -> u64 {{ SAID.with_borrow(|it| if it.is_none() {{ 3 }} else {{ 4 }}) }} +pub fn total() -> u64 {{ {total} }} +#[cfg(not(test))] pub fn different() -> u64 {{ 2 }} +#[cfg(test)] pub fn different() -> u64 {{ 1 }} +#[cfg(test)] mod tests {{ + {reports} + #[test] fn copies() {{ + assert!(super::Flag::check(1)); + let _ = super::Flag.clone(); + assert_eq!(super::generated_value(), 7); + assert_eq!(super::macro_dependency_value(), 7); + assert_eq!(super::macro_local(), 3); + assert_eq!(super::total(), 79800); + assert_eq!(super::total(), probe::total()); + assert_eq!(super::different(), 1); + assert_eq!(probe::different(), 2); + }} + #[test] fn policy_values() {{ + let local = super::policy_value(); + let normal = probe::policy_value(); + println!("policy-values: {{local}}/{{normal}}"); + if local == 7 {{ assert_eq!(normal, 7); }} + else {{ assert_eq!(local, 1); assert_eq!(normal, 2); }} + }} +}} +''', + } + for name, text in files.items(): + path = pathlib.Path(ws, name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + store = os.path.join(work, "store-sharing-incremental-" + mode) + for prime in ("cold", "retained"): + env = dict(base_env(os.path.join(work, f"hi-{mode}-{prime[0]}")), RUSTD_CACHE=store, + RUSTD_STATS="1", RUSTD_TRACE="1", RUSTD_VERIFY_FASTPATH="1", RUSTD_CFG_SHARING="1" if mode == "on" else "0") + side = Side(ws, env, True) + def logs(): + return {path: path.read_text(errors="replace") for path in pathlib.Path(env["RUSTD_HOME"]).glob("**/log")} + try: + _, built = side.cargo("test", "--lib") + wait_warm(side, ws) + before = logs() + primed = "\n".join(before.values()) + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "let enabled = x != 2;\n enabled")) + _, edited = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + count = sum(map(int, re.findall(r"backend: (?:cranelift|llvm) compiled (\d+) functions,", fresh))) + counts[mode, prime] = count + name = f"cfg sharing: {mode} {prime}-prime body edit splices" + held = (built.returncode == edited.returncode == 0 and how_built(edited.stderr) == "daemon" + and "spliced:" in fresh and "the whole object was written again" not in fresh + and "cfg sharing decisions changed" not in fresh and "patch discard:" not in fresh) + detail = f"{count} backend functions; {fresh[-700:]}" + checks.append((name, held, detail)) + print(f" {'ok ' if held else 'NOT'} {name} {detail}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime body edit takes verified fast paths" + held = ("reach fast path: unchanged edges" in fresh and "reach full path" not in fresh + and (mode == "off" or ("sharing fast path: unchanged facts" in fresh and "sharing full path" not in fresh))) + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime edit keeps unchanged report shims" + held = "patch reports: 0 of 20" in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime reach checks only patch functions" + candidates = list(map(int, re.findall(r"reach patch candidates: (\d+)", fresh))) + held = bool(candidates) and max(candidates) <= 4 + checks.append((name, held, str(candidates))) + print(f" {'ok ' if held else 'NOT'} {name}: {candidates}", flush=True) + if mode == "on": + name = f"cfg sharing: {prime}-prime policy checks only edited function facts" + candidates = list(map(int, re.findall(r"sharing candidates: bodies=(\d+)", fresh))) + held = bool(candidates) and max(candidates) <= 2 + checks.append((name, held, str(candidates))) + print(f" {'ok ' if held else 'NOT'} {name}: {candidates}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime path diagnostics identify the program" + paths = re.findall(r"^(?:reach|sharing) (?:fast|full) path:?.*$", fresh, re.M) + held = bool(paths) and all("; program=" in line for line in paths) + checks.append((name, held, str(paths))) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + for query, limit in [("reach_topology", 0), ("reach_component_print", 2), ("reach_root_code", 1), ("loaded_crates", 0)] + ([("twin_index", 0), ("pair_safety", 0), ("pair_eligibility", 0), ("decision_serial", 0), ("applicable_adt_impls", 0), ("local_trait_impls", 0)] if mode == "on" else [("pair_eligibility", 0), ("eligibility_facts", 0)]): + name = f"cfg sharing: {mode} {prime}-prime {query} executions <= {limit}" + disabled_policy = mode == "off" and query in ("pair_eligibility", "eligibility_facts") + count_ran = sum(map(int, re.findall(rf"\b{query}=(\d+)", primed + fresh if disabled_policy else fresh))) + held = ("queries:" in fresh and ("queries:" in primed if disabled_policy else re.search(rf"\b{query}=\d+", primed) is not None) + and count_ran <= limit) + checks.append((name, held, f"executed {count_ran}")) + print(f" {'ok ' if held else 'NOT'} {name}: {count_ran}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime edited file dependency names execute and backdate" + projected = sum(map(int, re.findall(r"\bfile_dependency_names=(\d+)", fresh))) + held = projected > 0 and re.search(r"\bloaded_crates=\d+", primed) is not None + checks.append((name, held, f"executed {projected}")) + print(f" {'ok ' if held else 'NOT'} {name}: {projected}", flush=True) + for case, text in [("code-only edit", feature.replace("x != 0", "x != 3")), ("restored code", feature), ("relocated source", feature.replace("x != 0", "{\n x != 0\n}"))]: + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(text) + _, result = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime {case} verifies reach prints" + held = result.returncode == 0 and "reach fast path: unchanged edges" in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + if case == "restored code": + name = f"cfg sharing: {mode} {prime}-prime body edits skip impl prewarm" + prewarm = re.findall(r"check impls prewarm: (\w+);", fresh) + held = bool(prewarm) and all(value == "skipped" for value in prewarm) + checks.append((name, held, str(prewarm))) + print(f" {'ok ' if held else 'NOT'} {name}: {prewarm}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime root checks follow changed functions" + candidates = re.findall(r"patch root candidates: (\d+) of (\d+)", fresh) + held = bool(candidates) and all(int(selected) <= 4 for selected, total in candidates) + checks.append((name, held, str(candidates))) + print(f" {'ok ' if held else 'NOT'} {name}: {candidates}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime instance checks follow changed functions" + candidates = re.findall(r"patch instance candidates: (\d+) of (\d+)", fresh) + held = bool(candidates) and all(int(selected) < int(total) or int(total) == 0 for selected, total in candidates) + checks.append((name, held, str(candidates))) + print(f" {'ok ' if held else 'NOT'} {name}: {candidates}", flush=True) + name = f"cfg sharing: {mode} {prime}-prime retains diagnostics and harness text" + held = "write text: retained skipped=true harness=true" in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + for case, text in [("added", feature.replace("macro_value!(0)", "macro_value!(1)")), ("removed", feature)]: + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(text) + _, result = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime {case} macro dependency invalidates loaded crates" + ran = sum(map(int, re.findall(r"\bloaded_crates=(\d+)", fresh))) + held = result.returncode == 0 and ran > 0 + checks.append((name, held, f"executed {ran}")) + print(f" {'ok ' if held else 'NOT'} {name}: {ran}", flush=True) + if mode == "on": + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "let _ = std::any::TypeId::of::(); let _ = named_dependency::VALUE; !std::ops::Not::not(x != 0)")) + _, foreign_edit = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {prime}-prime foreign trait reference keeps policy fast path" + held = (foreign_edit.returncode == 0 and "field=references.items" in fresh + and "sharing fast path" in fresh and "sharing full path" not in fresh + and "sharing incremental path" not in fresh) + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + for case, text in [ + ("rejection", feature.replace("if x == 0 { 7 }", "if x == 0 { super::different() }")), + ("un-rejection", feature), + ]: + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(text) + _, policy_edit = side.cargo("test", "--lib", "--", "--nocapture") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + updates = re.findall(r"sharing incremental path: bodies=(\d+) affected=(\d+) rejected=(\d+)->(\d+)", fresh) + name = f"cfg sharing: {prime}-prime SCC incremental {case}" + values = "policy-values: 1/2" if case == "rejection" else "policy-values: 7/7" + held = (policy_edit.returncode == 0 and values in policy_edit.stdout + and bool(updates) and "sharing full path" not in fresh + and all(int(bodies) == 1 and 2 <= int(affected) <= 8 + and (int(new) > int(old) if case == "rejection" else int(new) < int(old)) + for bodies, affected, old, new in updates)) + checks.append((name, held, str(updates) + fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}: {updates}", flush=True) + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "identity(x) != 0")) + _, edge_edit = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime edge edit leaves unrelated roots cached" + topology_ran = sum(map(int, re.findall(r"\breach_topology=(\d+)", fresh))) + roots_ran = sum(map(int, re.findall(r"\breach_root_code=(\d+)", fresh))) + held = (edge_edit.returncode == 0 and topology_ran == 1 and roots_ran == 1 + and "reach fast path: updated edges" in fresh and "reach full path" not in fresh) + detail = f"topology {topology_ran}, roots {roots_ran}" + checks.append((name, held, detail)) + print(f" {'ok ' if held else 'NOT'} {name}: {detail}", flush=True) + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "identity(x) != 0").replace("fn identity(x: u64) -> u64 { x }", "fn identity(x: u64) -> u64 { x + 1 }")) + _, new_callee_edit = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime edit of callee absent from patch base verifies reach" + held = new_callee_edit.returncode == 0 and "reach fast path: unchanged edges" in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + for case, text in [ + ("remove a call", feature), + ("create a cycle", feature.replace("x != 0", "identity(x) != 0").replace( + "fn identity(x: u64) -> u64 { x }", + "fn identity(x: u64) -> u64 { if x == 0 { enabled(1) as u64 } else { x } }")), + ("break a cycle", feature.replace("x != 0", "identity(x) != 0")), + ]: + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(text) + _, edge_change = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime verified reach {case}" + held = (edge_change.returncode == 0 and "reach fast path: updated edges" in fresh + and "reach full path" not in fresh) + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + nested_feature = feature.replace("impl Flag {", "const _: () = { impl Flag {").replace("enabled(x) } }", "enabled(x) } } };") + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(nested_feature) + _, nested_prime = side.cargo("test", "--lib") + broad = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime impl edits retain impl prewarm" + held = nested_prime.returncode == 0 and "check impls prewarm: enabled;" in broad + checks.append((name, held, broad[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(nested_feature.replace("x != 0", "x != 2")) + _, nested_edit = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime unrelated nested impl permits verified reach fast path" + held = nested_prime.returncode == nested_edit.returncode == 0 and "reach fast path: unchanged edges" in fresh and "reach full path" not in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + if mode == "on": + name = f"cfg sharing: {prime}-prime unrelated nested impl permits one verified policy fast path" + held = fresh.count("sharing fast path: unchanged facts") == 1 and "sharing full path" not in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "let _ = cfg!(test); identity(x) != 0")) + _, cfg_edit = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {prime}-prime body cfg fact invalidates policy" + held = cfg_edit.returncode == 0 and "sharing full path" in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "const _: u64 = 0; x != 0")) + _, inventory_edit = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + impls_ran = sum(map(int, re.findall(r"\blocal_impls=(\d+)", fresh))) + applicable_ran = sum(map(int, re.findall(r"\bapplicable_adt_impls=(\d+)", fresh))) + name = f"cfg sharing: {prime}-prime body item preserves ADT impl facts" + held = inventory_edit.returncode == 0 and impls_ran > 0 and applicable_ran == 0 + detail = f"impl inventory {impls_ran}, ADT impl facts {applicable_ran}" + checks.append((name, held, detail)) + print(f" {'ok ' if held else 'NOT'} {name}: {detail}", flush=True) + for case, text in [("added static", feature.replace("x != 0", "static ON: u64 = 1; x != 0 && ON == 1")), ("removed static", feature)]: + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(text) + _, result = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {mode} {prime}-prime {case} rebuilds reach" + held = result.returncode == 0 and "reach full path" in fresh + checks.append((name, held, fresh[-700:])) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + if mode == "on" and prime == "cold": + source = files["src/lib.rs"] + cases = [ + ("error result", source.replace("fn report0() -> Result<(), &'static str> { Ok(()) }", "fn report0() -> Result<(), &'static str> { Err(\"expected\") }"), "0 of 20", False), + ("changed return", source.replace("fn report0() -> Result<(), &'static str> { Ok(()) }", "fn report0() -> std::process::ExitCode { std::process::ExitCode::SUCCESS }"), "1 of 20", True), + ("unit return", source.replace("fn report0() -> Result<(), &'static str> { Ok(()) }", "fn report0() {}"), "0 of 19", True), + ("new return", source.replace("fn copies() {", "fn copies() -> Result<(), &'static str> {").replace("assert_eq!(probe::different(), 2);", "assert_eq!(probe::different(), 2); Ok(())"), "1 of 21", True), + ] + for case, text, expected, succeeds in cases: + before = logs() + pathlib.Path(ws, "src/lib.rs").write_text(text) + _, result = side.cargo("test", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + name = f"cfg sharing: {case} report shim" + held = (result.returncode == 0) == succeeds and f"patch reports: {expected}" in fresh + checks.append((name, held, f"{result.stdout[-300:]} {fresh[-700:]}")) + print(f" {'ok ' if held else 'NOT'} {name}", flush=True) + if mode == "on": + pathlib.Path(ws, "src/features.rs").write_text(feature) + pathlib.Path(ws, "src/lib.rs").write_text(files["src/lib.rs"]) + _, schema_prime = side.cargo("test", "-p", "schema_probe", "--lib") + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "x != 2")) + _, schema_edit = side.cargo("test", "-p", "schema_probe", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + policy = [line for line in fresh.splitlines() if "sharing " in line and "/schema_probe-" in line] + name = f"cfg sharing: {prime}-prime dependency body retains schema policy facts" + held = schema_prime.returncode == schema_edit.returncode == 0 and any("sharing fast path" in line for line in policy) and not any("sharing full path" in line for line in policy) + checks.append((name, held, "\n".join(policy))) + print(f" {'ok ' if held else 'NOT'} {name}: {policy}", flush=True) + before = logs() + pathlib.Path(ws, "src/features.rs").write_text(feature.replace("x != 0", "let _ = cfg!(test); x != 0")) + _, schema_cfg = side.cargo("test", "-p", "schema_probe", "--lib") + fresh = "\n".join(text[len(before.get(path, "")):] for path, text in logs().items()) + policy = [line for line in fresh.splitlines() if "sharing " in line and "/schema_probe-" in line] + name = f"cfg sharing: {prime}-prime dependency cfg fact invalidates schema policy" + held = schema_cfg.returncode == 0 and any("sharing full path" in line for line in policy) and any("field=cfg." in line for line in policy) + checks.append((name, held, "\n".join(policy))) + print(f" {'ok ' if held else 'NOT'} {name}: {policy}", flush=True) + finally: + side.cargo("stop") + pathlib.Path(ws, "src/features.rs").write_text(feature) + pathlib.Path(ws, "src/lib.rs").write_text(files["src/lib.rs"]) + for prime in ("cold", "retained"): + name = f"cfg sharing: {prime}-prime edit backend count <= off + 2" + on, off = counts["on", prime], counts["off", prime] + held = on <= off + 2 and (on > 0 or prime == "retained") + detail = f"on {on}, off {off}" + checks.append((name, held, detail)) + print(f" {'ok ' if held else 'NOT'} {name} {detail}", flush=True) + return checks + + def daemon(work): """The life of the daemon. The result is each check with whether it held.""" ws = os.path.realpath(os.path.join(work, "life")) @@ -1608,7 +2464,7 @@ def main(): result["flags"], result["flags_status"] = rows, counted result["wrong"] += wrong print("daemon:", flush=True) - result["daemon"] = daemon(work) + incremental(work) + result["daemon"] = daemon(work) + incremental(work) + test_graph(work) result["wrong"] += [f"daemon: {name} {detail}" for name, held, detail in result["daemon"] if not held] print("plain cargo:", flush=True) result["plain"] = plain(work) diff --git a/harness/codex_warm.py b/harness/codex_warm.py new file mode 100644 index 0000000000..58b6735ca8 --- /dev/null +++ b/harness/codex_warm.py @@ -0,0 +1,177 @@ +"""Prime Codex, then measure five alternating features.rs edits (on|off). + +RUSTD_BENCH_WORKSPACE, RUSTD_BENCH_REPORTS, RUSTD_BENCH_RUN, RUSTD_BENCH_BIN +and RUSTD_BENCH_STORE override the local workspace, output and cache locations. +""" +import hashlib +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import threading +import time + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from warmstats import summarize + +repo = Path(__file__).resolve().parent.parent +workspace = Path(os.environ.get('RUSTD_BENCH_WORKSPACE', str(Path.home() / 'codex/codex-rs'))) +reports = Path(os.environ.get('RUSTD_BENCH_REPORTS', str(Path.home() / 'bench/profile7'))) +mode = sys.argv[1] +assert mode in ('on', 'off', 'local', 'cli') +head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=repo).strip() +diff = subprocess.check_output(['git', 'diff'], cwd=repo) +root = reports / os.environ.get('RUSTD_BENCH_RUN', 'sharing-perf-' + mode) +root.mkdir() +(root / 'measured.patch').write_bytes(diff) +home, store = root / 'daemon', root / 'store' +store = Path(os.environ.get('RUSTD_BENCH_STORE', str(store))) +binary = Path(os.environ.get('RUSTD_BENCH_BIN', str(repo / 'target/release/rustd'))) +env = dict(os.environ, PATH=str(binary.parent) + os.pathsep + os.environ['PATH'], + RUSTUP_TOOLCHAIN='1.99.0', RUSTD_HOME=str(home), RUSTD_CACHE=str(store), + RUSTD_STATS='1', RUSTD_CFG_SHARING='local' if mode == 'local' else ('0' if mode == 'off' else '1'), + CARGO_PROFILE_DEV_SPLIT_DEBUGINFO='unpacked', + CARGO_PROFILE_TEST_SPLIT_DEBUGINFO='unpacked', TMPDIR='/home/ubuntu/b') +for key in list(env): + if key.endswith('API_KEY') or key in ('_RJEM_MALLOC_CONF', 'RUSTD_MEMORY', + 'RUSTD_CFG_TWINS', 'RUSTD_CFG_ANALYSIS_TRACE'): + env.pop(key) +command = (['cargo', 'rustd', 'build', '--bin', 'codex'] if mode == 'cli' else + ['cargo', 'rustd', 'test', '-p', 'codex-core', '--no-run']) +(root / 'command.json').write_text(json.dumps(dict(command=command, + commit=head.decode(), mode=mode, split_debuginfo='unpacked', tmpdir=env['TMPDIR']), indent=2)) +offsets = {} +edits = [] + +def run(label): + perf = None + if label.startswith('edit') and os.environ.get('RUSTD_BENCH_PERF') == '1': + pids = [] + for proc in Path('/proc').iterdir(): + if not proc.name.isdigit(): + continue + try: + args = (proc / 'cmdline').read_bytes().split(b'\0') + except OSError: + continue + if any(str(home).encode() in arg for arg in args) and (b'--socket' in args or b'--spare' in args): + pids.append(proc.name) + assert pids + perf_log = (root / (label + '.perf.log')).open('w') + perf = subprocess.Popen(['sudo', '-n', 'perf', 'record', '-F', '199', + '--call-graph', 'dwarf,8192', '-p', ','.join(pids), + '-o', str(root / (label + '.perf.data'))], stdout=perf_log, stderr=subprocess.STDOUT) + start = time.monotonic() + peak, total_peak = 0, 0 + with (root / (label + '.command.log')).open('w') as log, \ + (root / (label + '.samples.jsonl')).open('w') as samples: + child = subprocess.Popen(command, cwd=workspace, env=env, stdout=log, stderr=subprocess.STDOUT) + done = threading.Event() + finished = [] + def wait_child(): + child.wait() + finished.append(time.monotonic()) + done.set() + waiter = threading.Thread(target=wait_child) + waiter.start() + while True: + workers = [] + for proc in Path('/proc').iterdir(): + if not proc.name.isdigit(): + continue + try: + args = (proc / 'cmdline').read_bytes().split(b'\0') + if not any(str(home).encode() in arg for arg in args): + continue + if not (b'--socket' in args or b'--spare' in args): + continue + status = dict(line.split(':', 1) for line in + (proc / 'status').read_text().splitlines() if ':' in line) + workers.append(dict(pid=proc.name, rss_kib=int(status['VmRSS'].split()[0]))) + except (OSError, KeyError, ValueError): + continue + peak = max(peak, max((w['rss_kib'] for w in workers), default=0)) + total_peak = max(total_peak, sum(w['rss_kib'] for w in workers)) + samples.write(json.dumps(dict(seconds=time.monotonic() - start, workers=workers)) + '\n') + samples.flush() + if done.wait(1): + break + waiter.join() + result = dict(wall_seconds=finished[0] - start, exit_code=child.returncode, + worker_peak_rss_kib=peak, total_peak_rss_kib=total_peak) + (root / (label + '.summary.json')).write_text(json.dumps(result, indent=2)) + logs = [] + for path in sorted(home.rglob('log')): + data = path.read_text(errors='replace') + logs.append(f'\n=== {path} ===\n' + data[offsets.get(str(path), 0):]) + offsets[str(path)] = len(data) + worker_log = ''.join(logs) + (root / (label + '.worker.log')).write_text(worker_log) + if label.startswith('edit'): + edits.append(dict(result, log=worker_log)) + print(mode, label, json.dumps(result), flush=True) + if perf: + pending, targets = [perf.pid], [] + while pending: + pid = pending.pop() + proc = Path('/proc') / str(pid) + try: + if (proc / 'comm').read_text().strip() == 'perf': + targets.append(str(pid)) + else: + pending.extend(map(int, (proc / 'task' / str(pid) / 'children').read_text().split())) + except FileNotFoundError: + pass + if targets: + subprocess.run(['sudo', '-n', 'kill', '-TERM', *targets], check=True) + perf.wait(timeout=120) + perf_log.close() + assert child.returncode == 0, (mode, label, child.returncode) + +source = workspace / 'core/src/features.rs' +original = source.read_bytes() +old, new = b' self.enabled.contains(&f)', b' !self.enabled.contains(&f)' +assert original.count(old) == 1 +(root / 'features.rs.before').write_bytes(original) +try: + run('prime') + if mode != 'cli' and os.environ.get('RUSTD_BENCH_SAVE_TEST_BINARIES') == '1': + programs = [p for state in home.glob('states/*.json') if '.kept.' not in state.name + for p in json.loads(state.read_text()).get('programs', [])] + for name in ('codex_core',): + matches = [p for p in programs if p['name'] == name and p['test']] + assert len(matches) == 1, (name, matches) + target = root / (name + '.bin') + shutil.copy2(matches[0]['executable'], target) + (root / (name + '.origin.json')).write_text(json.dumps(matches[0], indent=2)) + for repeat in range(5): + source.write_bytes(original.replace(old, new) if repeat % 2 == 0 else original) + run('edit' if repeat == 0 else 'edit-' + str(repeat + 1)) + summary = summarize(edits) + (root / 'five-edits.summary.json').write_text(json.dumps(summary, indent=2)) + print(mode, 'five-edits', json.dumps(summary), flush=True) + if os.environ.get('RUSTD_BENCH_REFERENCE_EDIT') == '1': + source.write_bytes(original) + run('restore-before-reference') + replacement = b' self.enabled.contains(&f) && !f.key().is_empty()' + (root / 'reference-edit.patch').write_text( + '- ' + old.decode() + '\n+ ' + replacement.decode() + '\n') + references = [] + for repeat in range(5): + source.write_bytes(original.replace(old, replacement) if repeat % 2 == 0 else original) + label = 'reference-edit' if repeat == 0 else 'reference-edit-' + str(repeat + 1) + run(label) + references.append(dict(json.loads((root / (label + '.summary.json')).read_text()), + log=(root / (label + '.worker.log')).read_text())) + reference_summary = summarize(references) + (root / 'reference-edits.summary.json').write_text(json.dumps(reference_summary, indent=2)) + print(mode, 'reference-edits', json.dumps(reference_summary), flush=True) +finally: + source.write_bytes(original) + assert source.read_bytes() == original + print('features.rs restored byte-identically:', hashlib.sha256(original).hexdigest(), flush=True) + with (root / 'stop.log').open('w') as log: + subprocess.run([str(binary), 'stop'], cwd=workspace, env=env, stdout=log, stderr=subprocess.STDOUT) diff --git a/harness/dxedits.py b/harness/dxedits.py index 855f399efd..7b148de243 100644 --- a/harness/dxedits.py +++ b/harness/dxedits.py @@ -146,7 +146,7 @@ def wait(pattern, seen): perf.wait() print(f"{label:38} {'edit-to-binary' if '--patch' in args else 'edit-to-object'} {b[8]:>8} recompiled {b[5]:>5} of {b[6]}" + (f" save-to-binary-on-disk {wall:.0f}ms" if wall else ""), flush=True) - steps = dict((k, int(v) / 1000) for k, v in (kv.split("=") for kv in b[7].split())) + steps = dict((k, int(v) / 1000) for k, v in (kv.split("=") for kv in b[7].split(";", 1)[0].split())) samples.append({"edit": name, "files": len(paths), "revert": label != name, "wall_ms": wall, "steps_ms": steps, "recompiled": int(b[5]), "profiled": perf is not None}) if "--patch" in args and "doctor" in name: ran = subprocess.run([binary, "doctor", "--help"], capture_output=True, text=True) diff --git a/harness/scaling.py b/harness/scaling.py index 0b721bad0d..bc44eb67ea 100644 --- a/harness/scaling.py +++ b/harness/scaling.py @@ -137,7 +137,7 @@ def wait(pattern, seen): done = len(found) b = found[-1] patched = "\npatch: " in log_tail(log) - steps = {k: int(v) / 1000 for k, v in (kv.split("=") for kv in b[7].split())} + steps = {k: int(v) / 1000 for k, v in (kv.split("=") for kv in b[7].split(";", 1)[0].split())} samples.append({"edit": kind, "file": os.path.relpath(where, fx.ws) if where != "all" else where, "revert": revert, "turn": turn, "steps_ms": steps, "recompiled": int(b[5]), "cached": int(b[6]), "patched": patched}) if kind == "body: println" and where == fx.touch: diff --git a/harness/test_bench.py b/harness/test_bench.py new file mode 100644 index 0000000000..88521fadec --- /dev/null +++ b/harness/test_bench.py @@ -0,0 +1,43 @@ +import ast +from pathlib import Path +import unittest + +from bench import daemon_steps + + +class DaemonStepsTests(unittest.TestCase): + def test_program_labels(self): + text = ("timeline-us: notice=1000 apply=200 roots=300 link=4; program=unit\n" + "timeline-us: notice=2000 apply=400 roots=500 link=6; program=other\n") + self.assertEqual(daemon_steps(text), dict(apply=.6, roots=.8, link=.01)) + + def test_unlabeled(self): + self.assertEqual(daemon_steps("timeline-us: notice=1000 apply=200 roots=300\n"), + dict(apply=.2, roots=.3)) + + def test_empty(self): + self.assertEqual(daemon_steps(""), {}) + + def test_invalid_timing(self): + with self.assertRaises(ValueError): + daemon_steps("timeline-us: roots=invalid; program=unit\n") + + def test_inline_timeline_consumers(self): + for name in ("scaling.py", "dxedits.py"): + tree = ast.parse(Path(__file__).with_name(name).read_text()) + assignment = next(node for node in ast.walk(tree) + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "steps" + for target in node.targets)) + expression = compile(ast.Expression(assignment.value), name, "eval") + for suffix in ("", "; program=unit"): + with self.subTest(name=name, suffix=suffix): + self.assertEqual(eval(expression, {"b": [""] * 7 + ["roots=200" + suffix]}), + dict(roots=.2)) + self.assertEqual(eval(expression, {"b": [""] * 8}), {}) + with self.assertRaises(ValueError): + eval(expression, {"b": [""] * 7 + ["roots=invalid; program=unit"]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/test_warmstats.py b/harness/test_warmstats.py new file mode 100644 index 0000000000..9b8d04f2e4 --- /dev/null +++ b/harness/test_warmstats.py @@ -0,0 +1,40 @@ +import unittest + +from warmstats import distribution, summarize, timelines + + +class WarmStatsTests(unittest.TestCase): + def edits(self, label="; program=unit"): + return [dict(wall_seconds=i, exit_code=0, + log=f"timeline-us: roots={i * 1000} check=100{label}\n") + for i in range(1, 6)] + + def test_five_samples(self): + result = summarize(self.edits()) + self.assertEqual(result["wall_seconds"], dict(median=3, min=1, max=5)) + self.assertEqual(result["program_phase_seconds"]["unit"]["roots"], + dict(median=.003, min=.001, max=.005)) + + def test_unlabeled_old_binary(self): + result = summarize(self.edits("")) + self.assertIsNone(result["program_phase_seconds"]) + self.assertEqual(result["longest_program_phase_seconds"]["roots"]["median"], .003) + + def test_overlapping_programs(self): + edits = self.edits() + for edit in edits: + edit["log"] += "timeline-us: roots=9000 check=200; program=other\n" + self.assertEqual(summarize(edits)["longest_program_phase_seconds"]["roots"]["median"], .009) + + def test_empty_and_failed(self): + self.assertEqual(timelines(""), []) + with self.assertRaises(AssertionError): + summarize([]) + with self.assertRaises(AssertionError): + summarize([dict(wall_seconds=1, exit_code=1, log="")] * 5) + with self.assertRaises(AssertionError): + summarize([dict(wall_seconds=1, exit_code=0, log="")] * 5) + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/warmstats.py b/harness/warmstats.py new file mode 100644 index 0000000000..c0738a19ea --- /dev/null +++ b/harness/warmstats.py @@ -0,0 +1,41 @@ +"""Summarize overlapping worker phases without summing concurrent programs.""" +import re +import statistics + + +def distribution(values): + return dict(median=statistics.median(values), min=min(values), max=max(values)) + + +def timelines(log): + rows = [] + for line in log.splitlines(): + if not line.startswith("timeline-us: "): + continue + phases = {name: int(value) / 1_000_000 + for name, value in re.findall(r"([\w-]+)=(\d+)", line)} + program = re.search(r"; program=(.*)$", line) + rows.append((program[1] if program else None, phases)) + return rows + + +def summarize(edits): + assert len(edits) == 5, len(edits) + assert all(edit["exit_code"] == 0 for edit in edits) + rows = [timelines(edit["log"]) for edit in edits] + assert all(row for row in rows), "missing worker timelines" + assert len({len(row) for row in rows}) == 1, "program count changed" + phases = set.intersection(*(set(p) for row in rows for _, p in row)) + result = dict(edits=5, wall_seconds=distribution([e["wall_seconds"] for e in edits]), + longest_program_phase_seconds={ + phase: distribution([max(p[phase] for _, p in row) for row in rows]) + for phase in sorted(phases)}) + programs = [{name for name, _ in row} for row in rows] + if None not in programs[0] and all(names == programs[0] for names in programs): + assert all(len(names) == len(row) for names, row in zip(programs, rows)) + result["program_phase_seconds"] = { + name: {phase: distribution([dict(row)[name][phase] for row in rows]) + for phase in sorted(phases)} for name in sorted(programs[0])} + else: + result["program_phase_seconds"] = None + return result diff --git a/harness/worktrees.py b/harness/worktrees.py index 089d72a73b..6c5d8d17a0 100644 --- a/harness/worktrees.py +++ b/harness/worktrees.py @@ -181,6 +181,7 @@ def ask(at): print(open(log_path, errors="replace").read()[-6000:], flush=True) reply = "no answer in 120 s" took = (time.time() - began) * 1000 + outputs = None cargo = subprocess.run(["cargo", "build", "-q", "-p", "sapp", *TARGET_ARGS], cwd=checkout["ws"], capture_output=True, text=True, env={**os.environ, "CARGO_TARGET_DIR": checkout["reference"]}) assert (cargo.returncode != 0) == (at in broken), cargo.stderr[-2000:] @@ -193,11 +194,13 @@ def ask(at): elif not reply.startswith("ok"): verdict = "REFUSED" else: - same = run(checkout["bin"]) == run(built(checkout["reference"], "sapp")) - verdict = "same" if same else "DIFFERENT OUTPUT" + outputs = run(checkout["bin"]), run(built(checkout["reference"], "sapp")) + verdict = "same" if outputs[0] == outputs[1] else "DIFFERENT OUTPUT" asked += 1 wrong += verdict.isupper() print(f" checkout {at} {verdict:16} {took:6.0f} ms {reply[:90]}", flush=True) + if verdict == "DIFFERENT OUTPUT": + print(f" rustd: {outputs[0]!r}\n cargo: {outputs[1]!r}", flush=True) if daemon.poll() is not None: print(open(log_path, errors="replace").read()[-6000:], flush=True) raise SystemExit(f"the daemon died: {log_path}") diff --git a/vendor/rust-analyzer b/vendor/rust-analyzer index 3648d25a45..dedaa205f8 160000 --- a/vendor/rust-analyzer +++ b/vendor/rust-analyzer @@ -1 +1 @@ -Subproject commit 3648d25a45f8fdb7688173d155baa6c1056a8ffb +Subproject commit dedaa205f80a6141f29138cf058171e28dc74fff