Build a new incr comp session dir from scratch every time

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -140,7 +140,6 @@ fn emit_module(
         bytecode: None,
         assembly: None,
         llvm_ir: None,
-        links_from_incr_cache: Vec::new(),
     })
 }
 

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -463,23 +463,33 @@ fn thin_lto(
 
         info!("thin LTO data created");
 
-        let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) =
-            cgcx.incr_comp_session_dir
+        let new_key_map_path = cgcx
+            .new_incr_comp_session_dir
+            .as_ref()
+            .map(|dir| dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME));
+
+        let (prev_key_map, curr_key_map) = if let Some(ref old_incr_comp_session_dir) =
+            cgcx.old_incr_comp_session_dir
         {
-            let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME);
+            let old_path = old_incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME);
+
             // If the previous file was deleted, or we get an IO error
             // reading the file, then we'll just use `None` as the
             // prev_key_map, which will force the code to be recompiled.
-            let prev =
-                if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None };
+            let prev = if old_path.exists() {
+                ThinLTOKeysMap::load_from_file(&old_path).ok()
+            } else {
+                None
+            };
             let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names);
-            (Some(path), prev, curr)
+
+            (prev, curr)
         } else {
             // If we don't compile incrementally, we don't need to load the
             // import data from LLVM.
             assert!(green_modules.is_empty());
             let curr = ThinLTOKeysMap::default();
-            (None, None, curr)
+            (None, curr)
         };
         info!("thin LTO cache key map loaded");
         info!("prev_key_map: {:#?}", prev_key_map);
@@ -500,15 +510,15 @@ fn thin_lto(
             if let (Some(prev_key_map), true) =
                 (prev_key_map.as_ref(), green_modules.contains_key(module_name))
             {
-                assert!(cgcx.incr_comp_session_dir.is_some());
+                assert!(cgcx.old_incr_comp_session_dir.is_some());
+                assert!(cgcx.new_incr_comp_session_dir.is_some());
 
                 // If a module exists in both the current and the previous session,
                 // and has the same LTO cache key in both sessions, then we can re-use it
                 if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) {
                     let work_product = green_modules[module_name].clone();
                     copy_jobs.push(work_product);
                     info!(" - {}: re-used", module_name);
-                    assert!(cgcx.incr_comp_session_dir.is_some());
                     continue;
                 }
             }
@@ -518,8 +528,8 @@ fn thin_lto(
         }
 
         // Save the current ThinLTO import information for the next compilation
-        // session, overwriting the previous serialized data (if any).
-        if let Some(path) = key_map_path
+        // session.
+        if let Some(path) = new_key_map_path
             && let Err(err) = curr_key_map.save_to_file(&path)
         {
             write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err });

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -16,7 +16,9 @@ use rustc_errors::{
 };
 use rustc_fs_util::link_or_copy;
 use rustc_hir::find_attr;
-use rustc_incremental::{copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir_sess};
+use rustc_incremental::{
+    copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir_sess, in_old_incr_comp_dir_sess,
+};
 use rustc_macros::{Decodable, Encodable};
 use rustc_metadata::fs::copy_to_stdout;
 use rustc_middle::bug;
@@ -354,9 +356,12 @@ pub struct CodegenContext {
     /// Directory into which should the LLVM optimization remarks be written.
     /// If `None`, they will be written to stderr.
     pub remark_dir: Option<PathBuf>,
+    /// The previous incremental compilation session directory, or None if we
+    /// are not compiling incrementally or there is no previous session.
+    pub old_incr_comp_session_dir: Option<PathBuf>,
     /// The incremental compilation session directory, or None if we are not
     /// compiling incrementally
-    pub incr_comp_session_dir: Option<PathBuf>,
+    pub new_incr_comp_session_dir: Option<PathBuf>,
     /// `Some(limit)` if the codegen should be run in parallel.
     ///
     /// Depends on [`WriteBackendMethods::supports_parallel()`] and `--jobs-backend`.
@@ -503,7 +508,6 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
             incr_comp_session.unwrap(),
             &module.name,
             files.as_slice(),
-            &module.links_from_incr_cache,
         );
         work_products.insert(id, product);
     }
@@ -845,7 +849,7 @@ fn execute_optimize_work_item<B: WriteBackendMethods>(
     // save our module to disk first.
     let bitcode = if cgcx.module_config.emit_pre_lto_bc {
         let filename = pre_lto_bitcode_filename(&module.name);
-        cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
+        cgcx.new_incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
     } else {
         None
     };
@@ -892,11 +896,9 @@ fn execute_copy_from_cache_work_item(
     let dcx = DiagCtxt::new(Box::new(shared_emitter));
     let dcx = dcx.handle();
 
-    let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
-
-    let mut links_from_incr_cache = Vec::new();
+    let incr_comp_session_dir = cgcx.old_incr_comp_session_dir.as_ref().unwrap();
 
-    let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
+    let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
         let source_file_in_incr_comp_dir = incr_comp_session_dir.join(saved_path);
         debug!(
             "copying preexisting module `{}` from {:?} to {}",
@@ -905,10 +907,7 @@ fn execute_copy_from_cache_work_item(
             output_path.display()
         );
         match link_or_copy(&source_file_in_incr_comp_dir, &output_path) {
-            Ok(_) => {
-                links_from_incr_cache.push(source_file_in_incr_comp_dir);
-                Some(output_path)
-            }
+            Ok(_) => Some(output_path),
             Err(error) => {
                 dcx.emit_err(diagnostics::CopyPathBuf {
                     source_file: source_file_in_incr_comp_dir,
@@ -931,7 +930,7 @@ fn execute_copy_from_cache_work_item(
             load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
         });
 
-    let mut load_from_incr_cache = |perform, output_type: OutputType| {
+    let load_from_incr_cache = |perform, output_type: OutputType| {
         if perform {
             let saved_file = module.source.saved_files.get(output_type.extension())?;
             let output_path = cgcx.output_filenames.temp_path_for_cgu(output_type, &module.name);
@@ -959,7 +958,6 @@ fn execute_copy_from_cache_work_item(
     }
 
     CompiledModule {
-        links_from_incr_cache,
         kind: ModuleKind::Regular,
         name: module.name,
         object,
@@ -1301,10 +1299,14 @@ fn start_executing_work<B: WriteBackendMethods>(
         time_trace: sess.opts.unstable_opts.llvm_time_trace,
         remark: sess.opts.cg.remark.clone(),
         remark_dir,
-        incr_comp_session_dir: tcx
+        old_incr_comp_session_dir: tcx
+            .incr_comp_session
+            .as_ref()
+            .and_then(|incr_comp_session| incr_comp_session.old_session_directory.clone()),
+        new_incr_comp_session_dir: tcx
             .incr_comp_session
             .as_ref()
-            .map(|incr_comp_session| incr_comp_session.session_directory.clone()),
+            .map(|incr_comp_session| incr_comp_session.new_session_directory.clone()),
         output_filenames: Arc::clone(tcx.output_filenames(())),
         module_config: regular_config,
         opt_level,
@@ -2278,7 +2280,22 @@ pub(crate) fn submit_pre_lto_module_to_llvm<B: WriteBackendMethods>(
     module: CachedModuleCodegen,
 ) {
     let filename = pre_lto_bitcode_filename(&module.name);
+    let old_bitcode_path =
+        in_old_incr_comp_dir_sess(tcx.incr_comp_session.unwrap(), &filename).unwrap();
     let bitcode_path = in_incr_comp_dir_sess(tcx.incr_comp_session.unwrap(), &filename);
+
+    match link_or_copy(&old_bitcode_path, &bitcode_path) {
+        Ok(_) => {}
+        Err(error) => {
+            tcx.sess.dcx().emit_err(diagnostics::CopyPathBuf {
+                source_file: old_bitcode_path,
+                output_path: bitcode_path,
+                error,
+            });
+            return;
+        }
+    }
+
     // Schedule the module to be loaded
     drop(
         coordinator

compiler/rustc_codegen_ssa/src/lib.rs

@@ -115,7 +115,6 @@ impl<M> ModuleCodegen<M> {
             bytecode,
             assembly,
             llvm_ir,
-            links_from_incr_cache: Vec::new(),
         }
     }
 }
@@ -130,7 +129,6 @@ pub struct CompiledModule {
     pub bytecode: Option<PathBuf>,
     pub assembly: Option<PathBuf>, // --emit=asm
     pub llvm_ir: Option<PathBuf>,  // --emit=llvm-ir, llvm-bc is in bytecode
-    pub links_from_incr_cache: Vec<PathBuf>,
 }
 
 impl CompiledModule {

compiler/rustc_incremental/src/diagnostics.rs

@@ -169,21 +169,6 @@ pub(crate) struct DeleteLock<'a> {
     pub err: std::io::Error,
 }
 
-#[derive(Diagnostic)]
-#[diag(
-    "hard linking files in the incremental compilation cache failed. copying files instead. consider moving the cache directory to a file system which supports hard linking in session dir `{$path}`"
-)]
-pub(crate) struct HardLinkFailed<'a> {
-    pub path: &'a Path,
-}
-
-#[derive(Diagnostic)]
-#[diag("failed to delete partly initialized session dir `{$path}`: {$err}")]
-pub(crate) struct DeletePartial<'a> {
-    pub path: &'a Path,
-    pub err: std::io::Error,
-}
-
 #[derive(Diagnostic)]
 #[diag("did not finalize incremental compilation session directory `{$path}`: {$err}")]
 #[help("the next build will not be able to reuse work from this compilation")]
@@ -266,13 +251,6 @@ pub(crate) struct CopyWorkProductToCache<'a> {
     pub err: std::io::Error,
 }
 
-#[derive(Diagnostic)]
-#[diag("file-system error deleting outdated file `{$path}`: {$err}")]
-pub(crate) struct DeleteWorkProduct<'a> {
-    pub path: &'a Path,
-    pub err: std::io::Error,
-}
-
 #[derive(Diagnostic)]
 #[diag(
     "corrupt incremental compilation artifact found at `{$path}`. This file will automatically be ignored and deleted. If you see this message repeatedly or can provoke it without manually manipulating the compiler's artifacts, please file an issue. The incremental compilation system relies on hardlinks and filesystem locks behaving correctly, and may not deal well with OS crashes, so whatever information you can provide about your filesystem or other state may be very relevant"

compiler/rustc_incremental/src/lib.rs

@@ -11,7 +11,7 @@ mod persist;
 
 pub use persist::{
     copy_cgu_workproduct_to_incr_comp_cache_dir, finalize_session_directory, in_incr_comp_dir_sess,
-    load_query_result_cache, save_work_product_index, setup_dep_graph,
+    in_old_incr_comp_dir_sess, load_query_result_cache, save_work_product_index, setup_dep_graph,
 };
 use rustc_middle::util::Providers;