feat: support for rustdoc mergeable cross-crate info

This is an unstable feature that we designed to fix several
performance problems with the old system:

1. You couldn't easily build crate docs in hermetic environments.
   This doesn't matter for Cargo, but it was one of the original
   reasons to implement the feature.
2. We have to build all the doc resources in their final form at
   every step, instead of delaying slow parts (mostly the
   search index) until the end and only doing them once.
3. It requires rustdoc to take a lock at the end. This reduces
   available concurrency for generating docs.

A nightly feature `-Zrustdoc-mergeable-info` is added.

Co-authored-by: Michael Howell <[email protected]>
Co-authored-by: Weihang Lo <[email protected]>

Author: notriddle

crates/cargo-test-support/src/compare.rs

@@ -338,6 +338,7 @@ static E2E_LITERAL_REDACTIONS: &[(&str, &str)] = &[
     ("[BLOCKING]", "    Blocking"),
     ("[GENERATED]", "   Generated"),
     ("[OPENING]", "     Opening"),
+    ("[MERGING]", "     Merging"),
 ];
 
 /// Checks that the given string contains the given contiguous lines

src/cargo/core/compiler/build_context/target_info.rs

@@ -74,6 +74,8 @@ pub enum FileFlavor {
     DebugInfo,
     /// SBOM (Software Bill of Materials pre-cursor) file (e.g. cargo-sbon.json).
     Sbom,
+    /// Cross-crate info JSON files generated by rustdoc.
+    DocParts,
 }
 
 /// Type of each file generated by a Unit.
@@ -1191,6 +1193,19 @@ impl RustDocFingerprint {
             })
             .filter(|path| path.exists())
             .try_for_each(|path| clean_doc(path))?;
+
+        // Clean docdeps directory as well for `-Zrustdoc-mergeable-info`.
+        //
+        // This could potentially has a rustdoc version prefix
+        // so we can retain between different toolchain versions.
+        build_runner
+            .bcx
+            .all_kinds
+            .iter()
+            .map(|kind| build_runner.files().layout(*kind).build_dir().docdeps())
+            .filter(|path| path.exists())
+            .try_for_each(std::fs::remove_dir_all)?;
+
         write_fingerprint()?;
         return Ok(());
 

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -328,6 +328,13 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
             .build_script(&dir)
     }
 
+    /// Returns the directory where mergeable cross crate info for docs is stored.
+    pub fn docdeps_dir(&self, unit: &Unit) -> &Path {
+        assert!(unit.mode.is_doc());
+        assert!(self.metas.contains_key(unit));
+        self.layout(unit.kind).build_dir().docdeps()
+    }
+
     /// Returns the directory for compiled artifacts files.
     /// `/path/to/target/{debug,release}/deps/artifact/KIND/PKG-HASH`
     fn artifact_dir(&self, unit: &Unit) -> PathBuf {
@@ -500,12 +507,26 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                         .join("index.html")
                 };
 
-                vec![OutputFile {
+                let mut outputs = vec![OutputFile {
                     path,
                     hardlink: None,
                     export_path: None,
                     flavor: FileFlavor::Normal,
-                }]
+                }];
+
+                if bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+                    outputs.push(OutputFile {
+                        path: self
+                            .docdeps_dir(unit)
+                            .join(unit.target.crate_name())
+                            .with_extension("json"),
+                        hardlink: None,
+                        export_path: None,
+                        flavor: FileFlavor::DocParts,
+                    })
+                }
+
+                outputs
             }
             CompileMode::RunCustomBuild => {
                 // At this time, this code path does not handle build script

src/cargo/core/compiler/build_runner/mod.rs

@@ -1,6 +1,7 @@
 //! [`BuildRunner`] is the mutable state used during the build process.
 
 use std::collections::{HashMap, HashSet};
+use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
@@ -230,6 +231,8 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             }
         }
 
+        self.collect_doc_merge_info()?;
+
         // Collect the result of the build into `self.compilation`.
         for unit in &self.bcx.roots {
             self.collect_tests_and_executables(unit)?;
@@ -335,6 +338,132 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         Ok(())
     }
 
+    fn collect_doc_merge_info(&mut self) -> CargoResult<()> {
+        if !self.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            return Ok(());
+        }
+
+        if !self.bcx.build_config.intent.is_doc() {
+            return Ok(());
+        }
+
+        if self.bcx.build_config.intent.wants_doc_json_output() {
+            // rustdoc JSON output doesn't support merge (yet?)
+            return Ok(());
+        }
+
+        let mut doc_merge_info = HashMap::new();
+
+        let unit_iter = if self.bcx.build_config.intent.wants_deps_docs() {
+            itertools::Either::Left(self.bcx.unit_graph.keys())
+        } else {
+            itertools::Either::Right(self.bcx.roots.iter())
+        };
+
+        for unit in unit_iter {
+            let has_doc_parts = unit.mode.is_doc()
+                && self
+                    .outputs(unit)?
+                    .iter()
+                    .any(|o| matches!(o.flavor, FileFlavor::DocParts));
+            if !has_doc_parts {
+                continue;
+            }
+
+            doc_merge_info.entry(unit.kind).or_insert_with(|| {
+                let out_dir = self
+                    .files()
+                    .layout(unit.kind)
+                    .artifact_dir()
+                    .expect("artifact-dir was not locked")
+                    .doc()
+                    .to_owned();
+                let docdeps_dir = self.files().docdeps_dir(unit);
+
+                let mut requires_merge = false;
+
+                // HACK: get mtime of crates.js to inform outside
+                // whether we need to merge cross-crate info.
+                // The content of `crates.js` looks like
+                //
+                // ```
+                // window.ALL_CRATES = ["cargo","cargo_util","cargo_util_schemas","crates_io"]
+                // ```
+                //
+                // and will be updated when any new crate got documented
+                // even with the legacy `--merge=shared` mode.
+                let crates_js = out_dir.join("crates.js");
+                let crates_js_mtime = paths::mtime(&crates_js);
+
+                let mut num_crates = 0;
+
+                for entry in walkdir::WalkDir::new(docdeps_dir).max_depth(1) {
+                    let Ok(entry) = entry else {
+                        tracing::debug!("failed to read entry at {}", docdeps_dir.display());
+                        continue;
+                    };
+
+                    if !entry.file_type().is_file()
+                        || entry.path().extension() != Some(OsStr::new("json"))
+                    {
+                        continue;
+                    }
+
+                    num_crates += 1;
+
+                    if requires_merge {
+                        continue;
+                    }
+
+                    let crates_js_mtime = match crates_js_mtime {
+                        Ok(mtime) => mtime,
+                        Err(ref err) => {
+                            tracing::debug!(
+                                ?err,
+                                "failed to read mtime of {}",
+                                crates_js.display()
+                            );
+                            requires_merge = true;
+                            continue;
+                        }
+                    };
+
+                    let parts_mtime = match paths::mtime(entry.path()) {
+                        Ok(mtime) => mtime,
+                        Err(err) => {
+                            tracing::debug!(
+                                ?err,
+                                "failed to read mtime of {}",
+                                entry.path().display()
+                            );
+                            requires_merge = true;
+                            continue;
+                        }
+                    };
+
+                    if parts_mtime > crates_js_mtime {
+                        requires_merge = true;
+                        continue;
+                    }
+                }
+
+                if requires_merge {
+                    compilation::DocMergeInfo::Merge {
+                        num_crates,
+                        parts_dir: docdeps_dir.to_owned(),
+                        out_dir,
+                    }
+                } else {
+                    compilation::DocMergeInfo::Fresh
+                }
+            });
+        }
+
+        self.compilation.doc_merge_info = doc_merge_info;
+
+        Ok(())
+    }
+
     /// Returns the executable for the specified unit (if any).
     pub fn get_executable(&mut self, unit: &Unit) -> CargoResult<Option<PathBuf>> {
         let is_binary = unit.target.is_executable();

src/cargo/core/compiler/compilation.rs

@@ -106,6 +106,11 @@ pub struct Compilation<'gctx> {
     /// Libraries to test with rustdoc.
     pub to_doc_test: Vec<Doctest>,
 
+    /// Compilation information for running `rustdoc --merge=finalize`.
+    ///
+    /// See `-Zrustdoc-mergeable-info` for more.
+    pub doc_merge_info: HashMap<CompileKind, DocMergeInfo>,
+
     /// The target host triple.
     pub host: String,
 
@@ -143,6 +148,7 @@ impl<'gctx> Compilation<'gctx> {
             root_crate_names: Vec::new(),
             extra_env: HashMap::new(),
             to_doc_test: Vec::new(),
+            doc_merge_info: Default::default(),
             gctx: bcx.gctx,
             host: bcx.host_triple().to_string(),
             rustc_process,
@@ -383,6 +389,25 @@ impl<'gctx> Compilation<'gctx> {
     }
 }
 
+/// Compilation information for running `rustdoc --merge=finalize`.
+#[derive(Default)]
+pub enum DocMergeInfo {
+    /// Doc merge disabled.
+    #[default]
+    None,
+    /// Nothing is stale.
+    Fresh,
+    /// Doc merge is required
+    Merge {
+        /// Number of crates to merge.
+        num_crates: u64,
+        /// Output directory holding every cross-crate info JSON file.
+        parts_dir: PathBuf,
+        /// Output directory for rustdoc.
+        out_dir: PathBuf,
+    },
+}
+
 /// Prepares a `rustc_tool` process with additional environment variables
 /// that are only relevant in a context that has a unit
 fn fill_rustc_tool_env(mut cmd: ProcessBuilder, unit: &Unit) -> ProcessBuilder {

src/cargo/core/compiler/layout.rs

@@ -192,6 +192,7 @@ impl Layout {
                 incremental: build_dest.join("incremental"),
                 fingerprint: build_dest.join(".fingerprint"),
                 examples: build_dest.join("examples"),
+                docdeps: build_dest.join("docdeps"),
                 tmp: build_root.join("tmp"),
                 _lock: build_dir_lock,
                 is_new_layout,
@@ -273,6 +274,8 @@ pub struct BuildDirLayout {
     fingerprint: PathBuf,
     /// The directory for pre-uplifted examples: `build-dir/debug/examples`
     examples: PathBuf,
+    /// The directory with intermediate artifacts from rustdoc.
+    docdeps: PathBuf,
     /// The directory for temporary data of integration tests and benches
     tmp: PathBuf,
     /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
@@ -290,6 +293,7 @@ impl BuildDirLayout {
         if !self.is_new_layout {
             paths::create_dir_all(&self.deps)?;
             paths::create_dir_all(&self.fingerprint)?;
+            paths::create_dir_all(&self.docdeps)?;
         }
         paths::create_dir_all(&self.incremental)?;
         paths::create_dir_all(&self.examples)?;
@@ -344,6 +348,12 @@ impl BuildDirLayout {
             self.build().join(pkg_dir)
         }
     }
+    /// Fetch the path storing intermediate artifacts from rustdoc.
+    pub fn docdeps(&self) -> &Path {
+        // This doesn't need to consider new build-dir layout (yet?)
+        // because rustdoc artifacts must be appendable.
+        &self.docdeps
+    }
     /// Fetch the build script execution path.
     pub fn build_script_execution(&self, pkg_dir: &str) -> PathBuf {
         if self.is_new_layout {

src/cargo/core/compiler/mod.rs

@@ -76,6 +76,7 @@ pub use self::build_context::{
     BuildContext, FileFlavor, FileType, RustDocFingerprint, RustcTargetData, TargetInfo,
 };
 pub use self::build_runner::{BuildRunner, Metadata, UnitHash};
+pub use self::compilation::DocMergeInfo;
 pub use self::compilation::{Compilation, Doctest, UnitOutput};
 pub use self::compile_kind::{CompileKind, CompileKindFallback, CompileTarget};
 pub use self::crate_type::CrateType;
@@ -830,8 +831,13 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     if build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo {
         // toolchain-shared-resources is required for keeping the shared styling resources
         // invocation-specific is required for keeping the original rustdoc emission
-        let mut arg =
-            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=");
+        let mut arg = if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            // toolchain resources are written at the end, at the same time as merging
+            OsString::from("--emit=invocation-specific,dep-info=")
+        } else {
+            // if not using mergeable CCI, everything is written every time
+            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=")
+        };
         arg.push(rustdoc_dep_info_loc(build_runner, unit));
         rustdoc.arg(arg);
 
@@ -840,6 +846,18 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
         }
 
         rustdoc.arg("-Zunstable-options");
+    } else if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // toolchain resources are written at the end, at the same time as merging
+        rustdoc.arg("--emit=invocation-specific");
+        rustdoc.arg("-Zunstable-options");
+    }
+
+    if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // write out mergeable data to be imported
+        rustdoc.arg("--merge=none");
+        let mut arg = OsString::from("--parts-out-dir=");
+        arg.push(build_runner.files().docdeps_dir(unit));
+        rustdoc.arg(arg);
     }
 
     if let Some(trim_paths) = unit.profile.trim_paths.as_ref() {

src/cargo/core/features.rs

@@ -884,6 +884,7 @@ unstable_cli_options!(
     rustc_unicode: bool = ("Enable `rustc`'s unicode error format in Cargo's error messages"),
     rustdoc_depinfo: bool = ("Use dep-info files in rustdoc rebuild detection"),
     rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"),
+    rustdoc_mergeable_info: bool = ("Use rustdoc mergeable cross-crate-info files"),
     rustdoc_scrape_examples: bool = ("Allows Rustdoc to scrape code examples from reverse-dependencies"),
     sbom: bool = ("Enable the `sbom` option in build config in .cargo/config.toml file"),
     script: bool = ("Enable support for single-file, `.rs` packages"),
@@ -1415,6 +1416,7 @@ impl CliUnstable {
             "rustc-unicode" => self.rustc_unicode = parse_empty(k, v)?,
             "rustdoc-depinfo" => self.rustdoc_depinfo = parse_empty(k, v)?,
             "rustdoc-map" => self.rustdoc_map = parse_empty(k, v)?,
+            "rustdoc-mergeable-info" => self.rustdoc_mergeable_info = parse_empty(k, v)?,
             "rustdoc-scrape-examples" => self.rustdoc_scrape_examples = parse_empty(k, v)?,
             "sbom" => self.sbom = parse_empty(k, v)?,
             "section-timings" => self.section_timings = parse_empty(k, v)?,