feat: support for rustdoc mergeable cross-crate info (#16309)

### What does this PR try to resolve?

This is an unstable feature that we designed to fix several performance
problems with the old system:

1. You couldn't easily build crate docs in hermetic environments. This
doesn't matter for Cargo, but it was one of the original reasons to
implement the feature.
2. We have to build all the doc resources in their final form at every
step, instead of delaying slow parts (mostly the search index) until the
end and only doing them once.
3. It requires rustdoc to take a lock at the end. This reduces available
concurrency for generating docs.

Part of

* rust-lang/rust#130676
* #16306

### How to test and review this PR?

Design decisions and questions:

* [ ] Doc parts are always stored in the new build dir layout:
#16309 (comment)
* [ ] Doesn't take `cargo clean` into account yet:
#16309 (comment)
* [x] A new `FileFlavor::DocParts` is added:
#16309 (comment)
* [ ] One doc-merge per target platform (and run in serial)
#16309 (comment)
* A new status "Merging" is added. Example output:
	```
	    Documenting ...
	    Documenting cargo v0.94.0 (/Users/frodo/cargo)
Finished `dev` profile [unoptimized + debuginfo] target(s) in 29.68s
	     Merging 300 docs for host
	    Finished documentation merge in 2.74s
	   Generated /Users/frodo/cargo/target/doc/cargo/index.html
	```
* Who should garbage collect unneeded crate docs under mergeable CCI
mode, rustdoc or cargo? (or don't, we shouldn't do any garbage
collection?)
#16309 (comment)
* `.rustdoc_fingerprint.json` now stores doc part file paths in previous
builds.

### Simple benchmark

* Tested against this PR on top of
15fde07
* rustdoc 1.93.0-nightly (1be6b13be 2025-11-26)
* macOS Sequoia 15.7.1
* Apple M1 Pro 2021
* 32GB RAM; 10 cores
* 300 crate doc merge
* Test steps:
	```bash
	cargo check
	cargo doc
	cargo clean --doc && rm -rf target/debug/build/**/deps/*.json
	cargo doc -Zrustdoc-mergeable-info
	cargo clean --doc && rm -rf target/debug/build/**/deps/*.json
	```

**With `-Zrustdoc-mergeable-info`**:
  * <40s: ~30s documenting time + 3-5s merge time
  * 76262 files, 970.5MiB total

**Without `-Zrustdoc-mergeable-info`**:
  * ~500s: 8m10s - 8m30s
  * 75242 files, 932.8MiB total

Author: epage

crates/cargo-test-support/src/compare.rs

@@ -338,6 +338,7 @@ static E2E_LITERAL_REDACTIONS: &[(&str, &str)] = &[
     ("[BLOCKING]", "    Blocking"),
     ("[GENERATED]", "   Generated"),
     ("[OPENING]", "     Opening"),
+    ("[MERGING]", "     Merging"),
 ];
 
 /// Checks that the given string contains the given contiguous lines

src/cargo/core/compiler/build_context/target_info.rs

@@ -79,6 +79,8 @@ pub enum FileFlavor {
     DebugInfo,
     /// SBOM (Software Bill of Materials pre-cursor) file (e.g. cargo-sbon.json).
     Sbom,
+    /// Cross-crate info JSON files generated by rustdoc.
+    DocParts,
 }
 
 /// Type of each file generated by a Unit.

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -275,6 +275,15 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         self.layout(unit.kind).build_dir().deps(&dir)
     }
 
+    /// Returns the directories where Rust crate dependencies are found for the
+    /// specified unit. (new layout)
+    ///
+    /// New features should consider using this so we can avoid their migrations.
+    pub fn deps_dir_new_layout(&self, unit: &Unit) -> PathBuf {
+        let dir = self.pkg_dir(unit);
+        self.layout(unit.kind).build_dir().deps_new_layout(&dir)
+    }
+
     /// Directory where the fingerprint for the given unit should go.
     pub fn fingerprint_dir(&self, unit: &Unit) -> PathBuf {
         let dir = self.pkg_dir(unit);
@@ -495,12 +504,27 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                         .join("index.html")
                 };
 
-                vec![OutputFile {
+                let mut outputs = vec![OutputFile {
                     path,
                     hardlink: None,
                     export_path: None,
                     flavor: FileFlavor::Normal,
-                }]
+                }];
+
+                if bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+                    // `-Zrustdoc-mergeable-info` always uses the new layout.
+                    outputs.push(OutputFile {
+                        path: self
+                            .deps_dir_new_layout(unit)
+                            .join(unit.target.crate_name())
+                            .with_extension("json"),
+                        hardlink: None,
+                        export_path: None,
+                        flavor: FileFlavor::DocParts,
+                    })
+                }
+
+                outputs
             }
             CompileMode::RunCustomBuild => {
                 // At this time, this code path does not handle build script

src/cargo/core/compiler/build_runner/mod.rs

@@ -16,15 +16,14 @@ use filetime::FileTime;
 use itertools::Itertools;
 use jobserver::Client;
 
+use super::RustdocFingerprint;
 use super::custom_build::{self, BuildDeps, BuildScriptOutputs, BuildScripts};
 use super::fingerprint::{Checksum, Fingerprint};
 use super::job_queue::JobQueue;
 use super::layout::Layout;
 use super::lto::Lto;
 use super::unit_graph::UnitDep;
-use super::{
-    BuildContext, Compilation, CompileKind, CompileMode, Executor, FileFlavor, RustDocFingerprint,
-};
+use super::{BuildContext, Compilation, CompileKind, CompileMode, Executor, FileFlavor};
 
 mod compilation_files;
 use self::compilation_files::CompilationFiles;
@@ -178,7 +177,7 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         // they were compiled with the same Rustc version that we're currently using.
         // See the function doc comment for more.
         if self.bcx.build_config.intent.is_doc() {
-            RustDocFingerprint::check_rustdoc_fingerprint(&self)?
+            RustdocFingerprint::check_rustdoc_fingerprint(&self)?
         }
 
         for unit in &self.bcx.roots {
@@ -225,6 +224,8 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             }
         }
 
+        self.collect_doc_merge_info()?;
+
         // Collect the result of the build into `self.compilation`.
         for unit in &self.bcx.roots {
             self.collect_tests_and_executables(unit)?;
@@ -330,6 +331,58 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         Ok(())
     }
 
+    fn collect_doc_merge_info(&mut self) -> CargoResult<()> {
+        if !self.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            return Ok(());
+        }
+
+        if !self.bcx.build_config.intent.is_doc() {
+            return Ok(());
+        }
+
+        if self.bcx.build_config.intent.wants_doc_json_output() {
+            // rustdoc JSON output doesn't support merge (yet?)
+            return Ok(());
+        }
+
+        let mut doc_parts_map: HashMap<_, Vec<_>> = HashMap::new();
+
+        let unit_iter = if self.bcx.build_config.intent.wants_deps_docs() {
+            itertools::Either::Left(self.bcx.unit_graph.keys())
+        } else {
+            itertools::Either::Right(self.bcx.roots.iter())
+        };
+
+        for unit in unit_iter {
+            if !unit.mode.is_doc() {
+                continue;
+            }
+            // Assumption: one `rustdoc` call generates only one cross-crate info JSON.
+            let outputs = self.outputs(unit)?;
+
+            let Some(doc_parts) = outputs
+                .iter()
+                .find(|o| matches!(o.flavor, FileFlavor::DocParts))
+            else {
+                continue;
+            };
+
+            doc_parts_map
+                .entry(unit.kind)
+                .or_default()
+                .push(doc_parts.path.to_owned());
+        }
+
+        self.compilation.rustdoc_fingerprints = Some(
+            doc_parts_map
+                .into_iter()
+                .map(|(kind, doc_parts)| (kind, RustdocFingerprint::new(self, kind, doc_parts)))
+                .collect(),
+        );
+
+        Ok(())
+    }
+
     /// Returns the executable for the specified unit (if any).
     pub fn get_executable(&mut self, unit: &Unit) -> CargoResult<Option<PathBuf>> {
         let is_binary = unit.target.is_executable();

src/cargo/core/compiler/compilation.rs

@@ -9,6 +9,7 @@ use cargo_util::{ProcessBuilder, paths};
 
 use crate::core::Package;
 use crate::core::compiler::BuildContext;
+use crate::core::compiler::RustdocFingerprint;
 use crate::core::compiler::apply_env_config;
 use crate::core::compiler::{CompileKind, Unit, UnitHash};
 use crate::util::{CargoResult, GlobalContext, context};
@@ -106,6 +107,11 @@ pub struct Compilation<'gctx> {
     /// Libraries to test with rustdoc.
     pub to_doc_test: Vec<Doctest>,
 
+    /// Rustdoc fingerprint files to determine whether we need to run `rustdoc --merge=finalize`.
+    ///
+    /// See `-Zrustdoc-mergeable-info` for more.
+    pub rustdoc_fingerprints: Option<HashMap<CompileKind, RustdocFingerprint>>,
+
     /// The target host triple.
     pub host: String,
 
@@ -143,6 +149,7 @@ impl<'gctx> Compilation<'gctx> {
             root_crate_names: Vec::new(),
             extra_env: HashMap::new(),
             to_doc_test: Vec::new(),
+            rustdoc_fingerprints: None,
             gctx: bcx.gctx,
             host: bcx.host_triple().to_string(),
             rustc_process,

src/cargo/core/compiler/fingerprint/mod.rs

@@ -410,7 +410,7 @@ pub use self::dep_info::parse_dep_info;
 pub use self::dep_info::parse_rustc_dep_info;
 pub use self::dep_info::translate_dep_info;
 pub use self::dirty_reason::DirtyReason;
-pub use self::rustdoc::RustDocFingerprint;
+pub use self::rustdoc::RustdocFingerprint;
 
 /// Determines if a [`Unit`] is up-to-date, and if not prepares necessary work to
 /// update the persisted fingerprint.