Make create_def a query.

Author: cjgillot

compiler/rustc_ast_lowering/src/lib.rs

@@ -532,11 +532,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             self.tcx.hir_def_key(self.local_def_id(node_id)),
         );
 
-        let def_id = self
-            .tcx
-            .at(span)
-            .create_def(parent, name, def_kind, None, &mut self.resolver.disambiguator)
-            .def_id();
+        let def_id = self.tcx.at(span).create_def(parent, name, def_kind, None).def_id();
 
         debug!("create_def: def_id_to_node_id[{:?}] <-> {:?}", def_id, node_id);
         self.resolver.node_id_to_def_id.insert(node_id, def_id);

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -17,7 +17,7 @@ use hir::def::DefKind;
 use rustc_ast::Mutability;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_hir as hir;
-use rustc_hir::definitions::{DefPathData, DisambiguatorState};
+use rustc_hir::definitions::DefPathData;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::mir::interpret::{ConstAllocation, CtfeProvenance, InterpResult};
 use rustc_middle::query::TyCtxtAt;
@@ -66,7 +66,6 @@ fn intern_shallow<'tcx, T, M: CompileTimeMachine<'tcx, T>>(
     ecx: &mut InterpCx<'tcx, M>,
     alloc_id: AllocId,
     mutability: Mutability,
-    disambiguator: Option<&mut DisambiguatorState>,
 ) -> Result<impl Iterator<Item = CtfeProvenance> + 'tcx, ()> {
     trace!("intern_shallow {:?}", alloc_id);
     // remove allocation
@@ -89,13 +88,7 @@ fn intern_shallow<'tcx, T, M: CompileTimeMachine<'tcx, T>>(
     // link the alloc id to the actual allocation
     let alloc = ecx.tcx.mk_const_alloc(alloc);
     if let Some(static_id) = ecx.machine.static_def_id() {
-        intern_as_new_static(
-            ecx.tcx,
-            static_id,
-            alloc_id,
-            alloc,
-            disambiguator.expect("disambiguator needed"),
-        );
+        intern_as_new_static(ecx.tcx, static_id, alloc_id, alloc);
     } else {
         ecx.tcx.set_alloc_id_memory(alloc_id, alloc);
     }
@@ -109,18 +102,16 @@ fn intern_as_new_static<'tcx>(
     static_id: LocalDefId,
     alloc_id: AllocId,
     alloc: ConstAllocation<'tcx>,
-    disambiguator: &mut DisambiguatorState,
 ) {
-    // `intern_const_alloc_recursive` is called once per static and it contains the `DisambiguatorState`.
-    //  The `<static_id>::{{nested}}` path is thus unique to `intern_const_alloc_recursive` and the
-    // `DisambiguatorState` ensures the generated path is unique for this call as we generate
-    // `<static_id>::{{nested#n}}` where `n` is the `n`th `intern_as_new_static` call.
+    // `intern_const_alloc_recursive` is called once per static. The `<static_id>::{{nested}}` path
+    // is thus unique to `intern_const_alloc_recursive`. If there are several calls to
+    // `intern_as_new_static`, `create_def` generates a path that is unique for this call as
+    // generate `<static_id>::{{nested#n}}` where `n` is the `n`th `intern_as_new_static` call.
     let feed = tcx.create_def(
         static_id,
         None,
         DefKind::Static { safety: hir::Safety::Safe, mutability: alloc.0.mutability, nested: true },
         Some(DefPathData::NestedStatic),
-        disambiguator,
     );
     tcx.set_nested_alloc_id_static(alloc_id, feed.def_id());
 
@@ -168,8 +159,6 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx, const_eval
     intern_kind: InternKind,
     ret: &MPlaceTy<'tcx>,
 ) -> Result<(), InternResult> {
-    let mut disambiguator = DisambiguatorState::new();
-
     // We are interning recursively, and for mutability we are distinguishing the "root" allocation
     // that we are starting in, and all other allocations that we are encountering recursively.
     let (base_mutability, inner_mutability, is_static) = match intern_kind {
@@ -213,9 +202,7 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx, const_eval
         alloc.1.mutability = base_mutability;
         alloc.1.provenance().ptrs().iter().map(|&(_, prov)| prov).collect()
     } else {
-        intern_shallow(ecx, base_alloc_id, base_mutability, Some(&mut disambiguator))
-            .unwrap()
-            .collect()
+        intern_shallow(ecx, base_alloc_id, base_mutability).unwrap().collect()
     };
     // We need to distinguish "has just been interned" from "was already in `tcx`",
     // so we track this in a separate set.
@@ -308,7 +295,7 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx, const_eval
         // okay with losing some potential for immutability here. This can anyway only affect
         // `static mut`.
         just_interned.insert(alloc_id);
-        match intern_shallow(ecx, alloc_id, inner_mutability, Some(&mut disambiguator)) {
+        match intern_shallow(ecx, alloc_id, inner_mutability) {
             Ok(nested) => todo.extend(nested),
             Err(()) => {
                 ecx.tcx.dcx().delayed_bug("found dangling pointer during const interning");
@@ -330,9 +317,8 @@ pub fn intern_const_alloc_for_constprop<'tcx, T, M: CompileTimeMachine<'tcx, T>>
         return interp_ok(());
     }
     // Move allocation to `tcx`.
-    if let Some(_) = intern_shallow(ecx, alloc_id, Mutability::Not, None)
-        .map_err(|()| err_ub!(DeadLocal))?
-        .next()
+    if let Some(_) =
+        intern_shallow(ecx, alloc_id, Mutability::Not).map_err(|()| err_ub!(DeadLocal))?.next()
     {
         // We are not doing recursive interning, so we don't currently support provenance.
         // (If this assertion ever triggers, we should just implement a
@@ -358,7 +344,7 @@ impl<'tcx> InterpCx<'tcx, DummyMachine> {
         let dest = self.allocate(layout, MemoryKind::Stack)?;
         f(self, &dest.clone().into())?;
         let alloc_id = dest.ptr().provenance.unwrap().alloc_id(); // this was just allocated, it must have provenance
-        for prov in intern_shallow(self, alloc_id, Mutability::Not, None).unwrap() {
+        for prov in intern_shallow(self, alloc_id, Mutability::Not).unwrap() {
             // We are not doing recursive interning, so we don't currently support provenance.
             // (If this assertion ever triggers, we should just implement a
             // proper recursive interning loop -- or just call `intern_const_alloc_recursive`.

compiler/rustc_hir/src/definitions.rs

@@ -11,7 +11,7 @@ use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::unord::UnordMap;
 use rustc_hashes::Hash64;
 use rustc_index::IndexVec;
-use rustc_macros::{Decodable, Encodable};
+use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 use rustc_span::{Symbol, kw, sym};
 use tracing::{debug, instrument};
 
@@ -274,7 +274,7 @@ impl DefPath {
 }
 
 /// New variants should only be added in synchronization with `enum DefKind`.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable, HashStable_Generic)]
 pub enum DefPathData {
     // Root: these should only be used for the root nodes, because
     // they are treated specially by the `def_path` function.

compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs

@@ -14,7 +14,7 @@ use rustc_ast::visit::walk_list;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::definitions::{DefPathData, DisambiguatorState};
+use rustc_hir::definitions::DefPathData;
 use rustc_hir::intravisit::{self, InferKind, Visitor, VisitorExt};
 use rustc_hir::{
     self as hir, AmbigArg, GenericArg, GenericParam, GenericParamKind, HirId, LifetimeKind, Node,
@@ -64,7 +64,6 @@ impl ResolvedArg {
 struct BoundVarContext<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     rbv: &'a mut ResolveBoundVars,
-    disambiguator: &'a mut DisambiguatorState,
     scope: ScopeRef<'a>,
 }
 
@@ -247,12 +246,8 @@ pub(crate) fn provide(providers: &mut Providers) {
 #[instrument(level = "debug", skip(tcx))]
 fn resolve_bound_vars(tcx: TyCtxt<'_>, local_def_id: hir::OwnerId) -> ResolveBoundVars {
     let mut rbv = ResolveBoundVars::default();
-    let mut visitor = BoundVarContext {
-        tcx,
-        rbv: &mut rbv,
-        scope: &Scope::Root { opt_parent_item: None },
-        disambiguator: &mut DisambiguatorState::new(),
-    };
+    let mut visitor =
+        BoundVarContext { tcx, rbv: &mut rbv, scope: &Scope::Root { opt_parent_item: None } };
     match tcx.hir_owner_node(local_def_id) {
         hir::OwnerNode::Item(item) => visitor.visit_item(item),
         hir::OwnerNode::ForeignItem(item) => visitor.visit_foreign_item(item),
@@ -1098,8 +1093,8 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
     where
         F: for<'b> FnOnce(&mut BoundVarContext<'b, 'tcx>),
     {
-        let BoundVarContext { tcx, rbv, disambiguator, .. } = self;
-        let mut this = BoundVarContext { tcx: *tcx, rbv, disambiguator, scope: &wrap_scope };
+        let BoundVarContext { tcx, rbv, .. } = self;
+        let mut this = BoundVarContext { tcx: *tcx, rbv, scope: &wrap_scope };
         let span = debug_span!("scope", scope = ?this.scope.debug_truncated());
         {
             let _enter = span.enter();
@@ -1472,13 +1467,12 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                 // `opaque_def_id` is unique to the `BoundVarContext` pass which is executed once
                 // per `resolve_bound_vars` query. This is the only location that creates
                 // `OpaqueLifetime` paths. `<opaque_def_id>::OpaqueLifetime(..)` is thus unique
-                // to this query and duplicates within the query are handled by `self.disambiguator`.
+                // to this query.
                 let feed = self.tcx.create_def(
                     opaque_def_id,
                     None,
                     DefKind::LifetimeParam,
                     Some(DefPathData::OpaqueLifetime(ident.name)),
-                    &mut self.disambiguator,
                 );
                 feed.def_span(ident.span);
                 feed.def_ident_span(Some(ident.span));

compiler/rustc_middle/src/query/keys.rs

@@ -3,11 +3,13 @@
 use std::ffi::OsStr;
 
 use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId, ModDefId};
+use rustc_hir::definitions::DefPathData;
 use rustc_hir::hir_id::{HirId, OwnerId};
 use rustc_query_system::dep_graph::DepNodeIndex;
 use rustc_query_system::query::{DefIdCache, DefaultCache, SingleCache, VecCache};
 use rustc_span::{DUMMY_SP, Ident, Span, Symbol};
 
+use crate::dep_graph::DepNode;
 use crate::infer::canonical::CanonicalQueryInput;
 use crate::mir::mono::CollectionMode;
 use crate::ty::fast_reject::SimplifiedType;
@@ -632,3 +634,11 @@ impl<'tcx> Key for (ty::Instance<'tcx>, CollectionMode) {
         self.0.default_span(tcx)
     }
 }
+
+impl Key for (LocalDefId, DefPathData, Option<DepNode>, usize) {
+    type Cache<V> = DefaultCache<Self, V>;
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}

compiler/rustc_middle/src/query/mod.rs

@@ -25,6 +25,7 @@ use rustc_hir::def::{DefKind, DocLinkResMap};
 use rustc_hir::def_id::{
     CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap, LocalDefIdSet, LocalModDefId,
 };
+use rustc_hir::definitions::DefPathData;
 use rustc_hir::lang_items::{LangItem, LanguageItems};
 use rustc_hir::{Crate, ItemLocalId, ItemLocalMap, PreciseCapturingArgKind, TraitCandidate};
 use rustc_index::IndexVec;
@@ -46,6 +47,7 @@ use rustc_span::{DUMMY_SP, Span, Symbol};
 use rustc_target::spec::PanicStrategy;
 use {rustc_abi as abi, rustc_ast as ast, rustc_attr_data_structures as attr, rustc_hir as hir};
 
+use crate::dep_graph::DepNode;
 use crate::infer::canonical::{self, Canonical};
 use crate::lint::LintExpectation;
 use crate::metadata::ModChild;
@@ -159,6 +161,23 @@ rustc_queries! {
         desc { "getting the source span" }
     }
 
+    /// Create a new definition.
+    query create_def_raw(key: (
+        LocalDefId, // parent
+        DefPathData, // def_path_data
+        Option<DepNode>, // caller query
+        usize, // counter of calls to `create_def_raw` by the caller query
+    )) -> LocalDefId {
+        // Accesses untracked data
+        eval_always
+        desc { |tcx|
+            "create a new definition for `{}::{:?}`, {}th call",
+            tcx.def_path_str(key.0),
+            key.1,
+            key.3,
+        }
+    }
+
     /// Represents crate as a whole (as distinct from the top-level crate module).
     ///
     /// If you call `tcx.hir_crate(())` we will have to assume that any change

compiler/rustc_middle/src/ty/context.rs

@@ -60,7 +60,7 @@ use rustc_type_ir::{
 use tracing::{debug, instrument};
 
 use crate::arena::Arena;
-use crate::dep_graph::{DepGraph, DepKindStruct};
+use crate::dep_graph::{DepGraph, DepKindStruct, DepNode, TaskDepsRef};
 use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarKind, CanonicalVarKinds};
 use crate::lint::lint_level;
 use crate::metadata::ModChild;
@@ -1454,6 +1454,9 @@ pub struct GlobalCtxt<'tcx> {
     pub(crate) hooks: crate::hooks::Providers,
 
     untracked: Untracked,
+    /// This is shared untracked state for creating new definitions.
+    /// It should only be accessed by the `create_def_raw` query.
+    untracked_disambiguator_state: Lock<DisambiguatorState>,
 
     pub query_system: QuerySystem<'tcx>,
     pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
@@ -1727,6 +1730,7 @@ impl<'tcx> TyCtxt<'tcx> {
             lifetimes: common_lifetimes,
             consts: common_consts,
             untracked,
+            untracked_disambiguator_state: Lock::new(DisambiguatorState::new()),
             query_system,
             query_kinds,
             ty_rcache: Default::default(),
@@ -2017,6 +2021,30 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 }
 
+#[instrument(level = "trace", skip(tcx), ret)]
+fn create_def_raw_provider<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    (parent, data, query, index): (LocalDefId, DefPathData, Option<DepNode>, usize),
+) -> LocalDefId {
+    // `query` and `index` are guaranteed to change for each successive call to
+    // `create_def_raw`, but in a predictable manner.
+    let _ = (query, index);
+
+    // This query is `eval_always`, so we can access untracked data.
+    let mut disambiguator_state = tcx.untracked_disambiguator_state.lock();
+
+    // The following call has the side effect of modifying the tables inside `definitions`.
+    // These very tables are relied on by the incr. comp. engine to decode DepNodes and to
+    // decode the on-disk cache.
+    //
+    // Any LocalDefId which is used within queries, either as key or result, either:
+    // - has been created before the construction of the TyCtxt;
+    // - has been created by this call to `create_def`.
+    // As a consequence, this LocalDefId is always re-created before it is needed by the incr.
+    // comp. engine itself.
+    tcx.untracked.definitions.write().create_def(parent, data, &mut disambiguator_state)
+}
+
 impl<'tcx> TyCtxtAt<'tcx> {
     /// Create a new definition within the incr. comp. engine.
     pub fn create_def(
@@ -2025,11 +2053,8 @@ impl<'tcx> TyCtxtAt<'tcx> {
         name: Option<Symbol>,
         def_kind: DefKind,
         override_def_path_data: Option<DefPathData>,
-        disambiguator: &mut DisambiguatorState,
     ) -> TyCtxtFeed<'tcx, LocalDefId> {
-        let feed =
-            self.tcx.create_def(parent, name, def_kind, override_def_path_data, disambiguator);
-
+        let feed = self.tcx.create_def(parent, name, def_kind, override_def_path_data);
         feed.def_span(self.span);
         feed
     }
@@ -2043,28 +2068,39 @@ impl<'tcx> TyCtxt<'tcx> {
         name: Option<Symbol>,
         def_kind: DefKind,
         override_def_path_data: Option<DefPathData>,
-        disambiguator: &mut DisambiguatorState,
     ) -> TyCtxtFeed<'tcx, LocalDefId> {
         let data = override_def_path_data.unwrap_or_else(|| def_kind.def_path_data(name));
-        // The following call has the side effect of modifying the tables inside `definitions`.
-        // These very tables are relied on by the incr. comp. engine to decode DepNodes and to
-        // decode the on-disk cache.
-        //
-        // Any LocalDefId which is used within queries, either as key or result, either:
-        // - has been created before the construction of the TyCtxt;
-        // - has been created by this call to `create_def`.
-        // As a consequence, this LocalDefId is always re-created before it is needed by the incr.
-        // comp. engine itself.
-        let def_id = self.untracked.definitions.write().create_def(parent, data, disambiguator);
-
-        // This function modifies `self.definitions` using a side-effect.
-        // We need to ensure that these side effects are re-run by the incr. comp. engine.
-        // Depending on the forever-red node will tell the graph that the calling query
-        // needs to be re-evaluated.
-        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+
+        // `create_def_raw` is a query, so it can be replayed by the dep-graph engine. However, we
+        // may invoke it multiple times with the same `(parent, data)` pair, and we expect to
+        // create *different* defintions from them. In order to make this compatible with the
+        // general model of queries, we add additional information which must change at each call.
+        let (dep_node, query_local_index) =
+            ty::tls::with_related_context(self, |icx| match icx.task_deps {
+                // If we are inside a query, we can only use local information, and no global
+                // mutable state. The current query's name and the number of calls to `create_def`
+                // are local to the current query, so are ok to use.
+                TaskDepsRef::Allow(deps) => {
+                    let opt_dep_node_and_index = deps.lock().next_query_local_index();
+                    if let Some((dep_node, index)) = opt_dep_node_and_index {
+                        (Some(dep_node), index)
+                    } else {
+                        // No idea how to support this for now...
+                        bug!("trying to create a definition from an anonymous query")
+                    }
+                }
+                // If we are not tracking dependencies, we can use global mutable state,
+                // so we use the total number of definitions as a proxy.
+                TaskDepsRef::EvalAlways | TaskDepsRef::Forbid | TaskDepsRef::Ignore => {
+                    let global_count = self.untracked.definitions.read().def_index_count();
+                    (None, global_count)
+                }
+            });
+        let def_id = self.create_def_raw((parent, data, dep_node, query_local_index));
 
         let feed = TyCtxtFeed { tcx: self, key: def_id };
         feed.def_kind(def_kind);
+
         // Unique types created for closures participate in type privacy checking.
         // They have visibilities inherited from the module they are defined in.
         // Visibilities for opaque types are meaningless, but still provided
@@ -3475,6 +3511,7 @@ pub fn provide(providers: &mut Providers) {
         tcx.lang_items().panic_impl().is_some_and(|did| did.is_local())
     };
     providers.source_span = |tcx, def_id| tcx.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP);
+    providers.create_def_raw = create_def_raw_provider;
 }
 
 pub fn contains_name(attrs: &[Attribute], name: Symbol) -> bool {