From eb934f809f15a8b50f9d035d3d922c49acfa95de Mon Sep 17 00:00:00 2001 From: aerooneqq Date: Thu, 8 Oct 2026 19:05:34 +0300 Subject: [PATCH 1/7] Remap local def ids for deterministic metadata encoding --- compiler/rustc_hir_id/src/definitions.rs | 15 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 158 ++++++++++++++++-- compiler/rustc_metadata/src/rmeta/mod.rs | 2 +- .../async-fns-issue-162202.rs | 3 + .../parallel-reproducible-build/rmake.rs | 33 ++++ .../rpit-issue-162202.rs | 19 +++ .../static-muts-lib-issue-162203.rs | 4 + 7 files changed, 216 insertions(+), 18 deletions(-) create mode 100644 tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs create mode 100644 tests/run-make/parallel-reproducible-build/rpit-issue-162202.rs create mode 100644 tests/run-make/parallel-reproducible-build/static-muts-lib-issue-162203.rs diff --git a/compiler/rustc_hir_id/src/definitions.rs b/compiler/rustc_hir_id/src/definitions.rs index 92c8aeb43f283..65840b4c59430 100644 --- a/compiler/rustc_hir_id/src/definitions.rs +++ b/compiler/rustc_hir_id/src/definitions.rs @@ -98,6 +98,7 @@ pub struct Definitions { def_id_to_key: IndexVec, // We do only store the local hash, as all the definitions are from the current crate. def_path_hashes: IndexVec, + last_deterministic_index: Option, def_path_hash_to_index: DefPathToIndexMap, } @@ -302,13 +303,22 @@ pub enum DefPathData { } impl Definitions { - /// This function indicates that the order of def id allocations + /// This function indicates that the def id allocations /// may be non-deterministic after it was called. pub fn commit_end_of_determinism(&mut self) { assert!( self.def_path_hash_to_index.after_parallel_alloc.replace(Default::default()).is_none(), "this function should be called only once" - ) + ); + + // `self.def_id_to_key.len()` can't be empty, at least crate root should + // be present. + let def_index = DefIndex::from_usize(self.def_id_to_key.len() - 1); + self.last_deterministic_index.replace(def_index); + } + + pub fn last_deterministic_index(&self) -> DefIndex { + self.last_deterministic_index.expect("must contain index") } #[inline(always)] @@ -362,6 +372,7 @@ impl Definitions { def_path_hashes: Default::default(), def_id_to_key: Default::default(), def_path_hash_to_index: Default::default(), + last_deterministic_index: Default::default(), }; // Create the root definition. diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 38dbf8d428a20..2ffb520ba8f62 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -78,6 +78,15 @@ pub(super) struct EncodeContext<'a, 'tcx> { hygiene_ctxt: Rc>, // Used for both `Symbol`s and `ByteSymbol`s. symbol_index_table: FxHashMap, + // Remapping of non-deterministic local def ids for stable encoding + // during parallel compilation. + local_def_ids_remapping: FxHashMap, + // DefIndex of a last deterministically allocated local def id + // (see `Definitions::commit_end_of_determinism`). + last_deterministic_index: u32, + // Stably sorted non-deterministic local def ids that are used + // when iterating over all local def ids. + non_det_sorted_ids: Vec, } /// If the current crate is a proc-macro, returns early with `LazyArray::default()`. @@ -147,6 +156,18 @@ impl<'a, 'tcx> Encodable> for ExpnIndex { } } +impl Encodable> for RawDefId { + fn encode(&self, e: &mut EncodeContext<'_, '_>) { + self.krate.encode(e); + + if self.krate == 0 { + e.emit_u32(e.map_index(DefIndex::from_u32(self.index)).as_u32()); + } else { + self.index.encode(e); + } + } +} + impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> { fn encode_crate_num(&mut self, crate_num: CrateNum) { if crate_num != LOCAL_CRATE && self.is_proc_macro { @@ -164,6 +185,8 @@ impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> { } fn encode_def_id(&mut self, def_id: DefId) { + let def_id = self.map_def_id(def_id); + def_id.krate.encode(self); self.emit_u32(def_id.index.as_u32()); } @@ -235,7 +258,7 @@ impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> { /// `LocalDefId` as a wrapper around def index we want to encode /// def index only, so we override the encode implementation. fn encode_local_def_id(&mut self, def_id: LocalDefId) { - self.emit_u32(def_id.local_def_index.as_u32()); + self.emit_u32(self.map_local_def_id(def_id).local_def_index.as_u32()); } /// We override encoding logic for `DefKey` during metadata encoding, @@ -434,20 +457,23 @@ macro_rules! record_some_lazy { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ let value = $value; let lazy = $self.lazy(value); - $self.$tables.$table.set_some($def_id, lazy); + let def_id = $self.map_local_def_id($def_id); + $self.$tables.$table.set_some(def_id, lazy); }}; } macro_rules! record_some { - ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => { - $self.$tables.$table.set_some($def_id, $value) - }; + ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ + let def_id = $self.map_local_def_id($def_id); + $self.$tables.$table.set_some(def_id, $value) + }}; } macro_rules! record_value { - ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => { - $self.$tables.$table.set($def_id, $value) - }; + ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ + let def_id = $self.map_local_def_id($def_id); + $self.$tables.$table.set(def_id, $value) + }}; } // Shorthand for `$self.$tables.$table.set_some($def_id.index, $self.lazy_array($value))`, which would @@ -456,7 +482,8 @@ macro_rules! record_array { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ let value = $value; let lazy = $self.lazy_array(value); - $self.$tables.$table.set_some($def_id, lazy); + let def_id = $self.map_local_def_id($def_id); + $self.$tables.$table.set_some(def_id, lazy); }}; } @@ -464,7 +491,8 @@ macro_rules! record_defaulted_array { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ let value = $value; let lazy = $self.lazy_array(value); - $self.$tables.$table.set($def_id, lazy); + let def_id = $self.map_local_def_id($def_id); + $self.$tables.$table.set(def_id, lazy); }}; } @@ -566,10 +594,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { record_value!(self.tables.def_path_hashes[def_id] <- def_path_hash.local_hash().as_u64()) } } else { - for (def_index, def_key, def_path_hash) in defs.enumerated_keys_and_path_hashes() { - let def_id = LocalDefId { local_def_index: def_index }; + for def_id in self.iter_sorted_local_ids() { + let def_key = defs.def_key(def_id); + let hash = defs.def_path_hash(def_id).local_hash().as_u64(); + record_some_lazy!(self.tables.def_keys[def_id] <- def_key); - record_value!(self.tables.def_path_hashes[def_id] <- def_path_hash.local_hash().as_u64()) + record_value!(self.tables.def_path_hashes[def_id] <- hash); } } } @@ -1457,6 +1487,48 @@ fn should_encode_const(def_kind: DefKind) -> bool { } } +impl EncodeContext<'_, '_> { + #[inline] + fn map_local_def_id(&self, local_def_id: LocalDefId) -> LocalDefId { + LocalDefId { local_def_index: self.map_index(local_def_id.local_def_index) } + } + + #[inline] + fn map_def_id(&self, def_id: DefId) -> DefId { + if def_id.is_local() { + DefId { krate: LOCAL_CRATE, index: self.map_index(def_id.index) } + } else { + def_id + } + } + + #[inline] + fn map_index(&self, def_index: DefIndex) -> DefIndex { + let index = def_index.as_u32(); + + if index <= self.last_deterministic_index { + def_index + } else { + self.local_def_ids_remapping.get(&def_index).copied().unwrap_or(def_index) + } + } + + fn iter_sorted_local_ids(&self) -> impl Iterator + use<> { + let non_det_part = self.non_det_sorted_ids.clone(); + let last_det_index = self.last_deterministic_index; + + gen move { + for i in 0..last_det_index + 1 { + yield LocalDefId { local_def_index: DefIndex::from_u32(i) } + } + + for def_id in non_det_part { + yield def_id + } + } + } +} + impl<'a, 'tcx> EncodeContext<'a, 'tcx> { fn encode_attrs(&mut self, def_id: LocalDefId) { let tcx = self.tcx; @@ -1489,8 +1561,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { } let tcx = self.tcx; - - for local_id in tcx.iter_local_def_id() { + for local_id in self.iter_sorted_local_ids() { let def_id = local_id.to_def_id(); let def_kind = tcx.def_kind(local_id); record_some!(self.tables.def_kind[local_id] <- def_kind); @@ -2626,6 +2697,10 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { tcx, || { with_encode_metadata_header(tcx, path, |ecx| { + ecx.local_def_ids_remapping = create_local_def_ids_remapping(tcx); + ecx.non_det_sorted_ids = + create_sorted_non_det_local_ids(tcx, &ecx.local_def_ids_remapping); + // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. let (root, unhashed) = ecx.encode_crate_root(); @@ -2661,6 +2736,56 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { } } +/// Creates remapping of non determinsitic local def ids +/// (i.e., ids that start from `Definitions::last_deterministic_index + 1`). +fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> FxHashMap { + let defs = tcx.untracked().definitions.read(); + + let mut non_det_ids = vec![]; + let start = tcx.definitions().last_deterministic_index().as_usize() + 1; + + for idx in start..defs.num_definitions() { + let def_id = LocalDefId { local_def_index: idx.into() }; + non_det_ids.push((def_id, defs.def_path_hash(def_id).local_hash())); + } + + non_det_ids.sort_by_key(|(_, hash)| *hash); + + let mut remapping = FxHashMap::default(); + for (idx, (id, _)) in non_det_ids.into_iter().enumerate() { + let from = id.local_def_index; + let to = DefIndex::from_usize(start + idx); + + if from != to { + remapping.insert(from, to); + } + } + + remapping +} + +/// Creates stably sorted list of non determinsitic local def ids +/// (i.e., ids that start from `Definitions::last_deterministic_index + 1`). +fn create_sorted_non_det_local_ids( + tcx: TyCtxt<'_>, + remapping: &FxHashMap, +) -> Vec { + let defs = tcx.untracked().definitions.read(); + let start = defs.last_deterministic_index().as_usize() + 1; + + let mut sorted_def_ids = (start..defs.num_definitions()) + .into_iter() + .map(|idx| LocalDefId { local_def_index: DefIndex::from(idx) }) + .collect::>(); + + sorted_def_ids.sort_by_key(|id| { + let def_index = id.local_def_index; + remapping.get(&def_index).copied().unwrap_or(def_index) + }); + + sorted_def_ids +} + fn with_encode_metadata_header( tcx: TyCtxt<'_>, path: &Path, @@ -2734,6 +2859,9 @@ fn with_encode_metadata_header( is_proc_macro: tcx.crate_types().contains(&CrateType::ProcMacro), hygiene_ctxt: Default::default(), symbol_index_table: Default::default(), + local_def_ids_remapping: Default::default(), + last_deterministic_index: tcx.definitions().last_deterministic_index().as_u32(), + non_det_sorted_ids: Default::default(), }; // Encode the rustc version string in a predictable location. diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index eef513382b57e..5ee0c43de0b6b 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -343,7 +343,7 @@ pub(crate) struct CrateRootUnhashed { /// On-disk representation of `DefId`. /// This creates a type-safe way to enforce that we remap the CrateNum between the on-disk /// representation and the compilation session. -#[derive(Copy, Clone, Hash, PartialEq, Eq, Encodable, BlobDecodable)] +#[derive(Copy, Clone, Hash, PartialEq, Eq, BlobDecodable)] pub(crate) struct RawDefId { krate: u32, index: u32, diff --git a/tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs b/tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs new file mode 100644 index 0000000000000..1e959cfd62543 --- /dev/null +++ b/tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs @@ -0,0 +1,3 @@ +async fn test1(_: &'_ u8) {} +async fn test2<'s>(_: &'s u8) {} +fn main() {} diff --git a/tests/run-make/parallel-reproducible-build/rmake.rs b/tests/run-make/parallel-reproducible-build/rmake.rs index caaee1b47c583..01de08e0551f3 100644 --- a/tests/run-make/parallel-reproducible-build/rmake.rs +++ b/tests/run-make/parallel-reproducible-build/rmake.rs @@ -14,6 +14,39 @@ fn main() { const TESTS: &[(&str, &[&str])] = &[ ("static-muts-issue-140413", &["-Zthreads=50"]), ("derives-issue-129094", &["-Zthreads=16", "-Copt-level=3"]), + ( + "async-fns-issue-162202", + &[ + "-Zthreads=30", + "--crate-type=lib", + "--crate-name=test", + "-Ccodegen-units=1", + "-Zremap-cwd-prefix=reproducible_dir", + "--edition=2024", + ], + ), + ( + "rpit-issue-162202", + &[ + "-Zthreads=30", + "--crate-type=lib", + "--crate-name=test", + "-Ccodegen-units=1", + "-Zremap-cwd-prefix=reproducible_dir", + "--edition=2024", + ], + ), + ( + "static-muts-lib-issue-162203", + &[ + "-Zthreads=30", + "--edition=2024", + "-Zremap-cwd-prefix=reproducible_dir", + "--cap-lints=warn", + "-Ccodegen-units=1", + "-Clink-dead-code=true", + ], + ), ]; for (file, args) in TESTS { diff --git a/tests/run-make/parallel-reproducible-build/rpit-issue-162202.rs b/tests/run-make/parallel-reproducible-build/rpit-issue-162202.rs new file mode 100644 index 0000000000000..1147f4d450d7d --- /dev/null +++ b/tests/run-make/parallel-reproducible-build/rpit-issue-162202.rs @@ -0,0 +1,19 @@ +trait Foo { + fn test() -> impl IntoIterator + Send; +} + +struct A; +impl Foo for A { + fn test() -> impl IntoIterator + Send { + [] + } +} + +struct B; +impl Foo for B { + fn test() -> impl IntoIterator + Send { + [] + } +} + +fn main() {} diff --git a/tests/run-make/parallel-reproducible-build/static-muts-lib-issue-162203.rs b/tests/run-make/parallel-reproducible-build/static-muts-lib-issue-162203.rs new file mode 100644 index 0000000000000..adda309c8cd39 --- /dev/null +++ b/tests/run-make/parallel-reproducible-build/static-muts-lib-issue-162203.rs @@ -0,0 +1,4 @@ +#![crate_type = "lib"] +static mut TEST: &'static mut [isize] = &mut [1]; +static mut TEST_RAW: *mut [isize] = &mut [1isize] as *mut _; +pub fn main() {} From 5199056965f45ea31253b76c60d808bc0f40e450 Mon Sep 17 00:00:00 2001 From: aerooneqq Date: Thu, 8 Oct 2026 19:34:35 +0300 Subject: [PATCH 2/7] Fix typos --- compiler/rustc_metadata/src/rmeta/encoder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 2ffb520ba8f62..50933f35aced4 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2736,7 +2736,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { } } -/// Creates remapping of non determinsitic local def ids +/// Creates remapping of non deterministic local def ids /// (i.e., ids that start from `Definitions::last_deterministic_index + 1`). fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> FxHashMap { let defs = tcx.untracked().definitions.read(); @@ -2764,7 +2764,7 @@ fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> FxHashMap, From 69f556beea9d5a07356c708e2a28136501176bee Mon Sep 17 00:00:00 2001 From: aerooneqq Date: Fri, 9 Oct 2026 12:20:34 +0300 Subject: [PATCH 3/7] Review: remove remapping map, use Vec instead --- compiler/rustc_metadata/src/rmeta/encoder.rs | 28 +++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 50933f35aced4..577b8361fecde 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -18,7 +18,7 @@ use rustc_data_structures::sync::{par_for_each_in, par_join}; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_data_structures::thousands::usize_with_underscores; use rustc_hir as hir; -use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId, LocalDefIdSet}; +use rustc_hir::def_id::{CRATE_DEF_ID, CRATE_DEF_INDEX, LOCAL_CRATE, LocalDefId, LocalDefIdSet}; use rustc_hir::definitions::DefPathData; use rustc_hir_pretty::id_to_string; use rustc_index::IndexVec; @@ -80,7 +80,7 @@ pub(super) struct EncodeContext<'a, 'tcx> { symbol_index_table: FxHashMap, // Remapping of non-deterministic local def ids for stable encoding // during parallel compilation. - local_def_ids_remapping: FxHashMap, + local_def_ids_remapping: Vec, // DefIndex of a last deterministically allocated local def id // (see `Definitions::commit_end_of_determinism`). last_deterministic_index: u32, @@ -1509,7 +1509,8 @@ impl EncodeContext<'_, '_> { if index <= self.last_deterministic_index { def_index } else { - self.local_def_ids_remapping.get(&def_index).copied().unwrap_or(def_index) + let remapped_idx = index - self.last_deterministic_index - 1; + self.local_def_ids_remapping[remapped_idx as usize] } } @@ -2738,7 +2739,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { /// Creates remapping of non deterministic local def ids /// (i.e., ids that start from `Definitions::last_deterministic_index + 1`). -fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> FxHashMap { +fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> Vec { let defs = tcx.untracked().definitions.read(); let mut non_det_ids = vec![]; @@ -2751,14 +2752,9 @@ fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> FxHashMap) -> FxHashMap, - remapping: &FxHashMap, -) -> Vec { +fn create_sorted_non_det_local_ids(tcx: TyCtxt<'_>, remapping: &[DefIndex]) -> Vec { let defs = tcx.untracked().definitions.read(); let start = defs.last_deterministic_index().as_usize() + 1; @@ -2778,10 +2771,7 @@ fn create_sorted_non_det_local_ids( .map(|idx| LocalDefId { local_def_index: DefIndex::from(idx) }) .collect::>(); - sorted_def_ids.sort_by_key(|id| { - let def_index = id.local_def_index; - remapping.get(&def_index).copied().unwrap_or(def_index) - }); + sorted_def_ids.sort_by_key(|id| remapping[id.local_def_index.as_usize() - start]); sorted_def_ids } From 81fe2b42435f3f3ffc4a4700d1c51f5baf771c80 Mon Sep 17 00:00:00 2001 From: aerooneqq Date: Fri, 9 Oct 2026 12:33:05 +0300 Subject: [PATCH 4/7] Use `first_non_det_index` instead of `last_deterministic_index` --- compiler/rustc_hir_id/src/definitions.rs | 12 ++++++------ compiler/rustc_metadata/src/rmeta/encoder.rs | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_hir_id/src/definitions.rs b/compiler/rustc_hir_id/src/definitions.rs index 65840b4c59430..25734541ff03e 100644 --- a/compiler/rustc_hir_id/src/definitions.rs +++ b/compiler/rustc_hir_id/src/definitions.rs @@ -98,7 +98,7 @@ pub struct Definitions { def_id_to_key: IndexVec, // We do only store the local hash, as all the definitions are from the current crate. def_path_hashes: IndexVec, - last_deterministic_index: Option, + first_non_det_index: Option, def_path_hash_to_index: DefPathToIndexMap, } @@ -313,12 +313,12 @@ impl Definitions { // `self.def_id_to_key.len()` can't be empty, at least crate root should // be present. - let def_index = DefIndex::from_usize(self.def_id_to_key.len() - 1); - self.last_deterministic_index.replace(def_index); + let def_index = DefIndex::from_usize(self.def_id_to_key.len()); + self.first_non_det_index.replace(def_index); } - pub fn last_deterministic_index(&self) -> DefIndex { - self.last_deterministic_index.expect("must contain index") + pub fn first_non_det_index(&self) -> DefIndex { + self.first_non_det_index.expect("must contain index") } #[inline(always)] @@ -372,7 +372,7 @@ impl Definitions { def_path_hashes: Default::default(), def_id_to_key: Default::default(), def_path_hash_to_index: Default::default(), - last_deterministic_index: Default::default(), + first_non_det_index: Default::default(), }; // Create the root definition. diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 577b8361fecde..7556521b1338d 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -81,9 +81,9 @@ pub(super) struct EncodeContext<'a, 'tcx> { // Remapping of non-deterministic local def ids for stable encoding // during parallel compilation. local_def_ids_remapping: Vec, - // DefIndex of a last deterministically allocated local def id + // DefIndex of a first non-deterministically allocated local def id // (see `Definitions::commit_end_of_determinism`). - last_deterministic_index: u32, + first_non_det_index: u32, // Stably sorted non-deterministic local def ids that are used // when iterating over all local def ids. non_det_sorted_ids: Vec, @@ -1506,20 +1506,20 @@ impl EncodeContext<'_, '_> { fn map_index(&self, def_index: DefIndex) -> DefIndex { let index = def_index.as_u32(); - if index <= self.last_deterministic_index { + if index < self.first_non_det_index { def_index } else { - let remapped_idx = index - self.last_deterministic_index - 1; + let remapped_idx = index - self.first_non_det_index; self.local_def_ids_remapping[remapped_idx as usize] } } fn iter_sorted_local_ids(&self) -> impl Iterator + use<> { let non_det_part = self.non_det_sorted_ids.clone(); - let last_det_index = self.last_deterministic_index; + let first_non_det_index = self.first_non_det_index; gen move { - for i in 0..last_det_index + 1 { + for i in 0..first_non_det_index { yield LocalDefId { local_def_index: DefIndex::from_u32(i) } } @@ -2743,7 +2743,7 @@ fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> Vec { let defs = tcx.untracked().definitions.read(); let mut non_det_ids = vec![]; - let start = tcx.definitions().last_deterministic_index().as_usize() + 1; + let start = tcx.definitions().first_non_det_index().as_usize(); for idx in start..defs.num_definitions() { let def_id = LocalDefId { local_def_index: idx.into() }; @@ -2764,7 +2764,7 @@ fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> Vec { /// (i.e., ids that start from `Definitions::last_deterministic_index + 1`). fn create_sorted_non_det_local_ids(tcx: TyCtxt<'_>, remapping: &[DefIndex]) -> Vec { let defs = tcx.untracked().definitions.read(); - let start = defs.last_deterministic_index().as_usize() + 1; + let start = defs.first_non_det_index().as_usize(); let mut sorted_def_ids = (start..defs.num_definitions()) .into_iter() @@ -2850,7 +2850,7 @@ fn with_encode_metadata_header( hygiene_ctxt: Default::default(), symbol_index_table: Default::default(), local_def_ids_remapping: Default::default(), - last_deterministic_index: tcx.definitions().last_deterministic_index().as_u32(), + first_non_det_index: tcx.definitions().first_non_det_index().as_u32(), non_det_sorted_ids: Default::default(), }; From b8c5190f0bfb1fe6e764608ae391597e5ef3e4c8 Mon Sep 17 00:00:00 2001 From: aerooneqq Date: Fri, 9 Oct 2026 12:52:28 +0300 Subject: [PATCH 5/7] Review: sort ids by their level first, then by stable hash --- compiler/rustc_metadata/src/rmeta/encoder.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 7556521b1338d..ebd62ff2df697 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2746,14 +2746,28 @@ fn create_local_def_ids_remapping(tcx: TyCtxt<'_>) -> Vec { let start = tcx.definitions().first_non_det_index().as_usize(); for idx in start..defs.num_definitions() { + let def_id_level = |mut id| { + let mut level = 0; + while let Some(parent) = tcx.opt_local_parent(id) { + level += 1; + id = parent; + } + + level + }; + let def_id = LocalDefId { local_def_index: idx.into() }; - non_det_ids.push((def_id, defs.def_path_hash(def_id).local_hash())); + non_det_ids.push((def_id, def_id_level(def_id), defs.def_path_hash(def_id).local_hash())); } - non_det_ids.sort_by_key(|(_, hash)| *hash); + // Sort by level first in order to satisfy the invariant that child def id + // always has a higher def index than its parent (see `def_id_partial_cmp`), + // in context of one level sort by stable hash. Ids from higher level will + // get higher remapped def index. + non_det_ids.sort_by_key(|(_, level, hash)| (*level, *hash)); let mut remapping = vec![CRATE_DEF_INDEX; non_det_ids.len()]; - for (idx, (id, _)) in non_det_ids.into_iter().enumerate() { + for (idx, (id, ..)) in non_det_ids.into_iter().enumerate() { remapping[id.local_def_index.as_usize() - start] = DefIndex::from_usize(start + idx); } From 5e78c2cd88b7cee4bf5e63c90ab0f235e5bd9da8 Mon Sep 17 00:00:00 2001 From: aerooneqq Date: Tue, 13 Oct 2026 12:14:53 +0300 Subject: [PATCH 6/7] Document remapping methods/implementations --- compiler/rustc_metadata/src/rmeta/encoder.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index ebd62ff2df697..3e275bdbc2cc5 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -156,6 +156,8 @@ impl<'a, 'tcx> Encodable> for ExpnIndex { } } +/// A non-default implementation of `RawDefId` encoding as we need to +/// remap it (see `map_local_def_id` and `map_def_id` methods of `EncodeContext`). impl Encodable> for RawDefId { fn encode(&self, e: &mut EncodeContext<'_, '_>) { self.krate.encode(e); @@ -1488,11 +1490,21 @@ fn should_encode_const(def_kind: DefKind) -> bool { } impl EncodeContext<'_, '_> { + /// This method should be used whenever the `LocalDefId` is encoded, as it remaps + /// it's `local_def_index` for deterministic metadata encoding, this + /// remapping needs to be applied to all `LocalDefId`s during metadata encoding. + /// Note that when implementing custom structs that represent `LocalDefId` the + /// remapping should also be applied (see `RawDefId`). #[inline] fn map_local_def_id(&self, local_def_id: LocalDefId) -> LocalDefId { LocalDefId { local_def_index: self.map_index(local_def_id.local_def_index) } } + /// Remaps `DefId` only if it local for deterministic metadata encoding. External + /// def ids are not needed to be remapped as they are already deterministic. + /// Whenever a `DefId` is encoded this method should be called. + /// Note that when implementing custom structs that represent `DefId` the + /// remapping should also be applied (see `RawDefId`). #[inline] fn map_def_id(&self, def_id: DefId) -> DefId { if def_id.is_local() { @@ -1502,6 +1514,11 @@ impl EncodeContext<'_, '_> { } } + /// Shared part of `map_local_def_id` and `map_def_id` which remaps a def index. + /// Given only def index we do not know whether it belongs to local or external + /// def id, so this method should not be called on `DefIndex`es whose origin is unknown. + /// Prefer to use above-mentioned methods instead or use it when it is clear that + /// `DefIndex` is local. #[inline] fn map_index(&self, def_index: DefIndex) -> DefIndex { let index = def_index.as_u32(); From 3b70289974dd870448f5e7fbbe5fccc5338abe67 Mon Sep 17 00:00:00 2001 From: aerooneqq Date: Tue, 13 Oct 2026 12:45:47 +0300 Subject: [PATCH 7/7] Small cleanups --- compiler/rustc_hir_id/src/definitions.rs | 2 -- compiler/rustc_metadata/src/rmeta/encoder.rs | 13 ++++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_hir_id/src/definitions.rs b/compiler/rustc_hir_id/src/definitions.rs index 25734541ff03e..d22643688a324 100644 --- a/compiler/rustc_hir_id/src/definitions.rs +++ b/compiler/rustc_hir_id/src/definitions.rs @@ -311,8 +311,6 @@ impl Definitions { "this function should be called only once" ); - // `self.def_id_to_key.len()` can't be empty, at least crate root should - // be present. let def_index = DefIndex::from_usize(self.def_id_to_key.len()); self.first_non_det_index.replace(def_index); } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 3e275bdbc2cc5..bd025c3742603 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -162,7 +162,7 @@ impl Encodable> for RawDefId { fn encode(&self, e: &mut EncodeContext<'_, '_>) { self.krate.encode(e); - if self.krate == 0 { + if self.krate == LOCAL_CRATE.as_u32() { e.emit_u32(e.map_index(DefIndex::from_u32(self.index)).as_u32()); } else { self.index.encode(e); @@ -682,6 +682,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { fn encode_crate_root(&mut self) -> (LazyValue, LazyValue) { let tcx = self.tcx; + + // We set those fields here as we need them only for metadata encoding, + // while `with_encode_metadata_header` is used in some other places too. + let remapping = create_local_def_ids_remapping(tcx); + self.non_det_sorted_ids = create_sorted_non_det_local_ids(tcx, &remapping); + self.local_def_ids_remapping = remapping; + let mut stats: Vec<(&'static str, usize)> = Vec::with_capacity(32); macro_rules! stat { @@ -2715,10 +2722,6 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { tcx, || { with_encode_metadata_header(tcx, path, |ecx| { - ecx.local_def_ids_remapping = create_local_def_ids_remapping(tcx); - ecx.non_det_sorted_ids = - create_sorted_non_det_local_ids(tcx, &ecx.local_def_ids_remapping); - // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. let (root, unhashed) = ecx.encode_crate_root();