From 47ea2ae933f4fda8e069a77f36cb6d8b21d93997 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 24 Jun 2021 21:02:09 +0200 Subject: [PATCH 01/12] Separate encoding paths. The two paths will be modified independently in the next few commits. --- compiler/rustc_metadata/src/rmeta/decoder.rs | 51 +++---- compiler/rustc_metadata/src/rmeta/encoder.rs | 9 +- .../src/ty/query/on_disk_cache.rs | 14 +- compiler/rustc_span/src/hygiene.rs | 139 +++++++++++------- 4 files changed, 116 insertions(+), 97 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index d28ebfe107cdd..d39ac588026b0 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -30,7 +30,6 @@ use rustc_middle::ty::codec::TyDecoder; use rustc_middle::ty::{self, Ty, TyCtxt, Visibility}; use rustc_serialize::{opaque, Decodable, Decoder}; use rustc_session::Session; -use rustc_span::hygiene::ExpnDataDecodeMode; use rustc_span::source_map::{respan, Spanned}; use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{self, hygiene::MacroKind, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP}; @@ -381,33 +380,29 @@ impl<'a, 'tcx> Decodable> for ExpnId { } }; - rustc_span::hygiene::decode_expn_id( - decoder, - ExpnDataDecodeMode::Metadata(get_ctxt), - |_this, index| { - let cnum = expn_cnum.get().unwrap(); - // Lookup local `ExpnData`s in our own crate data. Foreign `ExpnData`s - // are stored in the owning crate, to avoid duplication. - let crate_data = if cnum == LOCAL_CRATE { - local_cdata - } else { - local_cdata.cstore.get_crate_data(cnum) - }; - let expn_data = crate_data - .root - .expn_data - .get(&crate_data, index) - .unwrap() - .decode((&crate_data, sess)); - let expn_hash = crate_data - .root - .expn_hashes - .get(&crate_data, index) - .unwrap() - .decode((&crate_data, sess)); - Ok((expn_data, expn_hash)) - }, - ) + rustc_span::hygiene::decode_expn_id(decoder, get_ctxt, |_this, index| { + let cnum = expn_cnum.get().unwrap(); + // Lookup local `ExpnData`s in our own crate data. Foreign `ExpnData`s + // are stored in the owning crate, to avoid duplication. + let crate_data = if cnum == LOCAL_CRATE { + local_cdata + } else { + local_cdata.cstore.get_crate_data(cnum) + }; + let expn_data = crate_data + .root + .expn_data + .get(&crate_data, index) + .unwrap() + .decode((&crate_data, sess)); + let expn_hash = crate_data + .root + .expn_hashes + .get(&crate_data, index) + .unwrap() + .decode((&crate_data, sess)); + Ok((expn_data, expn_hash)) + }) } } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 5c7d84e2bc97f..ba6d2d74aa7ff 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -31,7 +31,7 @@ use rustc_session::config::CrateType; use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{self, ExternalSource, FileName, SourceFile, Span, SyntaxContext}; use rustc_span::{ - hygiene::{ExpnDataEncodeMode, HygieneEncodeContext, MacroKind}, + hygiene::{HygieneEncodeContext, MacroKind}, RealFileName, }; use rustc_target::abi::VariantIdx; @@ -176,12 +176,7 @@ impl<'a, 'tcx> Encodable> for SyntaxContext { impl<'a, 'tcx> Encodable> for ExpnId { fn encode(&self, s: &mut EncodeContext<'a, 'tcx>) -> opaque::EncodeResult { - rustc_span::hygiene::raw_encode_expn_id( - *self, - &s.hygiene_ctxt, - ExpnDataEncodeMode::Metadata, - s, - ) + rustc_span::hygiene::raw_encode_expn_id(*self, &s.hygiene_ctxt, s) } } diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index e3db0d2cf30a6..85e84d6a0f487 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -20,8 +20,7 @@ use rustc_serialize::{ }; use rustc_session::Session; use rustc_span::hygiene::{ - ExpnDataDecodeMode, ExpnDataEncodeMode, ExpnId, HygieneDecodeContext, HygieneEncodeContext, - SyntaxContext, SyntaxContextData, + ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData, }; use rustc_span::source_map::{SourceMap, StableSourceFileId}; use rustc_span::CachingSourceMapView; @@ -793,9 +792,9 @@ impl<'a, 'tcx> Decodable> for SyntaxContext { impl<'a, 'tcx> Decodable> for ExpnId { fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Result { let expn_data = decoder.expn_data; - rustc_span::hygiene::decode_expn_id( + rustc_span::hygiene::decode_expn_id_incrcomp( decoder, - ExpnDataDecodeMode::incr_comp(decoder.hygiene_context), + decoder.hygiene_context, |this, index| { // This closure is invoked if we haven't already decoded the data for the `ExpnId` we are deserializing. // We look up the position of the associated `ExpnData` and decode it. @@ -983,12 +982,7 @@ where E: 'a + OpaqueEncoder, { fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> { - rustc_span::hygiene::raw_encode_expn_id( - *self, - s.hygiene_context, - ExpnDataEncodeMode::IncrComp, - s, - ) + rustc_span::hygiene::raw_encode_expn_id_incrcomp(*self, s.hygiene_context, s) } } diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index d292f652896f1..ddf9e7b4255c6 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -1076,22 +1076,74 @@ pub struct HygieneDecodeContext { remapped_expns: Lock>>, } -pub fn decode_expn_id<'a, D: Decoder, G>( +pub fn decode_expn_id_incrcomp( d: &mut D, - mode: ExpnDataDecodeMode<'a, G>, + context: &HygieneDecodeContext, decode_data: impl FnOnce(&mut D, u32) -> Result<(ExpnData, ExpnHash), D::Error>, -) -> Result -where - G: FnOnce(CrateNum) -> &'a HygieneDecodeContext, -{ +) -> Result { let index = u32::decode(d)?; - let context = match mode { - ExpnDataDecodeMode::IncrComp(context) => context, - ExpnDataDecodeMode::Metadata(get_context) => { - let krate = CrateNum::decode(d)?; - get_context(krate) + + // Do this after decoding, so that we decode a `CrateNum` + // if necessary + if index == ExpnId::root().as_u32() { + debug!("decode_expn_id: deserialized root"); + return Ok(ExpnId::root()); + } + + let outer_expns = &context.remapped_expns; + + // Ensure that the lock() temporary is dropped early + { + if let Some(expn_id) = outer_expns.lock().get(index as usize).copied().flatten() { + return Ok(expn_id); } - }; + } + + // Don't decode the data inside `HygieneData::with`, since we need to recursively decode + // other ExpnIds + let (mut expn_data, hash) = decode_data(d, index)?; + + let expn_id = HygieneData::with(|hygiene_data| { + if let Some(&expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) { + return expn_id; + } + + let expn_id = ExpnId(hygiene_data.expn_data.len() as u32); + + // If we just deserialized an `ExpnData` owned by + // the local crate, its `orig_id` will be stale, + // so we need to update it to its own value. + // This only happens when we deserialize the incremental cache, + // since a crate will never decode its own metadata. + if expn_data.krate == LOCAL_CRATE { + expn_data.orig_id = Some(expn_id.0); + } + + hygiene_data.expn_data.push(Some(expn_data)); + hygiene_data.expn_hashes.push(hash); + let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); + debug_assert!(_old_id.is_none()); + + let mut expns = outer_expns.lock(); + let new_len = index as usize + 1; + if expns.len() < new_len { + expns.resize(new_len, None); + } + expns[index as usize] = Some(expn_id); + drop(expns); + expn_id + }); + Ok(expn_id) +} + +pub fn decode_expn_id<'a, D: Decoder>( + d: &mut D, + get_context: impl FnOnce(CrateNum) -> &'a HygieneDecodeContext, + decode_data: impl FnOnce(&mut D, u32) -> Result<(ExpnData, ExpnHash), D::Error>, +) -> Result { + let index = u32::decode(d)?; + let krate = CrateNum::decode(d)?; + let context = get_context(krate); // Do this after decoding, so that we decode a `CrateNum` // if necessary @@ -1274,56 +1326,39 @@ pub fn raw_encode_syntax_context( ctxt.0.encode(e) } -pub fn raw_encode_expn_id( +pub fn raw_encode_expn_id_incrcomp( expn: ExpnId, context: &HygieneEncodeContext, - mode: ExpnDataEncodeMode, e: &mut E, ) -> Result<(), E::Error> { // Record the fact that we need to serialize the corresponding // `ExpnData` - let needs_data = || { - if !context.serialized_expns.lock().contains(&expn) { - context.latest_expns.lock().insert(expn); - } - }; - - match mode { - ExpnDataEncodeMode::IncrComp => { - // Always serialize the `ExpnData` in incr comp mode - needs_data(); - expn.0.encode(e) - } - ExpnDataEncodeMode::Metadata => { - let data = expn.expn_data(); - // We only need to serialize the ExpnData - // if it comes from this crate. - // We currently don't serialize any hygiene information data for - // proc-macro crates: see the `SpecializedEncoder` impl - // for crate metadata. - if data.krate == LOCAL_CRATE { - needs_data(); - } - data.orig_id.expect("Missing orig_id").encode(e)?; - data.krate.encode(e) - } + if !context.serialized_expns.lock().contains(&expn) { + context.latest_expns.lock().insert(expn); } + expn.0.encode(e) } -pub enum ExpnDataEncodeMode { - IncrComp, - Metadata, -} - -pub enum ExpnDataDecodeMode<'a, F: FnOnce(CrateNum) -> &'a HygieneDecodeContext> { - IncrComp(&'a HygieneDecodeContext), - Metadata(F), -} - -impl<'a> ExpnDataDecodeMode<'a, Box &'a HygieneDecodeContext>> { - pub fn incr_comp(ctxt: &'a HygieneDecodeContext) -> Self { - ExpnDataDecodeMode::IncrComp(ctxt) +pub fn raw_encode_expn_id( + expn: ExpnId, + context: &HygieneEncodeContext, + e: &mut E, +) -> Result<(), E::Error> { + let data = expn.expn_data(); + // We only need to serialize the ExpnData + // if it comes from this crate. + // We currently don't serialize any hygiene information data for + // proc-macro crates: see the `SpecializedEncoder` impl + // for crate metadata. + if data.krate == LOCAL_CRATE { + // Record the fact that we need to serialize the corresponding + // `ExpnData` + if !context.serialized_expns.lock().contains(&expn) { + context.latest_expns.lock().insert(expn); + } } + data.orig_id.expect("Missing orig_id").encode(e)?; + data.krate.encode(e) } impl Encodable for SyntaxContext { From c2d43e132975126dda1364346db8498fbc3917b0 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 27 Jun 2021 10:03:10 +0200 Subject: [PATCH 02/12] Simplify metadata decoding. --- compiler/rustc_metadata/src/rmeta/decoder.rs | 18 +++------- compiler/rustc_span/src/hygiene.rs | 38 +++++--------------- 2 files changed, 12 insertions(+), 44 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index d39ac588026b0..e2203fffd45ab 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -35,7 +35,6 @@ use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{self, hygiene::MacroKind, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP}; use proc_macro::bridge::client::ProcMacro; -use std::cell::Cell; use std::io; use std::mem; use std::num::NonZeroUsize; @@ -370,21 +369,12 @@ impl<'a, 'tcx> Decodable> for ExpnId { fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Result { let local_cdata = decoder.cdata(); let sess = decoder.sess.unwrap(); - let expn_cnum = Cell::new(None); - let get_ctxt = |cnum| { - expn_cnum.set(Some(cnum)); - if cnum == LOCAL_CRATE { - &local_cdata.hygiene_context - } else { - &local_cdata.cstore.get_crate_data(cnum).cdata.hygiene_context - } - }; - rustc_span::hygiene::decode_expn_id(decoder, get_ctxt, |_this, index| { - let cnum = expn_cnum.get().unwrap(); + rustc_span::hygiene::decode_expn_id(decoder, |cnum, index| { // Lookup local `ExpnData`s in our own crate data. Foreign `ExpnData`s // are stored in the owning crate, to avoid duplication. - let crate_data = if cnum == LOCAL_CRATE { + debug_assert_ne!(cnum, LOCAL_CRATE); + let crate_data = if cnum == local_cdata.cnum { local_cdata } else { local_cdata.cstore.get_crate_data(cnum) @@ -401,7 +391,7 @@ impl<'a, 'tcx> Decodable> for ExpnId { .get(&crate_data, index) .unwrap() .decode((&crate_data, sess)); - Ok((expn_data, expn_hash)) + (expn_data, expn_hash) }) } } diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index ddf9e7b4255c6..e3174b47f8d0c 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -1136,14 +1136,12 @@ pub fn decode_expn_id_incrcomp( Ok(expn_id) } -pub fn decode_expn_id<'a, D: Decoder>( +pub fn decode_expn_id( d: &mut D, - get_context: impl FnOnce(CrateNum) -> &'a HygieneDecodeContext, - decode_data: impl FnOnce(&mut D, u32) -> Result<(ExpnData, ExpnHash), D::Error>, + decode_data: impl FnOnce(CrateNum, u32) -> (ExpnData, ExpnHash), ) -> Result { let index = u32::decode(d)?; let krate = CrateNum::decode(d)?; - let context = get_context(krate); // Do this after decoding, so that we decode a `CrateNum` // if necessary @@ -1152,18 +1150,14 @@ pub fn decode_expn_id<'a, D: Decoder>( return Ok(ExpnId::root()); } - let outer_expns = &context.remapped_expns; - - // Ensure that the lock() temporary is dropped early - { - if let Some(expn_id) = outer_expns.lock().get(index as usize).copied().flatten() { - return Ok(expn_id); - } - } + // This function is used to decode metadata, so it cannot decode information about LOCAL_CRATE. + debug_assert_ne!(krate, LOCAL_CRATE); // Don't decode the data inside `HygieneData::with`, since we need to recursively decode // other ExpnIds - let (mut expn_data, hash) = decode_data(d, index)?; + let (expn_data, hash) = decode_data(krate, index); + debug_assert_eq!(krate, expn_data.krate); + debug_assert_eq!(expn_data.orig_id, Some(index)); let expn_id = HygieneData::with(|hygiene_data| { if let Some(&expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) { @@ -1171,30 +1165,14 @@ pub fn decode_expn_id<'a, D: Decoder>( } let expn_id = ExpnId(hygiene_data.expn_data.len() as u32); - - // If we just deserialized an `ExpnData` owned by - // the local crate, its `orig_id` will be stale, - // so we need to update it to its own value. - // This only happens when we deserialize the incremental cache, - // since a crate will never decode its own metadata. - if expn_data.krate == LOCAL_CRATE { - expn_data.orig_id = Some(expn_id.0); - } - hygiene_data.expn_data.push(Some(expn_data)); hygiene_data.expn_hashes.push(hash); let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); debug_assert!(_old_id.is_none()); - let mut expns = outer_expns.lock(); - let new_len = index as usize + 1; - if expns.len() < new_len { - expns.resize(new_len, None); - } - expns[index as usize] = Some(expn_id); - drop(expns); expn_id }); + Ok(expn_id) } From 6e78d6c9d697217998322ab626cbeee24cdfe228 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 10 Jul 2021 23:34:41 +0200 Subject: [PATCH 03/12] Make the CrateNum part of the ExpnId. --- compiler/rustc_ast/src/node_id.rs | 6 +- compiler/rustc_metadata/src/rmeta/decoder.rs | 9 +- .../src/rmeta/decoder/cstore_impl.rs | 19 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 14 +- compiler/rustc_metadata/src/rmeta/mod.rs | 6 +- compiler/rustc_middle/src/middle/cstore.rs | 6 +- .../src/ty/query/on_disk_cache.rs | 12 +- compiler/rustc_span/src/hygiene.rs | 345 +++++++++++++----- compiler/rustc_span/src/lib.rs | 2 +- 9 files changed, 307 insertions(+), 112 deletions(-) diff --git a/compiler/rustc_ast/src/node_id.rs b/compiler/rustc_ast/src/node_id.rs index d20bace608882..e1e7d757d7e62 100644 --- a/compiler/rustc_ast/src/node_id.rs +++ b/compiler/rustc_ast/src/node_id.rs @@ -1,4 +1,4 @@ -use rustc_span::ExpnId; +use rustc_span::{ExpnId, LocalExpnId}; use std::fmt; rustc_index::newtype_index! { @@ -25,11 +25,11 @@ pub const DUMMY_NODE_ID: NodeId = NodeId::MAX; impl NodeId { pub fn placeholder_from_expn_id(expn_id: ExpnId) -> Self { - NodeId::from_u32(expn_id.as_u32()) + NodeId::from_u32(expn_id.expect_local().as_u32()) } pub fn placeholder_to_expn_id(self) -> ExpnId { - ExpnId::from_u32(self.as_u32()) + LocalExpnId::from_u32(self.as_u32()).to_expn_id() } } diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index e2203fffd45ab..4b72ac8695794 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -30,9 +30,10 @@ use rustc_middle::ty::codec::TyDecoder; use rustc_middle::ty::{self, Ty, TyCtxt, Visibility}; use rustc_serialize::{opaque, Decodable, Decoder}; use rustc_session::Session; +use rustc_span::hygiene::{ExpnIndex, MacroKind}; use rustc_span::source_map::{respan, Spanned}; use rustc_span::symbol::{sym, Ident, Symbol}; -use rustc_span::{self, hygiene::MacroKind, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP}; +use rustc_span::{self, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP}; use proc_macro::bridge::client::ProcMacro; use std::io; @@ -348,6 +349,12 @@ impl<'a, 'tcx> Decodable> for DefIndex { } } +impl<'a, 'tcx> Decodable> for ExpnIndex { + fn decode(d: &mut DecodeContext<'a, 'tcx>) -> Result { + Ok(ExpnIndex::from_u32(d.read_u32()?)) + } +} + impl<'a, 'tcx> Decodable> for SyntaxContext { fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Result { let cdata = decoder.cdata(); diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index ce8dfeae076e6..67023e9e84e22 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -18,11 +18,11 @@ use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, Visibility}; use rustc_session::utils::NativeLibKind; use rustc_session::{Session, StableCrateId}; +use rustc_span::hygiene::{ExpnData, ExpnHash, ExpnId}; use rustc_span::source_map::{Span, Spanned}; use rustc_span::symbol::Symbol; use rustc_data_structures::sync::Lrc; -use rustc_span::ExpnId; use smallvec::SmallVec; use std::any::Any; @@ -494,6 +494,23 @@ impl CrateStore for CStore { fn as_any(&self) -> &dyn Any { self } + fn decode_expn_data(&self, sess: &Session, expn_id: ExpnId) -> (ExpnData, ExpnHash) { + let crate_data = self.get_crate_data(expn_id.krate); + ( + crate_data + .root + .expn_data + .get(&crate_data, expn_id.local_id) + .unwrap() + .decode((&crate_data, sess)), + crate_data + .root + .expn_hashes + .get(&crate_data, expn_id.local_id) + .unwrap() + .decode((&crate_data, sess)), + ) + } fn crate_name(&self, cnum: CrateNum) -> Symbol { self.get_crate_data(cnum).root.name diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index ba6d2d74aa7ff..4684daef4a18c 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -31,7 +31,7 @@ use rustc_session::config::CrateType; use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{self, ExternalSource, FileName, SourceFile, Span, SyntaxContext}; use rustc_span::{ - hygiene::{HygieneEncodeContext, MacroKind}, + hygiene::{ExpnIndex, HygieneEncodeContext, MacroKind}, RealFileName, }; use rustc_target::abi::VariantIdx; @@ -168,6 +168,12 @@ impl<'a, 'tcx> Encodable> for DefIndex { } } +impl<'a, 'tcx> Encodable> for ExpnIndex { + fn encode(&self, s: &mut EncodeContext<'a, 'tcx>) -> opaque::EncodeResult { + s.emit_u32(self.as_u32()) + } +} + impl<'a, 'tcx> Encodable> for SyntaxContext { fn encode(&self, s: &mut EncodeContext<'a, 'tcx>) -> opaque::EncodeResult { rustc_span::hygiene::raw_encode_syntax_context(*self, &s.hygiene_ctxt, s) @@ -1588,8 +1594,10 @@ impl EncodeContext<'a, 'tcx> { Ok(()) }, |(this, _, expn_data_table, expn_hash_table), index, expn_data, hash| { - expn_data_table.set(index, this.lazy(expn_data)); - expn_hash_table.set(index, this.lazy(hash)); + if let Some(index) = index.as_local() { + expn_data_table.set(index.as_raw(), this.lazy(expn_data)); + expn_hash_table.set(index.as_raw(), this.lazy(hash)); + } Ok(()) }, ); diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index f2ebfb9b725ba..a487753f4628a 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -19,7 +19,7 @@ use rustc_middle::ty::{self, ReprOptions, Ty}; use rustc_serialize::opaque::Encoder; use rustc_session::config::SymbolManglingVersion; use rustc_span::edition::Edition; -use rustc_span::hygiene::MacroKind; +use rustc_span::hygiene::{ExpnIndex, MacroKind}; use rustc_span::symbol::{Ident, Symbol}; use rustc_span::{self, ExpnData, ExpnHash, ExpnId, Span}; use rustc_target::spec::{PanicStrategy, TargetTriple}; @@ -170,8 +170,8 @@ macro_rules! Lazy { } type SyntaxContextTable = Lazy>>; -type ExpnDataTable = Lazy>>; -type ExpnHashTable = Lazy>>; +type ExpnDataTable = Lazy>>; +type ExpnHashTable = Lazy>>; #[derive(MetadataEncodable, MetadataDecodable)] crate struct ProcMacroData { diff --git a/compiler/rustc_middle/src/middle/cstore.rs b/compiler/rustc_middle/src/middle/cstore.rs index 7efe8e061e885..c10fcc2e90c7e 100644 --- a/compiler/rustc_middle/src/middle/cstore.rs +++ b/compiler/rustc_middle/src/middle/cstore.rs @@ -6,12 +6,13 @@ use crate::ty::TyCtxt; use rustc_ast as ast; use rustc_data_structures::sync::{self, MetadataRef}; -use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; +use rustc_hir::def_id::{CrateNum, DefId, StableCrateId, LOCAL_CRATE}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_macros::HashStable; use rustc_session::search_paths::PathKind; use rustc_session::utils::NativeLibKind; -use rustc_session::StableCrateId; +use rustc_session::Session; +use rustc_span::hygiene::{ExpnData, ExpnHash, ExpnId}; use rustc_span::symbol::Symbol; use rustc_span::Span; use rustc_target::spec::Target; @@ -187,6 +188,7 @@ pub type MetadataLoaderDyn = dyn MetadataLoader + Sync; /// during resolve) pub trait CrateStore: std::fmt::Debug { fn as_any(&self) -> &dyn Any; + fn decode_expn_data(&self, sess: &Session, expn_id: ExpnId) -> (ExpnData, ExpnHash); // Foreign definitions. // This information is safe to access, since it's hashed as part of the DefPathHash, which incr. diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index 85e84d6a0f487..358d016368ee0 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -364,9 +364,12 @@ impl<'sess> OnDiskCache<'sess> { Ok(()) }, |encoder, index, expn_data, hash| -> FileEncodeResult { - let pos = AbsoluteBytePos::new(encoder.position()); - encoder.encode_tagged(TAG_EXPN_DATA, &(expn_data, hash))?; - expn_ids.insert(index, pos); + if index.krate == LOCAL_CRATE { + let pos = AbsoluteBytePos::new(encoder.position()); + encoder.encode_tagged(TAG_EXPN_DATA, &(expn_data, hash))?; + expn_ids.insert(index.local_id.as_u32(), pos); + } + // TODO Handle foreign expansions. Ok(()) }, )?; @@ -807,6 +810,9 @@ impl<'a, 'tcx> Decodable> for ExpnId { Ok(data) }) }, + |this, expn_id| { + Ok(this.tcx.untracked_resolutions.cstore.decode_expn_data(this.tcx.sess, expn_id)) + }, ) } } diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index e3174b47f8d0c..14b6811298261 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -29,12 +29,13 @@ use crate::symbol::{kw, sym, Symbol}; use crate::with_session_globals; use crate::{HashStableContext, Span, DUMMY_SP}; -use crate::def_id::{CrateNum, DefId, CRATE_DEF_INDEX, LOCAL_CRATE}; +use crate::def_id::{CrateNum, DefId, CRATE_DEF_ID, LOCAL_CRATE}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{Lock, Lrc}; use rustc_data_structures::unhash::UnhashMap; +use rustc_index::vec::IndexVec; use rustc_macros::HashStable_Generic; use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use std::fmt; @@ -58,9 +59,34 @@ pub struct SyntaxContextData { dollar_crate_name: Symbol, } +rustc_index::newtype_index! { + /// A unique ID associated with a macro invocation and expansion. + pub struct ExpnIndex { + ENCODABLE = custom + } +} + /// A unique ID associated with a macro invocation and expansion. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct ExpnId(u32); +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct ExpnId { + pub krate: CrateNum, + pub local_id: ExpnIndex, +} + +impl fmt::Debug for ExpnId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Generate crate_::{{expn_}}. + write!(f, "{:?}::{{{{expn{}}}}}", self.krate, self.local_id.private) + } +} + +rustc_index::newtype_index! { + /// A unique ID associated with a macro invocation and expansion. + pub struct LocalExpnId { + ENCODABLE = custom + DEBUG_FORMAT = "expn{}" + } +} /// A unique hash value associated to an expansion. #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encodable, Decodable, HashStable_Generic)] @@ -86,31 +112,103 @@ pub enum Transparency { Opaque, } -impl ExpnId { - pub fn fresh_empty() -> Self { +impl LocalExpnId { + /// The ID of the theoretical expansion that generates freshly parsed, unexpanded AST. + pub const ROOT: LocalExpnId = LocalExpnId::from_u32(0); + + pub fn from_raw(idx: ExpnIndex) -> LocalExpnId { + LocalExpnId::from_u32(idx.as_u32()) + } + + pub fn as_raw(self) -> ExpnIndex { + ExpnIndex::from_u32(self.as_u32()) + } + + pub fn fresh_empty() -> LocalExpnId { HygieneData::with(|data| data.fresh_expn(None)) } - pub fn fresh(expn_data: ExpnData, ctx: impl HashStableContext) -> Self { + pub fn fresh(expn_data: ExpnData, ctx: impl HashStableContext) -> LocalExpnId { + debug_assert_eq!(expn_data.parent.krate, LOCAL_CRATE); let expn_id = HygieneData::with(|data| data.fresh_expn(Some(expn_data))); update_disambiguator(expn_id, ctx); expn_id } - /// The ID of the theoretical expansion that generates freshly parsed, unexpanded AST. #[inline] - pub fn root() -> Self { - ExpnId(0) + pub fn expn_hash(self) -> ExpnHash { + HygieneData::with(|data| data.local_expn_hash(self)) } #[inline] - pub fn as_u32(self) -> u32 { - self.0 + pub fn expn_data(self) -> ExpnData { + HygieneData::with(|data| data.local_expn_data(self).clone()) + } + + #[inline] + pub fn to_expn_id(self) -> ExpnId { + ExpnId { krate: LOCAL_CRATE, local_id: self.as_raw() } + } + + #[inline] + pub fn set_expn_data(self, mut expn_data: ExpnData, ctx: impl HashStableContext) { + debug_assert_eq!(expn_data.parent.krate, LOCAL_CRATE); + HygieneData::with(|data| { + let old_expn_data = &mut data.local_expn_data[self]; + assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID"); + assert_eq!(expn_data.orig_id, None); + debug_assert_eq!(expn_data.krate, LOCAL_CRATE); + expn_data.orig_id = Some(self.as_u32()); + *old_expn_data = Some(expn_data); + }); + update_disambiguator(self, ctx) + } + + #[inline] + pub fn is_descendant_of(self, ancestor: LocalExpnId) -> bool { + self.to_expn_id().is_descendant_of(ancestor.to_expn_id()) } + /// `expn_id.outer_expn_is_descendant_of(ctxt)` is equivalent to but faster than + /// `expn_id.is_descendant_of(ctxt.outer_expn())`. #[inline] - pub fn from_u32(raw: u32) -> ExpnId { - ExpnId(raw) + pub fn outer_expn_is_descendant_of(self, ctxt: SyntaxContext) -> bool { + self.to_expn_id().outer_expn_is_descendant_of(ctxt) + } + + /// Returns span for the macro which originally caused this expansion to happen. + /// + /// Stops backtracing at include! boundary. + #[inline] + pub fn expansion_cause(self) -> Option { + self.to_expn_id().expansion_cause() + } + + #[inline] + #[track_caller] + pub fn parent(self) -> LocalExpnId { + self.expn_data().parent.as_local().unwrap() + } +} + +impl ExpnId { + /// The ID of the theoretical expansion that generates freshly parsed, unexpanded AST. + /// Invariant: we do not create any ExpnId with local_id == 0 and krate != 0. + pub const fn root() -> ExpnId { + ExpnId { krate: LOCAL_CRATE, local_id: ExpnIndex::from_u32(0) } + } + + pub fn fresh_empty() -> ExpnId { + LocalExpnId::fresh_empty().to_expn_id() + } + + pub fn fresh(expn_data: ExpnData, ctx: impl HashStableContext) -> ExpnId { + LocalExpnId::fresh(expn_data, ctx).to_expn_id() + } + + #[inline] + pub fn set_expn_data(self, expn_data: ExpnData, ctx: impl HashStableContext) { + self.expect_local().set_expn_data(expn_data, ctx) } #[inline] @@ -124,20 +222,19 @@ impl ExpnId { } #[inline] - pub fn expn_data(self) -> ExpnData { - HygieneData::with(|data| data.expn_data(self).clone()) + pub fn as_local(self) -> Option { + if self.krate == LOCAL_CRATE { Some(LocalExpnId::from_raw(self.local_id)) } else { None } } #[inline] - pub fn set_expn_data(self, mut expn_data: ExpnData, ctx: impl HashStableContext) { - HygieneData::with(|data| { - let old_expn_data = &mut data.expn_data[self.0 as usize]; - assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID"); - assert_eq!(expn_data.orig_id, None); - expn_data.orig_id = Some(self.as_u32()); - *old_expn_data = Some(expn_data); - }); - update_disambiguator(self, ctx) + #[track_caller] + pub fn expect_local(self) -> LocalExpnId { + self.as_local().unwrap() + } + + #[inline] + pub fn expn_data(self) -> ExpnData { + HygieneData::with(|data| data.expn_data(self).clone()) } pub fn is_descendant_of(self, ancestor: ExpnId) -> bool { @@ -175,8 +272,10 @@ pub struct HygieneData { /// Each expansion should have an associated expansion data, but sometimes there's a delay /// between creation of an expansion ID and obtaining its data (e.g. macros are collected /// first and then resolved later), so we use an `Option` here. - expn_data: Vec>, - expn_hashes: Vec, + local_expn_data: IndexVec>, + local_expn_hashes: IndexVec, + foreign_expn_data: FxHashMap, + foreign_expn_hashes: FxHashMap, expn_hash_to_expn_id: UnhashMap, syntax_context_data: Vec, syntax_context_map: FxHashMap<(SyntaxContext, ExpnId, Transparency), SyntaxContext>, @@ -194,15 +293,17 @@ impl HygieneData { ExpnKind::Root, DUMMY_SP, edition, - Some(DefId::local(CRATE_DEF_INDEX)), + Some(CRATE_DEF_ID.to_def_id()), None, ); root_data.orig_id = Some(0); HygieneData { - expn_data: vec![Some(root_data)], - expn_hashes: vec![ExpnHash(Fingerprint::ZERO)], - expn_hash_to_expn_id: std::iter::once((ExpnHash(Fingerprint::ZERO), ExpnId(0))) + local_expn_data: IndexVec::from_elem_n(Some(root_data), 1), + local_expn_hashes: IndexVec::from_elem_n(ExpnHash(Fingerprint::ZERO), 1), + foreign_expn_data: FxHashMap::default(), + foreign_expn_hashes: FxHashMap::default(), + expn_hash_to_expn_id: std::iter::once((ExpnHash(Fingerprint::ZERO), ExpnId::root())) .collect(), syntax_context_data: vec![SyntaxContextData { outer_expn: ExpnId::root(), @@ -221,24 +322,42 @@ impl HygieneData { with_session_globals(|session_globals| f(&mut *session_globals.hygiene_data.borrow_mut())) } - fn fresh_expn(&mut self, mut expn_data: Option) -> ExpnId { - let raw_id = self.expn_data.len() as u32; + fn fresh_expn(&mut self, mut expn_data: Option) -> LocalExpnId { + let expn_id = self.local_expn_data.next_index(); if let Some(data) = expn_data.as_mut() { + debug_assert_eq!(data.krate, LOCAL_CRATE); assert_eq!(data.orig_id, None); - data.orig_id = Some(raw_id); + data.orig_id = Some(expn_id.as_u32()); } - self.expn_data.push(expn_data); - self.expn_hashes.push(ExpnHash(Fingerprint::ZERO)); - ExpnId(raw_id) + self.local_expn_data.push(expn_data); + let _eid = self.local_expn_hashes.push(ExpnHash(Fingerprint::ZERO)); + debug_assert_eq!(expn_id, _eid); + expn_id + } + + #[inline] + fn local_expn_hash(&self, expn_id: LocalExpnId) -> ExpnHash { + self.local_expn_hashes[expn_id] } #[inline] fn expn_hash(&self, expn_id: ExpnId) -> ExpnHash { - self.expn_hashes[expn_id.0 as usize] + match expn_id.as_local() { + Some(expn_id) => self.local_expn_hashes[expn_id], + None => self.foreign_expn_hashes[&expn_id], + } + } + + fn local_expn_data(&self, expn_id: LocalExpnId) -> &ExpnData { + self.local_expn_data[expn_id].as_ref().expect("no expansion data for an expansion ID") } fn expn_data(&self, expn_id: ExpnId) -> &ExpnData { - self.expn_data[expn_id.0 as usize].as_ref().expect("no expansion data for an expansion ID") + if let Some(expn_id) = expn_id.as_local() { + self.local_expn_data[expn_id].as_ref().expect("no expansion data for an expansion ID") + } else { + &self.foreign_expn_data[&expn_id] + } } fn is_descendant_of(&self, mut expn_id: ExpnId, ancestor: ExpnId) -> bool { @@ -453,17 +572,21 @@ pub fn debug_hygiene_data(verbose: bool) -> String { } else { let mut s = String::from(""); s.push_str("Expansions:"); - data.expn_data.iter().enumerate().for_each(|(id, expn_info)| { - let expn_info = expn_info.as_ref().expect("no expansion data for an expansion ID"); + let mut debug_expn_data = |(id, expn_info): (&ExpnId, &ExpnData)| { s.push_str(&format!( - "\n{}: parent: {:?}, call_site_ctxt: {:?}, def_site_ctxt: {:?}, kind: {:?}", + "\n{:?}: parent: {:?}, call_site_ctxt: {:?}, def_site_ctxt: {:?}, kind: {:?}", id, expn_info.parent, expn_info.call_site.ctxt(), expn_info.def_site.ctxt(), expn_info.kind, - )); + )) + }; + data.local_expn_data.iter_enumerated().for_each(|(id, expn_info)| { + let expn_info = expn_info.as_ref().expect("no expansion data for an expansion ID"); + debug_expn_data((&id.to_expn_id(), expn_info)) }); + data.foreign_expn_data.iter().for_each(debug_expn_data); s.push_str("\n\nSyntaxContexts:"); data.syntax_context_data.iter().enumerate().for_each(|(id, ctxt)| { s.push_str(&format!( @@ -1024,7 +1147,7 @@ impl HygieneEncodeContext { &self, encoder: &mut T, mut encode_ctxt: impl FnMut(&mut T, u32, &SyntaxContextData) -> Result<(), R>, - mut encode_expn: impl FnMut(&mut T, u32, ExpnData, ExpnHash) -> Result<(), R>, + mut encode_expn: impl FnMut(&mut T, ExpnId, ExpnData, ExpnHash) -> Result<(), R>, ) -> Result<(), R> { // When we serialize a `SyntaxContextData`, we may end up serializing // a `SyntaxContext` that we haven't seen before @@ -1051,9 +1174,9 @@ impl HygieneEncodeContext { let latest_expns = { std::mem::take(&mut *self.latest_expns.lock()) }; - for_all_expns_in(latest_expns.into_iter(), |index, expn, data, hash| { + for_all_expns_in(latest_expns.into_iter(), |expn, data, hash| { if self.serialized_expns.lock().insert(expn) { - encode_expn(encoder, index, data, hash)?; + encode_expn(encoder, expn, data, hash)?; } Ok(()) })?; @@ -1073,56 +1196,75 @@ pub struct HygieneDecodeContext { // `SyntaxContext` remapped_ctxts: Lock>>, // The same as `remapepd_ctxts`, but for `ExpnId`s - remapped_expns: Lock>>, + remapped_expns: Lock>>, } pub fn decode_expn_id_incrcomp( d: &mut D, context: &HygieneDecodeContext, decode_data: impl FnOnce(&mut D, u32) -> Result<(ExpnData, ExpnHash), D::Error>, + decode_foreign: impl FnOnce(&mut D, ExpnId) -> Result<(ExpnData, ExpnHash), D::Error>, ) -> Result { + let krate = CrateNum::decode(d)?; let index = u32::decode(d)?; // Do this after decoding, so that we decode a `CrateNum` // if necessary - if index == ExpnId::root().as_u32() { + if index == 0 { debug!("decode_expn_id: deserialized root"); return Ok(ExpnId::root()); } + if krate != LOCAL_CRATE { + let expn_id = ExpnId { krate, local_id: ExpnIndex::from_u32(index) }; + if HygieneData::with(|hygiene_data| hygiene_data.foreign_expn_data.contains_key(&expn_id)) { + return Ok(expn_id); + } + let (expn_data, hash) = decode_foreign(d, expn_id)?; + debug_assert_eq!(krate, expn_data.krate); + debug_assert_eq!(expn_data.orig_id, Some(index)); + let expn_id = HygieneData::with(|hygiene_data| { + debug_assert_eq!(expn_data.orig_id, Some(index)); + let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, expn_data); + debug_assert!(_old_data.is_none()); + let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash); + debug_assert!(_old_hash.is_none()); + let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); + debug_assert!(_old_id.is_none()); + expn_id + }); + return Ok(expn_id); + } + let outer_expns = &context.remapped_expns; // Ensure that the lock() temporary is dropped early { if let Some(expn_id) = outer_expns.lock().get(index as usize).copied().flatten() { - return Ok(expn_id); + return Ok(expn_id.to_expn_id()); } } // Don't decode the data inside `HygieneData::with`, since we need to recursively decode // other ExpnIds let (mut expn_data, hash) = decode_data(d, index)?; + debug_assert_eq!(krate, expn_data.krate); let expn_id = HygieneData::with(|hygiene_data| { - if let Some(&expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) { - return expn_id; + if let Some(expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) { + return *expn_id; } - let expn_id = ExpnId(hygiene_data.expn_data.len() as u32); - // If we just deserialized an `ExpnData` owned by // the local crate, its `orig_id` will be stale, // so we need to update it to its own value. // This only happens when we deserialize the incremental cache, // since a crate will never decode its own metadata. - if expn_data.krate == LOCAL_CRATE { - expn_data.orig_id = Some(expn_id.0); - } - - hygiene_data.expn_data.push(Some(expn_data)); - hygiene_data.expn_hashes.push(hash); - let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); - debug_assert!(_old_id.is_none()); + let expn_id = hygiene_data.local_expn_data.next_index(); + expn_data.orig_id = Some(expn_id.as_u32()); + hygiene_data.local_expn_data.push(Some(expn_data)); + let _eid = hygiene_data.local_expn_hashes.push(hash); + debug_assert_eq!(expn_id, _eid); let mut expns = outer_expns.lock(); let new_len = index as usize + 1; @@ -1131,6 +1273,10 @@ pub fn decode_expn_id_incrcomp( } expns[index as usize] = Some(expn_id); drop(expns); + let expn_id = expn_id.to_expn_id(); + + let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); + debug_assert!(_old_id.is_none()); expn_id }); Ok(expn_id) @@ -1138,39 +1284,42 @@ pub fn decode_expn_id_incrcomp( pub fn decode_expn_id( d: &mut D, - decode_data: impl FnOnce(CrateNum, u32) -> (ExpnData, ExpnHash), + decode_data: impl FnOnce(CrateNum, ExpnIndex) -> (ExpnData, ExpnHash), ) -> Result { - let index = u32::decode(d)?; let krate = CrateNum::decode(d)?; + let index = u32::decode(d)?; // Do this after decoding, so that we decode a `CrateNum` // if necessary - if index == ExpnId::root().as_u32() { + if index == 0 { debug!("decode_expn_id: deserialized root"); return Ok(ExpnId::root()); } + let index = ExpnIndex::from_u32(index); + // This function is used to decode metadata, so it cannot decode information about LOCAL_CRATE. debug_assert_ne!(krate, LOCAL_CRATE); + let expn_id = ExpnId { krate, local_id: index }; + + // Fast path if the expansion has already been decoded. + if HygieneData::with(|hygiene_data| hygiene_data.foreign_expn_data.contains_key(&expn_id)) { + return Ok(expn_id); + } // Don't decode the data inside `HygieneData::with`, since we need to recursively decode // other ExpnIds let (expn_data, hash) = decode_data(krate, index); debug_assert_eq!(krate, expn_data.krate); - debug_assert_eq!(expn_data.orig_id, Some(index)); - - let expn_id = HygieneData::with(|hygiene_data| { - if let Some(&expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) { - return expn_id; - } + debug_assert_eq!(Some(index.as_u32()), expn_data.orig_id); - let expn_id = ExpnId(hygiene_data.expn_data.len() as u32); - hygiene_data.expn_data.push(Some(expn_data)); - hygiene_data.expn_hashes.push(hash); + HygieneData::with(|hygiene_data| { + let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, expn_data); + debug_assert!(_old_data.is_none()); + let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash); + debug_assert!(_old_hash.is_none()); let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); debug_assert!(_old_id.is_none()); - - expn_id }); Ok(expn_id) @@ -1264,29 +1413,37 @@ fn for_all_ctxts_in Resul fn for_all_expns_in( expns: impl Iterator, - mut f: impl FnMut(u32, ExpnId, ExpnData, ExpnHash) -> Result<(), E>, + mut f: impl FnMut(ExpnId, ExpnData, ExpnHash) -> Result<(), E>, ) -> Result<(), E> { let all_data: Vec<_> = HygieneData::with(|data| { expns - .map(|expn| { - let idx = expn.0 as usize; - (expn, data.expn_data[idx].clone(), data.expn_hashes[idx].clone()) - }) + .map(|expn| (expn, data.expn_data(expn).clone(), data.expn_hash(expn).clone())) .collect() }); for (expn, data, hash) in all_data.into_iter() { - let data = data.unwrap_or_else(|| panic!("Missing data for {:?}", expn)); - f(expn.0, expn, data, hash)?; + f(expn, data, hash)?; } Ok(()) } +impl Encodable for LocalExpnId { + fn encode(&self, e: &mut E) -> Result<(), E::Error> { + self.to_expn_id().encode(e) + } +} + impl Encodable for ExpnId { default fn encode(&self, _: &mut E) -> Result<(), E::Error> { panic!("cannot encode `ExpnId` with `{}`", std::any::type_name::()); } } +impl Decodable for LocalExpnId { + fn decode(d: &mut D) -> Result { + ExpnId::decode(d).map(ExpnId::expect_local) + } +} + impl Decodable for ExpnId { default fn decode(_: &mut D) -> Result { panic!("cannot decode `ExpnId` with `{}`", std::any::type_name::()); @@ -1309,12 +1466,12 @@ pub fn raw_encode_expn_id_incrcomp( context: &HygieneEncodeContext, e: &mut E, ) -> Result<(), E::Error> { - // Record the fact that we need to serialize the corresponding - // `ExpnData` + // Record the fact that we need to serialize the corresponding `ExpnData` if !context.serialized_expns.lock().contains(&expn) { context.latest_expns.lock().insert(expn); } - expn.0.encode(e) + expn.krate.encode(e)?; + expn.local_id.as_u32().encode(e) } pub fn raw_encode_expn_id( @@ -1322,21 +1479,19 @@ pub fn raw_encode_expn_id( context: &HygieneEncodeContext, e: &mut E, ) -> Result<(), E::Error> { - let data = expn.expn_data(); // We only need to serialize the ExpnData // if it comes from this crate. // We currently don't serialize any hygiene information data for // proc-macro crates: see the `SpecializedEncoder` impl // for crate metadata. - if data.krate == LOCAL_CRATE { - // Record the fact that we need to serialize the corresponding - // `ExpnData` + // Record the fact that we need to serialize the corresponding `ExpnData` + if expn.krate == LOCAL_CRATE { if !context.serialized_expns.lock().contains(&expn) { context.latest_expns.lock().insert(expn); } } - data.orig_id.expect("Missing orig_id").encode(e)?; - data.krate.encode(e) + expn.krate.encode(e)?; + expn.local_id.as_u32().encode(e) } impl Encodable for SyntaxContext { @@ -1360,7 +1515,7 @@ impl Decodable for SyntaxContext { /// `set_expn_data`). It is *not* called for foreign `ExpnId`s deserialized /// from another crate's metadata - since `ExpnData` includes a `krate` field, /// collisions are only possible between `ExpnId`s within the same crate. -fn update_disambiguator(expn_id: ExpnId, mut ctx: impl HashStableContext) { +fn update_disambiguator(expn_id: LocalExpnId, mut ctx: impl HashStableContext) { let mut expn_data = expn_id.expn_data(); // This disambiguator should not have been set yet. assert_eq!( @@ -1399,10 +1554,10 @@ fn update_disambiguator(expn_id: ExpnId, mut ctx: impl HashStableContext) { let expn_hash = ExpnHash(expn_hash); HygieneData::with(|data| { - data.expn_data[expn_id.0 as usize].as_mut().unwrap().disambiguator = disambiguator; - debug_assert_eq!(data.expn_hashes[expn_id.0 as usize].0, Fingerprint::ZERO); - data.expn_hashes[expn_id.0 as usize] = expn_hash; - let _old_id = data.expn_hash_to_expn_id.insert(expn_hash, expn_id); + data.local_expn_data[expn_id].as_mut().unwrap().disambiguator = disambiguator; + debug_assert_eq!(data.local_expn_hashes[expn_id].0, Fingerprint::ZERO); + data.local_expn_hashes[expn_id] = expn_hash; + let _old_id = data.expn_hash_to_expn_id.insert(expn_hash, expn_id.to_expn_id()); debug_assert!(_old_id.is_none()); }); } diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 7a1ee20ee7951..1c95cc91208d3 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -38,7 +38,7 @@ use edition::Edition; pub mod hygiene; use hygiene::Transparency; pub use hygiene::{DesugaringKind, ExpnKind, ForLoopLoc, MacroKind}; -pub use hygiene::{ExpnData, ExpnHash, ExpnId, SyntaxContext}; +pub use hygiene::{ExpnData, ExpnHash, ExpnId, LocalExpnId, SyntaxContext}; pub mod def_id; use def_id::{CrateNum, DefId, DefPathHash, LOCAL_CRATE}; pub mod lev_distance; From 078dd37f882a59d4277d80b829d729bfd2b5a5c2 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 25 Jun 2021 20:43:04 +0200 Subject: [PATCH 04/12] Use LocalExpnId where possible. --- compiler/rustc_ast/src/node_id.rs | 10 ++-- .../src/deriving/clone.rs | 5 +- .../src/deriving/generic/mod.rs | 2 +- .../src/proc_macro_harness.rs | 2 +- .../src/standard_library_imports.rs | 4 +- .../rustc_builtin_macros/src/test_harness.rs | 5 +- compiler/rustc_expand/src/base.rs | 40 +++++++++------- compiler/rustc_expand/src/expand.rs | 6 +-- compiler/rustc_expand/src/mbe/transcribe.rs | 6 +-- .../rustc_resolve/src/build_reduced_graph.rs | 37 +++++++++------ compiler/rustc_resolve/src/def_collector.rs | 10 ++-- compiler/rustc_resolve/src/imports.rs | 15 +++--- compiler/rustc_resolve/src/lib.rs | 46 ++++++++++--------- compiler/rustc_resolve/src/macros.rs | 35 ++++++++------ compiler/rustc_span/src/hygiene.rs | 15 +----- src/test/ui/hygiene/unpretty-debug.stdout | 8 ++-- .../ui/proc-macro/meta-macro-hygiene.stdout | 34 +++++++------- .../nonterminal-token-hygiene.stdout | 32 ++++++------- src/tools/rustfmt/src/utils.rs | 4 +- 19 files changed, 169 insertions(+), 147 deletions(-) diff --git a/compiler/rustc_ast/src/node_id.rs b/compiler/rustc_ast/src/node_id.rs index e1e7d757d7e62..7f928cb576180 100644 --- a/compiler/rustc_ast/src/node_id.rs +++ b/compiler/rustc_ast/src/node_id.rs @@ -1,4 +1,4 @@ -use rustc_span::{ExpnId, LocalExpnId}; +use rustc_span::LocalExpnId; use std::fmt; rustc_index::newtype_index! { @@ -24,12 +24,12 @@ pub const CRATE_NODE_ID: NodeId = NodeId::from_u32(0); pub const DUMMY_NODE_ID: NodeId = NodeId::MAX; impl NodeId { - pub fn placeholder_from_expn_id(expn_id: ExpnId) -> Self { - NodeId::from_u32(expn_id.expect_local().as_u32()) + pub fn placeholder_from_expn_id(expn_id: LocalExpnId) -> Self { + NodeId::from_u32(expn_id.as_u32()) } - pub fn placeholder_to_expn_id(self) -> ExpnId { - LocalExpnId::from_u32(self.as_u32()).to_expn_id() + pub fn placeholder_to_expn_id(self) -> LocalExpnId { + LocalExpnId::from_u32(self.as_u32()) } } diff --git a/compiler/rustc_builtin_macros/src/deriving/clone.rs b/compiler/rustc_builtin_macros/src/deriving/clone.rs index ca1226b445d97..90cdd62144d72 100644 --- a/compiler/rustc_builtin_macros/src/deriving/clone.rs +++ b/compiler/rustc_builtin_macros/src/deriving/clone.rs @@ -36,8 +36,9 @@ pub fn expand_deriving_clone( Annotatable::Item(ref annitem) => match annitem.kind { ItemKind::Struct(_, Generics { ref params, .. }) | ItemKind::Enum(_, Generics { ref params, .. }) => { - let container_id = cx.current_expansion.id.expn_data().parent; - if cx.resolver.has_derive_copy(container_id) + let container_id = cx.current_expansion.id.expn_data().parent.expect_local(); + let has_derive_copy = cx.resolver.has_derive_copy(container_id); + if has_derive_copy && !params .iter() .any(|param| matches!(param.kind, ast::GenericParamKind::Type { .. })) diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs index a3decff3ae7e1..417dedab60d08 100644 --- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs +++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs @@ -410,7 +410,7 @@ impl<'a> TraitDef<'a> { .any(|param| matches!(param.kind, ast::GenericParamKind::Type { .. })), _ => unreachable!(), }; - let container_id = cx.current_expansion.id.expn_data().parent; + let container_id = cx.current_expansion.id.expn_data().parent.expect_local(); let always_copy = has_no_type_params && cx.resolver.has_derive_copy(container_id); let use_temporaries = is_packed && always_copy; diff --git a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs index a8c61d53346de..f83329ecba824 100644 --- a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs +++ b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs @@ -304,7 +304,7 @@ fn mk_decls( &[sym::rustc_attrs, sym::proc_macro_internals], None, ); - let span = DUMMY_SP.with_def_site_ctxt(expn_id); + let span = DUMMY_SP.with_def_site_ctxt(expn_id.to_expn_id()); let proc_macro = Ident::new(sym::proc_macro, span); let krate = cx.item(span, proc_macro, Vec::new(), ast::ItemKind::ExternCrate(None)); diff --git a/compiler/rustc_builtin_macros/src/standard_library_imports.rs b/compiler/rustc_builtin_macros/src/standard_library_imports.rs index fbd8be22a9de2..e0d57267525d9 100644 --- a/compiler/rustc_builtin_macros/src/standard_library_imports.rs +++ b/compiler/rustc_builtin_macros/src/standard_library_imports.rs @@ -34,8 +34,8 @@ pub fn inject( &[sym::prelude_import], None, ); - let span = DUMMY_SP.with_def_site_ctxt(expn_id); - let call_site = DUMMY_SP.with_call_site_ctxt(expn_id); + let span = DUMMY_SP.with_def_site_ctxt(expn_id.to_expn_id()); + let call_site = DUMMY_SP.with_call_site_ctxt(expn_id.to_expn_id()); let ecfg = ExpansionConfig::default("std_lib_injection".to_string()); let cx = ExtCtxt::new(sess, ecfg, resolver, None); diff --git a/compiler/rustc_builtin_macros/src/test_harness.rs b/compiler/rustc_builtin_macros/src/test_harness.rs index c8a7ff67b4d50..74a97a4058fac 100644 --- a/compiler/rustc_builtin_macros/src/test_harness.rs +++ b/compiler/rustc_builtin_macros/src/test_harness.rs @@ -126,7 +126,8 @@ impl<'a> MutVisitor for TestHarnessGenerator<'a> { for test in &mut tests { // See the comment on `mk_main` for why we're using // `apply_mark` directly. - test.ident.span = test.ident.span.apply_mark(expn_id, Transparency::Opaque); + test.ident.span = + test.ident.span.apply_mark(expn_id.to_expn_id(), Transparency::Opaque); } self.cx.test_cases.extend(tests); } @@ -223,7 +224,7 @@ fn generate_test_harness( &[sym::test, sym::rustc_attrs], None, ); - let def_site = DUMMY_SP.with_def_site_ctxt(expn_id); + let def_site = DUMMY_SP.with_def_site_ctxt(expn_id.to_expn_id()); // Remove the entry points let mut cleaner = EntryPointCleaner { sess, depth: 0, def_site }; diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index 0183add495777..497be2d931872 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -16,7 +16,7 @@ use rustc_parse::{self, nt_to_tokenstream, parser, MACRO_ARGUMENTS}; use rustc_session::{parse::ParseSess, Limit, Session}; use rustc_span::def_id::{CrateNum, DefId}; use rustc_span::edition::Edition; -use rustc_span::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind}; +use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId}; use rustc_span::source_map::SourceMap; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{FileName, MultiSpan, Span, DUMMY_SP}; @@ -813,7 +813,7 @@ impl SyntaxExtension { pub fn expn_data( &self, - parent: ExpnId, + parent: LocalExpnId, call_site: Span, descr: Symbol, macro_def_id: Option, @@ -821,7 +821,7 @@ impl SyntaxExtension { ) -> ExpnData { ExpnData::new( ExpnKind::Macro(self.macro_kind(), descr), - parent, + parent.to_expn_id(), call_site, self.span, self.allow_internal_unstable.clone(), @@ -843,7 +843,11 @@ pub trait ResolverExpand { fn next_node_id(&mut self) -> NodeId; fn resolve_dollar_crates(&mut self); - fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment); + fn visit_ast_fragment_with_placeholders( + &mut self, + expn_id: LocalExpnId, + fragment: &AstFragment, + ); fn register_builtin_macro(&mut self, name: Symbol, ext: SyntaxExtensionKind); fn expansion_for_ast_pass( @@ -852,37 +856,41 @@ pub trait ResolverExpand { pass: AstPass, features: &[Symbol], parent_module_id: Option, - ) -> ExpnId; + ) -> LocalExpnId; fn resolve_imports(&mut self); fn resolve_macro_invocation( &mut self, invoc: &Invocation, - eager_expansion_root: ExpnId, + eager_expansion_root: LocalExpnId, force: bool, ) -> Result, Indeterminate>; fn check_unused_macros(&mut self); /// Some parent node that is close enough to the given macro call. - fn lint_node_id(&self, expn_id: ExpnId) -> NodeId; + fn lint_node_id(&self, expn_id: LocalExpnId) -> NodeId; // Resolver interfaces for specific built-in macros. /// Does `#[derive(...)]` attribute with the given `ExpnId` have built-in `Copy` inside it? - fn has_derive_copy(&self, expn_id: ExpnId) -> bool; + fn has_derive_copy(&self, expn_id: LocalExpnId) -> bool; /// Resolve paths inside the `#[derive(...)]` attribute with the given `ExpnId`. fn resolve_derives( &mut self, - expn_id: ExpnId, + expn_id: LocalExpnId, force: bool, derive_paths: &dyn Fn() -> DeriveResolutions, ) -> Result<(), Indeterminate>; /// Take resolutions for paths inside the `#[derive(...)]` attribute with the given `ExpnId` /// back from resolver. - fn take_derive_resolutions(&mut self, expn_id: ExpnId) -> Option; + fn take_derive_resolutions(&mut self, expn_id: LocalExpnId) -> Option; /// Path resolution logic for `#[cfg_accessible(path)]`. - fn cfg_accessible(&mut self, expn_id: ExpnId, path: &ast::Path) -> Result; + fn cfg_accessible( + &mut self, + expn_id: LocalExpnId, + path: &ast::Path, + ) -> Result; /// Decodes the proc-macro quoted span in the specified crate, with the specified id. /// No caching is performed. @@ -913,7 +921,7 @@ impl ModuleData { #[derive(Clone)] pub struct ExpansionData { - pub id: ExpnId, + pub id: LocalExpnId, pub depth: usize, pub module: Rc, pub dir_ownership: DirOwnership, @@ -958,7 +966,7 @@ impl<'a> ExtCtxt<'a> { extern_mod_loaded, root_path: PathBuf::new(), current_expansion: ExpansionData { - id: ExpnId::root(), + id: LocalExpnId::ROOT, depth: 0, module: Default::default(), dir_ownership: DirOwnership::Owned { relative: None }, @@ -995,19 +1003,19 @@ impl<'a> ExtCtxt<'a> { /// Equivalent of `Span::def_site` from the proc macro API, /// except that the location is taken from the span passed as an argument. pub fn with_def_site_ctxt(&self, span: Span) -> Span { - span.with_def_site_ctxt(self.current_expansion.id) + span.with_def_site_ctxt(self.current_expansion.id.to_expn_id()) } /// Equivalent of `Span::call_site` from the proc macro API, /// except that the location is taken from the span passed as an argument. pub fn with_call_site_ctxt(&self, span: Span) -> Span { - span.with_call_site_ctxt(self.current_expansion.id) + span.with_call_site_ctxt(self.current_expansion.id.to_expn_id()) } /// Equivalent of `Span::mixed_site` from the proc macro API, /// except that the location is taken from the span passed as an argument. pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { - span.with_mixed_site_ctxt(self.current_expansion.id) + span.with_mixed_site_ctxt(self.current_expansion.id.to_expn_id()) } /// Returns span for the macro which originally caused the current expansion to happen. diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 1ab7e15019eff..b9d4096241142 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -31,7 +31,7 @@ use rustc_session::lint::BuiltinLintDiagnostics; use rustc_session::parse::{feature_err, ParseSess}; use rustc_session::Limit; use rustc_span::symbol::{sym, Ident}; -use rustc_span::{ExpnId, FileName, Span}; +use rustc_span::{FileName, LocalExpnId, Span}; use smallvec::{smallvec, SmallVec}; use std::ops::DerefMut; @@ -508,7 +508,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { .map(|(path, item, _exts)| { // FIXME: Consider using the derive resolutions (`_exts`) // instead of enqueuing the derives to be resolved again later. - let expn_id = ExpnId::fresh_empty(); + let expn_id = LocalExpnId::fresh_empty(); derive_invocations.push(( Invocation { kind: InvocationKind::Derive { path, item }, @@ -993,7 +993,7 @@ struct InvocationCollector<'a, 'b> { impl<'a, 'b> InvocationCollector<'a, 'b> { fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { - let expn_id = ExpnId::fresh_empty(); + let expn_id = LocalExpnId::fresh_empty(); let vis = kind.placeholder_visibility(); self.invocations.push(( Invocation { diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index f9e7c4254bc49..9ed5c8b8ffba5 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -8,7 +8,7 @@ use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::{pluralize, PResult}; -use rustc_span::hygiene::{ExpnId, Transparency}; +use rustc_span::hygiene::{LocalExpnId, Transparency}; use rustc_span::symbol::MacroRulesNormalizedIdent; use rustc_span::Span; @@ -16,7 +16,7 @@ use smallvec::{smallvec, SmallVec}; use std::mem; // A Marker adds the given mark to the syntax context. -struct Marker(ExpnId, Transparency); +struct Marker(LocalExpnId, Transparency); impl MutVisitor for Marker { fn token_visiting_enabled(&self) -> bool { @@ -24,7 +24,7 @@ impl MutVisitor for Marker { } fn visit_span(&mut self, span: &mut Span) { - *span = span.apply_mark(self.0, self.1) + *span = span.apply_mark(self.0.to_expn_id(), self.1) } } diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs index 35e7688fbe45a..178d727418d74 100644 --- a/compiler/rustc_resolve/src/build_reduced_graph.rs +++ b/compiler/rustc_resolve/src/build_reduced_graph.rs @@ -31,7 +31,7 @@ use rustc_middle::bug; use rustc_middle::hir::exports::Export; use rustc_middle::middle::cstore::CrateStore; use rustc_middle::ty; -use rustc_span::hygiene::{ExpnId, MacroKind}; +use rustc_span::hygiene::{ExpnId, LocalExpnId, MacroKind}; use rustc_span::source_map::{respan, Spanned}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::Span; @@ -42,7 +42,7 @@ use tracing::debug; type Res = def::Res; -impl<'a> ToNameBinding<'a> for (Module<'a>, ty::Visibility, Span, ExpnId) { +impl<'a> ToNameBinding<'a> for (Module<'a>, ty::Visibility, Span, LocalExpnId) { fn to_name_binding(self, arenas: &'a ResolverArenas<'a>) -> &'a NameBinding<'a> { arenas.alloc_name_binding(NameBinding { kind: NameBindingKind::Module(self.0), @@ -54,7 +54,7 @@ impl<'a> ToNameBinding<'a> for (Module<'a>, ty::Visibility, Span, ExpnId) { } } -impl<'a> ToNameBinding<'a> for (Res, ty::Visibility, Span, ExpnId) { +impl<'a> ToNameBinding<'a> for (Res, ty::Visibility, Span, LocalExpnId) { fn to_name_binding(self, arenas: &'a ResolverArenas<'a>) -> &'a NameBinding<'a> { arenas.alloc_name_binding(NameBinding { kind: NameBindingKind::Res(self.0, false), @@ -68,7 +68,7 @@ impl<'a> ToNameBinding<'a> for (Res, ty::Visibility, Span, ExpnId) { struct IsMacroExport; -impl<'a> ToNameBinding<'a> for (Res, ty::Visibility, Span, ExpnId, IsMacroExport) { +impl<'a> ToNameBinding<'a> for (Res, ty::Visibility, Span, LocalExpnId, IsMacroExport) { fn to_name_binding(self, arenas: &'a ResolverArenas<'a>) -> &'a NameBinding<'a> { arenas.alloc_name_binding(NameBinding { kind: NameBindingKind::Res(self.0, true), @@ -157,7 +157,12 @@ impl<'a> Resolver<'a> { crate fn macro_def_scope(&mut self, expn_id: ExpnId) -> Module<'a> { let def_id = match expn_id.expn_data().macro_def_id { Some(def_id) => def_id, - None => return self.ast_transform_scopes.get(&expn_id).unwrap_or(&self.graph_root), + None => { + return expn_id + .as_local() + .and_then(|expn_id| self.ast_transform_scopes.get(&expn_id)) + .unwrap_or(&self.graph_root); + } }; self.macro_def_scope_from_def_id(def_id) } @@ -739,7 +744,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { if ptr::eq(parent, self.r.graph_root) { if let Some(entry) = self.r.extern_prelude.get(&ident.normalize_to_macros_2_0()) { - if expansion != ExpnId::root() + if expansion != LocalExpnId::ROOT && orig_name.is_some() && entry.extern_crate_item.is_none() { @@ -769,7 +774,13 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { no_implicit_prelude: parent.no_implicit_prelude || { self.r.session.contains_name(&item.attrs, sym::no_implicit_prelude) }, - ..ModuleData::new(Some(parent), module_kind, def_id, expansion, item.span) + ..ModuleData::new( + Some(parent), + module_kind, + def_id, + expansion.to_expn_id(), + item.span, + ) }); self.r.define(parent, ident, TypeNS, (module, vis, sp, expansion)); self.r.module_map.insert(local_def_id, module); @@ -808,7 +819,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { parent, module_kind, parent.nearest_parent_mod, - expansion, + expansion.to_expn_id(), item.span, ); self.r.define(parent, ident, TypeNS, (module, vis, sp, expansion)); @@ -883,7 +894,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { parent, module_kind, parent.nearest_parent_mod, - expansion, + expansion.to_expn_id(), item.span, ); self.r.define(parent, ident, TypeNS, (module, vis, sp, expansion)); @@ -926,7 +937,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { parent, ModuleKind::Block(block.id), parent.nearest_parent_mod, - expansion, + expansion.to_expn_id(), block.span, ); self.r.block_map.insert(block.id, module); @@ -946,7 +957,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { parent, ModuleKind::Def(kind, def_id, ident.name), def_id, - expansion, + expansion.to_expn_id(), span, ); self.r.define(parent, ident, TypeNS, (module, vis, span, expansion)); @@ -1112,7 +1123,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { }) }; - let allow_shadowing = self.parent_scope.expansion == ExpnId::root(); + let allow_shadowing = self.parent_scope.expansion == LocalExpnId::ROOT; if let Some(span) = import_all { let import = macro_use_import(self, span); self.r.potentially_unused_imports.push(import); @@ -1175,7 +1186,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { false } - fn visit_invoc(&mut self, id: NodeId) -> ExpnId { + fn visit_invoc(&mut self, id: NodeId) -> LocalExpnId { let invoc_id = id.placeholder_to_expn_id(); let old_parent_scope = self.r.invocation_parent_scopes.insert(invoc_id, self.parent_scope); assert!(old_parent_scope.is_none(), "invocation data is reset for an invocation"); diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs index 17f0c39e39735..6f4f1bdaea1b7 100644 --- a/compiler/rustc_resolve/src/def_collector.rs +++ b/compiler/rustc_resolve/src/def_collector.rs @@ -6,7 +6,7 @@ use rustc_ast_lowering::ResolverAstLowering; use rustc_expand::expand::AstFragment; use rustc_hir::def_id::LocalDefId; use rustc_hir::definitions::*; -use rustc_span::hygiene::ExpnId; +use rustc_span::hygiene::LocalExpnId; use rustc_span::symbol::{kw, sym}; use rustc_span::Span; use tracing::debug; @@ -14,7 +14,7 @@ use tracing::debug; crate fn collect_definitions( resolver: &mut Resolver<'_>, fragment: &AstFragment, - expansion: ExpnId, + expansion: LocalExpnId, ) { let (parent_def, impl_trait_context) = resolver.invocation_parents[&expansion]; fragment.visit_with(&mut DefCollector { resolver, parent_def, expansion, impl_trait_context }); @@ -25,14 +25,14 @@ struct DefCollector<'a, 'b> { resolver: &'a mut Resolver<'b>, parent_def: LocalDefId, impl_trait_context: ImplTraitContext, - expansion: ExpnId, + expansion: LocalExpnId, } impl<'a, 'b> DefCollector<'a, 'b> { fn create_def(&mut self, node_id: NodeId, data: DefPathData, span: Span) -> LocalDefId { let parent_def = self.parent_def; debug!("create_def(node_id={:?}, data={:?}, parent_def={:?})", node_id, data, parent_def); - self.resolver.create_def(parent_def, node_id, data, self.expansion, span) + self.resolver.create_def(parent_def, node_id, data, self.expansion.to_expn_id(), span) } fn with_parent(&mut self, parent_def: LocalDefId, f: F) { @@ -285,7 +285,7 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> { item_def, node_id, DefPathData::ImplTrait, - self.expansion, + self.expansion.to_expn_id(), ty.span, ), ImplTraitContext::Existential => { diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs index 26858915f45a0..acfa389fed58a 100644 --- a/compiler/rustc_resolve/src/imports.rs +++ b/compiler/rustc_resolve/src/imports.rs @@ -22,7 +22,7 @@ use rustc_middle::span_bug; use rustc_middle::ty; use rustc_session::lint::builtin::{PUB_USE_OF_PRIVATE_EXTERN_CRATE, UNUSED_IMPORTS}; use rustc_session::lint::BuiltinLintDiagnostics; -use rustc_span::hygiene::ExpnId; +use rustc_span::hygiene::LocalExpnId; use rustc_span::lev_distance::find_best_match_for_name; use rustc_span::symbol::{kw, Ident, Symbol}; use rustc_span::{MultiSpan, Span}; @@ -237,8 +237,9 @@ impl<'a> Resolver<'a> { if ns == TypeNS { if ident.name == kw::Crate || ident.name == kw::DollarCrate { let module = self.resolve_crate_root(ident); - let binding = (module, ty::Visibility::Public, module.span, ExpnId::root()) - .to_name_binding(self.arenas); + let binding = + (module, ty::Visibility::Public, module.span, LocalExpnId::ROOT) + .to_name_binding(self.arenas); return Ok(binding); } else if ident.name == kw::Super || ident.name == kw::SelfLower { // FIXME: Implement these with renaming requirements so that e.g. @@ -265,7 +266,7 @@ impl<'a> Resolver<'a> { self.resolution(module, key).try_borrow_mut().map_err(|_| (Determined, Weak::No))?; // This happens when there is a cycle of imports. if let Some(binding) = resolution.binding { - if !restricted_shadowing && binding.expansion != ExpnId::root() { + if !restricted_shadowing && binding.expansion != LocalExpnId::ROOT { if let NameBindingKind::Res(_, true) = binding.kind { self.macro_expanded_macro_export_errors.insert((path_span, binding.span)); } @@ -307,7 +308,7 @@ impl<'a> Resolver<'a> { if let Some(shadowed_glob) = resolution.shadowed_glob { // Forbid expanded shadowing to avoid time travel. if restricted_shadowing - && binding.expansion != ExpnId::root() + && binding.expansion != LocalExpnId::ROOT && binding.res() != shadowed_glob.res() { self.ambiguity_errors.push(AmbiguityError { @@ -521,7 +522,7 @@ impl<'a> Resolver<'a> { if old_glob { (old_binding, binding) } else { (binding, old_binding) }; if glob_binding.res() != nonglob_binding.res() && key.ns == MacroNS - && nonglob_binding.expansion != ExpnId::root() + && nonglob_binding.expansion != LocalExpnId::ROOT { resolution.binding = Some(this.ambiguity( AmbiguityKind::GlobVsExpanded, @@ -1271,7 +1272,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> { target: Ident, ) { // Skip if the import was produced by a macro. - if import.parent_scope.expansion != ExpnId::root() { + if import.parent_scope.expansion != LocalExpnId::ROOT { return; } diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs index fb2eb749e118f..7114fd33188d9 100644 --- a/compiler/rustc_resolve/src/lib.rs +++ b/compiler/rustc_resolve/src/lib.rs @@ -53,7 +53,7 @@ use rustc_session::lint; use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer}; use rustc_session::Session; use rustc_span::edition::Edition; -use rustc_span::hygiene::{ExpnId, ExpnKind, MacroKind, SyntaxContext, Transparency}; +use rustc_span::hygiene::{ExpnId, ExpnKind, LocalExpnId, MacroKind, SyntaxContext, Transparency}; use rustc_span::source_map::{CachingSourceMapView, Spanned}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{Span, DUMMY_SP}; @@ -103,7 +103,7 @@ impl Determinacy { /// but not for late resolution yet. #[derive(Clone, Copy)] enum Scope<'a> { - DeriveHelpers(ExpnId), + DeriveHelpers(LocalExpnId), DeriveHelpersCompat, MacroRules(MacroRulesScopeRef<'a>), CrateRoot, @@ -143,7 +143,7 @@ enum ScopeSet<'a> { #[derive(Clone, Copy, Debug)] pub struct ParentScope<'a> { module: Module<'a>, - expansion: ExpnId, + expansion: LocalExpnId, macro_rules: MacroRulesScopeRef<'a>, derives: &'a [ast::Path], } @@ -154,7 +154,7 @@ impl<'a> ParentScope<'a> { pub fn module(module: Module<'a>, resolver: &Resolver<'a>) -> ParentScope<'a> { ParentScope { module, - expansion: ExpnId::root(), + expansion: LocalExpnId::ROOT, macro_rules: resolver.arenas.alloc_macro_rules_scope(MacroRulesScope::Empty), derives: &[], } @@ -515,7 +515,7 @@ pub struct ModuleData<'a> { populate_on_access: Cell, /// Macro invocations that can expand into items in this module. - unexpanded_invocations: RefCell>, + unexpanded_invocations: RefCell>, /// Whether `#[no_implicit_prelude]` is active. no_implicit_prelude: bool, @@ -645,7 +645,7 @@ impl<'a> fmt::Debug for ModuleData<'a> { pub struct NameBinding<'a> { kind: NameBindingKind<'a>, ambiguity: Option<(&'a NameBinding<'a>, AmbiguityKind)>, - expansion: ExpnId, + expansion: LocalExpnId, span: Span, vis: ty::Visibility, } @@ -829,7 +829,11 @@ impl<'a> NameBinding<'a> { // in some later round and screw up our previously found resolution. // See more detailed explanation in // https://github.com/rust-lang/rust/pull/53778#issuecomment-419224049 - fn may_appear_after(&self, invoc_parent_expansion: ExpnId, binding: &NameBinding<'_>) -> bool { + fn may_appear_after( + &self, + invoc_parent_expansion: LocalExpnId, + binding: &NameBinding<'_>, + ) -> bool { // self > max(invoc, binding) => !(self <= invoc || self <= binding) // Expansions are partially ordered, so "may appear after" is an inversion of // "certainly appears before or simultaneously" and includes unordered cases. @@ -966,7 +970,7 @@ pub struct Resolver<'a> { dummy_ext_derive: Lrc, non_macro_attrs: [Lrc; 2], local_macro_def_scopes: FxHashMap>, - ast_transform_scopes: FxHashMap>, + ast_transform_scopes: FxHashMap>, unused_macros: FxHashMap, proc_macro_stubs: FxHashSet, /// Traces collected during macro resolution and validated when it's complete. @@ -978,18 +982,18 @@ pub struct Resolver<'a> { /// `derive(Copy)` marks items they are applied to so they are treated specially later. /// Derive macros cannot modify the item themselves and have to store the markers in the global /// context, so they attach the markers to derive container IDs using this resolver table. - containers_deriving_copy: FxHashSet, + containers_deriving_copy: FxHashSet, /// Parent scopes in which the macros were invoked. /// FIXME: `derives` are missing in these parent scopes and need to be taken from elsewhere. - invocation_parent_scopes: FxHashMap>, + invocation_parent_scopes: FxHashMap>, /// `macro_rules` scopes *produced* by expanding the macro invocations, /// include all the `macro_rules` items and other invocations generated by them. - output_macro_rules_scopes: FxHashMap>, + output_macro_rules_scopes: FxHashMap>, /// Helper attributes that are in scope for the given expansion. - helper_attrs: FxHashMap>, + helper_attrs: FxHashMap>, /// Ready or in-progress results of resolving paths inside the `#[derive(...)]` attribute /// with the given `ExpnId`. - derive_data: FxHashMap, + derive_data: FxHashMap, /// Avoid duplicated errors for "name already defined". name_already_seen: FxHashMap, @@ -1018,7 +1022,7 @@ pub struct Resolver<'a> { /// When collecting definitions from an AST fragment produced by a macro invocation `ExpnId` /// we know what parent node that fragment should be attached to thanks to this table, /// and how the `impl Trait` fragments were introduced. - invocation_parents: FxHashMap, + invocation_parents: FxHashMap, next_disambiguator: FxHashMap<(LocalDefId, DefPathData), u32>, /// Some way to know that we are in a *trait* impl in `visit_assoc_item`. @@ -1268,7 +1272,7 @@ impl<'a> Resolver<'a> { node_id_to_def_id.insert(CRATE_NODE_ID, root); let mut invocation_parents = FxHashMap::default(); - invocation_parents.insert(ExpnId::root(), (root, ImplTraitContext::Existential)); + invocation_parents.insert(LocalExpnId::ROOT, (root, ImplTraitContext::Existential)); let mut extern_prelude: FxHashMap> = session .opts @@ -1342,7 +1346,7 @@ impl<'a> Resolver<'a> { dummy_binding: arenas.alloc_name_binding(NameBinding { kind: NameBindingKind::Res(Res::Err, false), ambiguity: None, - expansion: ExpnId::root(), + expansion: LocalExpnId::ROOT, span: DUMMY_SP, vis: ty::Visibility::Public, }), @@ -1392,7 +1396,7 @@ impl<'a> Resolver<'a> { }; let root_parent_scope = ParentScope::module(graph_root, &resolver); - resolver.invocation_parent_scopes.insert(ExpnId::root(), root_parent_scope); + resolver.invocation_parent_scopes.insert(LocalExpnId::ROOT, root_parent_scope); resolver } @@ -1810,7 +1814,8 @@ impl<'a> Resolver<'a> { } scope = match scope { - Scope::DeriveHelpers(expn_id) if expn_id != ExpnId::root() => { + Scope::DeriveHelpers(LocalExpnId::ROOT) => Scope::DeriveHelpersCompat, + Scope::DeriveHelpers(expn_id) => { // Derive helpers are not visible to code generated by bang or derive macros. let expn_data = expn_id.expn_data(); match expn_data.kind { @@ -1818,10 +1823,9 @@ impl<'a> Resolver<'a> { | ExpnKind::Macro(MacroKind::Bang | MacroKind::Derive, _) => { Scope::DeriveHelpersCompat } - _ => Scope::DeriveHelpers(expn_data.parent), + _ => Scope::DeriveHelpers(expn_data.parent.expect_local()), } } - Scope::DeriveHelpers(..) => Scope::DeriveHelpersCompat, Scope::DeriveHelpersCompat => Scope::MacroRules(parent_scope.macro_rules), Scope::MacroRules(macro_rules_scope) => match macro_rules_scope.get() { MacroRulesScope::Binding(binding) => { @@ -3248,7 +3252,7 @@ impl<'a> Resolver<'a> { }; let crate_root = self.get_module(DefId { krate: crate_id, index: CRATE_DEF_INDEX }); Some( - (crate_root, ty::Visibility::Public, DUMMY_SP, ExpnId::root()) + (crate_root, ty::Visibility::Public, DUMMY_SP, LocalExpnId::ROOT) .to_name_binding(self.arenas), ) } diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs index 8686704388fee..86f271fdeceb8 100644 --- a/compiler/rustc_resolve/src/macros.rs +++ b/compiler/rustc_resolve/src/macros.rs @@ -30,7 +30,7 @@ use rustc_session::lint::BuiltinLintDiagnostics; use rustc_session::parse::feature_err; use rustc_session::Session; use rustc_span::edition::Edition; -use rustc_span::hygiene::{self, ExpnData, ExpnId, ExpnKind}; +use rustc_span::hygiene::{self, ExpnData, ExpnKind, LocalExpnId}; use rustc_span::hygiene::{AstPass, MacroKind}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{Span, DUMMY_SP}; @@ -62,7 +62,7 @@ pub enum MacroRulesScope<'a> { Binding(&'a MacroRulesBinding<'a>), /// The scope introduced by a macro invocation that can potentially /// create a `macro_rules!` macro definition. - Invocation(ExpnId), + Invocation(LocalExpnId), } /// `macro_rules!` scopes are always kept by reference and inside a cell. @@ -190,7 +190,11 @@ impl<'a> ResolverExpand for Resolver<'a> { }); } - fn visit_ast_fragment_with_placeholders(&mut self, expansion: ExpnId, fragment: &AstFragment) { + fn visit_ast_fragment_with_placeholders( + &mut self, + expansion: LocalExpnId, + fragment: &AstFragment, + ) { // Integrate the new AST fragment into all the definition and module structures. // We are inside the `expansion` now, but other parent scope components are still the same. let parent_scope = ParentScope { expansion, ..self.invocation_parent_scopes[&expansion] }; @@ -216,9 +220,9 @@ impl<'a> ResolverExpand for Resolver<'a> { pass: AstPass, features: &[Symbol], parent_module_id: Option, - ) -> ExpnId { + ) -> LocalExpnId { let parent_module = parent_module_id.map(|module_id| self.local_def_id(module_id)); - let expn_id = ExpnId::fresh( + let expn_id = LocalExpnId::fresh( ExpnData::allow_unstable( ExpnKind::AstPass(pass), call_site, @@ -244,7 +248,7 @@ impl<'a> ResolverExpand for Resolver<'a> { fn resolve_macro_invocation( &mut self, invoc: &Invocation, - eager_expansion_root: ExpnId, + eager_expansion_root: LocalExpnId, force: bool, ) -> Result, Indeterminate> { let invoc_id = invoc.expansion_data.id; @@ -328,7 +332,7 @@ impl<'a> ResolverExpand for Resolver<'a> { | ExpnKind::Macro(MacroKind::Bang | MacroKind::Derive, _) => { break; } - _ => expn_id = expn_data.parent, + _ => expn_id = expn_data.parent.expect_local(), } } } @@ -344,7 +348,7 @@ impl<'a> ResolverExpand for Resolver<'a> { } } - fn lint_node_id(&self, expn_id: ExpnId) -> NodeId { + fn lint_node_id(&self, expn_id: LocalExpnId) -> NodeId { // FIXME - make this more precise. This currently returns the NodeId of the // nearest closing item - we should try to return the closest parent of the ExpnId self.invocation_parents @@ -352,13 +356,13 @@ impl<'a> ResolverExpand for Resolver<'a> { .map_or(ast::CRATE_NODE_ID, |id| self.def_id_to_node_id[id.0]) } - fn has_derive_copy(&self, expn_id: ExpnId) -> bool { + fn has_derive_copy(&self, expn_id: LocalExpnId) -> bool { self.containers_deriving_copy.contains(&expn_id) } fn resolve_derives( &mut self, - expn_id: ExpnId, + expn_id: LocalExpnId, force: bool, derive_paths: &dyn Fn() -> DeriveResolutions, ) -> Result<(), Indeterminate> { @@ -423,7 +427,7 @@ impl<'a> ResolverExpand for Resolver<'a> { Ok(()) } - fn take_derive_resolutions(&mut self, expn_id: ExpnId) -> Option { + fn take_derive_resolutions(&mut self, expn_id: LocalExpnId) -> Option { self.derive_data.remove(&expn_id).map(|data| data.resolutions) } @@ -431,7 +435,11 @@ impl<'a> ResolverExpand for Resolver<'a> { // Returns true if the path can certainly be resolved in one of three namespaces, // returns false if the path certainly cannot be resolved in any of the three namespaces. // Returns `Indeterminate` if we cannot give a certain answer yet. - fn cfg_accessible(&mut self, expn_id: ExpnId, path: &ast::Path) -> Result { + fn cfg_accessible( + &mut self, + expn_id: LocalExpnId, + path: &ast::Path, + ) -> Result { let span = path.span; let path = &Segment::from_path(path); let parent_scope = self.invocation_parent_scopes[&expn_id]; @@ -714,7 +722,8 @@ impl<'a> Resolver<'a> { let ident = Ident::new(orig_ident.name, orig_ident.span.with_ctxt(ctxt)); let ok = |res, span, arenas| { Ok(( - (res, ty::Visibility::Public, span, ExpnId::root()).to_name_binding(arenas), + (res, ty::Visibility::Public, span, LocalExpnId::ROOT) + .to_name_binding(arenas), Flags::empty(), )) }; diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index 14b6811298261..138ec061423bd 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -198,19 +198,6 @@ impl ExpnId { ExpnId { krate: LOCAL_CRATE, local_id: ExpnIndex::from_u32(0) } } - pub fn fresh_empty() -> ExpnId { - LocalExpnId::fresh_empty().to_expn_id() - } - - pub fn fresh(expn_data: ExpnData, ctx: impl HashStableContext) -> ExpnId { - LocalExpnId::fresh(expn_data, ctx).to_expn_id() - } - - #[inline] - pub fn set_expn_data(self, expn_data: ExpnData, ctx: impl HashStableContext) { - self.expect_local().set_expn_data(expn_data, ctx) - } - #[inline] pub fn expn_hash(self) -> ExpnHash { HygieneData::with(|data| data.expn_hash(self)) @@ -819,7 +806,7 @@ impl Span { transparency: Transparency, ctx: impl HashStableContext, ) -> Span { - let expn_id = ExpnId::fresh(expn_data, ctx); + let expn_id = LocalExpnId::fresh(expn_data, ctx).to_expn_id(); HygieneData::with(|data| { self.with_ctxt(data.apply_mark(SyntaxContext::root(), expn_id, transparency)) }) diff --git a/src/test/ui/hygiene/unpretty-debug.stdout b/src/test/ui/hygiene/unpretty-debug.stdout index 84ca046212dc5..ffb9f9eed41fb 100644 --- a/src/test/ui/hygiene/unpretty-debug.stdout +++ b/src/test/ui/hygiene/unpretty-debug.stdout @@ -19,10 +19,10 @@ fn y /* 0#0 */() { } /* Expansions: -0: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Root -1: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "foo") +crate0::{{expn0}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Root +crate0::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "foo") SyntaxContexts: -#0: parent: #0, outer_mark: (ExpnId(0), Opaque) -#1: parent: #0, outer_mark: (ExpnId(1), SemiTransparent) +#0: parent: #0, outer_mark: (crate0::{{expn0}}, Opaque) +#1: parent: #0, outer_mark: (crate0::{{expn1}}, SemiTransparent) */ diff --git a/src/test/ui/proc-macro/meta-macro-hygiene.stdout b/src/test/ui/proc-macro/meta-macro-hygiene.stdout index dc63d014451db..2524d8273b705 100644 --- a/src/test/ui/proc-macro/meta-macro-hygiene.stdout +++ b/src/test/ui/proc-macro/meta-macro-hygiene.stdout @@ -43,23 +43,23 @@ fn main /* 0#0 */() { ; } /* Expansions: -0: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Root -1: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) -2: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "produce_it") -3: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) -4: parent: ExpnId(2), call_site_ctxt: #4, def_site_ctxt: #0, kind: Macro(Bang, "meta_macro::print_def_site") -5: parent: ExpnId(4), call_site_ctxt: #5, def_site_ctxt: #0, kind: Macro(Bang, "$crate::dummy") +crate0::{{expn0}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Root +crate0::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) +crate0::{{expn2}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "produce_it") +crate0::{{expn3}}: parent: crate0::{{expn2}}, call_site_ctxt: #4, def_site_ctxt: #0, kind: Macro(Bang, "meta_macro::print_def_site") +crate0::{{expn4}}: parent: crate0::{{expn3}}, call_site_ctxt: #5, def_site_ctxt: #0, kind: Macro(Bang, "$crate::dummy") +crate2::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) SyntaxContexts: -#0: parent: #0, outer_mark: (ExpnId(0), Opaque) -#1: parent: #0, outer_mark: (ExpnId(1), Opaque) -#2: parent: #0, outer_mark: (ExpnId(1), Transparent) -#3: parent: #0, outer_mark: (ExpnId(3), Opaque) -#4: parent: #0, outer_mark: (ExpnId(2), SemiTransparent) -#5: parent: #0, outer_mark: (ExpnId(4), Opaque) -#6: parent: #4, outer_mark: (ExpnId(4), Transparent) -#7: parent: #0, outer_mark: (ExpnId(4), SemiTransparent) -#8: parent: #0, outer_mark: (ExpnId(5), Opaque) -#9: parent: #5, outer_mark: (ExpnId(5), Transparent) -#10: parent: #5, outer_mark: (ExpnId(5), SemiTransparent) +#0: parent: #0, outer_mark: (crate0::{{expn0}}, Opaque) +#1: parent: #0, outer_mark: (crate0::{{expn1}}, Opaque) +#2: parent: #0, outer_mark: (crate0::{{expn1}}, Transparent) +#3: parent: #0, outer_mark: (crate2::{{expn1}}, Opaque) +#4: parent: #0, outer_mark: (crate0::{{expn2}}, SemiTransparent) +#5: parent: #0, outer_mark: (crate0::{{expn3}}, Opaque) +#6: parent: #4, outer_mark: (crate0::{{expn3}}, Transparent) +#7: parent: #0, outer_mark: (crate0::{{expn3}}, SemiTransparent) +#8: parent: #0, outer_mark: (crate0::{{expn4}}, Opaque) +#9: parent: #5, outer_mark: (crate0::{{expn4}}, Transparent) +#10: parent: #5, outer_mark: (crate0::{{expn4}}, SemiTransparent) */ diff --git a/src/test/ui/proc-macro/nonterminal-token-hygiene.stdout b/src/test/ui/proc-macro/nonterminal-token-hygiene.stdout index 75e6a49b314df..b5ab82737e9b6 100644 --- a/src/test/ui/proc-macro/nonterminal-token-hygiene.stdout +++ b/src/test/ui/proc-macro/nonterminal-token-hygiene.stdout @@ -67,22 +67,22 @@ fn main /* 0#0 */() { } /* Expansions: -0: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Root -1: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) -2: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "outer") -3: parent: ExpnId(0), call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) -4: parent: ExpnId(2), call_site_ctxt: #4, def_site_ctxt: #4, kind: Macro(Bang, "inner") -5: parent: ExpnId(4), call_site_ctxt: #6, def_site_ctxt: #0, kind: Macro(Bang, "print_bang") +crate0::{{expn0}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Root +crate0::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) +crate0::{{expn2}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "outer") +crate0::{{expn3}}: parent: crate0::{{expn2}}, call_site_ctxt: #4, def_site_ctxt: #4, kind: Macro(Bang, "inner") +crate0::{{expn4}}: parent: crate0::{{expn3}}, call_site_ctxt: #6, def_site_ctxt: #0, kind: Macro(Bang, "print_bang") +crate2::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports) SyntaxContexts: -#0: parent: #0, outer_mark: (ExpnId(0), Opaque) -#1: parent: #0, outer_mark: (ExpnId(1), Opaque) -#2: parent: #0, outer_mark: (ExpnId(1), Transparent) -#3: parent: #0, outer_mark: (ExpnId(3), Opaque) -#4: parent: #0, outer_mark: (ExpnId(2), SemiTransparent) -#5: parent: #0, outer_mark: (ExpnId(4), Opaque) -#6: parent: #4, outer_mark: (ExpnId(4), Opaque) -#7: parent: #0, outer_mark: (ExpnId(5), Opaque) -#8: parent: #6, outer_mark: (ExpnId(5), Transparent) -#9: parent: #5, outer_mark: (ExpnId(5), SemiTransparent) +#0: parent: #0, outer_mark: (crate0::{{expn0}}, Opaque) +#1: parent: #0, outer_mark: (crate0::{{expn1}}, Opaque) +#2: parent: #0, outer_mark: (crate0::{{expn1}}, Transparent) +#3: parent: #0, outer_mark: (crate2::{{expn1}}, Opaque) +#4: parent: #0, outer_mark: (crate0::{{expn2}}, SemiTransparent) +#5: parent: #0, outer_mark: (crate0::{{expn3}}, Opaque) +#6: parent: #4, outer_mark: (crate0::{{expn3}}, Opaque) +#7: parent: #0, outer_mark: (crate0::{{expn4}}, Opaque) +#8: parent: #6, outer_mark: (crate0::{{expn4}}, Transparent) +#9: parent: #5, outer_mark: (crate0::{{expn4}}, SemiTransparent) */ diff --git a/src/tools/rustfmt/src/utils.rs b/src/tools/rustfmt/src/utils.rs index d3c349fb701e1..614cda5f911c2 100644 --- a/src/tools/rustfmt/src/utils.rs +++ b/src/tools/rustfmt/src/utils.rs @@ -6,7 +6,7 @@ use rustc_ast::ast::{ }; use rustc_ast::ptr; use rustc_ast_pretty::pprust; -use rustc_span::{sym, symbol, BytePos, ExpnId, Span, Symbol, SyntaxContext}; +use rustc_span::{sym, symbol, BytePos, LocalExpnId, Span, Symbol, SyntaxContext}; use unicode_width::UnicodeWidthStr; use crate::comment::{filter_normal_code, CharClasses, FullCodeCharKind, LineClasses}; @@ -675,7 +675,7 @@ pub(crate) trait NodeIdExt { impl NodeIdExt for NodeId { fn root() -> NodeId { - NodeId::placeholder_from_expn_id(ExpnId::root()) + NodeId::placeholder_from_expn_id(LocalExpnId::ROOT) } } From 2fe37c5bd1cc7d6cfa176f408c2b834e1b2617bf Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 27 Jun 2021 10:38:06 +0200 Subject: [PATCH 05/12] Choose encoding format in caller code. --- compiler/rustc_metadata/src/rmeta/decoder.rs | 9 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 10 +- .../src/ty/query/on_disk_cache.rs | 23 +++-- compiler/rustc_span/src/hygiene.rs | 98 +++++-------------- 4 files changed, 55 insertions(+), 85 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 4b72ac8695794..8b8ca987b380d 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -377,7 +377,11 @@ impl<'a, 'tcx> Decodable> for ExpnId { let local_cdata = decoder.cdata(); let sess = decoder.sess.unwrap(); - rustc_span::hygiene::decode_expn_id(decoder, |cnum, index| { + let cnum = CrateNum::decode(decoder)?; + let index = u32::decode(decoder)?; + + let expn_id = rustc_span::hygiene::decode_expn_id(cnum, index, |expn_id| { + let ExpnId { krate: cnum, local_id: index } = expn_id; // Lookup local `ExpnData`s in our own crate data. Foreign `ExpnData`s // are stored in the owning crate, to avoid duplication. debug_assert_ne!(cnum, LOCAL_CRATE); @@ -399,7 +403,8 @@ impl<'a, 'tcx> Decodable> for ExpnId { .unwrap() .decode((&crate_data, sess)); (expn_data, expn_hash) - }) + }); + Ok(expn_id) } } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 4684daef4a18c..6877d6ef542c7 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -182,7 +182,15 @@ impl<'a, 'tcx> Encodable> for SyntaxContext { impl<'a, 'tcx> Encodable> for ExpnId { fn encode(&self, s: &mut EncodeContext<'a, 'tcx>) -> opaque::EncodeResult { - rustc_span::hygiene::raw_encode_expn_id(*self, &s.hygiene_ctxt, s) + if self.krate == LOCAL_CRATE { + // We will only write details for local expansions. Non-local expansions will fetch + // data from the corresponding crate's metadata. + // FIXME(#43047) FIXME(#74731) We may eventually want to avoid relying on external + // metadata from proc-macro crates. + s.hygiene_ctxt.schedule_expn_data_for_encoding(*self); + } + self.krate.encode(s)?; + self.local_id.encode(s) } } diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index 358d016368ee0..685cb3a61ecab 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -794,25 +794,26 @@ impl<'a, 'tcx> Decodable> for SyntaxContext { impl<'a, 'tcx> Decodable> for ExpnId { fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Result { + let krate = CrateNum::decode(decoder)?; + let index = u32::decode(decoder)?; + let expn_data = decoder.expn_data; + let tcx = decoder.tcx; rustc_span::hygiene::decode_expn_id_incrcomp( - decoder, + krate, + index, decoder.hygiene_context, - |this, index| { + |index| -> Result<(ExpnData, ExpnHash), _> { // This closure is invoked if we haven't already decoded the data for the `ExpnId` we are deserializing. // We look up the position of the associated `ExpnData` and decode it. let pos = expn_data .get(&index) .unwrap_or_else(|| panic!("Bad index {:?} (map {:?})", index, expn_data)); - this.with_position(pos.to_usize(), |decoder| { - let data: (ExpnData, ExpnHash) = decode_tagged(decoder, TAG_EXPN_DATA)?; - Ok(data) - }) - }, - |this, expn_id| { - Ok(this.tcx.untracked_resolutions.cstore.decode_expn_data(this.tcx.sess, expn_id)) + decoder + .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA)) }, + |expn_id| tcx.untracked_resolutions.cstore.decode_expn_data(tcx.sess, expn_id), ) } } @@ -988,7 +989,9 @@ where E: 'a + OpaqueEncoder, { fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> { - rustc_span::hygiene::raw_encode_expn_id_incrcomp(*self, s.hygiene_context, s) + s.hygiene_context.schedule_expn_data_for_encoding(*self); + self.krate.encode(s)?; + self.local_id.as_u32().encode(s) } } diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index 138ec061423bd..198bb5881422f 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -261,6 +261,8 @@ pub struct HygieneData { /// first and then resolved later), so we use an `Option` here. local_expn_data: IndexVec>, local_expn_hashes: IndexVec, + /// Data and hash information from external crates. We may eventually want to remove these + /// maps, and fetch the information directly from the other crate's metadata like DefIds do. foreign_expn_data: FxHashMap, foreign_expn_hashes: FxHashMap, expn_hash_to_expn_id: UnhashMap, @@ -1130,6 +1132,13 @@ pub struct HygieneEncodeContext { } impl HygieneEncodeContext { + /// Record the fact that we need to serialize the corresponding `ExpnData`. + pub fn schedule_expn_data_for_encoding(&self, expn: ExpnId) { + if !self.serialized_expns.lock().contains(&expn) { + self.latest_expns.lock().insert(expn); + } + } + pub fn encode( &self, encoder: &mut T, @@ -1186,15 +1195,13 @@ pub struct HygieneDecodeContext { remapped_expns: Lock>>, } -pub fn decode_expn_id_incrcomp( - d: &mut D, +pub fn decode_expn_id_incrcomp( + krate: CrateNum, + index: u32, context: &HygieneDecodeContext, - decode_data: impl FnOnce(&mut D, u32) -> Result<(ExpnData, ExpnHash), D::Error>, - decode_foreign: impl FnOnce(&mut D, ExpnId) -> Result<(ExpnData, ExpnHash), D::Error>, -) -> Result { - let krate = CrateNum::decode(d)?; - let index = u32::decode(d)?; - + decode_data: impl FnOnce(u32) -> Result<(ExpnData, ExpnHash), E>, + decode_foreign: impl FnOnce(ExpnId) -> (ExpnData, ExpnHash), +) -> Result { // Do this after decoding, so that we decode a `CrateNum` // if necessary if index == 0 { @@ -1203,23 +1210,7 @@ pub fn decode_expn_id_incrcomp( } if krate != LOCAL_CRATE { - let expn_id = ExpnId { krate, local_id: ExpnIndex::from_u32(index) }; - if HygieneData::with(|hygiene_data| hygiene_data.foreign_expn_data.contains_key(&expn_id)) { - return Ok(expn_id); - } - let (expn_data, hash) = decode_foreign(d, expn_id)?; - debug_assert_eq!(krate, expn_data.krate); - debug_assert_eq!(expn_data.orig_id, Some(index)); - let expn_id = HygieneData::with(|hygiene_data| { - debug_assert_eq!(expn_data.orig_id, Some(index)); - let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, expn_data); - debug_assert!(_old_data.is_none()); - let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash); - debug_assert!(_old_hash.is_none()); - let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); - debug_assert!(_old_id.is_none()); - expn_id - }); + let expn_id = decode_expn_id(krate, index, decode_foreign); return Ok(expn_id); } @@ -1234,7 +1225,7 @@ pub fn decode_expn_id_incrcomp( // Don't decode the data inside `HygieneData::with`, since we need to recursively decode // other ExpnIds - let (mut expn_data, hash) = decode_data(d, index)?; + let (mut expn_data, hash) = decode_data(index)?; debug_assert_eq!(krate, expn_data.krate); let expn_id = HygieneData::with(|hygiene_data| { @@ -1269,18 +1260,14 @@ pub fn decode_expn_id_incrcomp( Ok(expn_id) } -pub fn decode_expn_id( - d: &mut D, - decode_data: impl FnOnce(CrateNum, ExpnIndex) -> (ExpnData, ExpnHash), -) -> Result { - let krate = CrateNum::decode(d)?; - let index = u32::decode(d)?; - - // Do this after decoding, so that we decode a `CrateNum` - // if necessary +pub fn decode_expn_id( + krate: CrateNum, + index: u32, + decode_data: impl FnOnce(ExpnId) -> (ExpnData, ExpnHash), +) -> ExpnId { if index == 0 { debug!("decode_expn_id: deserialized root"); - return Ok(ExpnId::root()); + return ExpnId::root(); } let index = ExpnIndex::from_u32(index); @@ -1291,12 +1278,12 @@ pub fn decode_expn_id( // Fast path if the expansion has already been decoded. if HygieneData::with(|hygiene_data| hygiene_data.foreign_expn_data.contains_key(&expn_id)) { - return Ok(expn_id); + return expn_id; } // Don't decode the data inside `HygieneData::with`, since we need to recursively decode // other ExpnIds - let (expn_data, hash) = decode_data(krate, index); + let (expn_data, hash) = decode_data(expn_id); debug_assert_eq!(krate, expn_data.krate); debug_assert_eq!(Some(index.as_u32()), expn_data.orig_id); @@ -1309,7 +1296,7 @@ pub fn decode_expn_id( debug_assert!(_old_id.is_none()); }); - Ok(expn_id) + expn_id } // Decodes `SyntaxContext`, using the provided `HygieneDecodeContext` @@ -1448,39 +1435,6 @@ pub fn raw_encode_syntax_context( ctxt.0.encode(e) } -pub fn raw_encode_expn_id_incrcomp( - expn: ExpnId, - context: &HygieneEncodeContext, - e: &mut E, -) -> Result<(), E::Error> { - // Record the fact that we need to serialize the corresponding `ExpnData` - if !context.serialized_expns.lock().contains(&expn) { - context.latest_expns.lock().insert(expn); - } - expn.krate.encode(e)?; - expn.local_id.as_u32().encode(e) -} - -pub fn raw_encode_expn_id( - expn: ExpnId, - context: &HygieneEncodeContext, - e: &mut E, -) -> Result<(), E::Error> { - // We only need to serialize the ExpnData - // if it comes from this crate. - // We currently don't serialize any hygiene information data for - // proc-macro crates: see the `SpecializedEncoder` impl - // for crate metadata. - // Record the fact that we need to serialize the corresponding `ExpnData` - if expn.krate == LOCAL_CRATE { - if !context.serialized_expns.lock().contains(&expn) { - context.latest_expns.lock().insert(expn); - } - } - expn.krate.encode(e)?; - expn.local_id.as_u32().encode(e) -} - impl Encodable for SyntaxContext { default fn encode(&self, _: &mut E) -> Result<(), E::Error> { panic!("cannot encode `SyntaxContext` with `{}`", std::any::type_name::()); From 37a13def486e2133db264ec9b83735c4c4546e58 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 27 Jun 2021 15:51:25 +0200 Subject: [PATCH 06/12] Encode ExpnId using ExpnHash for incr. comp. --- compiler/rustc_metadata/src/rmeta/decoder.rs | 38 ++++++ .../src/rmeta/decoder/cstore_impl.rs | 23 +--- compiler/rustc_middle/src/middle/cstore.rs | 5 +- .../src/ty/query/on_disk_cache.rs | 93 +++++++++----- compiler/rustc_span/src/def_id.rs | 2 +- compiler/rustc_span/src/hygiene.rs | 121 ++++++++---------- 6 files changed, 161 insertions(+), 121 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 8b8ca987b380d..388a8bd22fb32 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -79,6 +79,8 @@ crate struct CrateMetadata { /// `DefIndex`. See `raw_def_id_to_def_id` for more details about how /// this is used. def_path_hash_map: OnceCell>, + /// Likewise for ExpnHash. + expn_hash_map: OnceCell>, /// Used for decoding interpret::AllocIds in a cached & thread-safe manner. alloc_decoding_state: AllocDecodingState, /// Caches decoded `DefKey`s. @@ -1619,6 +1621,41 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { self.def_path_hash_unlocked(index, &mut def_path_hashes) } + fn expn_hash_to_expn_id(&self, index_guess: u32, hash: ExpnHash) -> ExpnId { + debug_assert_eq!(ExpnId::from_hash(hash), None); + let index_guess = ExpnIndex::from_u32(index_guess); + let old_hash = self.root.expn_hashes.get(self, index_guess).map(|lazy| lazy.decode(self)); + + let index = if old_hash == Some(hash) { + // Fast path: the expn and its index is unchanged from the + // previous compilation session. There is no need to decode anything + // else. + index_guess + } else { + // Slow path: We need to find out the new `DefIndex` of the provided + // `DefPathHash`, if its still exists. This requires decoding every `DefPathHash` + // stored in this crate. + let map = self.cdata.expn_hash_map.get_or_init(|| { + let end_id = self.root.expn_hashes.size() as u32; + let mut map = + UnhashMap::with_capacity_and_hasher(end_id as usize, Default::default()); + for i in 0..end_id { + let i = ExpnIndex::from_u32(i); + if let Some(hash) = self.root.expn_hashes.get(self, i) { + map.insert(hash.decode(self), i); + } else { + panic!("Missing expn_hash entry for {:?}", i); + } + } + map + }); + map[&hash] + }; + + let data = self.root.expn_data.get(self, index).unwrap().decode(self); + rustc_span::hygiene::register_expn_id(data, hash) + } + /// Imports the source_map from an external crate into the source_map of the crate /// currently being compiled (the "local crate"). /// @@ -1857,6 +1894,7 @@ impl CrateMetadata { raw_proc_macros, source_map_import_info: OnceCell::new(), def_path_hash_map: Default::default(), + expn_hash_map: Default::default(), alloc_decoding_state, cnum, cnum_map, diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 67023e9e84e22..41839c58021ab 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -18,7 +18,7 @@ use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, Visibility}; use rustc_session::utils::NativeLibKind; use rustc_session::{Session, StableCrateId}; -use rustc_span::hygiene::{ExpnData, ExpnHash, ExpnId}; +use rustc_span::hygiene::{ExpnHash, ExpnId}; use rustc_span::source_map::{Span, Spanned}; use rustc_span::symbol::Symbol; @@ -494,23 +494,6 @@ impl CrateStore for CStore { fn as_any(&self) -> &dyn Any { self } - fn decode_expn_data(&self, sess: &Session, expn_id: ExpnId) -> (ExpnData, ExpnHash) { - let crate_data = self.get_crate_data(expn_id.krate); - ( - crate_data - .root - .expn_data - .get(&crate_data, expn_id.local_id) - .unwrap() - .decode((&crate_data, sess)), - crate_data - .root - .expn_hashes - .get(&crate_data, expn_id.local_id) - .unwrap() - .decode((&crate_data, sess)), - ) - } fn crate_name(&self, cnum: CrateNum) -> Symbol { self.get_crate_data(cnum).root.name @@ -545,6 +528,10 @@ impl CrateStore for CStore { self.get_crate_data(cnum).def_path_hash_to_def_id(cnum, index_guess, hash) } + fn expn_hash_to_expn_id(&self, cnum: CrateNum, index_guess: u32, hash: ExpnHash) -> ExpnId { + self.get_crate_data(cnum).expn_hash_to_expn_id(index_guess, hash) + } + fn encode_metadata(&self, tcx: TyCtxt<'_>) -> EncodedMetadata { encoder::encode_metadata(tcx) } diff --git a/compiler/rustc_middle/src/middle/cstore.rs b/compiler/rustc_middle/src/middle/cstore.rs index c10fcc2e90c7e..4c35b49bcefd7 100644 --- a/compiler/rustc_middle/src/middle/cstore.rs +++ b/compiler/rustc_middle/src/middle/cstore.rs @@ -11,8 +11,7 @@ use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_macros::HashStable; use rustc_session::search_paths::PathKind; use rustc_session::utils::NativeLibKind; -use rustc_session::Session; -use rustc_span::hygiene::{ExpnData, ExpnHash, ExpnId}; +use rustc_span::hygiene::{ExpnHash, ExpnId}; use rustc_span::symbol::Symbol; use rustc_span::Span; use rustc_target::spec::Target; @@ -188,7 +187,6 @@ pub type MetadataLoaderDyn = dyn MetadataLoader + Sync; /// during resolve) pub trait CrateStore: std::fmt::Debug { fn as_any(&self) -> &dyn Any; - fn decode_expn_data(&self, sess: &Session, expn_id: ExpnId) -> (ExpnData, ExpnHash); // Foreign definitions. // This information is safe to access, since it's hashed as part of the DefPathHash, which incr. @@ -209,6 +207,7 @@ pub trait CrateStore: std::fmt::Debug { index_guess: u32, hash: DefPathHash, ) -> Option; + fn expn_hash_to_expn_id(&self, cnum: CrateNum, index_guess: u32, hash: ExpnHash) -> ExpnId; // utility functions fn encode_metadata(&self, tcx: TyCtxt<'_>) -> EncodedMetadata; diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index 685cb3a61ecab..03e3ca5ec6b8b 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -83,7 +83,7 @@ pub struct OnDiskCache<'sess> { // `ExpnData` (e.g `ExpnData.krate` may not be `LOCAL_CRATE`). Alternatively, // we could look up the `ExpnData` from the metadata of foreign crates, // but it seemed easier to have `OnDiskCache` be independent of the `CStore`. - expn_data: FxHashMap, + expn_data: UnhashMap, // Additional information used when decoding hygiene data. hygiene_context: HygieneDecodeContext, // Maps `DefPathHash`es to their `RawDefId`s from the *previous* @@ -91,6 +91,8 @@ pub struct OnDiskCache<'sess> { // we try to map a `DefPathHash` to its `DefId` in the current compilation // session. foreign_def_path_hashes: UnhashMap, + // Likewise for ExpnId. + foreign_expn_data: UnhashMap, // The *next* compilation sessison's `foreign_def_path_hashes` - at // the end of our current compilation session, this will get written @@ -118,8 +120,9 @@ struct Footer { // See `OnDiskCache.syntax_contexts` syntax_contexts: FxHashMap, // See `OnDiskCache.expn_data` - expn_data: FxHashMap, + expn_data: UnhashMap, foreign_def_path_hashes: UnhashMap, + foreign_expn_data: UnhashMap, } pub type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; @@ -217,6 +220,7 @@ impl<'sess> OnDiskCache<'sess> { alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index), syntax_contexts: footer.syntax_contexts, expn_data: footer.expn_data, + foreign_expn_data: footer.foreign_expn_data, hygiene_context: Default::default(), foreign_def_path_hashes: footer.foreign_def_path_hashes, latest_foreign_def_path_hashes: Default::default(), @@ -236,7 +240,8 @@ impl<'sess> OnDiskCache<'sess> { prev_diagnostics_index: Default::default(), alloc_decoding_state: AllocDecodingState::new(Vec::new()), syntax_contexts: FxHashMap::default(), - expn_data: FxHashMap::default(), + expn_data: UnhashMap::default(), + foreign_expn_data: UnhashMap::default(), hygiene_context: Default::default(), foreign_def_path_hashes: Default::default(), latest_foreign_def_path_hashes: Default::default(), @@ -350,7 +355,8 @@ impl<'sess> OnDiskCache<'sess> { }; let mut syntax_contexts = FxHashMap::default(); - let mut expn_ids = FxHashMap::default(); + let mut expn_data = UnhashMap::default(); + let mut foreign_expn_data = UnhashMap::default(); // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current // session. @@ -363,13 +369,14 @@ impl<'sess> OnDiskCache<'sess> { syntax_contexts.insert(index, pos); Ok(()) }, - |encoder, index, expn_data, hash| -> FileEncodeResult { - if index.krate == LOCAL_CRATE { + |encoder, expn_id, data, hash| -> FileEncodeResult { + if expn_id.krate == LOCAL_CRATE { let pos = AbsoluteBytePos::new(encoder.position()); - encoder.encode_tagged(TAG_EXPN_DATA, &(expn_data, hash))?; - expn_ids.insert(index.local_id.as_u32(), pos); + encoder.encode_tagged(TAG_EXPN_DATA, &data)?; + expn_data.insert(hash, pos); + } else { + foreign_expn_data.insert(hash, expn_id.local_id.as_u32()); } - // TODO Handle foreign expansions. Ok(()) }, )?; @@ -387,7 +394,8 @@ impl<'sess> OnDiskCache<'sess> { diagnostics_index, interpret_alloc_index, syntax_contexts, - expn_data: expn_ids, + expn_data, + foreign_expn_data, foreign_def_path_hashes, }, )?; @@ -549,6 +557,7 @@ impl<'sess> OnDiskCache<'sess> { alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(), syntax_contexts: &self.syntax_contexts, expn_data: &self.expn_data, + foreign_expn_data: &self.foreign_expn_data, hygiene_context: &self.hygiene_context, }; f(&mut decoder) @@ -643,7 +652,8 @@ pub struct CacheDecoder<'a, 'tcx> { file_index_to_stable_id: &'a FxHashMap, alloc_decoding_session: AllocDecodingSession<'a>, syntax_contexts: &'a FxHashMap, - expn_data: &'a FxHashMap, + expn_data: &'a UnhashMap, + foreign_expn_data: &'a UnhashMap, hygiene_context: &'a HygieneDecodeContext, } @@ -794,27 +804,43 @@ impl<'a, 'tcx> Decodable> for SyntaxContext { impl<'a, 'tcx> Decodable> for ExpnId { fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Result { - let krate = CrateNum::decode(decoder)?; - let index = u32::decode(decoder)?; - - let expn_data = decoder.expn_data; - let tcx = decoder.tcx; - rustc_span::hygiene::decode_expn_id_incrcomp( - krate, - index, - decoder.hygiene_context, - |index| -> Result<(ExpnData, ExpnHash), _> { - // This closure is invoked if we haven't already decoded the data for the `ExpnId` we are deserializing. - // We look up the position of the associated `ExpnData` and decode it. - let pos = expn_data - .get(&index) - .unwrap_or_else(|| panic!("Bad index {:?} (map {:?})", index, expn_data)); - - decoder - .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA)) - }, - |expn_id| tcx.untracked_resolutions.cstore.decode_expn_data(tcx.sess, expn_id), - ) + let hash = ExpnHash::decode(decoder)?; + if hash.is_root() { + return Ok(ExpnId::root()); + } + + if let Some(expn_id) = ExpnId::from_hash(hash) { + return Ok(expn_id); + } + + let krate = decoder.cnum_map[&hash.stable_crate_id()]; + + let expn_id = if krate == LOCAL_CRATE { + // We look up the position of the associated `ExpnData` and decode it. + let pos = decoder + .expn_data + .get(&hash) + .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data)); + + let data: ExpnData = decoder + .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA))?; + rustc_span::hygiene::register_local_expn_id(data, hash) + } else { + let index_guess = decoder.foreign_expn_data[&hash]; + decoder.tcx.untracked_resolutions.cstore.expn_hash_to_expn_id(krate, index_guess, hash) + }; + + #[cfg(debug_assertions)] + { + use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; + let mut hcx = decoder.tcx.create_stable_hashing_context(); + let mut hasher = StableHasher::new(); + expn_id.expn_data().hash_stable(&mut hcx, &mut hasher); + let local_hash: u64 = hasher.finish(); + debug_assert_eq!(hash.local_hash(), local_hash); + } + + Ok(expn_id) } } @@ -990,8 +1016,7 @@ where { fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> { s.hygiene_context.schedule_expn_data_for_encoding(*self); - self.krate.encode(s)?; - self.local_id.as_u32().encode(s) + self.expn_hash().encode(s) } } diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs index c5d605ab57801..64baf94cc0063 100644 --- a/compiler/rustc_span/src/def_id.rs +++ b/compiler/rustc_span/src/def_id.rs @@ -136,7 +136,7 @@ impl Borrow for DefPathHash { /// further trouble. #[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Debug)] #[derive(HashStable_Generic, Encodable, Decodable)] -pub struct StableCrateId(u64); +pub struct StableCrateId(pub(crate) u64); impl StableCrateId { pub fn to_u64(self) -> u64 { diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index 198bb5881422f..a6d77852c06a9 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -29,7 +29,7 @@ use crate::symbol::{kw, sym, Symbol}; use crate::with_session_globals; use crate::{HashStableContext, Span, DUMMY_SP}; -use crate::def_id::{CrateNum, DefId, CRATE_DEF_ID, LOCAL_CRATE}; +use crate::def_id::{CrateNum, DefId, StableCrateId, CRATE_DEF_ID, LOCAL_CRATE}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -92,6 +92,34 @@ rustc_index::newtype_index! { #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encodable, Decodable, HashStable_Generic)] pub struct ExpnHash(Fingerprint); +impl ExpnHash { + /// Returns the [StableCrateId] identifying the crate this [ExpnHash] + /// originates from. + #[inline] + pub fn stable_crate_id(self) -> StableCrateId { + StableCrateId(self.0.as_value().0) + } + + /// Returns the crate-local part of the [ExpnHash]. + /// + /// Used for tests. + #[inline] + pub fn local_hash(self) -> u64 { + self.0.as_value().1 + } + + #[inline] + pub fn is_root(self) -> bool { + self.0 == Fingerprint::ZERO + } + + /// Builds a new [ExpnHash] with the given [StableCrateId] and + /// `local_hash`, where `local_hash` must be unique within its crate. + fn new(stable_crate_id: StableCrateId, local_hash: u64) -> ExpnHash { + ExpnHash(Fingerprint::new(stable_crate_id.0, local_hash)) + } +} + /// A property of a macro expansion that determines how identifiers /// produced by that expansion are resolved. #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug, Encodable, Decodable)] @@ -268,12 +296,12 @@ pub struct HygieneData { expn_hash_to_expn_id: UnhashMap, syntax_context_data: Vec, syntax_context_map: FxHashMap<(SyntaxContext, ExpnId, Transparency), SyntaxContext>, - /// Maps the `Fingerprint` of an `ExpnData` to the next disambiguator value. + /// Maps the `local_hash` of an `ExpnData` to the next disambiguator value. /// This is used by `update_disambiguator` to keep track of which `ExpnData`s /// would have collisions without a disambiguator. /// The keys of this map are always computed with `ExpnData.disambiguator` /// set to 0. - expn_data_disambiguators: FxHashMap, + expn_data_disambiguators: FxHashMap, } impl HygieneData { @@ -981,7 +1009,7 @@ impl ExpnData { } #[inline] - fn hash_expn(&self, ctx: &mut impl HashStableContext) -> Fingerprint { + fn hash_expn(&self, ctx: &mut impl HashStableContext) -> u64 { let mut hasher = StableHasher::new(); self.hash_stable(ctx, &mut hasher); hasher.finish() @@ -1191,75 +1219,46 @@ pub struct HygieneDecodeContext { // so that multiple occurrences of the same serialized id are decoded to the same // `SyntaxContext` remapped_ctxts: Lock>>, - // The same as `remapepd_ctxts`, but for `ExpnId`s - remapped_expns: Lock>>, } -pub fn decode_expn_id_incrcomp( - krate: CrateNum, - index: u32, - context: &HygieneDecodeContext, - decode_data: impl FnOnce(u32) -> Result<(ExpnData, ExpnHash), E>, - decode_foreign: impl FnOnce(ExpnId) -> (ExpnData, ExpnHash), -) -> Result { - // Do this after decoding, so that we decode a `CrateNum` - // if necessary - if index == 0 { - debug!("decode_expn_id: deserialized root"); - return Ok(ExpnId::root()); - } - - if krate != LOCAL_CRATE { - let expn_id = decode_expn_id(krate, index, decode_foreign); - return Ok(expn_id); - } - - let outer_expns = &context.remapped_expns; - - // Ensure that the lock() temporary is dropped early - { - if let Some(expn_id) = outer_expns.lock().get(index as usize).copied().flatten() { - return Ok(expn_id.to_expn_id()); - } - } - - // Don't decode the data inside `HygieneData::with`, since we need to recursively decode - // other ExpnIds - let (mut expn_data, hash) = decode_data(index)?; - debug_assert_eq!(krate, expn_data.krate); - - let expn_id = HygieneData::with(|hygiene_data| { - if let Some(expn_id) = hygiene_data.expn_hash_to_expn_id.get(&hash) { - return *expn_id; - } - +/// Register an expansion which has been decoded from the on-disk-cache for the local crate. +pub fn register_local_expn_id(mut data: ExpnData, hash: ExpnHash) -> ExpnId { + HygieneData::with(|hygiene_data| { // If we just deserialized an `ExpnData` owned by // the local crate, its `orig_id` will be stale, // so we need to update it to its own value. // This only happens when we deserialize the incremental cache, // since a crate will never decode its own metadata. let expn_id = hygiene_data.local_expn_data.next_index(); - expn_data.orig_id = Some(expn_id.as_u32()); - hygiene_data.local_expn_data.push(Some(expn_data)); + data.orig_id = Some(expn_id.as_u32()); + hygiene_data.local_expn_data.push(Some(data)); let _eid = hygiene_data.local_expn_hashes.push(hash); debug_assert_eq!(expn_id, _eid); - let mut expns = outer_expns.lock(); - let new_len = index as usize + 1; - if expns.len() < new_len { - expns.resize(new_len, None); - } - expns[index as usize] = Some(expn_id); - drop(expns); let expn_id = expn_id.to_expn_id(); let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); debug_assert!(_old_id.is_none()); expn_id + }) +} + +/// Register an expansion which has been decoded from the metadata of a foreign crate. +pub fn register_expn_id(data: ExpnData, hash: ExpnHash) -> ExpnId { + let expn_id = + ExpnId { krate: data.krate, local_id: ExpnIndex::from_u32(data.orig_id.unwrap()) }; + HygieneData::with(|hygiene_data| { + let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, data); + debug_assert!(_old_data.is_none()); + let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash); + debug_assert!(_old_hash.is_none()); + let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); + debug_assert!(_old_id.is_none()); }); - Ok(expn_id) + expn_id } +/// Decode an expansion from the metadata of a foreign crate. pub fn decode_expn_id( krate: CrateNum, index: u32, @@ -1287,16 +1286,7 @@ pub fn decode_expn_id( debug_assert_eq!(krate, expn_data.krate); debug_assert_eq!(Some(index.as_u32()), expn_data.orig_id); - HygieneData::with(|hygiene_data| { - let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, expn_data); - debug_assert!(_old_data.is_none()); - let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash); - debug_assert!(_old_hash.is_none()); - let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); - debug_assert!(_old_id.is_none()); - }); - - expn_id + register_expn_id(expn_data, hash) } // Decodes `SyntaxContext`, using the provided `HygieneDecodeContext` @@ -1493,7 +1483,8 @@ fn update_disambiguator(expn_id: LocalExpnId, mut ctx: impl HashStableContext) { }); } - let expn_hash = ExpnHash(expn_hash); + let expn_hash = + ExpnHash::new(ctx.def_path_hash(LOCAL_CRATE.as_def_id()).stable_crate_id(), expn_hash); HygieneData::with(|data| { data.local_expn_data[expn_id].as_mut().unwrap().disambiguator = disambiguator; debug_assert_eq!(data.local_expn_hashes[expn_id].0, Fingerprint::ZERO); From dbd2d77641e87ac841d12db1ca8e1f38d96f85d3 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 27 Jun 2021 19:14:29 +0200 Subject: [PATCH 07/12] Drop orig_id. --- compiler/rustc_metadata/src/rmeta/decoder.rs | 2 +- compiler/rustc_span/src/hygiene.rs | 42 +++++--------------- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 388a8bd22fb32..a3f8c7e771b85 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -1653,7 +1653,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { }; let data = self.root.expn_data.get(self, index).unwrap().decode(self); - rustc_span::hygiene::register_expn_id(data, hash) + rustc_span::hygiene::register_expn_id(index, data, hash) } /// Imports the source_map from an external crate into the source_map of the crate diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index a6d77852c06a9..32149f1550e30 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -179,14 +179,12 @@ impl LocalExpnId { } #[inline] - pub fn set_expn_data(self, mut expn_data: ExpnData, ctx: impl HashStableContext) { + pub fn set_expn_data(self, expn_data: ExpnData, ctx: impl HashStableContext) { debug_assert_eq!(expn_data.parent.krate, LOCAL_CRATE); HygieneData::with(|data| { let old_expn_data = &mut data.local_expn_data[self]; assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID"); - assert_eq!(expn_data.orig_id, None); debug_assert_eq!(expn_data.krate, LOCAL_CRATE); - expn_data.orig_id = Some(self.as_u32()); *old_expn_data = Some(expn_data); }); update_disambiguator(self, ctx) @@ -306,14 +304,13 @@ pub struct HygieneData { impl HygieneData { crate fn new(edition: Edition) -> Self { - let mut root_data = ExpnData::default( + let root_data = ExpnData::default( ExpnKind::Root, DUMMY_SP, edition, Some(CRATE_DEF_ID.to_def_id()), None, ); - root_data.orig_id = Some(0); HygieneData { local_expn_data: IndexVec::from_elem_n(Some(root_data), 1), @@ -339,13 +336,11 @@ impl HygieneData { with_session_globals(|session_globals| f(&mut *session_globals.hygiene_data.borrow_mut())) } - fn fresh_expn(&mut self, mut expn_data: Option) -> LocalExpnId { - let expn_id = self.local_expn_data.next_index(); - if let Some(data) = expn_data.as_mut() { + fn fresh_expn(&mut self, expn_data: Option) -> LocalExpnId { + if let Some(data) = &expn_data { debug_assert_eq!(data.krate, LOCAL_CRATE); - assert_eq!(data.orig_id, None); - data.orig_id = Some(expn_id.as_u32()); } + let expn_id = self.local_expn_data.next_index(); self.local_expn_data.push(expn_data); let _eid = self.local_expn_hashes.push(ExpnHash(Fingerprint::ZERO)); debug_assert_eq!(expn_id, _eid); @@ -884,14 +879,6 @@ pub struct ExpnData { /// foreign `ExpnId`s will have their `ExpnData` looked up /// from the crate specified by `Crate krate: CrateNum, - /// The raw that this `ExpnData` had in its original crate. - /// An `ExpnData` can be created before being assigned an `ExpnId`, - /// so this might be `None` until `set_expn_data` is called - // This is used only for serialization/deserialization purposes: - // two `ExpnData`s that differ only in their `orig_id` should - // be considered equivalent. - #[stable_hasher(ignore)] - orig_id: Option, /// Used to force two `ExpnData`s to have different `Fingerprint`s. /// Due to macro expansion, it's possible to end up with two `ExpnId`s /// that have identical `ExpnData`s. This violates the contract of `HashStable` @@ -930,7 +917,6 @@ pub struct ExpnData { pub parent_module: Option, } -// These would require special handling of `orig_id`. impl !PartialEq for ExpnData {} impl !Hash for ExpnData {} @@ -959,7 +945,6 @@ impl ExpnData { macro_def_id, parent_module, krate: LOCAL_CRATE, - orig_id: None, disambiguator: 0, } } @@ -984,7 +969,6 @@ impl ExpnData { macro_def_id, parent_module, krate: LOCAL_CRATE, - orig_id: None, disambiguator: 0, } } @@ -1222,15 +1206,9 @@ pub struct HygieneDecodeContext { } /// Register an expansion which has been decoded from the on-disk-cache for the local crate. -pub fn register_local_expn_id(mut data: ExpnData, hash: ExpnHash) -> ExpnId { +pub fn register_local_expn_id(data: ExpnData, hash: ExpnHash) -> ExpnId { HygieneData::with(|hygiene_data| { - // If we just deserialized an `ExpnData` owned by - // the local crate, its `orig_id` will be stale, - // so we need to update it to its own value. - // This only happens when we deserialize the incremental cache, - // since a crate will never decode its own metadata. let expn_id = hygiene_data.local_expn_data.next_index(); - data.orig_id = Some(expn_id.as_u32()); hygiene_data.local_expn_data.push(Some(data)); let _eid = hygiene_data.local_expn_hashes.push(hash); debug_assert_eq!(expn_id, _eid); @@ -1244,9 +1222,8 @@ pub fn register_local_expn_id(mut data: ExpnData, hash: ExpnHash) -> ExpnId { } /// Register an expansion which has been decoded from the metadata of a foreign crate. -pub fn register_expn_id(data: ExpnData, hash: ExpnHash) -> ExpnId { - let expn_id = - ExpnId { krate: data.krate, local_id: ExpnIndex::from_u32(data.orig_id.unwrap()) }; +pub fn register_expn_id(local_id: ExpnIndex, data: ExpnData, hash: ExpnHash) -> ExpnId { + let expn_id = ExpnId { krate: data.krate, local_id }; HygieneData::with(|hygiene_data| { let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, data); debug_assert!(_old_data.is_none()); @@ -1284,9 +1261,8 @@ pub fn decode_expn_id( // other ExpnIds let (expn_data, hash) = decode_data(expn_id); debug_assert_eq!(krate, expn_data.krate); - debug_assert_eq!(Some(index.as_u32()), expn_data.orig_id); - register_expn_id(expn_data, hash) + register_expn_id(index, expn_data, hash) } // Decodes `SyntaxContext`, using the provided `HygieneDecodeContext` From 41c1f39fa8317fc16779ceda7536fe93f1c89c34 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 27 Jun 2021 20:06:26 +0200 Subject: [PATCH 08/12] Drop ExpnData::krate. --- compiler/rustc_metadata/src/rmeta/decoder.rs | 2 +- compiler/rustc_span/src/hygiene.rs | 29 +++++++------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index a3f8c7e771b85..8bdd4313de4c3 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -1653,7 +1653,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { }; let data = self.root.expn_data.get(self, index).unwrap().decode(self); - rustc_span::hygiene::register_expn_id(index, data, hash) + rustc_span::hygiene::register_expn_id(self.cnum, index, data, hash) } /// Imports the source_map from an external crate into the source_map of the crate diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index 32149f1550e30..49ad40fdf4d5b 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -184,7 +184,6 @@ impl LocalExpnId { HygieneData::with(|data| { let old_expn_data = &mut data.local_expn_data[self]; assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID"); - debug_assert_eq!(expn_data.krate, LOCAL_CRATE); *old_expn_data = Some(expn_data); }); update_disambiguator(self, ctx) @@ -337,11 +336,7 @@ impl HygieneData { } fn fresh_expn(&mut self, expn_data: Option) -> LocalExpnId { - if let Some(data) = &expn_data { - debug_assert_eq!(data.krate, LOCAL_CRATE); - } - let expn_id = self.local_expn_data.next_index(); - self.local_expn_data.push(expn_data); + let expn_id = self.local_expn_data.push(expn_data); let _eid = self.local_expn_hashes.push(ExpnHash(Fingerprint::ZERO)); debug_assert_eq!(expn_id, _eid); expn_id @@ -873,12 +868,6 @@ pub struct ExpnData { /// call_site span would have its own ExpnData, with the call_site /// pointing to the `foo!` invocation. pub call_site: Span, - /// The crate that originally created this `ExpnData`. During - /// metadata serialization, we only encode `ExpnData`s that were - /// created locally - when our serialized metadata is decoded, - /// foreign `ExpnId`s will have their `ExpnData` looked up - /// from the crate specified by `Crate - krate: CrateNum, /// Used to force two `ExpnData`s to have different `Fingerprint`s. /// Due to macro expansion, it's possible to end up with two `ExpnId`s /// that have identical `ExpnData`s. This violates the contract of `HashStable` @@ -944,7 +933,6 @@ impl ExpnData { edition, macro_def_id, parent_module, - krate: LOCAL_CRATE, disambiguator: 0, } } @@ -968,7 +956,6 @@ impl ExpnData { edition, macro_def_id, parent_module, - krate: LOCAL_CRATE, disambiguator: 0, } } @@ -1222,8 +1209,13 @@ pub fn register_local_expn_id(data: ExpnData, hash: ExpnHash) -> ExpnId { } /// Register an expansion which has been decoded from the metadata of a foreign crate. -pub fn register_expn_id(local_id: ExpnIndex, data: ExpnData, hash: ExpnHash) -> ExpnId { - let expn_id = ExpnId { krate: data.krate, local_id }; +pub fn register_expn_id( + krate: CrateNum, + local_id: ExpnIndex, + data: ExpnData, + hash: ExpnHash, +) -> ExpnId { + let expn_id = ExpnId { krate, local_id }; HygieneData::with(|hygiene_data| { let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, data); debug_assert!(_old_data.is_none()); @@ -1260,9 +1252,8 @@ pub fn decode_expn_id( // Don't decode the data inside `HygieneData::with`, since we need to recursively decode // other ExpnIds let (expn_data, hash) = decode_data(expn_id); - debug_assert_eq!(krate, expn_data.krate); - register_expn_id(index, expn_data, hash) + register_expn_id(krate, index, expn_data, hash) } // Decodes `SyntaxContext`, using the provided `HygieneDecodeContext` @@ -1420,7 +1411,7 @@ impl Decodable for SyntaxContext { /// This method is called only when an `ExpnData` is first associated /// with an `ExpnId` (when the `ExpnId` is initially constructed, or via /// `set_expn_data`). It is *not* called for foreign `ExpnId`s deserialized -/// from another crate's metadata - since `ExpnData` includes a `krate` field, +/// from another crate's metadata - since `ExpnHash` includes the stable crate id, /// collisions are only possible between `ExpnId`s within the same crate. fn update_disambiguator(expn_id: LocalExpnId, mut ctx: impl HashStableContext) { let mut expn_data = expn_id.expn_data(); From a51b131fd186ffef240835e18eae3a73501f3090 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 27 Jun 2021 21:30:20 +0200 Subject: [PATCH 09/12] Always hash spans in expn. --- compiler/rustc_middle/src/ty/query/on_disk_cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index 03e3ca5ec6b8b..c4847ea16c562 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -835,7 +835,7 @@ impl<'a, 'tcx> Decodable> for ExpnId { use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; let mut hcx = decoder.tcx.create_stable_hashing_context(); let mut hasher = StableHasher::new(); - expn_id.expn_data().hash_stable(&mut hcx, &mut hasher); + hcx.while_hashing_spans(true, |hcx| expn_id.expn_data().hash_stable(hcx, &mut hasher)); let local_hash: u64 = hasher.finish(); debug_assert_eq!(hash.local_hash(), local_hash); } From 0f8573e57b62576e4acb8fa0d74346cc82645ee0 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 12 Jul 2021 16:56:13 +0200 Subject: [PATCH 10/12] Pass ExpnData by reference. --- compiler/rustc_middle/src/ty/query/on_disk_cache.rs | 2 +- compiler/rustc_span/src/hygiene.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index c4847ea16c562..0ac80ebd6367b 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -372,7 +372,7 @@ impl<'sess> OnDiskCache<'sess> { |encoder, expn_id, data, hash| -> FileEncodeResult { if expn_id.krate == LOCAL_CRATE { let pos = AbsoluteBytePos::new(encoder.position()); - encoder.encode_tagged(TAG_EXPN_DATA, &data)?; + encoder.encode_tagged(TAG_EXPN_DATA, data)?; expn_data.insert(hash, pos); } else { foreign_expn_data.insert(hash, expn_id.local_id.as_u32()); diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index 49ad40fdf4d5b..b4adf2d46bbcd 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -1142,7 +1142,7 @@ impl HygieneEncodeContext { &self, encoder: &mut T, mut encode_ctxt: impl FnMut(&mut T, u32, &SyntaxContextData) -> Result<(), R>, - mut encode_expn: impl FnMut(&mut T, ExpnId, ExpnData, ExpnHash) -> Result<(), R>, + mut encode_expn: impl FnMut(&mut T, ExpnId, &ExpnData, ExpnHash) -> Result<(), R>, ) -> Result<(), R> { // When we serialize a `SyntaxContextData`, we may end up serializing // a `SyntaxContext` that we haven't seen before @@ -1344,7 +1344,7 @@ fn for_all_ctxts_in Resul fn for_all_expns_in( expns: impl Iterator, - mut f: impl FnMut(ExpnId, ExpnData, ExpnHash) -> Result<(), E>, + mut f: impl FnMut(ExpnId, &ExpnData, ExpnHash) -> Result<(), E>, ) -> Result<(), E> { let all_data: Vec<_> = HygieneData::with(|data| { expns @@ -1352,7 +1352,7 @@ fn for_all_expns_in( .collect() }); for (expn, data, hash) in all_data.into_iter() { - f(expn, data, hash)?; + f(expn, &data, hash)?; } Ok(()) } From dddaa6d06801955dcbb7b1d1094932383ff36853 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 15 Jul 2021 13:34:33 +0200 Subject: [PATCH 11/12] Rename expn_info -> expn_data. --- compiler/rustc_span/src/hygiene.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index b4adf2d46bbcd..c020375fab04b 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -579,19 +579,19 @@ pub fn debug_hygiene_data(verbose: bool) -> String { } else { let mut s = String::from(""); s.push_str("Expansions:"); - let mut debug_expn_data = |(id, expn_info): (&ExpnId, &ExpnData)| { + let mut debug_expn_data = |(id, expn_data): (&ExpnId, &ExpnData)| { s.push_str(&format!( "\n{:?}: parent: {:?}, call_site_ctxt: {:?}, def_site_ctxt: {:?}, kind: {:?}", id, - expn_info.parent, - expn_info.call_site.ctxt(), - expn_info.def_site.ctxt(), - expn_info.kind, + expn_data.parent, + expn_data.call_site.ctxt(), + expn_data.def_site.ctxt(), + expn_data.kind, )) }; - data.local_expn_data.iter_enumerated().for_each(|(id, expn_info)| { - let expn_info = expn_info.as_ref().expect("no expansion data for an expansion ID"); - debug_expn_data((&id.to_expn_id(), expn_info)) + data.local_expn_data.iter_enumerated().for_each(|(id, expn_data)| { + let expn_data = expn_data.as_ref().expect("no expansion data for an expansion ID"); + debug_expn_data((&id.to_expn_id(), expn_data)) }); data.foreign_expn_data.iter().for_each(debug_expn_data); s.push_str("\n\nSyntaxContexts:"); From b35ceeeec706e957ee46166e0a8b6d912ac215a3 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 15 Jul 2021 13:45:38 +0200 Subject: [PATCH 12/12] Simplify Expn creation. --- compiler/rustc_span/src/hygiene.rs | 52 ++++++++++++++---------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index c020375fab04b..cb3a08439d129 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -153,14 +153,25 @@ impl LocalExpnId { } pub fn fresh_empty() -> LocalExpnId { - HygieneData::with(|data| data.fresh_expn(None)) + HygieneData::with(|data| { + let expn_id = data.local_expn_data.push(None); + let _eid = data.local_expn_hashes.push(ExpnHash(Fingerprint::ZERO)); + debug_assert_eq!(expn_id, _eid); + expn_id + }) } - pub fn fresh(expn_data: ExpnData, ctx: impl HashStableContext) -> LocalExpnId { + pub fn fresh(mut expn_data: ExpnData, ctx: impl HashStableContext) -> LocalExpnId { debug_assert_eq!(expn_data.parent.krate, LOCAL_CRATE); - let expn_id = HygieneData::with(|data| data.fresh_expn(Some(expn_data))); - update_disambiguator(expn_id, ctx); - expn_id + let expn_hash = update_disambiguator(&mut expn_data, ctx); + HygieneData::with(|data| { + let expn_id = data.local_expn_data.push(Some(expn_data)); + let _eid = data.local_expn_hashes.push(expn_hash); + debug_assert_eq!(expn_id, _eid); + let _old_id = data.expn_hash_to_expn_id.insert(expn_hash, expn_id.to_expn_id()); + debug_assert!(_old_id.is_none()); + expn_id + }) } #[inline] @@ -179,14 +190,18 @@ impl LocalExpnId { } #[inline] - pub fn set_expn_data(self, expn_data: ExpnData, ctx: impl HashStableContext) { + pub fn set_expn_data(self, mut expn_data: ExpnData, ctx: impl HashStableContext) { debug_assert_eq!(expn_data.parent.krate, LOCAL_CRATE); + let expn_hash = update_disambiguator(&mut expn_data, ctx); HygieneData::with(|data| { let old_expn_data = &mut data.local_expn_data[self]; assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID"); *old_expn_data = Some(expn_data); + debug_assert_eq!(data.local_expn_hashes[self].0, Fingerprint::ZERO); + data.local_expn_hashes[self] = expn_hash; + let _old_id = data.expn_hash_to_expn_id.insert(expn_hash, self.to_expn_id()); + debug_assert!(_old_id.is_none()); }); - update_disambiguator(self, ctx) } #[inline] @@ -335,13 +350,6 @@ impl HygieneData { with_session_globals(|session_globals| f(&mut *session_globals.hygiene_data.borrow_mut())) } - fn fresh_expn(&mut self, expn_data: Option) -> LocalExpnId { - let expn_id = self.local_expn_data.push(expn_data); - let _eid = self.local_expn_hashes.push(ExpnHash(Fingerprint::ZERO)); - debug_assert_eq!(expn_id, _eid); - expn_id - } - #[inline] fn local_expn_hash(&self, expn_id: LocalExpnId) -> ExpnHash { self.local_expn_hashes[expn_id] @@ -1413,8 +1421,7 @@ impl Decodable for SyntaxContext { /// `set_expn_data`). It is *not* called for foreign `ExpnId`s deserialized /// from another crate's metadata - since `ExpnHash` includes the stable crate id, /// collisions are only possible between `ExpnId`s within the same crate. -fn update_disambiguator(expn_id: LocalExpnId, mut ctx: impl HashStableContext) { - let mut expn_data = expn_id.expn_data(); +fn update_disambiguator(expn_data: &mut ExpnData, mut ctx: impl HashStableContext) -> ExpnHash { // This disambiguator should not have been set yet. assert_eq!( expn_data.disambiguator, 0, @@ -1433,8 +1440,7 @@ fn update_disambiguator(expn_id: LocalExpnId, mut ctx: impl HashStableContext) { }); if disambiguator != 0 { - debug!("Set disambiguator for {:?} (hash {:?})", expn_id, expn_hash); - debug!("expn_data = {:?}", expn_data); + debug!("Set disambiguator for expn_data={:?} expn_hash={:?}", expn_data, expn_hash); expn_data.disambiguator = disambiguator; expn_hash = expn_data.hash_expn(&mut ctx); @@ -1450,15 +1456,7 @@ fn update_disambiguator(expn_id: LocalExpnId, mut ctx: impl HashStableContext) { }); } - let expn_hash = - ExpnHash::new(ctx.def_path_hash(LOCAL_CRATE.as_def_id()).stable_crate_id(), expn_hash); - HygieneData::with(|data| { - data.local_expn_data[expn_id].as_mut().unwrap().disambiguator = disambiguator; - debug_assert_eq!(data.local_expn_hashes[expn_id].0, Fingerprint::ZERO); - data.local_expn_hashes[expn_id] = expn_hash; - let _old_id = data.expn_hash_to_expn_id.insert(expn_hash, expn_id.to_expn_id()); - debug_assert!(_old_id.is_none()); - }); + ExpnHash::new(ctx.def_path_hash(LOCAL_CRATE.as_def_id()).stable_crate_id(), expn_hash) } impl HashStable for SyntaxContext {