Switch token trees to use Symbols

rust-lang · Jul 16, 2024 · 93024ad · 93024ad
1 parent 0c95aaa
commit 93024ad
Show file tree

Hide file tree

Showing 51 changed files with 593 additions and 399 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/cfg/Cargo.toml b/crates/cfg/Cargo.toml
@@ -16,6 +16,7 @@ rustc-hash.workspace = true
 
 # locals deps
 tt.workspace = true
+smol_str.workspace = true
 
 [dev-dependencies]
 expect-test = "1.4.1"

diff --git a/crates/cfg/src/cfg_expr.rs b/crates/cfg/src/cfg_expr.rs
@@ -4,7 +4,7 @@
 
 use std::{fmt, slice::Iter as SliceIter};
 
-use tt::SmolStr;
+use smol_str::SmolStr;
 
 /// A simple configuration value passed in from the outside.
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
@@ -66,7 +66,7 @@ impl CfgExpr {
 fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
     let name = match it.next() {
         None => return None,
-        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),
+        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
         Some(_) => return Some(CfgExpr::Invalid),
     };
 
@@ -77,10 +77,9 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
                 Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
                     it.next();
                     it.next();
-                    // FIXME: escape? raw string?
-                    let value =
-                        SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"'));
-                    CfgAtom::KeyValue { key: name, value }.into()
+                    // FIXME: escape?
+                    let value = literal.symbol.as_str().into();
+                    CfgAtom::KeyValue { key: name.as_str().into(), value }.into()
                 }
                 _ => return Some(CfgExpr::Invalid),
             }
@@ -96,7 +95,7 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
                 _ => CfgExpr::Invalid,
             }
         }
-        _ => CfgAtom::Flag(name).into(),
+        _ => CfgAtom::Flag(name.as_str().into()).into(),
     };
 
     // Eat comma separator

diff --git a/crates/cfg/src/lib.rs b/crates/cfg/src/lib.rs
@@ -8,10 +8,10 @@ mod tests;
 use std::fmt;
 
 use rustc_hash::FxHashSet;
-use tt::SmolStr;
 
 pub use cfg_expr::{CfgAtom, CfgExpr};
 pub use dnf::DnfExpr;
+use smol_str::SmolStr;
 
 /// Configuration options used for conditional compilation on items with `cfg` attributes.
 /// We have two kind of options in different namespaces: atomic options like `unix`, and

diff --git a/crates/hir-def/src/attr.rs b/crates/hir-def/src/attr.rs
@@ -159,14 +159,14 @@ impl Attrs {
     pub fn has_doc_hidden(&self) -> bool {
         self.by_key("doc").tt_values().any(|tt| {
             tt.delimiter.kind == DelimiterKind::Parenthesis &&
-                matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
+                matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::hidden)
         })
     }
 
     pub fn has_doc_notable_trait(&self) -> bool {
         self.by_key("doc").tt_values().any(|tt| {
             tt.delimiter.kind == DelimiterKind::Parenthesis &&
-                matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "notable_trait")
+                matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::notable_trait)
         })
     }
 
@@ -267,21 +267,24 @@ impl DocExpr {
 fn next_doc_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<DocExpr> {
     let name = match it.next() {
         None => return None,
-        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),
+        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
         Some(_) => return Some(DocExpr::Invalid),
     };
 
     // Peek
     let ret = match it.as_slice().first() {
         Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
             match it.as_slice().get(1) {
-                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
+                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
+                    symbol: text,
+                    kind: tt::LitKind::Str,
+                    ..
+                }))) => {
                     it.next();
                     it.next();
                     // FIXME: escape? raw string?
-                    let value =
-                        SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"'));
-                    DocAtom::KeyValue { key: name, value }.into()
+                    let value = SmolStr::new(text.as_str());
+                    DocAtom::KeyValue { key: name.as_str().into(), value }.into()
                 }
                 _ => return Some(DocExpr::Invalid),
             }
@@ -294,7 +297,7 @@ fn next_doc_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<DocExpr>
                 _ => DocExpr::Invalid,
             }
         }
-        _ => DocAtom::Flag(name).into(),
+        _ => DocAtom::Flag(name.as_str().into()).into(),
     };
 
     // Eat comma separator
@@ -311,10 +314,11 @@ fn parse_comma_sep<S>(subtree: &tt::Subtree<S>) -> Vec<SmolStr> {
         .token_trees
         .iter()
         .filter_map(|tt| match tt {
-            tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
-                // FIXME: escape? raw string?
-                Some(SmolStr::new(lit.text.trim_start_matches('"').trim_end_matches('"')))
-            }
+            tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
+                kind: tt::LitKind::Str,
+                symbol: text,
+                ..
+            })) => Some(SmolStr::new(text.as_str())),
             _ => None,
         })
         .collect()
@@ -598,14 +602,14 @@ impl<'attr> AttrQuery<'attr> {
     /// #[doc(html_root_url = "url")]
     ///       ^^^^^^^^^^^^^ key
     /// ```
-    pub fn find_string_value_in_tt(self, key: &'attr str) -> Option<&SmolStr> {
+    pub fn find_string_value_in_tt(self, key: &'attr str) -> Option<&str> {
         self.tt_values().find_map(|tt| {
             let name = tt.token_trees.iter()
-                .skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text, ..} )) if text == key))
+                .skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { sym, ..} )) if sym.as_str() == key))
                 .nth(2);
 
             match name {
-                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{  text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text),
+                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{  symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text.as_str()),
                 _ => None
             }
         })

diff --git a/crates/hir-def/src/builtin_type.rs b/crates/hir-def/src/builtin_type.rs
@@ -6,7 +6,7 @@
 use std::fmt;
 
 use hir_expand::name::{AsName, Name};
-use intern::sym;
+use intern::{sym, Symbol};
 /// Different signed int types.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum BuiltinInt {
@@ -143,6 +143,18 @@ impl BuiltinInt {
         };
         Some(res)
     }
+    pub fn from_suffix_sym(suffix: &Symbol) -> Option<BuiltinInt> {
+        let res = match suffix {
+            s if *s == sym::isize => Self::Isize,
+            s if *s == sym::i8    => Self::I8,
+            s if *s == sym::i16   => Self::I16,
+            s if *s == sym::i32   => Self::I32,
+            s if *s == sym::i64   => Self::I64,
+            s if *s == sym::i128  => Self::I128,
+            _ => return None,
+        };
+        Some(res)
+    }
 }
 
 #[rustfmt::skip]
@@ -160,6 +172,19 @@ impl BuiltinUint {
         };
         Some(res)
     }
+    pub fn from_suffix_sym(suffix: &Symbol) -> Option<BuiltinUint> {
+        let res = match suffix {
+            s if *s == sym::usize => Self::Usize,
+            s if *s == sym::u8    => Self::U8,
+            s if *s == sym::u16   => Self::U16,
+            s if *s == sym::u32   => Self::U32,
+            s if *s == sym::u64   => Self::U64,
+            s if *s == sym::u128  => Self::U128,
+
+            _ => return None,
+        };
+        Some(res)
+    }
 }
 
 #[rustfmt::skip]

diff --git a/crates/hir-def/src/data.rs b/crates/hir-def/src/data.rs
@@ -150,7 +150,7 @@ fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
     let mut indices = Vec::new();
     for args in tt.token_trees.chunks(2) {
         match &args[0] {
-            tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => match lit.text.parse() {
+            tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => match lit.symbol.as_str().parse() {
                 Ok(index) => indices.push(index),
                 Err(_) => break,
             },

diff --git a/crates/hir-def/src/data/adt.rs b/crates/hir-def/src/data/adt.rs
@@ -9,7 +9,7 @@ use hir_expand::{
     name::{AsName, Name},
     HirFileId, InFile,
 };
-use intern::Interned;
+use intern::{sym, Interned};
 use la_arena::Arena;
 use rustc_abi::{Align, Integer, IntegerType, ReprFlags, ReprOptions};
 use syntax::ast::{self, HasName, HasVisibility};
@@ -112,12 +112,12 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
     let mut tts = tt.token_trees.iter().peekable();
     while let Some(tt) = tts.next() {
         if let TokenTree::Leaf(Leaf::Ident(ident)) = tt {
-            flags.insert(match &*ident.text {
-                "packed" => {
+            flags.insert(match &ident.sym {
+                s if *s == sym::packed => {
                     let pack = if let Some(TokenTree::Subtree(tt)) = tts.peek() {
                         tts.next();
                         if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() {
-                            lit.text.parse().unwrap_or_default()
+                            lit.symbol.as_str().parse().unwrap_or_default()
                         } else {
                             0
                         }
@@ -129,25 +129,25 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
                         Some(if let Some(min_pack) = min_pack { min_pack.min(pack) } else { pack });
                     ReprFlags::empty()
                 }
-                "align" => {
+                s if *s == sym::align => {
                     if let Some(TokenTree::Subtree(tt)) = tts.peek() {
                         tts.next();
                         if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() {
-                            if let Ok(align) = lit.text.parse() {
+                            if let Ok(align) = lit.symbol.as_str().parse() {
                                 let align = Align::from_bytes(align).ok();
                                 max_align = max_align.max(align);
                             }
                         }
                     }
                     ReprFlags::empty()
                 }
-                "C" => ReprFlags::IS_C,
-                "transparent" => ReprFlags::IS_TRANSPARENT,
-                "simd" => ReprFlags::IS_SIMD,
+                s if *s == sym::C => ReprFlags::IS_C,
+                s if *s == sym::transparent => ReprFlags::IS_TRANSPARENT,
+                s if *s == sym::simd => ReprFlags::IS_SIMD,
                 repr => {
-                    if let Some(builtin) = BuiltinInt::from_suffix(repr)
+                    if let Some(builtin) = BuiltinInt::from_suffix_sym(repr)
                         .map(Either::Left)
-                        .or_else(|| BuiltinUint::from_suffix(repr).map(Either::Right))
+                        .or_else(|| BuiltinUint::from_suffix_sym(repr).map(Either::Right))
                     {
                         int = Some(match builtin {
                             Either::Left(bi) => match bi {

diff --git a/crates/hir-def/src/db.rs b/crates/hir-def/src/db.rs
@@ -278,7 +278,7 @@ fn crate_supports_no_std(db: &dyn DefDatabase, crate_id: CrateId) -> bool {
             tt.split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(p)) if p.char == ','));
         for output in segments.skip(1) {
             match output {
-                [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "no_std" => {
+                [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::no_std => {
                     return true
                 }
                 _ => {}

diff --git a/crates/hir-def/src/nameres.rs b/crates/hir-def/src/nameres.rs
@@ -63,6 +63,7 @@ use base_db::{CrateId, FileId};
 use hir_expand::{
     name::Name, proc_macro::ProcMacroKind, ErasedAstId, HirFileId, InFile, MacroCallId, MacroDefId,
 };
+use intern::Symbol;
 use itertools::Itertools;
 use la_arena::Arena;
 use rustc_hash::{FxHashMap, FxHashSet};
@@ -148,11 +149,11 @@ struct DefMapCrateData {
     proc_macro_loading_error: Option<Box<str>>,
 
     /// Custom attributes registered with `#![register_attr]`.
-    registered_attrs: Vec<SmolStr>,
+    registered_attrs: Vec<Symbol>,
     /// Custom tool modules registered with `#![register_tool]`.
-    registered_tools: Vec<SmolStr>,
+    registered_tools: Vec<Symbol>,
     /// Unstable features of Rust enabled with `#![feature(A, B)]`.
-    unstable_features: FxHashSet<SmolStr>,
+    unstable_features: FxHashSet<Symbol>,
     /// #[rustc_coherence_is_core]
     rustc_coherence_is_core: bool,
     no_core: bool,
@@ -170,7 +171,7 @@ impl DefMapCrateData {
             fn_proc_macro_mapping: FxHashMap::default(),
             proc_macro_loading_error: None,
             registered_attrs: Vec::new(),
-            registered_tools: PREDEFINED_TOOLS.into(),
+            registered_tools: PREDEFINED_TOOLS.iter().map(|it| Symbol::intern(it)).collect(),
             unstable_features: FxHashSet::default(),
             rustc_coherence_is_core: false,
             no_core: false,
@@ -447,15 +448,15 @@ impl DefMap {
         self.derive_helpers_in_scope.get(&id.map(|it| it.upcast())).map(Deref::deref)
     }
 
-    pub fn registered_tools(&self) -> &[SmolStr] {
+    pub fn registered_tools(&self) -> &[Symbol] {
         &self.data.registered_tools
     }
 
-    pub fn registered_attrs(&self) -> &[SmolStr] {
+    pub fn registered_attrs(&self) -> &[Symbol] {
         &self.data.registered_attrs
     }
 
-    pub fn is_unstable_feature_enabled(&self, feature: &str) -> bool {
+    pub fn is_unstable_feature_enabled(&self, feature: &Symbol) -> bool {
         self.data.unstable_features.contains(feature)
     }