From 922ec171151176f41d9d1b3dd107960baf595636 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 14 Aug 2020 16:52:47 +0200 Subject: [PATCH] Move jointness info from TokenStream to Token Part of #63689. --- src/librustc_ast/attr/mod.rs | 20 +++-- src/librustc_ast/mut_visit.rs | 4 +- src/librustc_ast/token.rs | 29 ++++++- src/librustc_ast/tokenstream.rs | 79 ++++++------------- src/librustc_expand/mbe/quoted.rs | 52 ++++++------ src/librustc_expand/mbe/transcribe.rs | 4 +- src/librustc_expand/parse/tests.rs | 9 ++- src/librustc_expand/proc_macro_server.rs | 19 +++-- src/librustc_expand/tokenstream/tests.rs | 8 +- src/librustc_parse/lexer/tokentrees.rs | 31 ++++---- src/librustc_parse/parser/mod.rs | 57 +++++++------ .../ast-json/ast-json-noexpand-output.stdout | 2 +- src/test/ui/ast-json/ast-json-output.stdout | 2 +- 13 files changed, 160 insertions(+), 156 deletions(-) diff --git a/src/librustc_ast/attr/mod.rs b/src/librustc_ast/attr/mod.rs index edcbce3e2cfd3..a7345dbfe2d88 100644 --- a/src/librustc_ast/attr/mod.rs +++ b/src/librustc_ast/attr/mod.rs @@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment}; use crate::mut_visit::visit_clobber; use crate::ptr::P; use crate::token::{self, CommentKind, Token}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; +use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_index::bit_set::GrowableBitSet; use rustc_span::source_map::{BytePos, Spanned}; @@ -362,7 +362,7 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool { } impl MetaItem { - fn token_trees_and_joints(&self) -> Vec { + fn token_trees_and_joints(&self) -> Vec { let mut idents = vec![]; let mut last_pos = BytePos(0 as u32); for (i, segment) in self.path.segments.iter().enumerate() { @@ -370,9 +370,9 @@ impl MetaItem { if !is_first { let mod_sep_span = Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt()); - idents.push(TokenTree::token(token::ModSep, mod_sep_span).into()); + idents.push(TokenTree::token(token::ModSep, mod_sep_span)); } - idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); + idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident))); last_pos = segment.ident.span.hi(); } idents.extend(self.kind.token_trees_and_joints(self.span)); @@ -388,6 +388,7 @@ impl MetaItem { Some(TokenTree::Token(Token { kind: kind @ (token::Ident(..) | token::ModSep), span, + spacing: _, })) => 'arm: { let mut segments = if let token::Ident(name, _) = kind { if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() @@ -401,8 +402,11 @@ impl MetaItem { vec![PathSegment::path_root(span)] }; loop { - if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = - tokens.next().map(TokenTree::uninterpolate) + if let Some(TokenTree::Token(Token { + kind: token::Ident(name, _), + span, + spacing: _, + })) = tokens.next().map(TokenTree::uninterpolate) { segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { @@ -459,7 +463,7 @@ impl MetaItemKind { } } - fn token_trees_and_joints(&self, span: Span) -> Vec { + fn token_trees_and_joints(&self, span: Span) -> Vec { match *self { MetaItemKind::Word => vec![], MetaItemKind::NameValue(ref lit) => { @@ -554,7 +558,7 @@ impl NestedMetaItem { } } - fn token_trees_and_joints(&self) -> Vec { + fn token_trees_and_joints(&self) -> Vec { match *self { NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(), NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()], diff --git a/src/librustc_ast/mut_visit.rs b/src/librustc_ast/mut_visit.rs index df6e8218f6c3a..32b7b314876c8 100644 --- a/src/librustc_ast/mut_visit.rs +++ b/src/librustc_ast/mut_visit.rs @@ -640,14 +640,14 @@ pub fn noop_visit_tt(tt: &mut TokenTree, vis: &mut T) { pub fn noop_visit_tts(TokenStream(tts): &mut TokenStream, vis: &mut T) { let tts = Lrc::make_mut(tts); - visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree)); + visit_vec(tts, |tree| vis.visit_tt(tree)); } // Applies ident visitor if it's an ident; applies other visits to interpolated nodes. // In practice the ident part is not actually used by specific visitors right now, // but there's a test below checking that it works. pub fn noop_visit_token(t: &mut Token, vis: &mut T) { - let Token { kind, span } = t; + let Token { kind, span, spacing: _ } = t; match kind { token::Ident(name, _) | token::Lifetime(name) => { let mut ident = Ident::new(*name, *span); diff --git a/src/librustc_ast/token.rs b/src/librustc_ast/token.rs index bcce881ed48c5..5145612bc4490 100644 --- a/src/librustc_ast/token.rs +++ b/src/librustc_ast/token.rs @@ -267,10 +267,24 @@ pub enum TokenKind { #[cfg(target_arch = "x86_64")] rustc_data_structures::static_assert_size!(TokenKind, 16); +// FIXME: remove to Spacing, to match proc_macro +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] +pub enum Spacing { + Alone, + Joint, +} + #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] pub struct Token { pub kind: TokenKind, pub span: Span, + /// Is this operator token immediately followed by another operator token, + /// without whitespace? + /// + /// At the moment, this field is only used for proc_macros, and joint tokens + /// are usually represented by dedicated token kinds. We want to transition + /// to using jointness everywhere though (#63689). + pub spacing: Spacing, } impl TokenKind { @@ -324,7 +338,11 @@ impl TokenKind { impl Token { pub fn new(kind: TokenKind, span: Span) -> Self { - Token { kind, span } + Token::with_spacing(kind, span, Spacing::Alone) + } + + pub fn with_spacing(kind: TokenKind, span: Span, spacing: Spacing) -> Self { + Token { kind, span, spacing } } /// Some token that will be thrown away later. @@ -622,6 +640,9 @@ impl Token { } pub fn glue(&self, joint: &Token) -> Option { + if self.spacing == Spacing::Alone { + return None; + } let kind = match self.kind { Eq => match joint.kind { Eq => EqEq, @@ -678,7 +699,9 @@ impl Token { | Shebang(..) | Unknown(..) | Eof => return None, }; - Some(Token::new(kind, self.span.to(joint.span))) + let mut token = Token::new(kind, self.span.to(joint.span)); + token.spacing = joint.spacing; + Some(token) } } @@ -709,7 +732,7 @@ pub enum Nonterminal { // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -rustc_data_structures::static_assert_size!(Nonterminal, 40); +rustc_data_structures::static_assert_size!(Nonterminal, 48); #[derive(Debug, Copy, Clone, PartialEq, RustcEncodable, RustcDecodable)] pub enum NonterminalKind { diff --git a/src/librustc_ast/tokenstream.rs b/src/librustc_ast/tokenstream.rs index 9d0199078fa6a..41d1b69044b1e 100644 --- a/src/librustc_ast/tokenstream.rs +++ b/src/librustc_ast/tokenstream.rs @@ -13,7 +13,7 @@ //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking //! ownership of the original. -use crate::token::{self, DelimToken, Token, TokenKind}; +use crate::token::{self, DelimToken, Spacing, Token, TokenKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::Lrc; @@ -82,10 +82,6 @@ impl TokenTree { } } - pub fn joint(self) -> TokenStream { - TokenStream::new(vec![(self, Joint)]) - } - pub fn token(kind: TokenKind, span: Span) -> TokenTree { TokenTree::Token(Token::new(kind, span)) } @@ -125,22 +121,12 @@ where /// instead of a representation of the abstract syntax tree. /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat. #[derive(Clone, Debug, Default, RustcEncodable, RustcDecodable)] -pub struct TokenStream(pub Lrc>); - -pub type TreeAndJoint = (TokenTree, IsJoint); +pub struct TokenStream(pub Lrc>); // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] rustc_data_structures::static_assert_size!(TokenStream, 8); -#[derive(Clone, Copy, Debug, PartialEq, RustcEncodable, RustcDecodable)] -pub enum IsJoint { - Joint, - NonJoint, -} - -use IsJoint::*; - impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// separating the two arguments with a comma for diagnostic suggestions. @@ -151,10 +137,10 @@ impl TokenStream { while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { let sp = match (&ts, &next) { - (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, + (_, TokenTree::Token(Token { kind: token::Comma, .. })) => continue, ( - (TokenTree::Token(token_left), NonJoint), - (TokenTree::Token(token_right), _), + TokenTree::Token(token_left @ Token { spacing: Spacing::Alone, .. }), + TokenTree::Token(token_right), ) if ((token_left.is_ident() && !token_left.is_reserved_ident()) || token_left.is_lit()) && ((token_right.is_ident() && !token_right.is_reserved_ident()) @@ -162,11 +148,11 @@ impl TokenStream { { token_left.span } - ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(), + (TokenTree::Delimited(sp, ..), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); - let comma = (TokenTree::token(token::Comma, sp), NonJoint); + let comma = TokenTree::token(token::Comma, sp); suggestion = Some((pos, comma, sp)); } } @@ -184,19 +170,13 @@ impl TokenStream { impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream::new(vec![(tree, NonJoint)]) - } -} - -impl From for TreeAndJoint { - fn from(tree: TokenTree) -> TreeAndJoint { - (tree, NonJoint) + TokenStream::new(vec![(tree)]) } } impl iter::FromIterator for TokenStream { fn from_iter>(iter: I) -> Self { - TokenStream::new(iter.into_iter().map(Into::into).collect::>()) + TokenStream::new(iter.into_iter().map(Into::into).collect::>()) } } @@ -209,7 +189,7 @@ impl PartialEq for TokenStream { } impl TokenStream { - pub fn new(streams: Vec) -> TokenStream { + pub fn new(streams: Vec) -> TokenStream { TokenStream(Lrc::new(streams)) } @@ -224,8 +204,8 @@ impl TokenStream { pub fn span(&self) -> Option { match &**self.0 { [] => None, - [(tt, _)] => Some(tt.span()), - [(tt_start, _), .., (tt_end, _)] => Some(tt_start.span().to(tt_end.span())), + [tt] => Some(tt.span()), + [tt_start, .., tt_end] => Some(tt_start.span().to(tt_end.span())), } } @@ -290,18 +270,12 @@ impl TokenStream { pub fn map_enumerated TokenTree>(self, mut f: F) -> TokenStream { TokenStream(Lrc::new( - self.0 - .iter() - .enumerate() - .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint)) - .collect(), + self.0.iter().enumerate().map(|(i, tree)| f(i, tree.clone())).collect(), )) } pub fn map TokenTree>(self, mut f: F) -> TokenStream { - TokenStream(Lrc::new( - self.0.iter().map(|(tree, is_joint)| (f(tree.clone()), *is_joint)).collect(), - )) + TokenStream(Lrc::new(self.0.iter().map(|tree| f(tree.clone())).collect())) } } @@ -320,11 +294,11 @@ impl TokenStreamBuilder { // If `self` is not empty and the last tree within the last stream is a // token tree marked with `Joint`... if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() { - if let Some((TokenTree::Token(last_token), Joint)) = last_stream_lrc.last() { + if let Some(TokenTree::Token(last_token)) = last_stream_lrc.last() { // ...and `stream` is not empty and the first tree within it is // a token tree... let TokenStream(ref mut stream_lrc) = stream; - if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() { + if let Some(TokenTree::Token(token)) = stream_lrc.first() { // ...and the two tokens can be glued together... if let Some(glued_tok) = last_token.glue(&token) { // ...then do so, by overwriting the last token @@ -337,8 +311,7 @@ impl TokenStreamBuilder { // Overwrite the last token tree with the merged // token. let last_vec_mut = Lrc::make_mut(last_stream_lrc); - *last_vec_mut.last_mut().unwrap() = - (TokenTree::Token(glued_tok), *is_joint); + *last_vec_mut.last_mut().unwrap() = TokenTree::Token(glued_tok); // Remove the first token tree from `stream`. (This // is almost always the only tree in `stream`.) @@ -375,16 +348,6 @@ impl Iterator for Cursor { type Item = TokenTree; fn next(&mut self) -> Option { - self.next_with_joint().map(|(tree, _)| tree) - } -} - -impl Cursor { - fn new(stream: TokenStream) -> Self { - Cursor { stream, index: 0 } - } - - pub fn next_with_joint(&mut self) -> Option { if self.index < self.stream.len() { self.index += 1; Some(self.stream.0[self.index - 1].clone()) @@ -392,6 +355,12 @@ impl Cursor { None } } +} + +impl Cursor { + fn new(stream: TokenStream) -> Self { + Cursor { stream, index: 0 } + } pub fn append(&mut self, new_stream: TokenStream) { if new_stream.is_empty() { @@ -404,7 +373,7 @@ impl Cursor { } pub fn look_ahead(&self, n: usize) -> Option { - self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone()) + self.stream.0[self.index..].get(n).cloned() } } diff --git a/src/librustc_expand/mbe/quoted.rs b/src/librustc_expand/mbe/quoted.rs index 774cc84afdeb1..5786e007bb904 100644 --- a/src/librustc_expand/mbe/quoted.rs +++ b/src/librustc_expand/mbe/quoted.rs @@ -56,31 +56,31 @@ pub(super) fn parse( match tree { TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { - match trees.next() { - Some(tokenstream::TokenTree::Token(token)) => match token.ident() { - Some((frag, _)) => { - let span = token.span.with_lo(start_sp.lo()); - let kind = token::NonterminalKind::from_symbol(frag.name) - .unwrap_or_else(|| { - let msg = format!( - "invalid fragment specifier `{}`", - frag.name - ); - sess.span_diagnostic - .struct_span_err(span, &msg) - .help(VALID_FRAGMENT_NAMES_MSG) - .emit(); - token::NonterminalKind::Ident - }); - result.push(TokenTree::MetaVarDecl(span, ident, Some(kind))); - continue; - } - _ => token.span, - }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), - } - } + Some(tokenstream::TokenTree::Token(Token { + kind: token::Colon, + span, + spacing: _, + })) => match trees.next() { + Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some((frag, _)) => { + let span = token.span.with_lo(start_sp.lo()); + let kind = token::NonterminalKind::from_symbol(frag.name) + .unwrap_or_else(|| { + let msg = + format!("invalid fragment specifier `{}`", frag.name); + sess.span_diagnostic + .struct_span_err(span, &msg) + .help(VALID_FRAGMENT_NAMES_MSG) + .emit(); + token::NonterminalKind::Ident + }); + result.push(TokenTree::MetaVarDecl(span, ident, Some(kind))); + continue; + } + _ => token.span, + }, + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + }, tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), }; if node_id != DUMMY_NODE_ID { @@ -122,7 +122,7 @@ fn parse_tree( // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => { + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span, spacing: _ }) => { // FIXME: Handle `None`-delimited groups in a more systematic way // during parsing. let mut next = outer_trees.next(); diff --git a/src/librustc_expand/mbe/transcribe.rs b/src/librustc_expand/mbe/transcribe.rs index e2d3d5c4d644e..d0eabe7a59ec4 100644 --- a/src/librustc_expand/mbe/transcribe.rs +++ b/src/librustc_expand/mbe/transcribe.rs @@ -5,7 +5,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; use rustc_ast::ast::MacCall; use rustc_ast::mut_visit::{self, MutVisitor}; use rustc_ast::token::{self, NtTT, Token}; -use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; +use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::{pluralize, PResult}; @@ -111,7 +111,7 @@ pub(super) fn transcribe<'a>( // // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level // again, and we are done transcribing. - let mut result: Vec = Vec::new(); + let mut result: Vec = Vec::new(); let mut result_stack = Vec::new(); let mut marker = Marker(cx.current_expansion.id, transparency); diff --git a/src/librustc_expand/parse/tests.rs b/src/librustc_expand/parse/tests.rs index 5c9116b2f139d..35bca7cd34455 100644 --- a/src/librustc_expand/parse/tests.rs +++ b/src/librustc_expand/parse/tests.rs @@ -2,7 +2,7 @@ use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_pa use rustc_ast::ast::{self, PatKind}; use rustc_ast::ptr::P; -use rustc_ast::token::{self, Token}; +use rustc_ast::token::{self, Spacing, Token}; use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_ast::visit; use rustc_ast_pretty::pprust::item_to_string; @@ -116,7 +116,12 @@ fn string_to_tts_1() { DelimSpan::from_pair(sp(15, 16), sp(20, 21)), token::DelimToken::Brace, TokenStream::new(vec![ - TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(17, 18)).into(), + TokenTree::Token(Token::with_spacing( + token::Ident(Symbol::intern("b"), false), + sp(17, 18), + Spacing::Joint, + )) + .into(), TokenTree::token(token::Semi, sp(18, 19)).into(), ]) .into(), diff --git a/src/librustc_expand/proc_macro_server.rs b/src/librustc_expand/proc_macro_server.rs index 83a650443bcba..e3db34fc5b034 100644 --- a/src/librustc_expand/proc_macro_server.rs +++ b/src/librustc_expand/proc_macro_server.rs @@ -2,7 +2,7 @@ use crate::base::ExtCtxt; use rustc_ast::ast; use rustc_ast::token; -use rustc_ast::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; +use rustc_ast::tokenstream::{self, DelimSpan, TokenStream}; use rustc_ast_pretty::pprust; use rustc_data_structures::sync::Lrc; use rustc_errors::Diagnostic; @@ -47,22 +47,22 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> +impl FromInternal<(tokenstream::TokenTree, &'_ ParseSess, &'_ mut Vec)> for TokenTree { fn from_internal( - ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec), + (tree, sess, stack): (tokenstream::TokenTree, &ParseSess, &mut Vec), ) -> Self { use rustc_ast::token::*; - let joint = is_joint == Joint; - let Token { kind, span } = match tree { + let Token { kind, span, spacing } = match tree { tokenstream::TokenTree::Delimited(span, delim, tts) => { let delimiter = Delimiter::from_internal(delim); return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false }); } tokenstream::TokenTree::Token(token) => token, }; + let joint = spacing == token::Spacing::Joint; macro_rules! tt { ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( @@ -254,8 +254,11 @@ impl ToInternal for TokenTree { _ => unreachable!(), }; - let tree = tokenstream::TokenTree::token(kind, span); - TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) + let spacing = if joint { token::Spacing::Joint } else { token::Spacing::Alone }; + let token = token::Token::with_spacing(kind, span, spacing); + let tree = tokenstream::TokenTree::Token(token); + + TokenStream::from(tree) } } @@ -438,7 +441,7 @@ impl server::TokenStreamIter for Rustc<'_> { ) -> Option> { loop { let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_joint()?; + let next = iter.cursor.next()?; Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) })?; // A hack used to pass AST fragments to attribute and derive macros diff --git a/src/librustc_expand/tokenstream/tests.rs b/src/librustc_expand/tokenstream/tests.rs index 4e818e9feb08b..a602b813f81df 100644 --- a/src/librustc_expand/tokenstream/tests.rs +++ b/src/librustc_expand/tokenstream/tests.rs @@ -1,6 +1,6 @@ use crate::tests::string_to_stream; -use rustc_ast::token; +use rustc_ast::token::{self, Spacing, Token}; use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree}; use rustc_span::with_default_session_globals; use rustc_span::{BytePos, Span, Symbol}; @@ -99,9 +99,9 @@ fn test_is_empty() { fn test_dotdotdot() { with_default_session_globals(|| { let mut builder = TokenStreamBuilder::new(); - builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); - builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); - builder.push(TokenTree::token(token::Dot, sp(2, 3))); + builder.push(TokenTree::Token(Token::with_spacing(token::Dot, sp(0, 1), Spacing::Joint))); + builder.push(TokenTree::Token(Token::with_spacing(token::Dot, sp(1, 2), Spacing::Joint))); + builder.push(TokenTree::Token(Token::with_spacing(token::Dot, sp(2, 3), Spacing::Alone))); let stream = builder.build(); assert!(stream.eq_unspanned(&string_to_ts("..."))); assert_eq!(stream.trees().count(), 1); diff --git a/src/librustc_parse/lexer/tokentrees.rs b/src/librustc_parse/lexer/tokentrees.rs index c08659ec9f648..764d949b7922b 100644 --- a/src/librustc_parse/lexer/tokentrees.rs +++ b/src/librustc_parse/lexer/tokentrees.rs @@ -1,11 +1,11 @@ use super::{StringReader, UnmatchedBrace}; -use rustc_ast::token::{self, DelimToken, Token}; -use rustc_ast::tokenstream::{ - DelimSpan, - IsJoint::{self, *}, - TokenStream, TokenTree, TreeAndJoint, +use rustc_ast::token::{ + self, DelimToken, + Spacing::{self, *}, + Token, }; +use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_ast_pretty::pprust::token_to_string; use rustc_data_structures::fx::FxHashMap; use rustc_errors::PResult; @@ -32,7 +32,7 @@ impl<'a> StringReader<'a> { struct TokenTreesReader<'a> { string_reader: StringReader<'a>, token: Token, - joint_to_prev: IsJoint, + joint_to_prev: Spacing, /// Stack of open delimiters and their spans. Used for error message. open_braces: Vec<(token::DelimToken, Span)>, unmatched_braces: Vec, @@ -79,7 +79,7 @@ impl<'a> TokenTreesReader<'a> { } } - fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> { + fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> { let sm = self.string_reader.sess.source_map(); match self.token.kind { @@ -263,10 +263,11 @@ impl<'a> TokenTreesReader<'a> { Err(err) } _ => { - let tt = TokenTree::Token(self.token.take()); + let mut tt = self.token.take(); self.real_token(); let is_joint = self.joint_to_prev == Joint && self.token.is_op(); - Ok((tt, if is_joint { Joint } else { NonJoint })) + tt.spacing = if is_joint { Joint } else { Alone }; + Ok(TokenTree::Token(tt)) } } } @@ -277,7 +278,7 @@ impl<'a> TokenTreesReader<'a> { let token = self.string_reader.next_token(); match token.kind { token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => { - self.joint_to_prev = NonJoint; + self.joint_to_prev = Alone; } _ => { self.token = token; @@ -290,21 +291,21 @@ impl<'a> TokenTreesReader<'a> { #[derive(Default)] struct TokenStreamBuilder { - buf: Vec, + buf: Vec, } impl TokenStreamBuilder { - fn push(&mut self, (tree, joint): TreeAndJoint) { - if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() { + fn push(&mut self, tree: TokenTree) { + if let Some(TokenTree::Token(prev_token)) = self.buf.last() { if let TokenTree::Token(token) = &tree { if let Some(glued) = prev_token.glue(token) { self.buf.pop(); - self.buf.push((TokenTree::Token(glued), joint)); + self.buf.push(TokenTree::Token(glued)); return; } } } - self.buf.push((tree, joint)) + self.buf.push(tree) } fn into_token_stream(self) -> TokenStream { diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index b33ae4bed828b..fcc256ad91038 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -21,7 +21,7 @@ use rustc_ast::ast::{ }; use rustc_ast::ptr::P; use rustc_ast::token::{self, DelimToken, Token, TokenKind}; -use rustc_ast::tokenstream::{self, DelimSpan, TokenStream, TokenTree, TreeAndJoint}; +use rustc_ast::tokenstream::{self, DelimSpan, TokenStream, TokenTree}; use rustc_ast_pretty::pprust; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError, PResult}; use rustc_session::parse::ParseSess; @@ -120,7 +120,7 @@ impl<'a> Drop for Parser<'a> { struct TokenCursor { frame: TokenCursorFrame, stack: Vec, - cur_token: Option, + cur_token: Option, collecting: Option, } @@ -138,7 +138,7 @@ struct TokenCursorFrame { struct Collecting { /// Holds the current tokens captured during the most /// recent call to `collect_tokens` - buf: Vec, + buf: Vec, /// The depth of the `TokenCursor` stack at the time /// collection was started. When we encounter a `TokenTree::Delimited`, /// we want to record the `TokenTree::Delimited` itself, @@ -169,7 +169,7 @@ impl TokenCursor { let tree = if !self.frame.open_delim { self.frame.open_delim = true; TokenTree::open_tt(self.frame.span, self.frame.delim).into() - } else if let Some(tree) = self.frame.tree_cursor.next_with_joint() { + } else if let Some(tree) = self.frame.tree_cursor.next() { tree } else if !self.frame.close_delim { self.frame.close_delim = true; @@ -184,7 +184,7 @@ impl TokenCursor { // Don't set an open delimiter as our current token - we want // to leave it as the full `TokenTree::Delimited` from the previous // iteration of this loop - if !matches!(tree.0, TokenTree::Token(Token { kind: TokenKind::OpenDelim(_), .. })) { + if !matches!(tree, TokenTree::Token(Token { kind: TokenKind::OpenDelim(_), .. })) { self.cur_token = Some(tree.clone()); } @@ -199,7 +199,7 @@ impl TokenCursor { } } - match tree.0 { + match tree { TokenTree::Token(token) => return token, TokenTree::Delimited(sp, delim, tts) => { let frame = TokenCursorFrame::new(sp, delim, &tts); @@ -211,7 +211,7 @@ impl TokenCursor { fn next_desugared(&mut self) -> Token { let (data, attr_style, sp) = match self.next() { - Token { kind: token::DocComment(_, attr_style, data), span } => { + Token { kind: token::DocComment(_, attr_style, data), span, spacing: _ } => { (data, attr_style, span) } tok => return tok, @@ -1155,7 +1155,7 @@ impl<'a> Parser<'a> { f: impl FnOnce(&mut Self) -> PResult<'a, R>, ) -> PResult<'a, (R, TokenStream)> { // Record all tokens we parse when parsing this item. - let tokens: Vec = self.token_cursor.cur_token.clone().into_iter().collect(); + let tokens: Vec = self.token_cursor.cur_token.clone().into_iter().collect(); debug!("collect_tokens: starting with {:?}", tokens); // We need special handling for the case where `collect_tokens` is called @@ -1163,27 +1163,26 @@ impl<'a> Parser<'a> { // a new frame - however, we want to record the original `TokenTree::Delimited`, // for consistency with the case where we start recording one token earlier. // See `TokenCursor::next` to see how `cur_token` is set up. - let prev_depth = - if matches!(self.token_cursor.cur_token, Some((TokenTree::Delimited(..), _))) { - if self.token_cursor.stack.is_empty() { - // There is nothing below us in the stack that - // the function could consume, so the only thing it can legally - // capture is the entire contents of the current frame. - return Ok((f(self)?, TokenStream::new(tokens))); - } - // We have already recorded the full `TokenTree::Delimited` when we created - // our `tokens` vector at the start of this function. We are now inside - // a new frame corresponding to the `TokenTree::Delimited` we already recoreded. - // We don't want to record any of the tokens inside this frame, since they - // will be duplicates of the tokens nested inside the `TokenTree::Delimited`. - // Therefore, we set our recording depth to the *previous* frame. This allows - // us to record a sequence like: `(foo).bar()`: the `(foo)` will be recored - // as our initial `cur_token`, while the `.bar()` will be recored after we - // pop the `(foo)` frame. - self.token_cursor.stack.len() - 1 - } else { - self.token_cursor.stack.len() - }; + let prev_depth = if matches!(self.token_cursor.cur_token, Some(TokenTree::Delimited(..))) { + if self.token_cursor.stack.is_empty() { + // There is nothing below us in the stack that + // the function could consume, so the only thing it can legally + // capture is the entire contents of the current frame. + return Ok((f(self)?, TokenStream::new(tokens))); + } + // We have already recorded the full `TokenTree::Delimited` when we created + // our `tokens` vector at the start of this function. We are now inside + // a new frame corresponding to the `TokenTree::Delimited` we already recoreded. + // We don't want to record any of the tokens inside this frame, since they + // will be duplicates of the tokens nested inside the `TokenTree::Delimited`. + // Therefore, we set our recording depth to the *previous* frame. This allows + // us to record a sequence like: `(foo).bar()`: the `(foo)` will be recored + // as our initial `cur_token`, while the `.bar()` will be recored after we + // pop the `(foo)` frame. + self.token_cursor.stack.len() - 1 + } else { + self.token_cursor.stack.len() + }; let prev_collecting = self.token_cursor.collecting.replace(Collecting { buf: tokens, depth: prev_depth }); diff --git a/src/test/ui/ast-json/ast-json-noexpand-output.stdout b/src/test/ui/ast-json/ast-json-noexpand-output.stdout index c7b0fbeb0e39b..ba989c225b5c5 100644 --- a/src/test/ui/ast-json/ast-json-noexpand-output.stdout +++ b/src/test/ui/ast-json/ast-json-noexpand-output.stdout @@ -1 +1 @@ -{"module":{"inner":{"lo":0,"hi":0},"items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"_field0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"NonJoint"]]}]}}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} +{"module":{"inner":{"lo":0,"hi":0},"items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"_field0":[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0},"spacing":"Alone"}]}]}]}}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} diff --git a/src/test/ui/ast-json/ast-json-output.stdout b/src/test/ui/ast-json/ast-json-output.stdout index 59ed68c2a773f..43860753ef857 100644 --- a/src/test/ui/ast-json/ast-json-output.stdout +++ b/src/test/ui/ast-json/ast-json-output.stdout @@ -1 +1 @@ -{"module":{"inner":{"lo":0,"hi":0},"items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":"Empty"}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":"Empty"}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"_field0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"NonJoint"]]}]}}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} +{"module":{"inner":{"lo":0,"hi":0},"items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":"Empty"}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":"Empty"}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"_field0":[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0},"spacing":"Alone"}]}]}]}}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}