Remove NtIdent and NtLifetime.

There's no need to use Interpolated for these because they both consist
of a single token, and so there's no operator precedence problems with
inserting them directly into the token stream.

Author: nnethercote

compiler/rustc_ast/src/ast_like.rs

@@ -51,9 +51,7 @@ impl AstLike for crate::token::Nonterminal {
             | Nonterminal::NtMeta(_)
             | Nonterminal::NtPath(_)
             | Nonterminal::NtVis(_)
-            | Nonterminal::NtBlock(_)
-            | Nonterminal::NtIdent(..)
-            | Nonterminal::NtLifetime(_) => &[],
+            | Nonterminal::NtBlock(_) => &[],
         }
     }
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
@@ -66,9 +64,7 @@ impl AstLike for crate::token::Nonterminal {
             | Nonterminal::NtMeta(_)
             | Nonterminal::NtPath(_)
             | Nonterminal::NtVis(_)
-            | Nonterminal::NtBlock(_)
-            | Nonterminal::NtIdent(..)
-            | Nonterminal::NtLifetime(_) => {}
+            | Nonterminal::NtBlock(_) => {}
         }
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
@@ -82,7 +78,6 @@ impl AstLike for crate::token::Nonterminal {
             Nonterminal::NtPath(path) => path.tokens_mut(),
             Nonterminal::NtVis(vis) => vis.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),
-            Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }
 }

compiler/rustc_ast/src/attr/mod.rs

@@ -408,7 +408,7 @@ impl MetaItem {
         I: Iterator<Item = TokenTree>,
     {
         // FIXME: Share code with `parse_path`.
-        let path = match tokens.next().map(TokenTree::uninterpolate) {
+        let path = match tokens.next() {
             Some(TokenTree::Token(Token {
                 kind: kind @ (token::Ident(..) | token::ModSep),
                 span,
@@ -426,7 +426,7 @@ impl MetaItem {
                 };
                 loop {
                     if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
-                        tokens.next().map(TokenTree::uninterpolate)
+                        tokens.next()
                     {
                         segments.push(PathSegment::from_ident(Ident::new(name, span)));
                     } else {
@@ -449,6 +449,7 @@ impl MetaItem {
             },
             _ => return None,
         };
+        //eprintln!("A1 {:?}", path);
         let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());
         let kind = MetaItemKind::from_tokens(tokens)?;
         let hi = match kind {

compiler/rustc_ast/src/mut_visit.rs

@@ -787,8 +787,6 @@ pub fn visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut
         token::NtPat(pat) => vis.visit_pat(pat),
         token::NtExpr(expr) => vis.visit_expr(expr),
         token::NtTy(ty) => vis.visit_ty(ty),
-        token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
-        token::NtLifetime(ident) => vis.visit_ident(ident),
         token::NtLiteral(expr) => vis.visit_expr(expr),
         token::NtMeta(item) => {
             let AttrItem { path, args, tokens } = item.deref_mut();

compiler/rustc_ast/src/token.rs

@@ -12,7 +12,6 @@ use rustc_macros::HashStable_Generic;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{self, edition::Edition, Span, DUMMY_SP};
-use std::borrow::Cow;
 use std::{fmt, mem};
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@@ -227,14 +226,8 @@ pub enum TokenKind {
     Literal(Lit),
 
     /// Identifier token.
-    /// Do not forget about `NtIdent` when you want to match on identifiers.
-    /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
-    /// treat regular and interpolated identifiers in the same way.
     Ident(Symbol, /* is_raw */ bool),
     /// Lifetime identifier token.
-    /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
-    /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
-    /// treat regular and interpolated lifetime identifiers in the same way.
     Lifetime(Symbol),
 
     Interpolated(Lrc<Nonterminal>),
@@ -364,7 +357,7 @@ impl Token {
 
     /// Returns `true` if the token can appear at the start of an expression.
     pub fn can_begin_expr(&self) -> bool {
-        match self.uninterpolate().kind {
+        match self.kind {
             Ident(name, is_raw)              =>
                 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
             OpenDelim(..)                     | // tuple, array or block
@@ -391,7 +384,7 @@ impl Token {
 
     /// Returns `true` if the token can appear at the start of a type.
     pub fn can_begin_type(&self) -> bool {
-        match self.uninterpolate().kind {
+        match self.kind {
             Ident(name, is_raw)        =>
                 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
             OpenDelim(Delimiter::Parenthesis) | // tuple
@@ -439,7 +432,7 @@ impl Token {
     ///
     /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
     pub fn can_begin_literal_maybe_minus(&self) -> bool {
-        match self.uninterpolate().kind {
+        match self.kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,
             Interpolated(ref nt) => match &**nt {
@@ -457,37 +450,18 @@ impl Token {
         }
     }
 
-    // A convenience function for matching on identifiers during parsing.
-    // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
-    // into the regular identifier or lifetime token it refers to,
-    // otherwise returns the original token.
-    pub fn uninterpolate(&self) -> Cow<'_, Token> {
-        match &self.kind {
-            Interpolated(nt) => match **nt {
-                NtIdent(ident, is_raw) => {
-                    Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
-                }
-                NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
-                _ => Cow::Borrowed(self),
-            },
-            _ => Cow::Borrowed(self),
-        }
-    }
-
     /// Returns an identifier if this token is an identifier.
     pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
-        let token = self.uninterpolate();
-        match token.kind {
-            Ident(name, is_raw) => Some((Ident::new(name, token.span), is_raw)),
+        match self.kind {
+            Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
             _ => None,
         }
     }
 
     /// Returns a lifetime identifier if this token is a lifetime.
     pub fn lifetime(&self) -> Option<Ident> {
-        let token = self.uninterpolate();
-        match token.kind {
-            Lifetime(name) => Some(Ident::new(name, token.span)),
+        match self.kind {
+            Lifetime(name) => Some(Ident::new(name, self.span)),
             _ => None,
         }
     }
@@ -521,7 +495,7 @@ impl Token {
     /// (which happens while parsing the result of macro expansion)?
     pub fn is_whole_expr(&self) -> bool {
         if let Interpolated(ref nt) = self.kind
-            && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt
+            && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
         {
             return true;
         }
@@ -679,8 +653,6 @@ pub enum Nonterminal {
     NtPat(P<ast::Pat>),
     NtExpr(P<ast::Expr>),
     NtTy(P<ast::Ty>),
-    NtIdent(Ident, /* is_raw */ bool),
-    NtLifetime(Ident),
     NtLiteral(P<ast::Expr>),
     /// Stuff inside brackets for attributes
     NtMeta(P<ast::AttrItem>),
@@ -779,7 +751,6 @@ impl Nonterminal {
             NtPat(pat) => pat.span,
             NtExpr(expr) | NtLiteral(expr) => expr.span,
             NtTy(ty) => ty.span,
-            NtIdent(ident, _) | NtLifetime(ident) => ident.span,
             NtMeta(attr_item) => attr_item.span(),
             NtPath(path) => path.span,
             NtVis(vis) => vis.span,
@@ -790,11 +761,7 @@ impl Nonterminal {
 impl PartialEq for Nonterminal {
     fn eq(&self, rhs: &Self) -> bool {
         match (self, rhs) {
-            (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
-                ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
-            }
-            (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
-            // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
+            // FIXME: Assume that all nonterminals are not equal, because we can't compare them
             // correctly based on data from AST. This will prevent them from matching each other
             // in macros. The comparison will become possible only when each nonterminal has an
             // attached token stream from which it was parsed.
@@ -812,12 +779,10 @@ impl fmt::Debug for Nonterminal {
             NtPat(..) => f.pad("NtPat(..)"),
             NtExpr(..) => f.pad("NtExpr(..)"),
             NtTy(..) => f.pad("NtTy(..)"),
-            NtIdent(..) => f.pad("NtIdent(..)"),
             NtLiteral(..) => f.pad("NtLiteral(..)"),
             NtMeta(..) => f.pad("NtMeta(..)"),
             NtPath(..) => f.pad("NtPath(..)"),
             NtVis(..) => f.pad("NtVis(..)"),
-            NtLifetime(..) => f.pad("NtLifetime(..)"),
         }
     }
 }

compiler/rustc_ast/src/tokenstream.rs

@@ -93,13 +93,6 @@ impl TokenTree {
     pub fn token(kind: TokenKind, span: Span) -> TokenTree {
         TokenTree::Token(Token::new(kind, span))
     }
-
-    pub fn uninterpolate(self) -> TokenTree {
-        match self {
-            TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
-            tt => tt,
-        }
-    }
 }
 
 impl<CTX> HashStable<CTX> for TokenStream

compiler/rustc_ast/src/util/literal.rs

@@ -216,7 +216,7 @@ impl Lit {
     ///
     /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
     pub fn from_token(token: &Token) -> Result<Lit, LitError> {
-        let lit = match token.uninterpolate().kind {
+        let lit = match token.kind {
             token::Ident(name, false) if name.is_bool_lit() => {
                 token::Lit::new(token::Bool, name, None)
             }

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -713,8 +713,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::NtBlock(ref e) => self.block_to_string(e),
             token::NtStmt(ref e) => self.stmt_to_string(e),
             token::NtPat(ref e) => self.pat_to_string(e),
-            token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw).to_string(),
-            token::NtLifetime(e) => e.to_string(),
             token::NtLiteral(ref e) => self.expr_to_string(e),
             token::NtVis(ref e) => self.vis_to_string(e),
         }

compiler/rustc_builtin_macros/src/asm.rs

@@ -502,7 +502,7 @@ fn parse_reg<'a>(
     explicit_reg: &mut bool,
 ) -> PResult<'a, ast::InlineAsmRegOrRegClass> {
     p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
-    let result = match p.token.uninterpolate().kind {
+    let result = match p.token.kind {
         token::Ident(name, false) => ast::InlineAsmRegOrRegClass::RegClass(name),
         token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
             *explicit_reg = true;

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -344,6 +344,9 @@ crate enum NamedMatch {
     // A metavar match of type `tt`.
     MatchedTokenTree(rustc_ast::tokenstream::TokenTree),
 
+    // njn: comment
+    MatchedToken(token::Token),
+
     // A metavar match of any type other than `tt`.
     MatchedNonterminal(Lrc<Nonterminal>),
 }
@@ -624,6 +627,7 @@ impl TtParser {
                         let m = match nt {
                             NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)),
                             NtOrTt::Tt(tt) => MatchedTokenTree(tt),
+                            NtOrTt::Token(token) => MatchedToken(token),
                         };
                         mp.push_match(next_metavar, seq_depth, m);
                         mp.idx += 1;

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -1,5 +1,5 @@
 use crate::base::ExtCtxt;
-use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch};
+use crate::mbe::macro_parser::{NamedMatch, NamedMatch::*};
 use crate::mbe::{self, MetaVarExpr};
 use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
@@ -226,16 +226,22 @@ pub(super) fn transcribe<'a>(
                         MatchedTokenTree(ref tt) => {
                             // `tt`s are emitted into the output stream directly as "raw tokens",
                             // without wrapping them into groups.
-                            let token = tt.clone();
-                            result.push(token.into());
+                            let tt = tt.clone();
+                            result.push(tt.into());
+                        }
+                        MatchedToken(ref token) => {
+                            // njn: this doesn't quite work, but adding
+                            // visit_token makes things worse
+                            let tt = TokenTree::Token(token.clone());
+                            result.push(tt.into());
                         }
                         MatchedNonterminal(ref nt) => {
                             // Other variables are emitted into the output stream as groups with
                             // `Delimiter::Invisible` to maintain parsing priorities.
                             // `Interpolated` is currently used for such groups in rustc parser.
                             marker.visit_span(&mut sp);
-                            let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
-                            result.push(token.into());
+                            let tt = TokenTree::token(token::Interpolated(nt.clone()), sp);
+                            result.push(tt.into());
                         }
                         MatchedSeq(..) => {
                             // We were unable to descend far enough. This is an error.
@@ -306,8 +312,8 @@ fn lookup_cur_matched<'a>(
         let mut matched = matched;
         for &(idx, _) in repeats {
             match matched {
-                MatchedTokenTree(_) | MatchedNonterminal(_) => break,
                 MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(),
+                _ => break,
             }
         }
 
@@ -396,8 +402,8 @@ fn lockstep_iter_size(
             let name = MacroRulesNormalizedIdent::new(name);
             match lookup_cur_matched(name, interpolations, repeats) {
                 Some(matched) => match matched {
-                    MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained,
                     MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name),
+                    _ => LockstepIterSize::Unconstrained,
                 },
                 _ => LockstepIterSize::Unconstrained,
             }
@@ -443,18 +449,6 @@ fn count_repetitions<'a>(
         sp: &DelimSpan,
     ) -> PResult<'a, usize> {
         match matched {
-            MatchedTokenTree(_) | MatchedNonterminal(_) => {
-                if declared_lhs_depth == 0 {
-                    return Err(cx.struct_span_err(
-                        sp.entire(),
-                        "`count` can not be placed inside the inner-most repetition",
-                    ));
-                }
-                match depth_opt {
-                    None => Ok(1),
-                    Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")),
-                }
-            }
             MatchedSeq(ref named_matches) => {
                 let new_declared_lhs_depth = declared_lhs_depth + 1;
                 match depth_opt {
@@ -469,6 +463,18 @@ fn count_repetitions<'a>(
                         .sum(),
                 }
             }
+            _ => {
+                if declared_lhs_depth == 0 {
+                    return Err(cx.struct_span_err(
+                        sp.entire(),
+                        "`count` can not be placed inside the inner-most repetition",
+                    ));
+                }
+                match depth_opt {
+                    None => Ok(1),
+                    Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")),
+                }
+            }
         }
     }
     // `repeats` records all of the nested levels at which we are currently