From f8ad40c8542810d5c74d3270173825c5b9f9e947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= Date: Tue, 25 Aug 2026 19:13:31 +0200 Subject: [PATCH 01/11] Replace `IdentIsRaw` with `IdentKind` --- compiler/rustc_ast/src/token.rs | 78 ++++++++-------- compiler/rustc_ast/src/tokenstream.rs | 2 +- compiler/rustc_ast_pretty/src/pprust/state.rs | 22 ++--- .../src/assert/context.rs | 4 +- .../src/deriving/generic/mod.rs | 6 +- compiler/rustc_expand/src/mbe/macro_check.rs | 6 +- compiler/rustc_expand/src/mbe/macro_rules.rs | 10 +- compiler/rustc_expand/src/mbe/metavar_expr.rs | 6 +- compiler/rustc_expand/src/mbe/quoted.rs | 8 +- compiler/rustc_expand/src/mbe/transcribe.rs | 16 ++-- .../rustc_expand/src/proc_macro_server.rs | 20 ++-- compiler/rustc_lint/src/builtin.rs | 4 +- compiler/rustc_parse/src/lexer/mod.rs | 16 ++-- .../rustc_parse/src/lexer/unicode_chars.rs | 2 +- compiler/rustc_parse/src/parser/asm.rs | 4 +- .../rustc_parse/src/parser/diagnostics.rs | 14 +-- compiler/rustc_parse/src/parser/expr.rs | 37 ++++---- compiler/rustc_parse/src/parser/function.rs | 4 +- compiler/rustc_parse/src/parser/item.rs | 18 ++-- compiler/rustc_parse/src/parser/mod.rs | 18 ++-- .../rustc_parse/src/parser/nonterminal.rs | 6 +- compiler/rustc_parse/src/parser/pat.rs | 6 +- compiler/rustc_parse/src/parser/path.rs | 4 +- compiler/rustc_parse/src/parser/stmt.rs | 2 +- compiler/rustc_parse/src/parser/tests.rs | 92 +++++++++---------- .../src/parser/tokenstream/tests.rs | 8 +- compiler/rustc_parse/src/parser/ty.rs | 6 +- src/librustdoc/clean/render_macro_matchers.rs | 4 +- 28 files changed, 215 insertions(+), 208 deletions(-) diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 0144aabb24413..f9ef464400f6c 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -231,7 +231,9 @@ impl Lit { /// `Parser::eat_token_lit` (excluding unary negation). pub fn from_token(token: &Token) -> Option { match token.uninterpolate().kind { - Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)), + Ident(name, IdentKind::Normal) if name.is_bool_lit() => { + Some(Lit::new(Bool, name, None)) + } Literal(token_lit) => Some(token_lit), OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Literal | MetaVarKind::Expr { .. }, @@ -307,11 +309,11 @@ impl LitKind { } } -pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool { +pub fn ident_can_begin_expr(name: Symbol, span: Span, kind: IdentKind) -> bool { // WARNING: Take care when modifying this function! It will change the stable(!) set of // tokens that are allowed to match an `expr` nonterminal which is user observable. - let ident_token = Token::new(Ident(name, is_raw), span); + let ident_token = Token::new(Ident(name, kind), span); // FIXME: Remove `box` from this list given we officially no longer support box expressions // (#108471) (needs lang FCP as it affects stable macro matching behavior). @@ -344,11 +346,11 @@ pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: IdentIsRaw) -> boo .contains(&name) } -fn ident_can_begin_type(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool { +fn ident_can_begin_type(name: Symbol, span: Span, kind: IdentKind) -> bool { // WARNING: Take care when modifying this function! It will change the stable(!) set of // tokens that are allowed to match an `ty` nonterminal which is user observable. - let ident_token = Token::new(Ident(name, is_raw), span); + let ident_token = Token::new(Ident(name, kind), span); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() @@ -357,29 +359,29 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool { } #[derive(PartialEq, Eq, Encodable, Decodable, Hash, Debug, Copy, Clone, StableHash)] -pub enum IdentIsRaw { - No, - Yes, +pub enum IdentKind { + Normal, + Raw, } -impl IdentIsRaw { +impl IdentKind { pub fn to_print_mode_ident(self) -> IdentPrintMode { match self { - IdentIsRaw::No => IdentPrintMode::Normal, - IdentIsRaw::Yes => IdentPrintMode::RawIdent, + IdentKind::Normal => IdentPrintMode::Normal, + IdentKind::Raw => IdentPrintMode::RawIdent, } } pub fn to_print_mode_lifetime(self) -> IdentPrintMode { match self { - IdentIsRaw::No => IdentPrintMode::Normal, - IdentIsRaw::Yes => IdentPrintMode::RawLifetime, + IdentKind::Normal => IdentPrintMode::Normal, + IdentKind::Raw => IdentPrintMode::RawLifetime, } } } -impl From for IdentIsRaw { +impl From for IdentKind { fn from(b: bool) -> Self { - if b { Self::Yes } else { Self::No } + if b { Self::Raw } else { Self::Normal } } } @@ -507,22 +509,22 @@ pub enum TokenKind { /// It's recommended to use `Token::{ident,uninterpolate}` and /// `Parser::token_uninterpolated_span` to treat regular and interpolated /// identifiers in the same way. - Ident(Symbol, IdentIsRaw), + Ident(Symbol, IdentKind), /// This identifier (and its span) is the identifier passed to the /// declarative macro. The span in the surrounding `Token` is the span of /// the `ident` metavariable in the macro's RHS. - NtIdent(sp::Ident, IdentIsRaw), + NtIdent(sp::Ident, IdentKind), /// Lifetime identifier token. /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. /// It's recommended to use `Token::{ident,uninterpolate}` and /// `Parser::token_uninterpolated_span` to treat regular and interpolated /// identifiers in the same way. - Lifetime(Symbol, IdentIsRaw), + Lifetime(Symbol, IdentKind), /// This identifier (and its span) is the lifetime passed to the /// declarative macro. The span in the surrounding `Token` is the span of /// the `lifetime` metavariable in the macro's RHS. - NtLifetime(sp::Ident, IdentIsRaw), + NtLifetime(sp::Ident, IdentKind), /// A doc comment token. /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) @@ -674,8 +676,8 @@ impl Token { // tokens that are allowed to match an `expr` nonterminal which is user observable. match self.uninterpolate().kind { - Ident(name, is_raw) => - ident_can_begin_expr(name, self.span, is_raw), // value name or keyword + Ident(name, kind) => + ident_can_begin_expr(name, self.span, kind), // value name or keyword OpenParen | // tuple OpenBrace | // block OpenBracket | // array @@ -743,8 +745,8 @@ impl Token { // object types (consider `use<>+` and `use + Trait` for example). match self.uninterpolate().kind { - Ident(name, is_raw) => - ident_can_begin_type(name, self.span, is_raw), // type name or keyword + Ident(name, kind) => + ident_can_begin_type(name, self.span, kind), // type name or keyword OpenParen // tuple | OpenBracket // array | Bang // never @@ -768,7 +770,7 @@ impl Token { pub fn can_begin_const_arg(&self) -> bool { match self.kind { OpenBrace | Literal(..) | Minus => true, - Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, + Ident(name, IdentKind::Normal) if name.is_bool_lit() => true, OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal, )) => true, @@ -817,7 +819,7 @@ impl Token { pub fn can_begin_literal_maybe_minus(&self) -> bool { match self.uninterpolate().kind { Literal(..) | Minus => true, - Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, + Ident(name, IdentKind::Normal) if name.is_bool_lit() => true, OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind { MetaVarKind::Literal => true, MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => { @@ -847,9 +849,9 @@ impl Token { /// otherwise returns the original token. pub fn uninterpolate(&self) -> Cow<'_, Token> { match self.kind { - NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)), - NtLifetime(ident, is_raw) => { - Cow::Owned(Token::new(Lifetime(ident.name, is_raw), ident.span)) + NtIdent(ident, kind) => Cow::Owned(Token::new(Ident(ident.name, kind), ident.span)), + NtLifetime(ident, kind) => { + Cow::Owned(Token::new(Lifetime(ident.name, kind), ident.span)) } _ => Cow::Borrowed(self), } @@ -857,22 +859,22 @@ impl Token { /// Returns an identifier if this token is an identifier. #[inline] - pub fn ident(&self) -> Option<(sp::Ident, IdentIsRaw)> { + pub fn ident(&self) -> Option<(sp::Ident, IdentKind)> { // We avoid using `Token::uninterpolate` here because it's slow. match self.kind { - Ident(name, is_raw) => Some((sp::Ident::new(name, self.span), is_raw)), - NtIdent(ident, is_raw) => Some((ident, is_raw)), + Ident(name, kind) => Some((sp::Ident::new(name, self.span), kind)), + NtIdent(ident, kind) => Some((ident, kind)), _ => None, } } /// Returns a lifetime identifier if this token is a lifetime. #[inline] - pub fn lifetime(&self) -> Option<(sp::Ident, IdentIsRaw)> { + pub fn lifetime(&self) -> Option<(sp::Ident, IdentKind)> { // We avoid using `Token::uninterpolate` here because it's slow. match self.kind { - Lifetime(name, is_raw) => Some((sp::Ident::new(name, self.span), is_raw)), - NtLifetime(ident, is_raw) => Some((ident, is_raw)), + Lifetime(name, kind) => Some((sp::Ident::new(name, self.span), kind)), + NtLifetime(ident, kind) => Some((ident, kind)), _ => None, } } @@ -971,7 +973,7 @@ impl Token { } pub fn is_non_reserved_ident(&self) -> bool { - self.ident().is_some_and(|(id, raw)| raw == IdentIsRaw::Yes || !sp::Ident::is_reserved(id)) + self.ident().is_some_and(|(id, kind)| kind == IdentKind::Raw || !sp::Ident::is_reserved(id)) } /// Returns `true` if the token is the identifier `true` or `false`. @@ -994,7 +996,7 @@ impl Token { /// Returns `true` if the token is a non-raw identifier for which `pred` holds. pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(sp::Ident) -> bool) -> bool { match self.ident() { - Some((id, IdentIsRaw::No)) => pred(id), + Some((id, IdentKind::Normal)) => pred(id), _ => false, } } @@ -1072,8 +1074,8 @@ impl Token { (Colon, Colon) => PathSep, (Colon, _) => return None, - (SingleQuote, Ident(name, is_raw)) => { - Lifetime(Symbol::intern(&format!("'{name}")), *is_raw) + (SingleQuote, Ident(name, kind)) => { + Lifetime(Symbol::intern(&format!("'{name}")), *kind) } (SingleQuote, _) => return None, diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index df71aad0111cd..164d1cb8fd731 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -793,7 +793,7 @@ impl TokenStream { DelimSpacing::new(Spacing::JointHidden, Spacing::Alone), Delimiter::Bracket, [ - TokenTree::token_alone(token::Ident(sym::doc, token::IdentIsRaw::No), span), + TokenTree::token_alone(token::Ident(sym::doc, token::IdentKind::Normal), span), TokenTree::token_alone(token::Eq, span), TokenTree::token_alone( TokenKind::lit(token::StrRaw(num_of_hashes), data, None), diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 977eb0ee4592d..d97bf7a2a6db3 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -364,20 +364,18 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool { // IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if` ( - Tok(tk::Token { kind: tk::Ident(sym, is_raw), span }, _), + Tok(tk::Token { kind: tk::Ident(sym, kind), span }, _), Tok(tk::Token { kind: tk::Bang, .. }, _), - ) if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, tk::IdentIsRaw::Yes) => { - false - } + ) if !Ident::new(*sym, *span).is_reserved() || matches!(kind, tk::IdentKind::Raw) => false, // IDENT|`fn`|`Self`|`pub` + `(`: `f(3)`, `fn(x: u8)`, `Self()`, `pub(crate)`, // but `let (a, b) = (1, 2)` needs a space after the `let` - (Tok(tk::Token { kind: tk::Ident(sym, is_raw), span }, _), Del(_, _, Parenthesis, _)) + (Tok(tk::Token { kind: tk::Ident(sym, kind), span }, _), Del(_, _, Parenthesis, _)) if !Ident::new(*sym, *span).is_reserved() || *sym == kw::Fn || *sym == kw::SelfUpper || *sym == kw::Pub - || matches!(is_raw, tk::IdentIsRaw::Yes) => + || matches!(kind, tk::IdentKind::Raw) => { false } @@ -1076,17 +1074,17 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere tk::Literal(lit) => literal_to_string(lit).into(), /* Name components */ - tk::Ident(name, is_raw) => { - IdentPrinter::new(name, is_raw.to_print_mode_ident(), convert_dollar_crate) + tk::Ident(name, kind) => { + IdentPrinter::new(name, kind.to_print_mode_ident(), convert_dollar_crate) .to_string() .into() } - tk::NtIdent(ident, is_raw) => { - IdentPrinter::for_ast_ident(ident, is_raw.to_print_mode_ident()).to_string().into() + tk::NtIdent(ident, kind) => { + IdentPrinter::for_ast_ident(ident, kind.to_print_mode_ident()).to_string().into() } - tk::Lifetime(name, is_raw) | tk::NtLifetime(Ident { name, .. }, is_raw) => { - IdentPrinter::new(name, is_raw.to_print_mode_lifetime(), None).to_string().into() + tk::Lifetime(name, kind) | tk::NtLifetime(Ident { name, .. }, kind) => { + IdentPrinter::new(name, kind.to_print_mode_lifetime(), None).to_string().into() } /* Other */ diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs index 80ea2e3d877fc..a76d90b13c9d8 100644 --- a/compiler/rustc_builtin_macros/src/assert/context.rs +++ b/compiler/rustc_builtin_macros/src/assert/context.rs @@ -1,4 +1,4 @@ -use rustc_ast::token::{self, Delimiter, IdentIsRaw}; +use rustc_ast::token::{self, Delimiter, IdentKind}; use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_ast::{ BinOpKind, BorrowKind, DUMMY_NODE_ID, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MethodCall, @@ -165,7 +165,7 @@ impl<'cx, 'a> Context<'cx, 'a> { let captures = self.capture_decls.iter().flat_map(|cap| { [ TokenTree::token_joint( - token::Ident(cap.ident.name, IdentIsRaw::No), + token::Ident(cap.ident.name, IdentKind::Normal), cap.ident.span, ), TokenTree::token_alone(token::Comma, self.span), diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs index 6a53dafd396df..dc01a834a6d25 100644 --- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs +++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs @@ -179,7 +179,7 @@ use std::{iter, vec}; pub(crate) use StaticFields::*; pub(crate) use SubstructureFields::*; -use rustc_ast::token::{IdentIsRaw, LitKind, Token, TokenKind}; +use rustc_ast::token::{IdentKind, LitKind, Token, TokenKind}; use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenTree}; use rustc_ast::{ self as ast, AnonConst, AttrArgs, BindingMode, ByRef, DelimArgs, EnumDef, Expr, GenericArg, @@ -807,11 +807,11 @@ impl<'a> TraitDef<'a> { dspan: DelimSpan::from_single(self.span), delim: rustc_ast::token::Delimiter::Parenthesis, tokens: [ - TokenKind::Ident(sym::feature, IdentIsRaw::No), + TokenKind::Ident(sym::feature, IdentKind::Normal), TokenKind::Eq, TokenKind::lit(LitKind::Str, sym::derive_const, None), TokenKind::Comma, - TokenKind::Ident(sym::issue, IdentIsRaw::No), + TokenKind::Ident(sym::issue, IdentKind::Normal), TokenKind::Eq, TokenKind::lit(LitKind::Str, sym::derive_const_issue, None), ] diff --git a/compiler/rustc_expand/src/mbe/macro_check.rs b/compiler/rustc_expand/src/mbe/macro_check.rs index 7ab595dc73d89..e28044d7632a9 100644 --- a/compiler/rustc_expand/src/mbe/macro_check.rs +++ b/compiler/rustc_expand/src/mbe/macro_check.rs @@ -105,7 +105,7 @@ //! stored when entering a macro definition starting from the state in which the meta-variable is //! bound. -use rustc_ast::token::{Delimiter, IdentIsRaw, Token, TokenKind}; +use rustc_ast::token::{Delimiter, IdentKind, Token, TokenKind}; use rustc_ast::{DUMMY_NODE_ID, NodeId}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::DecorateDiagCompat; @@ -396,7 +396,9 @@ fn check_nested_occurrences( match (state, tt) { ( NestedMacroState::Empty, - &TokenTree::Token(Token { kind: TokenKind::Ident(name, IdentIsRaw::No), .. }), + &TokenTree::Token(Token { + kind: TokenKind::Ident(name, IdentKind::Normal), .. + }), ) => { if name == kw::MacroRules { state = NestedMacroState::MacroRules; diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index b268b8b767327..68c4eec89d7fd 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -3,7 +3,7 @@ use std::collections::hash_map::Entry; use std::sync::Arc; use std::{mem, slice}; -use ast::token::IdentIsRaw; +use ast::token::IdentKind; use rustc_ast::token::NtPatKind::*; use rustc_ast::token::TokenKind::*; use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind}; @@ -1753,7 +1753,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { match tok { TokenTree::Token(token) => match token.kind { FatArrow | Comma | Eq | Or => IsInFollow::Yes, - Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => { + Ident(name, IdentKind::Normal) if name == kw::If || name == kw::In => { IsInFollow::Yes } _ => IsInFollow::No(TOKENS), @@ -1767,7 +1767,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { match tok { TokenTree::Token(token) => match token.kind { FatArrow | Comma | Eq => IsInFollow::Yes, - Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => { + Ident(name, IdentKind::Normal) if name == kw::If || name == kw::In => { IsInFollow::Yes } _ => IsInFollow::No(TOKENS), @@ -1795,7 +1795,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { TokenTree::Token(token) => match token.kind { OpenBrace | OpenBracket | Comma | FatArrow | Colon | Eq | Gt | Shr | Semi | Or => IsInFollow::Yes, - Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => { + Ident(name, IdentKind::Normal) if name == kw::As || name == kw::Where => { IsInFollow::Yes } _ => IsInFollow::No(TOKENS), @@ -1823,7 +1823,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { match tok { TokenTree::Token(token) => match token.kind { Comma => IsInFollow::Yes, - Ident(_, IdentIsRaw::Yes) => IsInFollow::Yes, + Ident(_, IdentKind::Raw) => IsInFollow::Yes, Ident(name, _) if name != kw::Priv => IsInFollow::Yes, _ => { if token.can_begin_type() { diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs index a02b84204cb39..76a8d0497b1ad 100644 --- a/compiler/rustc_expand/src/mbe/metavar_expr.rs +++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs @@ -1,4 +1,4 @@ -use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, Token, TokenKind}; +use rustc_ast::token::{self, Delimiter, IdentKind, Lit, Token, TokenKind}; use rustc_ast::tokenstream::{TokenStream, TokenStreamIter, TokenTree}; use rustc_ast::{LitIntType, LitKind}; use rustc_ast_pretty::pprust; @@ -272,8 +272,8 @@ fn parse_ident_from_token<'psess>( psess: &'psess ParseSess, token: &Token, ) -> PResult<'psess, Ident> { - if let Some((elem, is_raw)) = token.ident() { - if let IdentIsRaw::Yes = is_raw { + if let Some((elem, kind)) = token.ident() { + if let IdentKind::Raw = kind { return Err(psess.dcx().struct_span_err(elem.span, RAW_IDENT_ERR)); } return Ok(elem); diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index 2779291abf361..aed69c9f5d938 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -1,4 +1,4 @@ -use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, Token}; +use rustc_ast::token::{self, Delimiter, IdentKind, NonterminalKind, Token}; use rustc_ast::tokenstream::TokenStreamIter; use rustc_ast::{NodeId, tokenstream}; use rustc_ast_pretty::pprust; @@ -325,10 +325,10 @@ fn parse_tree<'a>( // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` // special metavariable that names the crate of the invocation. Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => { - let (ident, is_raw) = token.ident().unwrap(); + let (ident, kind) = token.ident().unwrap(); let span = ident.span.with_lo(dollar_span.lo()); - if ident.name == kw::Crate && matches!(is_raw, IdentIsRaw::No) { - TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) + if ident.name == kw::Crate && matches!(kind, IdentKind::Normal) { + TokenTree::token(token::Ident(kw::DollarCrate, kind), span) } else { TokenTree::MetaVar(span, ident) } diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index eabec05cd66c6..b9573808caa8f 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -1,7 +1,7 @@ use std::mem; use rustc_ast::token::{ - self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Token, TokenKind, + self, Delimiter, IdentKind, InvisibleOrigin, Lit, LitKind, MetaVarKind, Token, TokenKind, }; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast::{ExprKind, StmtKind, TyKind, UnOp}; @@ -489,10 +489,10 @@ fn transcribe_pnr<'tx>( // parsing priorities. maybe_use_metavar_location(tscx.psess, &tscx.stack, sp, tt, &mut tscx.marker) } - ParseNtResult::Ident(ident, is_raw) => { + ParseNtResult::Ident(ident, kind) => { tscx.marker.mark_span(&mut sp); with_metavar_spans(|mspans| mspans.insert(ident.span, sp)); - let kind = token::NtIdent(*ident, *is_raw); + let kind = token::NtIdent(*ident, *kind); TokenTree::token_alone(kind, sp) } ParseNtResult::Lifetime(ident, is_raw) => { @@ -566,7 +566,7 @@ fn transcribe_pnr<'tx>( let leading_if_span = guard.span_with_leading_if.with_hi(guard.span_with_leading_if.lo() + BytePos(2)); let ts = std::iter::once(TokenTree::token_alone( - token::Ident(kw::If, IdentIsRaw::No), + token::Ident(kw::If, IdentKind::Normal), leading_if_span, )) .chain(TokenStream::from_ast(&guard.cond).iter().cloned()) @@ -995,18 +995,18 @@ fn extract_symbol_from_pnr<'a>( span_err: Span, ) -> PResult<'a, Symbol> { match pnr { - ParseNtResult::Ident(nt_ident, is_raw) => { - if let IdentIsRaw::Yes = is_raw { + ParseNtResult::Ident(nt_ident, kind) => { + if let IdentKind::Raw = kind { Err(dcx.struct_span_err(span_err, RAW_IDENT_ERR)) } else { Ok(nt_ident.name) } } ParseNtResult::Tt(TokenTree::Token( - Token { kind: TokenKind::Ident(symbol, is_raw), .. }, + Token { kind: TokenKind::Ident(symbol, kind), .. }, _, )) => { - if let IdentIsRaw::Yes = is_raw { + if let IdentKind::Raw = kind { Err(dcx.struct_span_err(span_err, RAW_IDENT_ERR)) } else { Ok(*symbol) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index c522626b39562..b648e1277a5e0 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -228,31 +228,31 @@ impl FromInternal for Vec> { tk::Question => op("?"), tk::SingleQuote => op("'"), - tk::Ident(sym, is_raw) => trees.push(TokenTree::Ident(Ident { + tk::Ident(sym, kind) => trees.push(TokenTree::Ident(Ident { sym, - is_raw: matches!(is_raw, tk::IdentIsRaw::Yes), + is_raw: matches!(kind, tk::IdentKind::Raw), span, })), - tk::NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident { + tk::NtIdent(ident, kind) => trees.push(TokenTree::Ident(Ident { sym: ident.name, - is_raw: matches!(is_raw, tk::IdentIsRaw::Yes), + is_raw: matches!(kind, tk::IdentKind::Raw), span: ident.span, })), - tk::Lifetime(name, is_raw) => { + tk::Lifetime(name, kind) => { let ident = rustc_span::Ident::new(name, span).without_first_quote(); trees.extend([ TokenTree::Punct(Punct { ch: b'\'', joint: true, span }), TokenTree::Ident(Ident { sym: ident.name, - is_raw: matches!(is_raw, tk::IdentIsRaw::Yes), + is_raw: matches!(kind, tk::IdentKind::Raw), span, }), ]); } - tk::NtLifetime(ident, is_raw) => { + tk::NtLifetime(ident, kind) => { let stream = - TokenStream::token_alone(tk::Lifetime(ident.name, is_raw), ident.span); + TokenStream::token_alone(tk::Lifetime(ident.name, kind), ident.span); trees.push(TokenTree::Group(Group { delimiter: rustc_proc_macro::Delimiter::None, stream: Some(stream), @@ -274,7 +274,7 @@ impl FromInternal for Vec> { escaped.extend(ch.escape_debug()); } let stream = [ - tk::Ident(sym::doc, tk::IdentIsRaw::No), + tk::Ident(sym::doc, tk::IdentKind::Normal), tk::Eq, tk::TokenKind::lit(tk::Str, Symbol::intern(&escaped), None), ] @@ -613,7 +613,7 @@ impl server::Server for Rustc<'_, '_> { match &expr.kind { ast::ExprKind::Lit(token_lit) if token_lit.kind == tk::Bool => { Ok(tokenstream::TokenStream::token_alone( - tk::Ident(token_lit.symbol, tk::IdentIsRaw::No), + tk::Ident(token_lit.symbol, tk::IdentKind::Normal), expr.span, )) } diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs index a2112293df204..46f9cb60c6f1f 100644 --- a/compiler/rustc_lint/src/builtin.rs +++ b/compiler/rustc_lint/src/builtin.rs @@ -1758,11 +1758,11 @@ impl KeywordIdents { match tt { // Only report non-raw idents. TokenTree::Token(token, _) => { - if let Some((ident, token::IdentIsRaw::No)) = token.ident() { + if let Some((ident, token::IdentKind::Normal)) = token.ident() { if !prev_dollar { self.check_ident_token(cx, UnderMacro(true), ident, ""); } - } else if let Some((ident, token::IdentIsRaw::No)) = token.lifetime() { + } else if let Some((ident, token::IdentKind::Normal)) = token.lifetime() { self.check_ident_token( cx, UnderMacro(true), diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 6ed61a9f4e01d..76750584912b2 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -1,6 +1,6 @@ use diagnostics::make_errors_for_mismatched_closing_delims; use rustc_ast::ast::{self, AttrStyle}; -use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Token, TokenKind}; +use rustc_ast::token::{self, CommentKind, Delimiter, IdentKind, Token, TokenKind}; use rustc_ast::tokenstream::TokenStream; use rustc_ast::util::unicode::{TEXT_FLOW_CONTROL_CHARS, contains_text_flow_control_chars}; use rustc_errors::codes::*; @@ -238,7 +238,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { self.dcx().emit_err(crate::diagnostics::CannotBeRawIdent { span, ident: sym }); } self.psess.raw_identifier_spans.push(span); - token::Ident(sym, IdentIsRaw::Yes) + token::Ident(sym, IdentKind::Raw) } rustc_lexer::TokenKind::UnknownPrefix => { self.report_unknown_prefix(start); @@ -252,7 +252,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { let lifetime_name = self.str_from(start); self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1))); let ident = Symbol::intern(lifetime_name); - token::Lifetime(ident, IdentIsRaw::No) + token::Lifetime(ident, IdentKind::Normal) } rustc_lexer::TokenKind::InvalidIdent // Do not recover an identifier with emoji if the codepoint is a confusable @@ -270,7 +270,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { .entry(sym) .or_default() .push(span); - token::Ident(sym, IdentIsRaw::No) + token::Ident(sym, IdentKind::Normal) } // split up (raw) c string literals to an ident and a string literal when edition < // 2021. @@ -337,7 +337,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { .with_span(span) .stash(span, StashKey::LifetimeIsChar); } - token::Lifetime(lifetime_name, IdentIsRaw::No) + token::Lifetime(lifetime_name, IdentKind::Normal) } rustc_lexer::TokenKind::RawLifetime => { self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1))); @@ -387,7 +387,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { // Make sure we mark this as a raw identifier. self.psess.raw_identifier_spans.push(span); - token::Lifetime(sym, IdentIsRaw::Yes) + token::Lifetime(sym, IdentKind::Raw) } else { // Reset the state so we just lex the `'r`. self.pos = start + BytePos(2); @@ -407,7 +407,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { ); let lifetime_name = nfc_normalize(self.str_from(start)); - token::Lifetime(lifetime_name, IdentIsRaw::No) + token::Lifetime(lifetime_name, IdentKind::Normal) } } rustc_lexer::TokenKind::Semi => token::Semi, @@ -497,7 +497,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { let sym = nfc_normalize(self.str_from(start)); let span = self.mk_sp(start, self.pos); self.psess.symbol_gallery.insert(sym, span); - token::Ident(sym, IdentIsRaw::No) + token::Ident(sym, IdentKind::Normal) } /// Detect usages of Unicode codepoints changing the direction of the text on screen and loudly diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs index 826d193634a45..a934bb62d6a9c 100644 --- a/compiler/rustc_parse/src/lexer/unicode_chars.rs +++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs @@ -307,7 +307,7 @@ pub(super) static UNICODE_ARRAY: &[(char, &str, &str)] = &[ // fancier error recovery to it, as there will be less overall work to do this way. const ASCII_ARRAY: &[(&str, &str, Option)] = &[ (" ", "Space", None), - ("_", "Underscore", Some(token::Ident(kw::Underscore, token::IdentIsRaw::No))), + ("_", "Underscore", Some(token::Ident(kw::Underscore, token::IdentKind::Normal))), ("-", "Minus/Hyphen", Some(token::Minus)), (",", "Comma", Some(token::Comma)), (";", "Semicolon", Some(token::Semi)), diff --git a/compiler/rustc_parse/src/parser/asm.rs b/compiler/rustc_parse/src/parser/asm.rs index 5c177af086565..92b0b96fab165 100644 --- a/compiler/rustc_parse/src/parser/asm.rs +++ b/compiler/rustc_parse/src/parser/asm.rs @@ -1,7 +1,7 @@ use rustc_ast::{self as ast, AsmMacro}; use rustc_span::{Span, Symbol, kw}; -use super::{ExpKeywordPair, ForceCollect, IdentIsRaw, Trailing, UsePreAttrPos}; +use super::{ExpKeywordPair, ForceCollect, IdentKind, Trailing, UsePreAttrPos}; use crate::{PResult, Parser, diagnostics, exp, token}; /// An argument to one of the `asm!` macros. The argument is syntactically valid, but is otherwise @@ -368,7 +368,7 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>) -> PResult<'a, Vec<(Symbol, Span)>> fn parse_reg<'a>(p: &mut Parser<'a>) -> PResult<'a, ast::InlineAsmRegOrRegClass> { p.expect(exp!(OpenParen))?; let result = match p.token.uninterpolate().kind { - token::Ident(name, IdentIsRaw::No) => ast::InlineAsmRegOrRegClass::RegClass(name), + token::Ident(name, IdentKind::Normal) => ast::InlineAsmRegOrRegClass::RegClass(name), token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => { ast::InlineAsmRegOrRegClass::Reg(symbol) } diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 0b91828639d36..2d8f52460542f 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -1,7 +1,7 @@ use std::mem::take; use std::ops::{Deref, DerefMut}; -use ast::token::IdentIsRaw; +use ast::token::IdentKind; use rustc_ast::token::{self, Lit, LitKind, Token, TokenKind}; use rustc_ast::util::parser::AssocOp; use rustc_ast::{ @@ -275,7 +275,7 @@ impl<'a> Parser<'a> { pub(super) fn expected_ident_found( &mut self, recover: bool, - ) -> PResult<'a, (Ident, IdentIsRaw)> { + ) -> PResult<'a, (Ident, IdentKind)> { let valid_follow = &[ TokenKind::Eq, TokenKind::Colon, @@ -303,11 +303,11 @@ impl<'a> Parser<'a> { let bad_token = self.token; // suggest prepending a keyword in identifier position with `r#` - let suggest_raw = if let Some((ident, IdentIsRaw::No)) = self.token.ident() + let suggest_raw = if let Some((ident, IdentKind::Normal)) = self.token.ident() && ident.is_raw_guess() && self.look_ahead(1, |t| valid_follow.contains(&t.kind)) { - recovered_ident = Some((ident, IdentIsRaw::Yes)); + recovered_ident = Some((ident, IdentKind::Raw)); // `Symbol::to_string()` is different from `Symbol::into_diag_arg()`, // which uses `Symbol::to_ident_string()` and "helpfully" adds an implicit `r#` @@ -333,7 +333,7 @@ impl<'a> Parser<'a> { let help_cannot_start_number = self.is_lit_bad_ident().map(|(len, valid_portion)| { let (invalid, valid) = self.token.span.split_at(len as u32); - recovered_ident = Some((Ident::new(valid_portion, valid), IdentIsRaw::No)); + recovered_ident = Some((Ident::new(valid_portion, valid), IdentKind::Normal)); HelpIdentifierStartsWithNumber { num_span: invalid } }); @@ -637,9 +637,9 @@ impl<'a> Parser<'a> { // positive for a `cr#` that wasn't intended to start a c-string literal, but identifying // that in the parser requires unbounded lookahead, so we only add a hint to the existing // error rather than replacing it entirely. - if ((self.prev_token == TokenKind::Ident(sym::character('c'), IdentIsRaw::No) + if ((self.prev_token == TokenKind::Ident(sym::character('c'), IdentKind::Normal) && matches!(&self.token.kind, TokenKind::Literal(token::Lit { kind: token::Str, .. }))) - || (self.prev_token == TokenKind::Ident(sym::cr, IdentIsRaw::No) + || (self.prev_token == TokenKind::Ident(sym::cr, IdentKind::Normal) && matches!( &self.token.kind, TokenKind::Literal(token::Lit { kind: token::Str, .. }) | token::Pound diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 12de4957e99c2..bf53457c07ce8 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -4,7 +4,7 @@ use core::mem; use core::ops::{Bound, ControlFlow}; use ast::mut_visit::{self, MutVisitor}; -use ast::token::IdentIsRaw; +use ast::token::IdentKind; use ast::{ForLoopKind, MatchKind, Pat, Path, PathSegment, Recovered}; use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, Token, TokenKind}; use rustc_ast::util::case::Case; @@ -374,7 +374,7 @@ impl<'a> Parser<'a> { return None; } (Some(op), _) => (op, self.token.span), - (None, Some((Ident { name: sym::and, span }, IdentIsRaw::No))) + (None, Some((Ident { name: sym::and, span }, IdentKind::Normal))) if self.may_recover() => { self.dcx().emit_err(diagnostics::InvalidLogicalOperator { @@ -384,7 +384,9 @@ impl<'a> Parser<'a> { }); (AssocOp::Binary(BinOpKind::And), span) } - (None, Some((Ident { name: sym::or, span }, IdentIsRaw::No))) if self.may_recover() => { + (None, Some((Ident { name: sym::or, span }, IdentKind::Normal))) + if self.may_recover() => + { self.dcx().emit_err(diagnostics::InvalidLogicalOperator { span: self.token.span, incorrect: "or".into(), @@ -593,7 +595,7 @@ impl<'a> Parser<'a> { let token_cannot_continue_expr = |t: &Token| match t.uninterpolate().kind { // These tokens can start an expression after `!`, but // can't continue an expression after an ident - token::Ident(name, is_raw) => token::ident_can_begin_expr(name, t.span, is_raw), + token::Ident(name, kind) => token::ident_can_begin_expr(name, t.span, kind), token::Literal(..) | token::Pound => true, _ => t.is_metavar_expr(), }; @@ -671,7 +673,7 @@ impl<'a> Parser<'a> { ( // `foo: ` ExprKind::Path(None, ast::Path { segments, .. }), - token::Ident(kw::For | kw::Loop | kw::While, IdentIsRaw::No), + token::Ident(kw::For | kw::Loop | kw::While, IdentKind::Normal), ) if let [segment] = segments.as_slice() => { let snapshot = self.create_snapshot_for_diagnostic(); let label = Label { @@ -875,7 +877,8 @@ impl<'a> Parser<'a> { lo: Span, ) -> PResult<'a, Box> { let mut res = loop { - let has_question = if self.prev_token == TokenKind::Ident(kw::Return, IdentIsRaw::No) { + let has_question = if self.prev_token == TokenKind::Ident(kw::Return, IdentKind::Normal) + { // We are using noexpect here because we don't expect a `?` directly after // a `return` which could be suggested otherwise. self.eat_noexpect(&token::Question) @@ -887,7 +890,7 @@ impl<'a> Parser<'a> { e = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Try(e)); continue; } - let has_dot = if self.prev_token == TokenKind::Ident(kw::Return, IdentIsRaw::No) { + let has_dot = if self.prev_token == TokenKind::Ident(kw::Return, IdentKind::Normal) { // We are using noexpect here because we don't expect a `.` directly after // a `return` which could be suggested otherwise. self.eat_noexpect(&token::Dot) @@ -957,7 +960,7 @@ impl<'a> Parser<'a> { // We end up with the `sym` (`1`) token in `self.prev_token` and a dot in // `self.token`. assert!(suffix.is_none()); - self.token = Token::new(token::Ident(sym, IdentIsRaw::No), ident_span); + self.token = Token::new(token::Ident(sym, IdentKind::Normal), ident_span); self.bump_with((Token::new(token::Dot, dot_span), self.token_spacing)); self.mk_expr_tuple_field_access(lo, ident_span, base, sym, None) } @@ -973,7 +976,7 @@ impl<'a> Parser<'a> { // the `sym2` (`2` or `2e3`) token in `self.prev_token` and the following // token in `self.token`. let next_token2 = - Token::new(token::Ident(sym2, IdentIsRaw::No), ident2_span); + Token::new(token::Ident(sym2, IdentKind::Normal), ident2_span); self.bump_with((next_token2, self.token_spacing)); self.bump(); let base1 = @@ -2027,7 +2030,7 @@ impl<'a> Parser<'a> { self.bump(); // `builtin` self.bump(); // `#` - let Some((ident, IdentIsRaw::No)) = self.token.ident() else { + let Some((ident, IdentKind::Normal)) = self.token.ident() else { let err = self.dcx().create_err(diagnostics::ExpectedBuiltinIdent { span: self.token.span }); return Err(err); @@ -2141,7 +2144,7 @@ impl<'a> Parser<'a> { }; // On an error path, eagerly consider a lifetime to be an unclosed character lit, if that // makes sense. - if let Some((ident, IdentIsRaw::No)) = self.token.lifetime() + if let Some((ident, IdentKind::Normal)) = self.token.lifetime() && could_be_unclosed_char_literal(ident) { let lt = self.expect_lifetime(); @@ -2213,7 +2216,7 @@ impl<'a> Parser<'a> { } }; match self.token.uninterpolate().kind { - token::Ident(name, IdentIsRaw::No) if name.is_bool_lit() => { + token::Ident(name, IdentKind::Normal) if name.is_bool_lit() => { self.bump(); Some(token::Lit::new(token::Bool, name, None)) } @@ -3163,9 +3166,9 @@ impl<'a> Parser<'a> { } pub(crate) fn eat_label(&mut self) -> Option