diff --git a/crates/parser/src/event.rs b/crates/parser/src/event.rs index 001adfb78093..9b000b7cec58 100644 --- a/crates/parser/src/event.rs +++ b/crates/parser/src/event.rs @@ -128,8 +128,6 @@ pub(super) fn process(mut events: Vec, mut errors: Vec) -> Output } Event::FloatSplitHack { ends_in_dot } => { res.float_split_hack(ends_in_dot); - let ev = mem::replace(&mut events[i + 1], Event::tombstone()); - assert!(matches!(ev, Event::Finish), "{ev:?}"); } Event::Error { err } => { // Move the string out of the side table; each index is visited diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index 3df47bf0a429..2e9cf33f0a7c 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -598,13 +598,21 @@ fn field_expr( if p.at_ts(PATH_NAME_REF_OR_INDEX_KINDS) { name_ref_mod_path_or_index(p); } else if p.at(FLOAT_NUMBER) { - return match p.split_float(m) { - (true, m) => { - let lhs = m.complete(p, FIELD_EXPR); - postfix_dot_expr::(p, lhs) - } - (false, m) => Ok(m.complete(p, FIELD_EXPR)), - }; + if p.float_has_dot() { + p.split_float(); + name_ref_mod_path_or_index(p); + let lhs = m.complete(p, FIELD_EXPR); + return postfix_dot_expr::(p, lhs); + } + + // No `.` in the float lexeme (e.g. `1e0`): recover without FloatSplit. + let (inner, outer) = p.nest_field_expr(m); + let err = p.start(); + p.error("illegal float literal"); + p.bump(FLOAT_NUMBER); + err.complete(p, ERROR); + inner.complete(p, FIELD_EXPR); + return Ok(outer.complete(p, FIELD_EXPR)); } else { p.error("expected field name or number"); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 275a79b96b14..cd7c7843a0eb 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -269,7 +269,19 @@ fn builtin_expr(p: &mut Parser<'_>) -> Option { // fn foo() { // builtin#offset_of(Foo, (bar.baz.0)); // } + + // test offset_of_tuple_fields + // fn foo() { + // builtin#offset_of(ComplexTup, 0.1); + // builtin#offset_of(ComplexTup, 0.1.1.1); + // builtin#offset_of(ComplexTup, 0. 1); + // builtin#offset_of(ComplexTup, 0 .1.1.1); + // } while !p.at(EOF) && !p.at(T![')']) { + // `0.1` is one FLOAT_NUMBER; split so the name/DOT loop sees INT/DOT/INT. + if p.at(FLOAT_NUMBER) && p.float_has_dot() { + p.split_float(); + } name_ref_mod_path_or_index(p); if !p.at(T![')']) { p.expect(T![.]); diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index 54272ea9c9a0..11ca89664c66 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -17,6 +17,8 @@ type bits = u64; pub struct Input { kind: Vec, joint: Vec, + /// Whether a `FLOAT_NUMBER` lexeme contains `'.'`. Indexed like [`Self::joint`]. + float_has_dot: Vec, contextual_kind: Vec, edition: Vec, } @@ -25,9 +27,11 @@ pub struct Input { impl Input { #[inline] pub fn with_capacity(capacity: usize) -> Self { + let bits_capacity = capacity.div_ceil(bits::BITS as usize); Self { kind: Vec::with_capacity(capacity), - joint: Vec::with_capacity(capacity.div_ceil(bits::BITS as usize)), + joint: Vec::with_capacity(bits_capacity), + float_has_dot: Vec::with_capacity(bits_capacity), contextual_kind: Vec::with_capacity(capacity), edition: Vec::with_capacity(capacity), } @@ -62,11 +66,19 @@ impl Input { let (idx, b_idx) = self.bit_index(n); self.joint[idx] |= 1 << b_idx; } + /// Marks the last pushed token as a `FLOAT_NUMBER` whose text contains `.`. + #[inline] + pub fn set_float_has_dot(&mut self) { + let n = self.len() - 1; + let (idx, b_idx) = self.bit_index(n); + self.float_has_dot[idx] |= 1 << b_idx; + } #[inline] fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind, edition: Edition) { let idx = self.len(); if idx.is_multiple_of(bits::BITS as usize) { self.joint.push(0); + self.float_has_dot.push(0); } self.kind.push(kind); self.contextual_kind.push(contextual_kind); @@ -89,6 +101,10 @@ impl Input { let (idx, b_idx) = self.bit_index(n); self.joint[idx] & (1 << b_idx) != 0 } + pub(crate) fn float_has_dot(&self, n: usize) -> bool { + let (idx, b_idx) = self.bit_index(n); + self.float_has_dot[idx] & (1 << b_idx) != 0 + } } impl Input { diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 5900d7cfeda9..9eb83b409043 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -123,10 +123,8 @@ impl TopEntryPoint { match step { Step::Enter { .. } => depth += 1, Step::Exit => depth -= 1, - Step::FloatSplit { ends_in_dot: has_pseudo_dot } => { - depth -= 1 + !has_pseudo_dot as usize - } - Step::Token { .. } | Step::Error { .. } => (), + // FloatSplit does not contribute to tree depth; nesting is Enter/Exit. + Step::Token { .. } | Step::FloatSplit { .. } | Step::Error { .. } => (), } } assert!(!first, "no tree at all"); diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 11a3a532285b..bedaaee62b3f 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -6,7 +6,7 @@ use drop_bomb::DropBomb; use crate::{ Edition, - SyntaxKind::{self, EOF, ERROR, TOMBSTONE}, + SyntaxKind::{self, EOF, ERROR, INT_NUMBER, TOMBSTONE}, T, TokenSet, event::Event, input::Input, @@ -20,6 +20,16 @@ fn fwd_parent(offset: u32) -> NonZeroU32 { NonZeroU32::new(offset).expect("forward-parent offset must be non-zero") } +/// Tracks an in-flight split of a `FLOAT_NUMBER` into synthetic +/// `INT_NUMBER` / `DOT` / `INT_NUMBER` tokens. At most one split is active. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FloatSplitStage { + None, + BeforeFirstInt, + BeforeDot, + BeforeSecondInt, +} + /// `Parser` struct provides the low-level API for /// navigating through the stream of tokens and /// constructing the parse tree. The actual parsing @@ -37,6 +47,8 @@ pub(crate) struct Parser<'t> { /// into this vec, keeping `Event` itself a flat 8-byte enum. errors: Vec, steps: Cell, + float_split_stage: FloatSplitStage, + float_split_ends_in_dot: bool, } const PARSER_STEP_LIMIT: usize = if cfg!(debug_assertions) { 150_000 } else { 15_000_000 }; @@ -49,6 +61,8 @@ impl<'t> Parser<'t> { events: Vec::with_capacity(2 * inp.len()), errors: Vec::new(), steps: Cell::new(0), + float_split_stage: FloatSplitStage::None, + float_split_ends_in_dot: false, } } @@ -72,7 +86,67 @@ impl<'t> Parser<'t> { assert!((steps as usize) < PARSER_STEP_LIMIT, "the parser seems stuck"); self.steps.set(steps + 1); - self.inp.kind(self.pos + n) + self.logical_kind(n) + } + + /// Kind at lookahead `n`, honoring an in-flight float split. + /// Does not touch the stuck-detection `steps` counter. + fn logical_kind(&self, n: usize) -> SyntaxKind { + let Some(suffix_len) = self.float_split_suffix_len() else { + return self.inp.kind(self.pos + n); + }; + if n < suffix_len { + self.float_split_synthetic_kind(n) + } else { + self.inp.kind(self.pos + 1 + (n - suffix_len)) + } + } + + /// Length of the remaining synthetic token suffix, or `None` if no split is active. + fn float_split_suffix_len(&self) -> Option { + match self.float_split_stage { + FloatSplitStage::None => None, + FloatSplitStage::BeforeFirstInt => { + Some(if self.float_split_ends_in_dot { 2 } else { 3 }) + } + FloatSplitStage::BeforeDot => Some(if self.float_split_ends_in_dot { 1 } else { 2 }), + FloatSplitStage::BeforeSecondInt => Some(1), + } + } + + fn float_split_synthetic_kind(&self, n: usize) -> SyntaxKind { + match self.float_split_stage { + FloatSplitStage::None => unreachable!(), + FloatSplitStage::BeforeFirstInt => match n { + 0 => INT_NUMBER, + 1 => T![.], + 2 if !self.float_split_ends_in_dot => INT_NUMBER, + _ => unreachable!(), + }, + FloatSplitStage::BeforeDot => match n { + 0 => T![.], + 1 if !self.float_split_ends_in_dot => INT_NUMBER, + _ => unreachable!(), + }, + FloatSplitStage::BeforeSecondInt => { + assert_eq!(n, 0); + INT_NUMBER + } + } + } + + /// Whether lookahead slot `n` is still a synthetic piece of the active float split. + fn is_float_split_synthetic(&self, n: usize) -> bool { + self.float_split_suffix_len().is_some_and(|len| n < len) + } + + /// Maps a logical lookahead index to an `Input` index when that slot is a real token. + fn logical_input_index(&self, n: usize) -> Option { + match self.float_split_suffix_len() { + None => Some(self.pos + n), + Some(suffix_len) if n < suffix_len => None, + Some(suffix_len) => Some(self.pos + 1 + (n - suffix_len)), + } } /// Checks if the current token is `kind`. @@ -108,7 +182,7 @@ impl<'t> Parser<'t> { T![<<=] => self.at_composite3(n, T![<], T![<], T![=]), T![>>=] => self.at_composite3(n, T![>], T![>], T![=]), - _ => self.inp.kind(self.pos + n) == kind, + _ => self.logical_kind(n) == kind, } } @@ -161,17 +235,48 @@ impl<'t> Parser<'t> { } fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool { - self.inp.kind(self.pos + n) == k1 - && self.inp.kind(self.pos + n + 1) == k2 - && self.inp.is_joint(self.pos + n) + if self.float_split_stage == FloatSplitStage::None { + return self.inp.kind(self.pos + n) == k1 + && self.inp.kind(self.pos + n + 1) == k2 + && self.inp.is_joint(self.pos + n); + } + + if self.logical_kind(n) != k1 || self.logical_kind(n + 1) != k2 { + return false; + } + // Synthetic pieces are never Input-joint; float jointness means + // "does not end with `.`" (`ends_in_dot`), not glue to the next token. + if self.is_float_split_synthetic(n) || self.is_float_split_synthetic(n + 1) { + return false; + } + let idx = self.logical_input_index(n).expect("real token after synthetic check"); + self.inp.is_joint(idx) } fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool { - self.inp.kind(self.pos + n) == k1 - && self.inp.kind(self.pos + n + 1) == k2 - && self.inp.kind(self.pos + n + 2) == k3 - && self.inp.is_joint(self.pos + n) - && self.inp.is_joint(self.pos + n + 1) + if self.float_split_stage == FloatSplitStage::None { + return self.inp.kind(self.pos + n) == k1 + && self.inp.kind(self.pos + n + 1) == k2 + && self.inp.kind(self.pos + n + 2) == k3 + && self.inp.is_joint(self.pos + n) + && self.inp.is_joint(self.pos + n + 1); + } + + if self.logical_kind(n) != k1 + || self.logical_kind(n + 1) != k2 + || self.logical_kind(n + 2) != k3 + { + return false; + } + if self.is_float_split_synthetic(n) + || self.is_float_split_synthetic(n + 1) + || self.is_float_split_synthetic(n + 2) + { + return false; + } + let idx0 = self.logical_input_index(n).expect("real token after synthetic check"); + let idx1 = self.logical_input_index(n + 1).expect("real token after synthetic check"); + self.inp.is_joint(idx0) && self.inp.is_joint(idx1) } /// Checks if the current token is in `kinds`. @@ -189,6 +294,11 @@ impl<'t> Parser<'t> { self.inp.contextual_kind(self.pos + n) == kw } + /// Whether the current `FLOAT_NUMBER` lexeme contains `'.'`. + pub(crate) fn float_has_dot(&self) -> bool { + self.inp.float_has_dot(self.pos) + } + /// Starts a new node in the syntax tree. All nodes and tokens /// consumed between the `start` and the corresponding `Marker::complete` /// belong to the same node. @@ -212,33 +322,38 @@ impl<'t> Parser<'t> { self.do_bump(kind, 1); } - /// Advances the parser by one token - pub(crate) fn split_float(&mut self, mut marker: Marker) -> (bool, Marker) { + /// Begin splitting the current `FLOAT_NUMBER` into synthetic + /// `INT_NUMBER` / `DOT` / optional `INT_NUMBER` tokens. + /// + /// Does not advance `pos`; subsequent `nth`/`bump` emulate the pieces. + pub(crate) fn split_float(&mut self) { + assert_eq!( + self.float_split_stage, + FloatSplitStage::None, + "cannot split more than one float at a time" + ); assert!(self.at(SyntaxKind::FLOAT_NUMBER)); - // we have parse `.` - // ``.0.1 - // here we need to insert an extra event - // - // ``. 0. 1; - // here we need to change the follow up parse, the return value will cause us to emulate a dot - // the actual splitting happens later let ends_in_dot = !self.inp.is_joint(self.pos); - if !ends_in_dot { - let new_marker = self.start(); - let idx = marker.pos as usize; - match &mut self.events[idx] { - Event::Start { forward_parent, kind } => { - *kind = SyntaxKind::FIELD_EXPR; - *forward_parent = Some(fwd_parent(new_marker.pos - marker.pos)); - } - _ => unreachable!(), - } - marker.bomb.defuse(); - marker = new_marker; - }; - self.pos += 1; + self.float_split_ends_in_dot = ends_in_dot; + self.float_split_stage = FloatSplitStage::BeforeFirstInt; self.push_event(Event::FloatSplitHack { ends_in_dot }); - (ends_in_dot, marker) + } + + /// Nest an outer `FIELD_EXPR` around `inner` via `forward_parent`. + /// + /// Used when recovering an unsplittable float field (e.g. `1e0`) so the CST + /// keeps a nested `FIELD_EXPR` around the `ERROR`/`FLOAT_NUMBER`. + pub(crate) fn nest_field_expr(&mut self, inner: Marker) -> (Marker, Marker) { + let outer = self.start(); + let idx = inner.pos as usize; + match &mut self.events[idx] { + Event::Start { forward_parent, kind } => { + *kind = SyntaxKind::FIELD_EXPR; + *forward_parent = Some(fwd_parent(outer.pos - inner.pos)); + } + _ => unreachable!(), + } + (inner, outer) } /// Advances the parser by one token, remapping its kind. @@ -305,9 +420,37 @@ impl<'t> Parser<'t> { } fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { - self.pos += n_raw_tokens as usize; + if self.float_split_stage == FloatSplitStage::None { + self.pos += n_raw_tokens as usize; + self.steps.set(0); + self.push_event(Event::Token { kind, n_raw_tokens }); + return; + } + + assert_eq!(kind, self.logical_kind(0), "bump kind must match synthetic token"); + // Ignore caller `n_raw_tokens`: synthetic pieces are not Input slots. self.steps.set(0); - self.push_event(Event::Token { kind, n_raw_tokens }); + self.push_event(Event::Token { kind, n_raw_tokens: 0 }); + + match self.float_split_stage { + FloatSplitStage::None => unreachable!(), + FloatSplitStage::BeforeFirstInt => { + self.float_split_stage = FloatSplitStage::BeforeDot; + } + FloatSplitStage::BeforeDot if self.float_split_ends_in_dot => { + self.float_split_stage = FloatSplitStage::None; + self.float_split_ends_in_dot = false; + self.pos += 1; + } + FloatSplitStage::BeforeDot => { + self.float_split_stage = FloatSplitStage::BeforeSecondInt; + } + FloatSplitStage::BeforeSecondInt => { + self.float_split_stage = FloatSplitStage::None; + self.float_split_ends_in_dot = false; + self.pos += 1; + } + } } fn push_event(&mut self, event: Event) { diff --git a/crates/parser/src/shortcuts.rs b/crates/parser/src/shortcuts.rs index 3c19e025453a..cdb2e84975b4 100644 --- a/crates/parser/src/shortcuts.rs +++ b/crates/parser/src/shortcuts.rs @@ -45,11 +45,14 @@ impl LexedStr<'_> { res.was_joint(); } res.push(kind, edition); - // Tag the token as joint if it is float with a fractional part - // we use this jointness to inform the parser about what token split - // event to emit when we encounter a float literal in a field access + // For floats: `float_has_dot` if the lexeme contains `.`; joint if it + // does not end with `.` (`ends_in_dot = !is_joint` when splitting). if kind == SyntaxKind::FLOAT_NUMBER { - if !self.text(i).ends_with('.') { + let text = self.text(i); + if text.contains('.') { + res.set_float_has_dot(); + } + if !text.ends_with('.') { res.was_joint(); } else { was_joint = false; @@ -68,7 +71,14 @@ impl LexedStr<'_> { output: &crate::Output, sink: &mut dyn FnMut(StrStep<'_>), ) -> bool { - let mut builder = Builder { lexed: self, pos: 0, state: State::PendingEnter, sink }; + let mut builder = Builder { + lexed: self, + pos: 0, + state: State::PendingEnter, + sink, + float_split_stage: FloatSplitStage::None, + float_split_ends_in_dot: false, + }; for event in output.iter() { match event { @@ -105,6 +115,8 @@ struct Builder<'a, 'b> { pos: usize, state: State, sink: &'b mut dyn FnMut(StrStep<'_>), + float_split_stage: FloatSplitStage, + float_split_ends_in_dot: bool, } enum State { @@ -113,6 +125,16 @@ enum State { PendingExit, } +/// Mirrors the parser's in-flight float split: one lexical `FLOAT_NUMBER` +/// becomes synthetic `INT_NUMBER` / `DOT` / optional `INT_NUMBER` leaves. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FloatSplitStage { + None, + BeforeFirstInt, + BeforeDot, + BeforeSecondInt, +} + impl Builder<'_, '_> { fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { match mem::replace(&mut self.state, State::Normal) { @@ -121,17 +143,34 @@ impl Builder<'_, '_> { State::Normal => (), } self.eat_trivias(); - self.do_token(kind, n_tokens as usize); + if n_tokens == 0 { + assert_ne!( + self.float_split_stage, + FloatSplitStage::None, + "Token with n_raw_tokens=0 is only valid during a float split" + ); + self.do_synthetic_float_token(kind); + } else { + assert_eq!( + self.float_split_stage, + FloatSplitStage::None, + "nonzero Token during an active float split" + ); + self.do_token(kind, n_tokens as usize); + } } - fn float_split(&mut self, has_pseudo_dot: bool) { + fn float_split(&mut self, ends_in_dot: bool) { match mem::replace(&mut self.state, State::Normal) { State::PendingEnter => unreachable!(), State::PendingExit => (self.sink)(StrStep::Exit), State::Normal => (), } self.eat_trivias(); - self.do_float_split(has_pseudo_dot); + assert_eq!(self.float_split_stage, FloatSplitStage::None); + assert_eq!(self.lexed.kind(self.pos), SyntaxKind::FLOAT_NUMBER); + self.float_split_ends_in_dot = ends_in_dot; + self.float_split_stage = FloatSplitStage::BeforeFirstInt; } fn enter(&mut self, kind: SyntaxKind) { @@ -190,50 +229,51 @@ impl Builder<'_, '_> { (self.sink)(StrStep::Token { kind, text }); } - fn do_float_split(&mut self, has_pseudo_dot: bool) { - let text = &self.lexed.range_text(self.pos..self.pos + 1); - - match text.split_once('.') { - Some((left, right)) => { - assert!(!left.is_empty()); - (self.sink)(StrStep::Enter { kind: SyntaxKind::NAME_REF }); - (self.sink)(StrStep::Token { kind: SyntaxKind::INT_NUMBER, text: left }); - (self.sink)(StrStep::Exit); - - // here we move the exit up, the original exit has been deleted in process - (self.sink)(StrStep::Exit); - - (self.sink)(StrStep::Token { kind: SyntaxKind::DOT, text: "." }); - - if has_pseudo_dot { - assert!(right.is_empty(), "{left}.{right}"); - self.state = State::Normal; - } else { - assert!(!right.is_empty(), "{left}.{right}"); - (self.sink)(StrStep::Enter { kind: SyntaxKind::NAME_REF }); - (self.sink)(StrStep::Token { kind: SyntaxKind::INT_NUMBER, text: right }); - (self.sink)(StrStep::Exit); + /// Emit one synthetic piece; advance the lexed cursor only on the last piece. + fn do_synthetic_float_token(&mut self, kind: SyntaxKind) { + assert_eq!(self.lexed.kind(self.pos), SyntaxKind::FLOAT_NUMBER); + let text = self.lexed.text(self.pos); + let (left, right) = text.split_once('.').unwrap_or((text, "")); + assert!(!left.is_empty(), "float split left part must be non-empty: {text:?}"); - // the parser creates an unbalanced start node, we are required to close it here - self.state = State::PendingExit; - } + let piece = match self.float_split_stage { + FloatSplitStage::None => unreachable!(), + FloatSplitStage::BeforeFirstInt => { + assert_eq!(kind, SyntaxKind::INT_NUMBER); + left } - None => { - // illegal float literal which doesn't have dot in form (like 1e0) - // we should emit an error node here - (self.sink)(StrStep::Error { msg: "illegal float literal", pos: self.pos }); - (self.sink)(StrStep::Enter { kind: SyntaxKind::ERROR }); - (self.sink)(StrStep::Token { kind: SyntaxKind::FLOAT_NUMBER, text }); - (self.sink)(StrStep::Exit); - - // move up - (self.sink)(StrStep::Exit); + FloatSplitStage::BeforeDot => { + assert_eq!(kind, SyntaxKind::DOT); + "." + } + FloatSplitStage::BeforeSecondInt => { + assert_eq!(kind, SyntaxKind::INT_NUMBER); + assert!(!self.float_split_ends_in_dot); + assert!(!right.is_empty(), "expected fractional part: {text:?}"); + right + } + }; + (self.sink)(StrStep::Token { kind, text: piece }); - self.state = if has_pseudo_dot { State::Normal } else { State::PendingExit }; + match self.float_split_stage { + FloatSplitStage::None => unreachable!(), + FloatSplitStage::BeforeFirstInt => { + self.float_split_stage = FloatSplitStage::BeforeDot; + } + FloatSplitStage::BeforeDot if self.float_split_ends_in_dot => { + self.float_split_stage = FloatSplitStage::None; + self.float_split_ends_in_dot = false; + self.pos += 1; + } + FloatSplitStage::BeforeDot => { + self.float_split_stage = FloatSplitStage::BeforeSecondInt; + } + FloatSplitStage::BeforeSecondInt => { + self.float_split_stage = FloatSplitStage::None; + self.float_split_ends_in_dot = false; + self.pos += 1; } } - - self.pos += 1; } } diff --git a/crates/parser/test_data/generated/runner.rs b/crates/parser/test_data/generated/runner.rs index 9e3de7517f22..49c6564783f6 100644 --- a/crates/parser/test_data/generated/runner.rs +++ b/crates/parser/test_data/generated/runner.rs @@ -502,6 +502,10 @@ mod ok { run_and_expect_no_errors("test_data/parser/inline/ok/offset_of_parens.rs"); } #[test] + fn offset_of_tuple_fields() { + run_and_expect_no_errors("test_data/parser/inline/ok/offset_of_tuple_fields.rs"); + } + #[test] fn or_pattern() { run_and_expect_no_errors("test_data/parser/inline/ok/or_pattern.rs"); } #[test] fn param_list() { run_and_expect_no_errors("test_data/parser/inline/ok/param_list.rs"); } diff --git a/crates/parser/test_data/parser/err/0054_float_split_scientific_notation.rast b/crates/parser/test_data/parser/err/0054_float_split_scientific_notation.rast index d6ad7334839d..00f8615826b1 100644 --- a/crates/parser/test_data/parser/err/0054_float_split_scientific_notation.rast +++ b/crates/parser/test_data/parser/err/0054_float_split_scientific_notation.rast @@ -85,4 +85,4 @@ SOURCE_FILE WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" -error 42: illegal float literal +error 64: illegal float literal diff --git a/crates/parser/test_data/parser/inline/ok/offset_of_tuple_fields.rast b/crates/parser/test_data/parser/inline/ok/offset_of_tuple_fields.rast new file mode 100644 index 000000000000..ba017c52cda1 --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/offset_of_tuple_fields.rast @@ -0,0 +1,114 @@ +SOURCE_FILE + FN + FN_KW "fn" + WHITESPACE " " + NAME + IDENT "foo" + PARAM_LIST + L_PAREN "(" + R_PAREN ")" + WHITESPACE " " + BLOCK_EXPR + STMT_LIST + L_CURLY "{" + WHITESPACE "\n " + EXPR_STMT + OFFSET_OF_EXPR + BUILTIN_KW "builtin" + POUND "#" + OFFSET_OF_KW "offset_of" + L_PAREN "(" + PATH_TYPE + PATH + PATH_SEGMENT + NAME_REF + IDENT "ComplexTup" + COMMA "," + WHITESPACE " " + NAME_REF + INT_NUMBER "0" + DOT "." + NAME_REF + INT_NUMBER "1" + R_PAREN ")" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + OFFSET_OF_EXPR + BUILTIN_KW "builtin" + POUND "#" + OFFSET_OF_KW "offset_of" + L_PAREN "(" + PATH_TYPE + PATH + PATH_SEGMENT + NAME_REF + IDENT "ComplexTup" + COMMA "," + WHITESPACE " " + NAME_REF + INT_NUMBER "0" + DOT "." + NAME_REF + INT_NUMBER "1" + DOT "." + NAME_REF + INT_NUMBER "1" + DOT "." + NAME_REF + INT_NUMBER "1" + R_PAREN ")" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + OFFSET_OF_EXPR + BUILTIN_KW "builtin" + POUND "#" + OFFSET_OF_KW "offset_of" + L_PAREN "(" + PATH_TYPE + PATH + PATH_SEGMENT + NAME_REF + IDENT "ComplexTup" + COMMA "," + WHITESPACE " " + NAME_REF + INT_NUMBER "0" + DOT "." + WHITESPACE " " + NAME_REF + INT_NUMBER "1" + R_PAREN ")" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + OFFSET_OF_EXPR + BUILTIN_KW "builtin" + POUND "#" + OFFSET_OF_KW "offset_of" + L_PAREN "(" + PATH_TYPE + PATH + PATH_SEGMENT + NAME_REF + IDENT "ComplexTup" + COMMA "," + WHITESPACE " " + NAME_REF + INT_NUMBER "0" + WHITESPACE " " + DOT "." + NAME_REF + INT_NUMBER "1" + DOT "." + NAME_REF + INT_NUMBER "1" + DOT "." + NAME_REF + INT_NUMBER "1" + R_PAREN ")" + SEMICOLON ";" + WHITESPACE "\n" + R_CURLY "}" + WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/offset_of_tuple_fields.rs b/crates/parser/test_data/parser/inline/ok/offset_of_tuple_fields.rs new file mode 100644 index 000000000000..d9ab6d95fcbd --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/offset_of_tuple_fields.rs @@ -0,0 +1,6 @@ +fn foo() { + builtin#offset_of(ComplexTup, 0.1); + builtin#offset_of(ComplexTup, 0.1.1.1); + builtin#offset_of(ComplexTup, 0. 1); + builtin#offset_of(ComplexTup, 0 .1.1.1); +} diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs index 3e6e5f804e26..39eb3b2d6d41 100644 --- a/crates/syntax-bridge/src/lib.rs +++ b/crates/syntax-bridge/src/lib.rs @@ -840,6 +840,17 @@ struct TtTreeSink<'a> { text_pos: TextSize, inner: SyntaxTreeBuilder, token_map: SpanMap, + float_split_stage: FloatSplitStage, + float_split_ends_in_dot: bool, +} + +/// Mirrors the parser's in-flight float split over one TT float literal leaf. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FloatSplitStage { + None, + BeforeFirstInt, + BeforeDot, + BeforeSecondInt, } impl<'a> TtTreeSink<'a> { @@ -850,6 +861,8 @@ impl<'a> TtTreeSink<'a> { text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), token_map: SpanMap::empty(), + float_split_stage: FloatSplitStage::None, + float_split_ends_in_dot: false, } } @@ -872,65 +885,35 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> { } impl TtTreeSink<'_> { - /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween. - /// This occurs when a float literal is used as a field access. - fn float_split(&mut self, has_pseudo_dot: bool) { - let token_tree = self.cursor.token_tree(); - let (text, span) = match &token_tree { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal( - lit @ tt::Literal { span, kind: tt::LitKind::Float, .. }, - ))) => (lit.text(), *span), - tt => unreachable!("{tt:?}"), - }; - // FIXME: Span splitting - match text.split_once('.') { - Some((left, right)) => { - assert!(!left.is_empty()); - - self.inner.start_node(SyntaxKind::NAME_REF); - self.inner.token(SyntaxKind::INT_NUMBER, left); - self.inner.finish_node(); - self.token_map.push(self.text_pos + TextSize::of(left), span); - - // here we move the exit up, the original exit has been deleted in process - self.inner.finish_node(); - - self.inner.token(SyntaxKind::DOT, "."); - self.token_map.push(self.text_pos + TextSize::of(left) + TextSize::of("."), span); - - if has_pseudo_dot { - assert!(right.is_empty(), "{left}.{right}"); - } else { - assert!(!right.is_empty(), "{left}.{right}"); - self.inner.start_node(SyntaxKind::NAME_REF); - self.inner.token(SyntaxKind::INT_NUMBER, right); - self.token_map.push(self.text_pos + TextSize::of(text), span); - self.inner.finish_node(); - - // the parser creates an unbalanced start node, we are required to close it here - self.inner.finish_node(); - } - self.text_pos += TextSize::of(text); - } - None => { - self.error("illegal float literal".to_owned()); - self.inner.start_node(SyntaxKind::ERROR); - self.inner.token(SyntaxKind::FLOAT_NUMBER, text); - self.token_map.push(self.text_pos + TextSize::of(text), span); - self.inner.finish_node(); - self.inner.finish_node(); - - if !has_pseudo_dot { - self.inner.finish_node(); - } - - self.text_pos += TextSize::of(text); + /// Begin an in-flight float split; tree shape still comes from Enter/Exit. + fn float_split(&mut self, ends_in_dot: bool) { + assert_eq!(self.float_split_stage, FloatSplitStage::None); + match self.cursor.token_tree() { + Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) if lit.kind == tt::LitKind::Float => { + debug_assert_eq!(ends_in_dot, lit.text().ends_with('.')); } + tt => unreachable!("{tt:?}"), } - self.cursor.bump(); + self.float_split_ends_in_dot = ends_in_dot; + self.float_split_stage = FloatSplitStage::BeforeFirstInt; } fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { + if n_tokens == 0 { + assert_ne!( + self.float_split_stage, + FloatSplitStage::None, + "Token with n_raw_tokens=0 is only valid during a float split" + ); + self.do_synthetic_float_token(kind); + return; + } + assert_eq!( + self.float_split_stage, + FloatSplitStage::None, + "nonzero Token during an active float split" + ); + if kind == LIFETIME_IDENT { n_tokens = 2; } @@ -1027,6 +1010,65 @@ impl TtTreeSink<'_> { } } + /// Emit one synthetic piece; advance the TT cursor only on the last piece. + fn do_synthetic_float_token(&mut self, kind: SyntaxKind) { + let token_tree = self.cursor.token_tree(); + let (text, span) = match &token_tree { + Some(tt::TokenTree::Leaf(tt::Leaf::Literal( + lit @ tt::Literal { span, kind: tt::LitKind::Float, .. }, + ))) => (lit.text().to_owned(), *span), + tt => unreachable!("{tt:?}"), + }; + // FIXME: Span splitting — each piece still maps to the whole float span. + let (left, right) = match text.split_once('.') { + Some((left, right)) => (left, right), + None => (text.as_str(), ""), + }; + assert!(!left.is_empty(), "float split left part must be non-empty: {text:?}"); + + let piece = match self.float_split_stage { + FloatSplitStage::None => unreachable!(), + FloatSplitStage::BeforeFirstInt => { + assert_eq!(kind, SyntaxKind::INT_NUMBER); + left + } + FloatSplitStage::BeforeDot => { + assert_eq!(kind, SyntaxKind::DOT); + "." + } + FloatSplitStage::BeforeSecondInt => { + assert_eq!(kind, SyntaxKind::INT_NUMBER); + assert!(!self.float_split_ends_in_dot); + assert!(!right.is_empty(), "expected fractional part: {text:?}"); + right + } + }; + + self.inner.token(kind, piece); + self.text_pos += TextSize::of(piece); + self.token_map.push(self.text_pos, span); + + match self.float_split_stage { + FloatSplitStage::None => unreachable!(), + FloatSplitStage::BeforeFirstInt => { + self.float_split_stage = FloatSplitStage::BeforeDot; + } + FloatSplitStage::BeforeDot if self.float_split_ends_in_dot => { + self.float_split_stage = FloatSplitStage::None; + self.float_split_ends_in_dot = false; + self.cursor.bump(); + } + FloatSplitStage::BeforeDot => { + self.float_split_stage = FloatSplitStage::BeforeSecondInt; + } + FloatSplitStage::BeforeSecondInt => { + self.float_split_stage = FloatSplitStage::None; + self.float_split_ends_in_dot = false; + self.cursor.bump(); + } + } + } + fn start_node(&mut self, kind: SyntaxKind) { self.inner.start_node(kind); } diff --git a/crates/syntax-bridge/src/to_parser_input.rs b/crates/syntax-bridge/src/to_parser_input.rs index 851a4af86439..9b1b99af61bb 100644 --- a/crates/syntax-bridge/src/to_parser_input.rs +++ b/crates/syntax-bridge/src/to_parser_input.rs @@ -52,11 +52,15 @@ pub fn to_parser_input( }; res.push(kind, ctx_edition(lit.span.ctx)); - if kind == FLOAT_NUMBER && !lit.text().ends_with('.') { - // Tag the token as joint if it is float with a fractional part - // we use this jointness to inform the parser about what token split - // event to emit when we encounter a float literal in a field access - res.was_joint(); + if kind == FLOAT_NUMBER { + let text = lit.text(); + if text.contains('.') { + res.set_float_has_dot(); + } + // A float is joint when it does not end with `.`. + if !text.ends_with('.') { + res.was_joint(); + } } } tt::Leaf::Ident(ident) => {