Trivia in tokens

Cargo.toml

@@ -35,7 +35,7 @@
 debug = 2
 
 [patch.'crates-io']
-# rowan = { path = "../rowan" }
+rowan = { git = "https://github.com/bit-aloo/rowan", branch = "2026-09-08-more-trivia-changes" }
 
 # line-index = { path = "lib/line-index" }
 # la-arena = { path = "lib/la-arena" }
@@ -68,10 +68,10 @@
 paths = { path = "./crates/paths", version = "0.0.0" }
 proc-macro-api = { path = "./crates/proc-macro-api", version = "0.0.0" }
 proc-macro-srv = { path = "./crates/proc-macro-srv", version = "0.0.0" }
 proc-macro-srv-cli = { path = "./crates/proc-macro-srv-cli", version = "0.0.0" }
 profile = { path = "./crates/profile", version = "0.0.0" }
 project-model = { path = "./crates/project-model", version = "0.0.0" }
 query-group = { package = "query-group-macro", path = "./crates/query-group-macro", version = "0.0.0" }
 span = { path = "./crates/span", version = "0.0.0" }
 stdx = { path = "./crates/stdx", version = "0.0.0" }
 syntax = { path = "./crates/syntax", version = "0.0.0" }

crates/hir-def/src/attrs/docs.rs

@@ -215,7 +215,8 @@ impl Docs {
 
     fn extend_with_doc_comment(&mut self, comment: ast::DocComment, indent: &mut Indent) {
         let doc = comment.text();
-        let offset = comment.syntax().text_range().start() + ast::DocComment::PREFIX_LEN;
+        let offset = comment.syntax().text_range_without_outer_trivia().start()
+            + ast::DocComment::PREFIX_LEN;
         self.extend_with_doc_str(doc, offset, DocCommentKind::Sugared(comment.shape()), indent);
     }
 

crates/hir-def/src/expr_store/lower.rs

@@ -3745,8 +3745,7 @@ impl<'db> ExprCollector<'db> {
 }
 
 fn comma_follows_token(t: Option<syntax::SyntaxToken>) -> bool {
-    (|| syntax::algo::skip_trivia_token(t?.next_token()?, syntax::Direction::Next))()
-        .is_some_and(|it| it.kind() == syntax::T![,])
+    t.and_then(|it| it.next_non_trivia_token()).is_some_and(|it| it.kind() == syntax::T![,])
 }
 
 /// This function find the AST fragment that corresponds to an `AssociatedTypeBinding` in the HIR.

crates/hir-def/src/expr_store/lower/asm.rs

@@ -53,7 +53,11 @@ impl ExprCollector<'_> {
                 }
                 ast::AsmPiece::AsmOptions(opt) => {
                     opt.asm_options().for_each(|opt| {
-                        options |= match opt.syntax().first_token().map_or(T![$], |it| it.kind()) {
+                        options |= match opt
+                            .syntax()
+                            .first_non_trivia_token()
+                            .map_or(T![$], |it| it.kind())
+                        {
                             T![att_syntax] => AsmOptions::ATT_SYNTAX,
                             T![may_unwind] => AsmOptions::MAY_UNWIND,
                             T![nomem] => AsmOptions::NOMEM,

crates/hir-def/src/macro_expansion_tests/builtin_fn_macro.rs

@@ -262,6 +262,7 @@ macro_rules! compile_error {
     ($msg:expr,) => ({ /* compiler built-in */ })
 }
 
+// This expands to nothing (since it's in item position), but emits an error.
 /* error: error, with an escaped quote: " */
 /* error: this is a raw string */
 "##]],

crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -35,7 +35,8 @@ macro_rules! f {
     };
 }
 
-struct#0:MacroRules[BE8F, 0]@58..64#1922# MyTraitMap2#0:MacroCall[BE8F, 0]@31..42#ROOT2024# {#0:MacroRules[BE8F, 0]@72..73#1922#
+// +spans+syntaxctxt
+struct#0:MacroRules[BE8F, 0]@58..64#1922# MyTraitMap2#0:MacroCall[BE8F, 0]@10..21#ROOT2024# {#0:MacroRules[BE8F, 0]@72..73#1922#
     map#0:MacroRules[BE8F, 0]@86..89#1922#:#0:MacroRules[BE8F, 0]@89..90#1922# #0:MacroRules[BE8F, 0]@89..90#1922#::#0:MacroRules[BE8F, 0]@91..93#1922#std#0:MacroRules[BE8F, 0]@93..96#1922#::#0:MacroRules[BE8F, 0]@96..98#1922#collections#0:MacroRules[BE8F, 0]@98..109#1922#::#0:MacroRules[BE8F, 0]@109..111#1922#HashSet#0:MacroRules[BE8F, 0]@111..118#1922#<#0:MacroRules[BE8F, 0]@118..119#1922#(#0:MacroRules[BE8F, 0]@119..120#1922#)#0:MacroRules[BE8F, 0]@120..121#1922#>#0:MacroRules[BE8F, 0]@121..122#1922#,#0:MacroRules[BE8F, 0]@122..123#1922#
 }#0:MacroRules[BE8F, 0]@132..133#1922#
 "#]],
@@ -75,12 +76,13 @@ macro_rules! f {
     };
 }
 
-fn#0:MacroCall[BE8F, 0]@30..32#ROOT2024# main#0:MacroCall[BE8F, 0]@33..37#ROOT2024#(#0:MacroCall[BE8F, 0]@37..38#ROOT2024#)#0:MacroCall[BE8F, 0]@38..39#ROOT2024# {#0:MacroCall[BE8F, 0]@40..41#ROOT2024#
-    1#0:MacroCall[BE8F, 0]@50..51#ROOT2024#;#0:MacroCall[BE8F, 0]@51..52#ROOT2024#
-    1.0#0:MacroCall[BE8F, 0]@61..64#ROOT2024#;#0:MacroCall[BE8F, 0]@64..65#ROOT2024#
-    (#0:MacroCall[BE8F, 0]@74..75#ROOT2024#(#0:MacroCall[BE8F, 0]@75..76#ROOT2024#1#0:MacroCall[BE8F, 0]@76..77#ROOT2024#,#0:MacroCall[BE8F, 0]@77..78#ROOT2024# )#0:MacroCall[BE8F, 0]@78..79#ROOT2024#,#0:MacroCall[BE8F, 0]@79..80#ROOT2024# )#0:MacroCall[BE8F, 0]@80..81#ROOT2024#.#0:MacroCall[BE8F, 0]@81..82#ROOT2024#0#0:MacroCall[BE8F, 0]@82..85#ROOT2024#.#0:MacroCall[BE8F, 0]@82..85#ROOT2024#0#0:MacroCall[BE8F, 0]@82..85#ROOT2024#;#0:MacroCall[BE8F, 0]@85..86#ROOT2024#
-    let#0:MacroCall[BE8F, 0]@95..98#ROOT2024# x#0:MacroCall[BE8F, 0]@99..100#ROOT2024# =#0:MacroCall[BE8F, 0]@101..102#ROOT2024# 1#0:MacroCall[BE8F, 0]@103..104#ROOT2024#;#0:MacroCall[BE8F, 0]@104..105#ROOT2024#
-}#0:MacroCall[BE8F, 0]@110..111#ROOT2024#
+// +spans+syntaxctxt
+fn#0:MacroCall[BE8F, 0]@9..11#ROOT2024# main#0:MacroCall[BE8F, 0]@12..16#ROOT2024#(#0:MacroCall[BE8F, 0]@16..17#ROOT2024#)#0:MacroCall[BE8F, 0]@17..18#ROOT2024# {#0:MacroCall[BE8F, 0]@19..20#ROOT2024#
+    1#0:MacroCall[BE8F, 0]@29..30#ROOT2024#;#0:MacroCall[BE8F, 0]@30..31#ROOT2024#
+    1.0#0:MacroCall[BE8F, 0]@40..43#ROOT2024#;#0:MacroCall[BE8F, 0]@43..44#ROOT2024#
+    (#0:MacroCall[BE8F, 0]@53..54#ROOT2024#(#0:MacroCall[BE8F, 0]@54..55#ROOT2024#1#0:MacroCall[BE8F, 0]@55..56#ROOT2024#,#0:MacroCall[BE8F, 0]@56..57#ROOT2024# )#0:MacroCall[BE8F, 0]@57..58#ROOT2024#,#0:MacroCall[BE8F, 0]@58..59#ROOT2024# )#0:MacroCall[BE8F, 0]@59..60#ROOT2024#.#0:MacroCall[BE8F, 0]@60..61#ROOT2024#0#0:MacroCall[BE8F, 0]@61..64#ROOT2024#.#0:MacroCall[BE8F, 0]@61..64#ROOT2024#0#0:MacroCall[BE8F, 0]@61..64#ROOT2024#;#0:MacroCall[BE8F, 0]@64..65#ROOT2024#
+    let#0:MacroCall[BE8F, 0]@74..77#ROOT2024# x#0:MacroCall[BE8F, 0]@78..79#ROOT2024# =#0:MacroCall[BE8F, 0]@80..81#ROOT2024# 1#0:MacroCall[BE8F, 0]@82..83#ROOT2024#;#0:MacroCall[BE8F, 0]@83..84#ROOT2024#
+}#0:MacroCall[BE8F, 0]@89..90#ROOT2024#
 
 
 "#]],
@@ -423,6 +425,7 @@ m! { foo, bar }
 macro_rules! m {
     ($($i:ident),*) => ( impl Bar { $(fn $i() {})* } );
 }
+// +syntaxctxt
 impl#\1922# Bar#\1922# {#\1922#
     fn#\1922# foo#\ROOT2024#(#\1922#)#\1922# {#\1922#}#\1922#
     fn#\1922# bar#\ROOT2024#(#\1922#)#\1922# {#\1922#}#\1922#
@@ -526,6 +529,7 @@ m! { 1 + 2 }
 macro_rules! m {
     ($ i:expr) => { fn bar() { $ i * 3; } }
 }
+// +tree
 fn bar() {
     (1+2)*3;
 }
@@ -646,6 +650,7 @@ macro_rules! structs {
     ($($i:ident),*) => { $(struct $i { field: u32 } )* }
 }
 
+// +tree
 struct Foo {
     field: u32
 }
@@ -1736,6 +1741,7 @@ m!(C("0"));
 macro_rules! m {
     ($k:expr) => { fn f() { K::$k; } }
 }
+// +tree +errors
 /* parse error: expected identifier, `self`, `super`, `crate`, or `Self` */
 /* parse error: expected SEMICOLON */
 /* parse error: expected SEMICOLON */

crates/hir-def/src/macro_expansion_tests/mbe/regression.rs

@@ -196,6 +196,7 @@ macro_rules! STRUCT {
     );
 }
 
+// from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/shared/d3d9caps.rs
 #[repr(C)]
 #[derive(Copy)] pub struct D3DVSHADERCAPS2_0 {
     pub Caps: u8,
@@ -920,6 +921,7 @@ macro_rules! rgb_color {
         }
     };
 }
+// +tree +errors
 /* parse error: expected type */
 /* parse error: expected R_PAREN */
 /* parse error: expected R_ANGLE */

crates/hir-def/src/macro_expansion_tests/mod.rs

@@ -33,7 +33,7 @@ use span::{
 use stdx::{format_to, format_to_acc};
 use syntax::{
     AstNode, AstPtr,
-    SyntaxKind::{COMMENT, EOF, IDENT, LIFETIME_IDENT},
+    SyntaxKind::{COMMENT, EOF, IDENT, LIFETIME_IDENT, TOKEN_TREE},
     SyntaxNode, T,
     ast::{self, edit::IndentLevel},
 };
@@ -76,7 +76,7 @@ fn check_errors(#[rust_analyzer::rust_fixture] ra_fixture: &str, expect: Expect)
             let ast = editioned_file_id.parse(&db).syntax_node();
             let ast_id_map = ast_id.file_id.ast_id_map(&db);
             let node = ast_id_map.get_erased(ast_id.value).to_node(&ast);
-            Some((node.text_range(), errors))
+            Some((node.text_range_without_outer_trivia(), errors))
         })
         .sorted_unstable_by_key(|(range, _)| range.start())
         .format_with("\n", |(range, err), format| format(&format_args!("{range:?}: {err}")))
@@ -133,11 +133,18 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream
         let mut expect_errors = false;
         let mut show_spans = false;
         let mut show_ctxt = false;
-        for comment in call.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) {
-            tree |= comment.to_string().contains("+tree");
-            expect_errors |= comment.to_string().contains("+errors");
-            show_spans |= comment.to_string().contains("+spans");
-            show_ctxt |= comment.to_string().contains("+syntaxctxt");
+        for comment in call
+            .syntax()
+            .descendants_with_tokens()
+            .filter_map(|it| it.into_token())
+            .filter(|it| it.parent_ancestors().all(|it| it.kind() != TOKEN_TREE))
+            .flat_map(|it| it.leading_trivia().chain(it.trailing_trivia()))
+            .filter(|it| it.kind() == COMMENT)
+        {
+            tree |= comment.text().contains("+tree");
+            expect_errors |= comment.text().contains("+errors");
+            show_spans |= comment.text().contains("+spans");
+            show_ctxt |= comment.text().contains("+syntaxctxt");
         }
 
         let mut expn_text = String::new();
@@ -174,14 +181,14 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream
                 .fold(String::new(), |mut acc, line| format_to_acc!(acc, "// {line}"));
             format_to!(expn_text, "\n{}", tree)
         }
-        let range = call.syntax().text_range();
+        let range = call.syntax().text_range_without_outer_trivia();
         let range: Range<usize> = range.into();
         text_edits.push((range, expn_text));
     }
 
     text_edits.sort_by_key(|(range, _)| range.start);
     text_edits.reverse();
-    let mut expanded_text = source_file.to_string();
+    let mut expanded_text = source_file.syntax().text().to_string();
     for (range, text) in text_edits {
         expanded_text.replace_range(range, &text);
     }
@@ -208,9 +215,21 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream
             let call = file_id.call_node(&db);
             let mut show_spans = false;
             let mut show_ctxt = false;
-            for comment in call.value.children_with_tokens().filter(|it| it.kind() == COMMENT) {
-                show_spans |= comment.to_string().contains("+spans");
-                show_ctxt |= comment.to_string().contains("+syntaxctxt");
+            for comment in call
+                .value
+                .first_non_trivia_token()
+                .into_iter()
+                .flat_map(|it| it.leading_trivia())
+                .chain(
+                    call.value
+                        .last_non_trivia_token()
+                        .into_iter()
+                        .flat_map(|it| it.trailing_trivia()),
+                )
+                .filter(|it| it.kind() == COMMENT)
+            {
+                show_spans |= comment.text().contains("+spans");
+                show_ctxt |= comment.text().contains("+syntaxctxt");
             }
             let pp = pretty_print_macro_expansion(
                 src.value,
@@ -314,13 +333,15 @@ fn pretty_print_macro_expansion(
 ) -> String {
     let mut res = String::new();
     let mut prev_kind = EOF;
+    let mut prev_is_trivia = false;
     let mut indent_level = 0;
-    for token in iter::successors(expn.first_token(), |t| t.next_token())
+    for token in iter::successors(expn.first_non_trivia_token(), |t| t.next_token())
         .take_while(|token| token.text_range().start() < expn.text_range().end())
     {
         let curr_kind = token.kind();
+        let trivia_token = token.is_trivia();
         let space = match (prev_kind, curr_kind) {
-            _ if prev_kind.is_trivia() || curr_kind.is_trivia() => "",
+            _ if prev_is_trivia || trivia_token => "",
             _ if prev_kind.is_literal() && !curr_kind.is_punct() => " ",
             (T!['{'], T!['}']) => "",
             (T![=], _) | (_, T![=]) => " ",
@@ -363,7 +384,8 @@ fn pretty_print_macro_expansion(
             res.push_str(&"    ".repeat(level));
         }
         prev_kind = curr_kind;
-        format_to!(res, "{}", token);
+        prev_is_trivia = trivia_token;
+        res.push_str(token.text());
         if show_spans || show_ctxt {
             let span = map.span_for_range(token.text_range());
             format_to!(res, "#");

crates/hir-def/src/macro_expansion_tests/proc_macros.rs

@@ -263,9 +263,9 @@ fn foo(&self) {
     self.0. 1;
 }
 
-fn#0:Fn[8A31, 0]@45..47#ROOT2024# foo#0:Fn[8A31, 0]@48..51#ROOT2024#(#0:Fn[8A31, 0]@51..52#ROOT2024#&#0:Fn[8A31, 0]@52..53#ROOT2024#self#0:Fn[8A31, 0]@53..57#ROOT2024# )#0:Fn[8A31, 0]@57..58#ROOT2024# {#0:Fn[8A31, 0]@59..60#ROOT2024#
-    self#0:Fn[8A31, 0]@65..69#ROOT2024# .#0:Fn[8A31, 0]@69..70#ROOT2024#0#0:Fn[8A31, 0]@70..71#ROOT2024#.#0:Fn[8A31, 0]@71..72#ROOT2024#1#0:Fn[8A31, 0]@73..74#ROOT2024#;#0:Fn[8A31, 0]@74..75#ROOT2024#
-}#0:Fn[8A31, 0]@76..77#ROOT2024#"#]],
+fn#0:Fn[8A31, 0]@25..27#ROOT2024# foo#0:Fn[8A31, 0]@28..31#ROOT2024#(#0:Fn[8A31, 0]@31..32#ROOT2024#&#0:Fn[8A31, 0]@32..33#ROOT2024#self#0:Fn[8A31, 0]@33..37#ROOT2024# )#0:Fn[8A31, 0]@37..38#ROOT2024# {#0:Fn[8A31, 0]@39..40#ROOT2024#
+    self#0:Fn[8A31, 0]@45..49#ROOT2024# .#0:Fn[8A31, 0]@49..50#ROOT2024#0#0:Fn[8A31, 0]@50..51#ROOT2024#.#0:Fn[8A31, 0]@51..52#ROOT2024#1#0:Fn[8A31, 0]@53..54#ROOT2024#;#0:Fn[8A31, 0]@54..55#ROOT2024#
+}#0:Fn[8A31, 0]@56..57#ROOT2024#"#]],
     );
 }
 

crates/hir-expand/src/attrs.rs

@@ -424,10 +424,10 @@ impl AttrId {
     ) -> TextRange {
         let (_, derive_attr) = self.find_attr_range(db, krate, owner);
         let ast::Meta::TokenTreeMeta(derive_attr) = derive_attr else {
-            return derive_attr.syntax().text_range();
+            return derive_attr.syntax().text_range_without_outer_trivia();
         };
         let Some(tt) = derive_attr.token_tree() else {
-            return derive_attr.syntax().text_range();
+            return derive_attr.syntax().text_range_without_outer_trivia();
         };
         // Fake the span map, as we don't really need spans here, just the offsets of the node in the file.
         let span_map = RealSpanMap::absolute(span::EditionedFileId::current_edition(
@@ -442,11 +442,11 @@ impl AttrId {
         let Some((_, _, derive_tts)) =
             parse_path_comma_token_tree(db, &tt).nth(derive_index as usize)
         else {
-            return derive_attr.syntax().text_range();
+            return derive_attr.syntax().text_range_without_outer_trivia();
         };
         let (Some(first_span), Some(last_span)) = (derive_tts.first_span(), derive_tts.last_span())
         else {
-            return derive_attr.syntax().text_range();
+            return derive_attr.syntax().text_range_without_outer_trivia();
         };
         let start = first_span.range.start();
         let end = last_span.range.end();

crates/hir-expand/src/cfg_process.rs

@@ -196,9 +196,8 @@ fn macro_input_callback(
                                         let kind = token.kind();
                                         if kind == T![,] {
                                             break;
-                                        } else if !kind.is_trivia() {
-                                            break 'eat_comma;
                                         }
+                                        break 'eat_comma;
                                     }
                                 }
                                 events_until_comma += 1;

crates/hir-expand/src/eager.rs

@@ -184,7 +184,7 @@ fn eager_macro_recur(
             WalkEvent::Leave(child) => {
                 if let SyntaxElement::Token(t) = child {
                     let start = t.text_range().start();
-                    offset += t.text_range().len();
+                    offset += t.text_range_including_trivia().len();
                     expanded_map.push(offset, span_map.span_at(start));
                 }
                 continue;