Merge pull request #22959 from github/tausbn/yeast-explicit-locationless-nodes

yeast: Add explicit locationless nodes

Author: tausbn

shared/yeast-macros/src/ast.rs

@@ -147,6 +147,8 @@ pub(crate) enum Field {
         /// A Rust block yielding the node or nodes whose locations should be combined.
         block: syn::Block,
     },
+    /// Explicitly omit source-location metadata: `location: none`.
+    LocationNone,
 }
 
 impl Rule {

shared/yeast-macros/src/lib.rs

@@ -67,6 +67,7 @@ pub fn query(input: TokenStream) -> TokenStream {
 /// field: {expr}                - extend a named field with `{expr}`'s ids
 /// location: {expr}             - override this node's source range with the
 ///                                span of one or more nodes; may be repeated
+/// location: none               - explicitly make this node locationless
 /// field: [(kind ...)]          - set the field only if the alternative can
 ///                                be constructed (see below)
 /// field: [(a) (b)]             - use the first constructible alternative

shared/yeast-macros/src/lower.rs

@@ -367,9 +367,7 @@ impl Template {
     fn lower_choice(alternatives: &[Alternative]) -> TokenStream {
         let pick = alternatives.iter().rev().fold(
             quote! { ::std::option::Option::None },
-            |fallback, alternative| {
-                alternative.lower_candidate(fallback)
-            },
+            |fallback, alternative| alternative.lower_candidate(fallback),
         );
         quote! {{ #pick }}
     }
@@ -402,25 +400,26 @@ impl Alternative {
         // evaluated in source order. Construction happens only after the
         // resulting option has been matched, keeping it in a separate branch
         // from the fallback.
-        let prepared = interpolations.into_iter().rev().fold(
-            prepared,
-            |candidate, interpolation| {
-                let GuardingInterpolation {
-                    value,
-                    source_range,
-                    expression,
-                } = interpolation;
-                // Do not use `map`/`and_then` here: `candidate` may contain
-                // embedded user Rust, and putting it in a closure would change
-                // the target of `return` and the result type seen by `?`.
-                quote! {
-                    match #expression {
-                        ::std::option::Option::Some((#value, #source_range)) => #candidate,
-                        ::std::option::Option::None => ::std::option::Option::None,
+        let prepared =
+            interpolations
+                .into_iter()
+                .rev()
+                .fold(prepared, |candidate, interpolation| {
+                    let GuardingInterpolation {
+                        value,
+                        source_range,
+                        expression,
+                    } = interpolation;
+                    // Do not use `map`/`and_then` here: `candidate` may contain
+                    // embedded user Rust, and putting it in a closure would change
+                    // the target of `return` and the result type seen by `?`.
+                    quote! {
+                        match #expression {
+                            ::std::option::Option::Some((#value, #source_range)) => #candidate,
+                            ::std::option::Option::None => ::std::option::Option::None,
+                        }
                     }
-                }
-            },
-        );
+                });
         let prepared = if let Some(guard) = &self.guard {
             let guard = rust_block(guard);
             quote! {
@@ -462,27 +461,27 @@ impl Node {
                             .interpolations
                             .borrow_mut()
                             .push(GuardingInterpolation {
-                            value: value.clone(),
-                            source_range: source_range.clone(),
-                            expression: quote! {
-                                {
-                                    let __expr = #block;
-                                    yeast::MaybeYeastValue::maybe_yeast_value(&__expr).map(
-                                        |__value_ref| {
-                                            (
-                                                yeast::YeastDisplay::yeast_to_string(
-                                                    __value_ref,
-                                                    &#ctx,
-                                                ),
-                                                yeast::YeastSourceRange::yeast_source_range(
-                                                    __value_ref,
-                                                    &#ctx,
-                                                ),
-                                            )
-                                        },
-                                    )
-                                }
-                            },
+                                value: value.clone(),
+                                source_range: source_range.clone(),
+                                expression: quote! {
+                                    {
+                                        let __expr = #block;
+                                        yeast::MaybeYeastValue::maybe_yeast_value(&__expr).map(
+                                            |__value_ref| {
+                                                (
+                                                    yeast::YeastDisplay::yeast_to_string(
+                                                        __value_ref,
+                                                        &#ctx,
+                                                    ),
+                                                    yeast::YeastSourceRange::yeast_source_range(
+                                                        __value_ref,
+                                                        &#ctx,
+                                                    ),
+                                                )
+                                            },
+                                        )
+                                    }
+                                },
                             });
                         quote! {
                             #ctx.literal_with_source_range(
@@ -517,6 +516,10 @@ impl Node {
 
         let mut stmts = Vec::new();
         let mut field_args = Vec::new();
+        let location_none = self
+            .fields
+            .iter()
+            .any(|field| matches!(field, Field::LocationNone));
 
         for (field_counter, field) in self.fields.iter().enumerate() {
             match field {
@@ -528,6 +531,7 @@ impl Node {
                         #ctx.extend_location_ranges(__value, &mut __location_ranges);
                     });
                 }
+                Field::LocationNone => {}
                 Field::Named { name, value } => {
                     let field_name = name.to_string();
                     let field_str = field_name
@@ -592,8 +596,13 @@ impl Node {
                 #node_init
                 #(#stmts)*
                 #build_node
-                let __source_range = __location_ranges.into_iter().reduce(yeast::Range::union);
-                #ctx.set_node_source_range(__node, __source_range)
+                if #location_none {
+                    #ctx.set_node_locationless(__node)
+                } else {
+                    let __source_range =
+                        __location_ranges.into_iter().reduce(yeast::Range::union);
+                    #ctx.set_node_source_range(__node, __source_range)
+                }
             }
         }
     }

shared/yeast-macros/src/template_parse.rs

@@ -71,6 +71,8 @@ fn parse_node(group: proc_macro2::Group) -> Result<Node> {
     };
 
     let mut fields = Vec::new();
+    let mut has_location = false;
+    let mut has_location_none = false;
     while peek_is_field(&mut tokens) {
         let name = expect_ident(&mut tokens, "expected field name")?;
         let is_location = name == "location";
@@ -82,21 +84,47 @@ fn parse_node(group: proc_macro2::Group) -> Result<Node> {
         }
         expect_punct(&mut tokens, ':', "expected `:` after field name")?;
         if is_location {
-            if !peek_is_group(&mut tokens, Delimiter::Brace) {
+            if matches!(tokens.peek(), Some(TokenTree::Ident(ident)) if ident == "none") {
+                let none = tokens.next().unwrap();
+                if has_location || has_location_none {
+                    return Err(syn::Error::new_spanned(
+                        none,
+                        "`location: none` cannot be combined with another `location` field",
+                    ));
+                }
+                has_location_none = true;
+                fields.push(Field::LocationNone);
+            } else if peek_is_group(&mut tokens, Delimiter::Brace) {
+                if has_location_none {
+                    return Err(syn::Error::new_spanned(
+                        tokens.peek().unwrap().clone(),
+                        "`location: none` cannot be combined with another `location` field",
+                    ));
+                }
+                has_location = true;
+                let block = parse_block(expect_group(&mut tokens, Delimiter::Brace)?)?;
+                fields.push(Field::Location { block });
+            } else {
                 let error = match tokens.peek() {
                     Some(token) => syn::Error::new_spanned(
                         token.clone(),
-                        "`location` field value must be a Rust block",
+                        "`location` field value must be a Rust block or `none`",
                     ),
                     None => syn::Error::new(
                         Span::call_site(),
-                        "`location` field value must be a Rust block",
+                        "`location` field value must be a Rust block or `none`",
                     ),
                 };
                 return Err(error);
             }
-            let block = parse_block(expect_group(&mut tokens, Delimiter::Brace)?)?;
-            fields.push(Field::Location { block });
+            if let Some(TokenTree::Punct(punctuation)) = tokens.peek() {
+                if punctuation.as_char() == '?' {
+                    return Err(syn::Error::new_spanned(
+                        punctuation.clone(),
+                        "`?` is not valid on a `location` field",
+                    ));
+                }
+            }
             continue;
         }
 
@@ -290,6 +318,19 @@ mod tests {
         assert!(matches!(node.fields.as_slice(), [Field::Location { .. }]));
     }
 
+    #[test]
+    fn parses_location_none() {
+        let template = parse_template(quote! {
+            (identifier "name" location: none)
+        })
+        .unwrap();
+
+        let Template::Node(node) = template else {
+            panic!("expected node template");
+        };
+        assert!(matches!(node.fields.as_slice(), [Field::LocationNone]));
+    }
+
     #[test]
     fn rejects_non_block_location_fields() {
         let result = parse_template(quote! {
@@ -300,7 +341,21 @@ mod tests {
         };
         assert_eq!(
             error.to_string(),
-            "`location` field value must be a Rust block"
+            "`location` field value must be a Rust block or `none`"
+        );
+    }
+
+    #[test]
+    fn rejects_location_none_with_other_locations() {
+        let result = parse_template(quote! {
+            (identifier "name" location: none location: {source})
+        });
+        let Err(error) = result else {
+            panic!("expected mixed location metadata to be rejected");
+        };
+        assert_eq!(
+            error.to_string(),
+            "`location: none` cannot be combined with another `location` field"
         );
     }
 

shared/yeast/doc/yeast.md

@@ -300,6 +300,17 @@ leaf nodes:
 The override applies only to the node carrying the `location` field; nested
 nodes still derive their locations normally.
 
+As an exception to the Rust-block requirement above, use `location: none` for a
+synthetic node that intentionally has no source location. The field-combination
+rules above apply only to block-valued fields: `location: none` cannot be
+combined with any other `location` field on the same node. A locationless node
+does not contribute to its parent's derived range, and a returned rule result
+marked this way does not inherit the matched input range:
+
+```rust
+(metadata_node location: none)
+```
+
 For input fields whose leading or trailing syntax should never belong to rule
 results, configure them once with
 `DesugaringConfig::with_ignored_location_fields(...)`. For example, ignoring

shared/yeast/src/build.rs

@@ -187,6 +187,14 @@ impl<'a, C> BuildCtx<'a, C> {
         node
     }
 
+    /// Explicitly make a newly-built node locationless.
+    #[doc(hidden)]
+    pub fn set_node_locationless(&mut self, node: OutputId) -> OutputId {
+        self.output.clear_source_range(node);
+        self.created_nodes_without_source_range.remove(&node);
+        node
+    }
+
     /// Return the immutable input AST. Available inside rule transforms.
     pub fn ast(&self) -> &Ast {
         self.ast

shared/yeast/src/lib.rs

@@ -816,6 +816,18 @@ impl<Id: TreeId> Ast<Id> {
         node.source_range = Some(source_range);
     }
 
+    /// Remove the source range from a synthetic node.
+    pub(crate) fn clear_source_range(&mut self, id: Id) {
+        let node = self
+            .nodes
+            .get_mut(id.index())
+            .unwrap_or_else(|| panic!("clear_source_range: invalid node id {}", id.index()));
+        if matches!(node.content, NodeContent::Range(_)) {
+            panic!("clear_source_range: cannot modify a parsed node");
+        }
+        node.source_range = None;
+    }
+
     /// Register a named node kind, returning its id (idempotent). Lets callers
     /// build an AST in a single pass, registering kinds as nodes are created
     /// rather than pre-populating the schema.

shared/yeast/tests/test.rs

@@ -1981,6 +1981,60 @@ fn test_location_field_overrides_derived_location() {
     assert_eq!(call.byte_range(), 4..5);
 }
 
+#[test]
+fn test_location_none_makes_result_root_locationless() {
+    let rule: Rule = rule!(
+        (call method: (identifier) @method)
+        =>
+        (call method: {method} location: none)
+    );
+
+    let ast = run_and_ast("foo.bar()", vec![rule]);
+    let call = ast
+        .reachable_node_ids()
+        .into_iter()
+        .filter_map(|id| ast.get_node(id))
+        .find(|node| node.kind_name() == "call")
+        .expect("call exists");
+    assert_eq!(call.source_range(), None);
+}
+
+/// A nested `location: none` node stays locationless and does not widen its
+/// parent's derived range with an empty range at the match start.
+#[test]
+fn test_location_none_nested_node_does_not_widen_parent() {
+    let rule: Rule = rule!(
+        (call method: (identifier) @method)
+        =>
+        (call
+            method: (identifier "outer")
+            receiver: (call
+                method: {method}
+                arguments: (argument_list location: none)))
+    );
+
+    let ast = run_and_ast("foo.bar()", vec![rule]);
+    let nodes: Vec<_> = ast
+        .reachable_node_ids()
+        .into_iter()
+        .filter_map(|id| ast.get_node(id))
+        .collect();
+    let arguments = nodes
+        .iter()
+        .find(|node| node.kind_name() == "argument_list")
+        .expect("argument list exists");
+    assert_eq!(arguments.source_range(), None);
+
+    let mut call_ranges: Vec<_> = nodes
+        .iter()
+        .filter(|node| node.kind_name() == "call")
+        .map(|node| node.byte_range())
+        .collect();
+    call_ranges.sort_by_key(|range| (range.start, range.end));
+    // Without `location: none`, the inner call would be widened to 0..7.
+    assert_eq!(call_ranges, [0..9, 4..7]);
+}
+
 #[test]
 fn test_multiple_location_fields_combine_iterable_locations() {
     let rule: Rule = rule!(