yeast: Track child source ranges in separate contexts

NB: This temporarily changes the test output for bulk imports, blocks,
and things like synthesised Array and Option literals.. Later commits
will fix these up again.

Author: tausbn

shared/yeast-macros/src/parse.rs

@@ -987,6 +987,7 @@ pub fn parse_rule_top(input: TokenStream) -> Result<TokenStream> {
                     #(#translated_bindings)*
                     let mut #ctx_ident = yeast::build::BuildCtx::with_translator(__ast, &__captures, __fresh, __source_range, __user_ctx, __translator);
                     let __result: Vec<yeast::Id> = { #transform_body };
+                    let __result = #ctx_ident.finish_rule(__result);
                     Ok(__result)
                 }),
             )

shared/yeast/doc/yeast.md

@@ -235,6 +235,36 @@ yeast::trees!(ctx,
 (identifier #{name})         // an identifier from a Rust variable
 ```
 
+### Source locations
+
+Captured nodes keep the locations assigned by their own translations. New
+nodes in an output template derive their locations from their children. After
+the transform completes, the matched input node's range is added only to
+locally-created nodes returned as rule results:
+
+```rust
+rule!(
+    (wrapper child: (_) @child)
+    =>
+    (outer nested: (inner value: {child}))
+)
+```
+
+Here `inner` derives its range from `child`, while the returned `outer` node
+also includes the full `wrapper` range. This lets replacement roots include
+elided keywords or delimiters without assigning the same broad range to every
+synthetic descendant. A transform that simply returns a translated capture
+does not widen that capture to the wrapper's range.
+
+Rust bodies can explicitly assign a default range to a synthetic subtree whose
+location cannot be derived from children:
+
+```rust
+let synthetic = ctx.with_node_source_range(ctx.matched_source_range, |ctx| {
+    tree!(ctx, (synthetic_node))
+});
+```
+
 ### Optional fields (`?`)
 
 A `?` on a field's value makes that field fallible. If a `#{expr}` anywhere

shared/yeast/src/build.rs

@@ -1,8 +1,8 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 use crate::captures::Captures;
 use crate::tree_builder::FreshScope;
-use crate::{Ast, FieldId, Id, NodeContent, Range, TranslatorHandle};
+use crate::{Ast, FieldId, Id, KindId, NodeContent, Range, TranslatorHandle};
 
 /// Context for building new AST nodes during a transformation.
 ///
@@ -33,13 +33,25 @@ pub struct BuildCtx<'a, C: 'a = ()> {
     pub ast: &'a mut Ast,
     pub captures: &'a Captures,
     pub fresh: &'a FreshScope,
-    /// Source range of the matched node, inherited by synthetic nodes.
+    /// Optional source range explicitly inherited by every synthetic node built
+    /// through this context.
     pub source_range: Option<Range>,
+    /// Source range of the node matched by the current rule.
+    ///
+    /// The `rule!` macro applies this range to locally-created result roots
+    /// after the transform completes. Nested synthetic nodes derive their
+    /// ranges from their children unless `source_range` is explicitly set.
+    pub matched_source_range: Option<Range>,
     /// User-supplied context, accessible directly via `ctx.field` (via Deref).
     pub user_ctx: &'a mut C,
     /// Optional translator handle, populated when the context is built by
     /// the framework's rule driver. None when the context is built by hand.
     pub(crate) translator: Option<TranslatorHandle<'a, C>>,
+    /// Nodes built directly through this context without an explicit source
+    /// range in either their content or constructor argument. Recursive
+    /// translations use their own context and therefore do not contribute to
+    /// this list.
+    created_nodes: BTreeSet<Id>,
 }
 
 impl<'a, C> BuildCtx<'a, C> {
@@ -54,8 +66,10 @@ impl<'a, C> BuildCtx<'a, C> {
             captures,
             fresh,
             source_range: None,
+            matched_source_range: None,
             user_ctx,
             translator: None,
+            created_nodes: BTreeSet::new(),
         }
     }
 
@@ -71,8 +85,10 @@ impl<'a, C> BuildCtx<'a, C> {
             captures,
             fresh,
             source_range,
+            matched_source_range: None,
             user_ctx,
             translator: None,
+            created_nodes: BTreeSet::new(),
         }
     }
 
@@ -90,10 +106,94 @@ impl<'a, C> BuildCtx<'a, C> {
             ast,
             captures,
             fresh,
-            source_range,
+            source_range: None,
+            matched_source_range: source_range,
             user_ctx,
             translator: Some(translator),
+            created_nodes: BTreeSet::new(),
+        }
+    }
+
+    /// Create a node and record it as constructed by this rule invocation.
+    pub fn create_node_with_range(
+        &mut self,
+        kind: KindId,
+        content: NodeContent,
+        fields: BTreeMap<FieldId, Vec<Id>>,
+        is_named: bool,
+        source_range: Option<Range>,
+    ) -> Id {
+        let has_explicit_source_range =
+            source_range.is_some() || matches!(&content, NodeContent::Range(_));
+        let id = self
+            .ast
+            .create_node_with_range(kind, content, fields, is_named, source_range);
+        if !has_explicit_source_range {
+            self.created_nodes.insert(id);
+        }
+        id
+    }
+
+    /// Create a node using this context's explicit default source range.
+    pub fn create_node(
+        &mut self,
+        kind: KindId,
+        content: NodeContent,
+        fields: BTreeMap<FieldId, Vec<Id>>,
+        is_named: bool,
+    ) -> Id {
+        self.create_node_with_range(kind, content, fields, is_named, self.source_range)
+    }
+
+    /// Create a named token and record it as constructed by this rule invocation.
+    pub fn create_named_token_with_range(
+        &mut self,
+        kind: &'static str,
+        content: String,
+        source_range: Option<Range>,
+    ) -> Id {
+        let has_explicit_source_range = source_range.is_some();
+        let id = self
+            .ast
+            .create_named_token_with_range(kind, content, source_range);
+        if !has_explicit_source_range {
+            self.created_nodes.insert(id);
+        }
+        id
+    }
+
+    /// Create a named token using this context's explicit default source range.
+    pub fn create_named_token(&mut self, kind: &'static str, content: String) -> Id {
+        self.create_named_token_with_range(kind, content, self.source_range)
+    }
+
+    /// Finish the current rule invocation by applying the matched source range
+    /// to locally-created result roots.
+    #[doc(hidden)]
+    pub fn finish_rule(self, results: Vec<Id>) -> Vec<Id> {
+        if let Some(source_range) = self.matched_source_range {
+            for &id in &results {
+                if self.created_nodes.contains(&id) {
+                    self.ast.extend_source_range(id, source_range);
+                }
+            }
         }
+        results
+    }
+
+    /// Build nodes with an explicit default source range.
+    ///
+    /// This is the Rust escape hatch for synthetic subtrees whose useful
+    /// location cannot be derived from children. The previous default is
+    /// restored after `f` returns.
+    pub fn with_node_source_range<F, R>(&mut self, source_range: Option<Range>, f: F) -> R
+    where
+        F: FnOnce(&mut Self) -> R,
+    {
+        let previous = std::mem::replace(&mut self.source_range, source_range);
+        let result = f(self);
+        self.source_range = previous;
+        result
     }
 
     /// Look up a capture variable, returning its node Id.
@@ -133,30 +233,29 @@ impl<'a, C> BuildCtx<'a, C> {
                 .unwrap_or_else(|| panic!("build: field '{name}' not found"));
             field_map.entry(field_id).or_default().extend(ids);
         }
-        self.ast.create_node_with_range(
+        self.create_node(
             kind_id,
             NodeContent::DynamicString(String::new()),
             field_map,
             true,
-            self.source_range,
         )
     }
 
     /// Create a leaf node with a fixed string content.
     pub fn literal(&mut self, kind: &'static str, value: &str) -> Id {
-        self.ast
-            .create_named_token_with_range(kind, value.to_string(), self.source_range)
+        self.create_named_token(kind, value.to_string())
     }
 
     /// Create a leaf node with fixed content and an optional preferred source range.
-    /// If `source_range` is `None`, falls back to this context's inherited range.
+    /// If `source_range` is `None`, falls back to this context's explicit
+    /// default range.
     pub fn literal_with_source_range(
         &mut self,
         kind: &'static str,
         value: &str,
         source_range: Option<Range>,
     ) -> Id {
-        self.ast.create_named_token_with_range(
+        self.create_named_token_with_range(
             kind,
             value.to_string(),
             source_range.or(self.source_range),
@@ -166,8 +265,7 @@ impl<'a, C> BuildCtx<'a, C> {
     /// Create a leaf node with an auto-generated unique name.
     pub fn fresh(&mut self, kind: &'static str, name: &str) -> Id {
         let generated = self.fresh.resolve(name);
-        self.ast
-            .create_named_token_with_range(kind, generated, self.source_range)
+        self.create_named_token(kind, generated)
     }
 }
 
@@ -203,8 +301,9 @@ impl<C: Clone> BuildCtx<'_, C> {
 
     /// Run `f` with a temporary child [`BuildCtx`] whose `user_ctx` is
     /// a fresh clone of the current one, sharing everything else
-    /// (`ast`, `captures`, `fresh`, `source_range`, `translator`) by
-    /// re-borrow. Any mutations `f` makes to the child's `user_ctx`
+    /// (`ast`, `captures`, `fresh`, source ranges, `translator`) by re-borrow.
+    /// Nodes constructed through the child remain part of the current rule
+    /// invocation. Any mutations `f` makes to the child's `user_ctx`
     /// are discarded when it returns — no restore needed, because the
     /// mutations only ever happened on a local clone.
     ///
@@ -236,10 +335,16 @@ impl<C: Clone> BuildCtx<'_, C> {
             captures: self.captures,
             fresh: self.fresh,
             source_range: self.source_range,
+            matched_source_range: self.matched_source_range,
             user_ctx: &mut child_user_ctx,
             translator: self.translator,
+            created_nodes: BTreeSet::new(),
         };
-        f(&mut child)
+        let result = f(&mut child);
+        let created_nodes = std::mem::take(&mut child.created_nodes);
+        drop(child);
+        self.created_nodes.extend(created_nodes);
+        result
         // child_user_ctx dropped; the outer `self` is unaffected.
     }
 }

shared/yeast/src/lib.rs

@@ -128,9 +128,10 @@ pub trait YeastDisplay {
 
 /// Optional source range for values used in `#{expr}` interpolations.
 ///
-/// By default this returns `None`, so synthesized leaves inherit the matched
-/// rule's source range. `Id` returns the referenced node's range, letting
-/// `(kind #{capture})` carry the captured node's location.
+/// By default this returns `None`, so synthesized leaves use the current
+/// [`crate::build::BuildCtx`] default range, if any. `Id` returns the
+/// referenced node's range, letting `(kind #{capture})` carry the captured
+/// node's location.
 pub trait YeastSourceRange {
     fn yeast_source_range(&self, ast: &Ast) -> Option<Range>;
 }
@@ -592,8 +593,7 @@ impl Ast {
             // Parsed nodes already carry an exact source range in their content.
             NodeContent::Range(_) => source_range,
             // Synthesized nodes derive location from both their children and
-            // the inherited rule-match range, so tokens matched by a rule but
-            // elided from its output still contribute to the replacement range.
+            // any explicitly supplied source range.
             _ => self
                 .union_source_range_of_children(&fields)
                 .map_or(source_range, |child_range| {
@@ -618,6 +618,24 @@ impl Ast {
         Id(id)
     }
 
+    /// Extend a synthetic node's source range to include `source_range`.
+    ///
+    /// Parsed nodes carry their exact range in [`NodeContent::Range`] and must
+    /// not be modified through this API.
+    pub fn extend_source_range(&mut self, id: Id, source_range: Range) {
+        let node = self
+            .nodes
+            .get_mut(id.0)
+            .unwrap_or_else(|| panic!("extend_source_range: invalid node id {}", id.0));
+        if matches!(node.content, NodeContent::Range(_)) {
+            panic!("extend_source_range: cannot modify a parsed node");
+        }
+        node.source_range = Some(match node.source_range {
+            Some(existing) => union_source_ranges(existing, source_range),
+            None => source_range,
+        });
+    }
+
     /// Register a named node kind, returning its id (idempotent). Lets callers
     /// build an AST in a single pass, registering kinds as nodes are created
     /// rather than pre-populating the schema.
@@ -1134,8 +1152,8 @@ impl<C> Rule<C> {
         }
     }
 
-    /// Run this rule's transform with the given captures, using `node`'s
-    /// source range as the source range of the produced nodes.
+    /// Run this rule's transform with the given captures, making `node`'s
+    /// source range available to the transform.
     fn run_transform(
         &self,
         ast: &mut Ast,