Merge pull request #22945 from github/tausbn/yeast-emit-tokens-for-leaf-nodes-on-ast

yeast: Emit tokens for leaf nodes on AST

Author: tausbn

Cargo.lock

@@ -8,5 +8,7 @@ rust_proc_macro(
     srcs = glob(["src/**/*.rs"]),
     aliases = aliases(),
     visibility = ["//visibility:public"],
-    deps = all_crate_deps(),
+    deps = all_crate_deps() + [
+        "//shared/yeast-schema",
+    ],
 )

shared/yeast-macros/BUILD.bazel

@@ -10,3 +10,4 @@ proc-macro = true
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = { version = "3.0", features = ["full"] }
+yeast-schema = { path = "../yeast-schema" }

shared/yeast-macros/Cargo.toml

@@ -1,3 +1,5 @@
+use std::collections::BTreeSet;
+
 use proc_macro2::{Ident, Literal};
 
 /// A complete rewrite rule such as `(input) where guard => (output)`.
@@ -126,6 +128,8 @@ pub(crate) enum Content {
     Static(Literal),
     /// Computed leaf content such as `#{name}` in `(identifier #{name})`.
     Computed(syn::Block),
+    /// Value inherited from the node matched by the enclosing rule.
+    Inherited,
 }
 
 pub(crate) enum Field {
@@ -144,3 +148,117 @@ pub(crate) enum Field {
         block: syn::Block,
     },
 }
+
+impl Rule {
+    pub(crate) fn resolve_bare_leaf_content(
+        &mut self,
+        leaf_kinds: &BTreeSet<String>,
+    ) -> syn::Result<()> {
+        let Replacement::Templates(templates) = &mut self.replacement else {
+            return Ok(());
+        };
+        for template in templates {
+            template.resolve_bare_leaf_content(leaf_kinds, true)?;
+        }
+        Ok(())
+    }
+}
+
+impl Template {
+    fn resolve_bare_leaf_content(
+        &mut self,
+        leaf_kinds: &BTreeSet<String>,
+        is_rule_result: bool,
+    ) -> syn::Result<()> {
+        let Template::Node(node) = self else {
+            return Ok(());
+        };
+
+        for field in &mut node.fields {
+            if let Field::Named { value, .. } = field {
+                value.resolve_bare_leaf_content(leaf_kinds, false)?;
+            }
+        }
+
+        if node.content.is_none()
+            && node.fields.is_empty()
+            && leaf_kinds.contains(node.kind.to_string().as_str())
+        {
+            if !is_rule_result {
+                return Err(syn::Error::new_spanned(
+                    &node.kind,
+                    format!(
+                        "bare leaf `{}` must be a top-level rule result or have explicit content",
+                        node.kind
+                    ),
+                ));
+            }
+            node.content = Some(Content::Inherited);
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use quote::quote;
+
+    fn output_leaf_kinds() -> BTreeSet<String> {
+        yeast_schema::node_types_yaml::leaf_kinds_from_yaml(
+            r#"
+named:
+  assignment:
+    left: identifier
+  identifier:
+"#,
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn marks_top_level_bare_leaves_as_inherited() {
+        let mut rule = crate::rule_parse::parse_rule(quote!((source) => (identifier))).unwrap();
+        rule.resolve_bare_leaf_content(&output_leaf_kinds())
+            .unwrap();
+
+        let Replacement::Templates(templates) = rule.replacement else {
+            panic!("expected template replacement");
+        };
+        let [Template::Node(node)] = templates.as_slice() else {
+            panic!("expected one output node");
+        };
+        assert!(matches!(node.content, Some(Content::Inherited)));
+    }
+
+    #[test]
+    fn rejects_nested_bare_leaves() {
+        let mut rule =
+            crate::rule_parse::parse_rule(quote!((source) => (assignment left: (identifier))))
+                .unwrap();
+        let error = rule
+            .resolve_bare_leaf_content(&output_leaf_kinds())
+            .unwrap_err();
+
+        assert_eq!(
+            error.to_string(),
+            "bare leaf `identifier` must be a top-level rule result or have explicit content"
+        );
+    }
+
+    #[test]
+    fn leaves_fieldless_structural_nodes_unchanged() {
+        let mut rule = crate::rule_parse::parse_rule(quote!((source) => (assignment))).unwrap();
+        rule.resolve_bare_leaf_content(&output_leaf_kinds())
+            .unwrap();
+
+        let Replacement::Templates(templates) = rule.replacement else {
+            panic!("expected template replacement");
+        };
+        let [Template::Node(node)] = templates.as_slice() else {
+            panic!("expected one output node");
+        };
+        assert!(node.content.is_none());
+    }
+}

shared/yeast-macros/src/ast.rs

@@ -210,8 +210,18 @@ pub fn rule(input: TokenStream) -> TokenStream {
 /// Paths are resolved relative to the consuming crate's `CARGO_MANIFEST_DIR`
 /// (the same convention `include_str!` uses for relative paths). The
 /// resolved paths are also emitted as `include_str!` references so the
-/// consuming crate gets invalidated when a schema YAML changes, prepping
-/// the ground for compile-time type-checking against those schemas.
+/// consuming crate gets invalidated when a schema YAML changes.
+///
+/// A bare output leaf used as a top-level rule result inherits the matched
+/// input node's value:
+///
+/// ```text
+/// rule!((superExpr) => (super_expr))
+/// ```
+///
+/// Whether the output kind is a leaf is determined from the output schema.
+/// Nested bare leaves are rejected at compile time because they have no
+/// unambiguous input node from which to inherit a value.
 #[proc_macro]
 pub fn rules(input: TokenStream) -> TokenStream {
     let input2: TokenStream2 = input.into();

shared/yeast-macros/src/lib.rs

@@ -508,6 +508,7 @@ impl Node {
                         }
                     }
                 }
+                Content::Inherited => quote! { #ctx.inherited_literal(#kind_str) },
             };
             quote! { let __node: yeast::OutputId = #node; }
         } else {

shared/yeast-macros/src/lower.rs

@@ -51,6 +51,25 @@ pub fn parse_rules_top(input: TokenStream) -> Result<TokenStream> {
     };
     let input_abs = resolve_path(&input_path.value);
     let output_abs = resolve_path(&output_path.value);
+    let output_yaml = std::fs::read_to_string(&output_abs).map_err(|error| {
+        syn::Error::new(
+            output_path.literal_span,
+            format!(
+                "rules!: failed to read output schema `{}`: {error}",
+                output_abs.display()
+            ),
+        )
+    })?;
+    let output_leaf_kinds = yeast_schema::node_types_yaml::leaf_kinds_from_yaml(&output_yaml)
+        .map_err(|error| {
+            syn::Error::new(
+                output_path.literal_span,
+                format!(
+                    "rules!: failed to parse output schema `{}`: {error}",
+                    output_abs.display()
+                ),
+            )
+        })?;
 
     let list = expect_group(&mut tokens, proc_macro2::Delimiter::Bracket)?;
     if let Some(token) = tokens.next() {
@@ -63,8 +82,15 @@ pub fn parse_rules_top(input: TokenStream) -> Result<TokenStream> {
     let emitted_items = split_top_level_commas(list.stream())
         .into_iter()
         .map(|item| {
-            if has_top_level_arrow(&item) {
-                Ok(crate::rule_parse::parse_rule(item)?.lower())
+            let rule_body = if has_top_level_arrow(&item) {
+                Some(item.clone())
+            } else {
+                explicit_rule_body(&item)
+            };
+            if let Some(rule_body) = rule_body {
+                let mut rule = crate::rule_parse::parse_rule(rule_body)?;
+                rule.resolve_bare_leaf_content(&output_leaf_kinds)?;
+                Ok(rule.lower())
             } else {
                 Ok(item)
             }
@@ -84,6 +110,7 @@ pub fn parse_rules_top(input: TokenStream) -> Result<TokenStream> {
 
 struct NamedString {
     value: String,
+    literal_span: Span,
 }
 
 fn parse_named_string_arg(
@@ -109,7 +136,10 @@ fn parse_named_string_arg(
             format!("`{expected_name}` must be a string literal path"),
         )
     })?;
-    Ok(NamedString { value })
+    Ok(NamedString {
+        value,
+        literal_span: literal.span(),
+    })
 }
 
 fn string_literal_value(literal: &proc_macro2::Literal) -> Option<String> {
@@ -152,6 +182,17 @@ fn has_top_level_arrow(stream: &TokenStream) -> bool {
     })
 }
 
+fn explicit_rule_body(stream: &TokenStream) -> Option<TokenStream> {
+    let expression = syn::parse2::<syn::ExprMacro>(stream.clone()).ok()?;
+    let is_rule = expression
+        .mac
+        .path
+        .segments
+        .last()
+        .is_some_and(|segment| segment.ident == "rule");
+    is_rule.then_some(expression.mac.tokens)
+}
+
 type Tokens = std::iter::Peekable<proc_macro2::token_stream::IntoIter>;
 
 fn expect_ident(tokens: &mut Tokens, message: &str) -> Result<syn::Ident> {
@@ -212,4 +253,12 @@ mod tests {
             &quote! { { match x { 1 => 2, _ => 3 } } }
         ));
     }
+
+    #[test]
+    fn recognizes_explicit_rule_macros() {
+        assert!(explicit_rule_body(&quote! { rule!((a) => (b)) }).is_some());
+        assert!(explicit_rule_body(&quote! { yeast::rule!((a) => (b)) }).is_some());
+        assert!(explicit_rule_body(&quote! { other!((a) => (b)) }).is_none());
+        assert!(explicit_rule_body(&quote! { make_rule() }).is_none());
+    }
 }

shared/yeast-macros/src/parse.rs

@@ -469,6 +469,22 @@ pub fn schema_from_yaml(yaml_input: &str) -> Result<crate::schema::Schema, Strin
     Ok(schema)
 }
 
+/// Return the named node kinds that declare no fields.
+pub fn leaf_kinds_from_yaml(yaml_input: &str) -> Result<BTreeSet<String>, String> {
+    let yaml: YamlNodeTypes =
+        serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?;
+    Ok(yaml
+        .named
+        .into_iter()
+        .filter_map(|(name, fields)| {
+            fields
+                .as_ref()
+                .map_or(true, BTreeMap::is_empty)
+                .then_some(name)
+        })
+        .collect())
+}
+
 // ---------------------------------------------------------------------------
 // JSON → YAML conversion
 // ---------------------------------------------------------------------------
@@ -810,6 +826,24 @@ unnamed:
         assert_eq!(end["named"], false);
     }
 
+    #[test]
+    fn test_leaf_kinds_from_yaml() {
+        let leaf_kinds = leaf_kinds_from_yaml(
+            r#"
+named:
+  identifier:
+  empty_fields: {}
+  call:
+    callee: identifier
+"#,
+        )
+        .unwrap();
+
+        assert!(leaf_kinds.contains("identifier"));
+        assert!(leaf_kinds.contains("empty_fields"));
+        assert!(!leaf_kinds.contains("call"));
+    }
+
     #[test]
     fn test_explicit_unnamed_disambiguation() {
         let yaml = r#"

shared/yeast-schema/src/node_types_yaml.rs

@@ -277,13 +277,8 @@ impl Schema {
             .insert((parent_kind.to_string(), field_id), node_types);
     }
 
-    pub fn field_types(
-        &self,
-        parent_kind: &str,
-        field_id: FieldId,
-    ) -> Option<&Vec<NodeType>> {
-        self.field_types
-            .get(&(parent_kind.to_string(), field_id))
+    pub fn field_types(&self, parent_kind: &str, field_id: FieldId) -> Option<&Vec<NodeType>> {
+        self.field_types.get(&(parent_kind.to_string(), field_id))
     }
 
     /// Record the declared (named) field order for a node kind, as authored in

shared/yeast-schema/src/schema.rs

@@ -236,6 +236,27 @@ impl<'a, C> BuildCtx<'a, C> {
         )
     }
 
+    /// Create a leaf whose value is inherited from the current rule's match.
+    #[doc(hidden)]
+    pub fn inherited_literal(&mut self, kind: &'static str) -> OutputId {
+        let source = self
+            .translator
+            .as_ref()
+            .expect("inherited leaf content requires a rule match")
+            .matched_root();
+        let node = self
+            .ast
+            .get_node(source)
+            .expect("rule source node must exist");
+        let stored = node.opt_string_content();
+        let has_children = node.fields.values().any(|children| !children.is_empty());
+        let content = match stored {
+            Some(value) if !value.is_empty() || !has_children => value,
+            _ => self.ast.source_text(source),
+        };
+        self.create_named_token_with_range(kind, content, None)
+    }
+
     /// Create a leaf node with a fixed string content.
     pub fn literal(&mut self, kind: &'static str, value: &str) -> OutputId {
         self.create_named_token_with_range(kind, value.to_string(), None)