yeast: Add internal AST for rules

Extends the previous work. We now explicitly split apart parsing of
rules from lowering them into Rust code.

There is some redundancy here -- a Pattern is basically the same as the
Query that we eventually execute, but cleaning that up would introduce a
lot of churn, so I've tried to leave it as-is for now.

Author: tausbn

shared/yeast-macros/src/ast.rs

@@ -1,5 +1,94 @@
 use proc_macro2::{Ident, Literal};
 
+/// A complete rewrite rule such as `(input) where guard => (output)`.
+pub(crate) struct Rule {
+    /// Input pattern before `where` or `=>`.
+    pub(crate) pattern: Pattern,
+    /// Optional Rust guard expression after `where`.
+    pub(crate) guard: Option<syn::Expr>,
+    /// Template or annotated Rust replacement after `=>`.
+    pub(crate) replacement: Replacement,
+}
+
+pub(crate) enum Replacement {
+    /// One or more output templates such as `(call ...) {extra_nodes}`.
+    Templates(Vec<Template>),
+    /// Annotated Rust body such as `expr? { ... }`.
+    Rust {
+        /// Retained in the rule AST for validation and analysis; runtime
+        /// lowering only needs the Rust body.
+        #[allow(dead_code)]
+        annotation: ReturnAnnotation,
+        body: syn::Block,
+    },
+}
+
+#[allow(dead_code)]
+pub(crate) struct ReturnAnnotation {
+    /// Declared output kind before the Rust body.
+    pub(crate) kind: Ident,
+    /// Declared output cardinality (`kind`, `kind?`, or `kind*`).
+    pub(crate) cardinality: Cardinality,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct Cardinality {
+    pub(crate) multiple: bool,
+    pub(crate) required: bool,
+}
+
+impl Cardinality {
+    pub(crate) const SINGLE: Self = Self {
+        multiple: false,
+        required: true,
+    };
+    pub(crate) const OPTIONAL: Self = Self {
+        multiple: false,
+        required: false,
+    };
+    pub(crate) const ZERO_OR_MORE: Self = Self {
+        multiple: true,
+        required: false,
+    };
+    pub(crate) const ONE_OR_MORE: Self = Self {
+        multiple: true,
+        required: true,
+    };
+}
+
+/// An input pattern such as `(call method: (identifier) @name)`.
+pub(crate) enum Pattern {
+    /// `_` or `(_)`, distinguished by whether unnamed nodes may match.
+    Any { match_unnamed: bool },
+    /// A named node and the pattern matched against each field.
+    Node {
+        kind: String,
+        fields: Vec<(String, Pattern)>,
+    },
+    /// An unnamed token pattern such as `"+"`.
+    Unnamed(String),
+    /// A captured pattern such as `(identifier) @name` or `_ @@raw`.
+    Capture {
+        capture: Capture,
+        pattern: Box<Pattern>,
+    },
+    /// An ordered sequence of patterns within a field.
+    Sequence(Vec<Pattern>),
+    /// A repeated or optional pattern.
+    Repeated {
+        pattern: Box<Pattern>,
+        cardinality: Cardinality,
+    },
+}
+
+#[derive(Clone)]
+pub(crate) struct Capture {
+    /// Capture identifier after `@` or `@@`.
+    pub(crate) name: Ident,
+    /// Whether `@@` keeps the captured input node untranslated.
+    pub(crate) raw: bool,
+}
+
 pub(crate) enum Template {
     /// A parenthesized node template such as `(call method: {name})`.
     Node(Node),

shared/yeast-macros/src/lib.rs

@@ -4,8 +4,10 @@ use proc_macro2::TokenStream as TokenStream2;
 mod ast;
 mod lower;
 mod parse;
+mod rule_parse;
+mod template_parse;
 
-/// Proc macro for constructing a `QueryNode` from a tree-sitter-inspired pattern.
+/// Proc macro for constructing a tree-sitter-inspired query `Pattern`.
 ///
 /// # Syntax
 ///

shared/yeast-macros/src/lower.rs

@@ -3,7 +3,9 @@ use quote::quote;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use syn::Lifetime;
 
-use crate::ast::{Content, Field, Node, Template};
+use crate::ast::{
+    Capture, Cardinality, Content, Field, Node, Pattern, Replacement, Rule, Template,
+};
 
 struct LoweringContext<'a> {
     /// Enclosing optional-field label to exit when a computed value is absent.
@@ -16,6 +18,209 @@ fn fresh_fallible_label() -> Lifetime {
     Lifetime::new(&format!("'__yeast_field_{n}"), Span::call_site())
 }
 
+#[derive(Clone)]
+struct BoundCapture {
+    capture: Capture,
+    cardinality: Cardinality,
+}
+
+impl Rule {
+    pub(crate) fn lower(&self) -> TokenStream {
+        let query = self.pattern.lower();
+        let captures = self.pattern.captures();
+        let (raw_captures, translated_captures): (Vec<_>, Vec<_>) =
+            captures.iter().partition(|capture| capture.capture.raw);
+        let raw_capture_names = raw_captures
+            .iter()
+            .map(|capture| capture.capture.name.to_string())
+            .collect::<Vec<_>>();
+        let raw_bindings = capture_bindings(raw_captures.into_iter());
+        let translated_bindings = capture_bindings(translated_captures.into_iter());
+        let replacement = self.replacement.lower();
+        let guard = self
+            .guard
+            .clone()
+            .unwrap_or_else(|| syn::parse_quote!(true));
+        let ctx = Ident::new("ctx", Span::call_site());
+
+        quote! {
+            {
+                let __query = #query;
+                yeast::Rule::guarded(
+                    __query,
+                    Box::new(|__ast: &yeast::Ast, __captures: &yeast::captures::Captures, __user_ctx: &mut _| {
+                        #(#raw_bindings)*
+                        #(#translated_bindings)*
+                        let ast = __ast;
+                        let #ctx = __user_ctx;
+                        Ok(#guard)
+                    }),
+                    Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __source_range: Option<yeast::Range>, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| {
+                        let __skip: &[&str] = &[#(#raw_capture_names),*];
+                        __translator.auto_translate_captures(&mut __captures, __ast, __user_ctx, __skip)?;
+                        #(#raw_bindings)*
+                        #(#translated_bindings)*
+                        let mut #ctx = yeast::build::BuildCtx::with_translator(__ast, &__captures, __source_range, __user_ctx, __translator);
+                        let __result: Vec<yeast::Id> = { #replacement };
+                        let __result = #ctx.finish_rule(__result);
+                        Ok(__result)
+                    }),
+                )
+            }
+        }
+    }
+}
+
+impl Replacement {
+    fn lower(&self) -> TokenStream {
+        match self {
+            Replacement::Templates(templates) => Template::lower_list(templates),
+            Replacement::Rust { body, .. } => {
+                let body = rust_block(body);
+                quote! {
+                    let __value = #body;
+                    let mut __ids: Vec<yeast::Id> = Vec::new();
+                    yeast::IntoFieldIds::extend_into(__value, &mut __ids);
+                    __ids
+                }
+            }
+        }
+    }
+}
+
+impl Pattern {
+    pub(crate) fn lower(&self) -> TokenStream {
+        match self {
+            Pattern::Any { match_unnamed } => {
+                quote! { yeast::query::QueryNode::Any { match_unnamed: #match_unnamed } }
+            }
+            Pattern::Node { kind, fields } => {
+                let fields = fields.iter().map(|(name, pattern)| {
+                    let elements = pattern.lower_list();
+                    quote! { (#name, vec![#(#elements),*]) }
+                });
+                quote! {
+                    yeast::query::QueryNode::Node {
+                        kind: #kind,
+                        children: vec![#(#fields),*],
+                    }
+                }
+            }
+            Pattern::Unnamed(kind) => {
+                quote! { yeast::query::QueryNode::UnnamedNode { kind: #kind } }
+            }
+            Pattern::Capture { capture, pattern } => {
+                let name = capture.name.to_string();
+                let pattern = pattern.lower();
+                quote! {
+                    yeast::query::QueryNode::Capture {
+                        capture: #name,
+                        node: Box::new(#pattern),
+                    }
+                }
+            }
+            Pattern::Sequence(_) | Pattern::Repeated { .. } => {
+                unreachable!("sequence and repeated patterns cannot match a single node")
+            }
+        }
+    }
+
+    fn lower_list(&self) -> Vec<TokenStream> {
+        match self {
+            Pattern::Sequence(patterns) => patterns.iter().flat_map(Pattern::lower_list).collect(),
+            Pattern::Repeated {
+                pattern,
+                cardinality,
+            } => {
+                let children = pattern.lower_list();
+                let repetition = match (cardinality.multiple, cardinality.required) {
+                    (true, false) => quote! { yeast::query::Rep::ZeroOrMore },
+                    (true, true) => quote! { yeast::query::Rep::OneOrMore },
+                    (false, false) => quote! { yeast::query::Rep::ZeroOrOne },
+                    (false, true) => unreachable!("single patterns are not wrapped as repeated"),
+                };
+                vec![quote! {
+                    yeast::query::QueryListElem::Repeated {
+                        children: vec![#(#children),*],
+                        rep: #repetition,
+                    }
+                }]
+            }
+            pattern => {
+                let pattern = pattern.lower();
+                vec![quote! { yeast::query::QueryListElem::SingleNode(#pattern) }]
+            }
+        }
+    }
+    fn captures(&self) -> Vec<BoundCapture> {
+        let mut captures = Vec::new();
+        self.collect_captures(Cardinality::SINGLE, &mut captures);
+        captures
+    }
+
+    fn collect_captures(&self, cardinality: Cardinality, captures: &mut Vec<BoundCapture>) {
+        match self {
+            Pattern::Any { .. } | Pattern::Unnamed(_) => {}
+            Pattern::Node { fields, .. } => {
+                for (_, pattern) in fields {
+                    pattern.collect_captures(cardinality, captures);
+                }
+            }
+            Pattern::Capture { capture, pattern } => {
+                pattern.collect_captures(cardinality, captures);
+                captures.push(BoundCapture {
+                    capture: capture.clone(),
+                    cardinality,
+                });
+            }
+            Pattern::Sequence(patterns) => {
+                for pattern in patterns {
+                    pattern.collect_captures(cardinality, captures);
+                }
+            }
+            Pattern::Repeated {
+                pattern,
+                cardinality: repeated,
+            } => {
+                pattern.collect_captures(combine_cardinality(cardinality, *repeated), captures);
+            }
+        }
+    }
+}
+
+fn combine_cardinality(parent: Cardinality, child: Cardinality) -> Cardinality {
+    Cardinality {
+        multiple: parent.multiple || child.multiple,
+        required: parent.required && child.required,
+    }
+}
+
+fn capture_bindings<'a>(captures: impl Iterator<Item = &'a BoundCapture>) -> Vec<TokenStream> {
+    captures
+        .map(|capture| {
+            let name_string = capture.capture.name.to_string();
+            let name = Ident::new(&name_string, Span::call_site());
+            match (capture.cardinality.multiple, capture.cardinality.required) {
+                (true, _) => {
+                    quote! {
+                        let #name: Vec<yeast::Id> = __captures.get_all(#name_string);
+                    }
+                }
+                (false, false) => {
+                    quote! {
+                        let #name: Option<yeast::Id> = __captures.get_opt(#name_string);
+                    }
+                }
+                (false, true) => {
+                    quote! {
+                        let #name: yeast::Id = __captures.get_var(#name_string).unwrap();
+                    }
+                }
+            }
+        })
+        .collect()
+}
+
 impl Template {
     pub(crate) fn lower_root(&self) -> TokenStream {
         self.lower(&LoweringContext {