1- //! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the
2- //! in-memory format the CodeQL desugaring rules operate on.
1+ //! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`].
32//!
4- //! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
5- //! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
6- //! so the extractor consumes swift-syntax output without pulling in the Swift
7- //! toolchain (the JSON is produced out-of-process).
3+ //! The JSON is produced out-of-process by `swift-syntax-rs`'s FFI shim, so
4+ //! this module is pure Rust (no Swift toolchain needed).
85//!
9- //! The mapping mirrors tree-sitter's node model, which is what yeast (and the
10- //! extractor's rewrite rules) expect:
6+ //! The mapping mirrors tree-sitter's node model:
117//!
12- //! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens** (identifiers,
13- //! literals, operators — the ones whose text is not determined by their kind)
14- //! become **named** nodes, keyed by their kind name.
15- //! * **Fixed tokens** (keywords and punctuation, whose text is fully determined
16- //! by their kind) become **anonymous** nodes, keyed by their text — exactly
17- //! how tree-sitter models anonymous tokens (e.g. `"func"`, `"->"`).
18- //! * Collection nodes are already elided to JSON arrays upstream, so a
19- //! list-valued field maps directly to that field holding several children.
8+ //! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens**
9+ //! (identifiers, literals, operators — text not determined by kind) become
10+ //! **named** nodes, keyed by their kind name.
11+ //! * **Fixed tokens** (keywords, punctuation) become **anonymous** nodes,
12+ //! keyed by their text (like tree-sitter's anonymous tokens).
13+ //! * Collection nodes are already elided to JSON arrays upstream.
2014//!
21- //! Note: this preserves swift-syntax's own kind/field names. Aligning those
22- //! names with the tree-sitter-swift schema (so the rewrite rules in
23- //! [`super::swift`] fire) is done incrementally in the rules.
15+ //! Kind/field names are preserved from swift-syntax; aligning them with the
16+ //! tree-sitter-swift schema is done incrementally in the rewrite rules.
2417
2518use std:: collections:: BTreeMap ;
2619
@@ -29,32 +22,29 @@ use yeast::schema::Schema;
2922use yeast:: { Ast , Id , NodeContent , Point , Range } ;
3023
3124/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia.
32- ///
33- /// These are collected into a side channel rather than embedded in the
34- /// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
35- /// nodes: they carry a location and text but are not attached to a parent.
25+ /// Collected into a side channel rather than embedded in the [`yeast::Ast`],
26+ /// mirroring how the extractor treats tree-sitter `extra` nodes.
3627#[ derive( Debug , Clone , PartialEq , Eq ) ]
3728pub struct TriviaToken {
38- /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`,
39- /// `docBlockComment`, `unexpectedText`).
29+ /// The trivia kind (e.g. `lineComment`, `docLineComment`, `unexpectedText`).
4030pub kind : String ,
41- /// The verbatim source text of the piece (e.g. `// comment`).
31+ /// The verbatim source text (e.g. `// comment`).
4232pub text : String ,
4333 /// The source range the piece occupies.
4434pub range : Range ,
4535}
4636
47- /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
48- /// comment/`unexpectedText` trivia harvested from it (in source order).
37+ /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus
38+ /// the comment/`unexpectedText` trivia harvested from it (in source order).
4939pub struct AdaptedTree {
5040 pub ast : Ast ,
5141 pub trivia : Vec < TriviaToken > ,
5242}
5343
5444/// swift-syntax `TokenKind` cases whose text is *not* determined by the kind
55- /// (i.e. `TokenKind.defaultText == nil`). These carry varying information and
56- /// are modelled as named leaf nodes; every other token is a fixed
57- /// keyword/punctuation token modelled as an anonymous token keyed by its text.
45+ /// (i.e. `TokenKind.defaultText == nil`). Modelled as named leaf nodes; every
46+ /// other token is a fixed keyword/punctuation, modelled as an anonymous token
47+ /// keyed by its text.
5848const VARYING_TOKEN_KINDS : & [ & str ] = & [
5949 "identifier" ,
6050 "integerLiteral" ,
@@ -116,7 +106,7 @@ fn classify(node: &Value) -> Result<KindInfo, String> {
116106 . unwrap_or ( "" )
117107 . to_string ( ) ;
118108
119- // The case name is the part before any `(payload)` in the debug rendering.
109+ // Case name is the part before any `(payload)` in the debug rendering.
120110 let case_name = token_kind. split ( '(' ) . next ( ) . unwrap_or ( token_kind) ;
121111
122112 if VARYING_TOKEN_KINDS . contains ( & case_name) {
@@ -142,8 +132,7 @@ fn classify(node: &Value) -> Result<KindInfo, String> {
142132 }
143133}
144134
145- /// Iterate over a node object's structural (field, value) pairs in a stable
146- /// order, skipping metadata keys.
135+ /// Iterate over a node's structural (field, value) pairs, skipping metadata.
147136fn field_entries ( node : & Value ) -> Vec < ( & str , & Value ) > {
148137 node. as_object ( )
149138 . map ( |map| {
@@ -155,8 +144,8 @@ fn field_entries(node: &Value) -> Vec<(&str, &Value)> {
155144 . unwrap_or_default ( )
156145}
157146
158- /// The child node objects held by a field value, which is either a single node
159- /// object or an array of them (an elided collection ).
147+ /// Child nodes held by a field value: either a single node or an elided
148+ /// collection (JSON array ).
160149fn children_of ( value : & Value ) -> Vec < & Value > {
161150 match value {
162151 Value :: Array ( items) => items. iter ( ) . collect ( ) ,
@@ -165,12 +154,8 @@ fn children_of(value: &Value) -> Vec<&Value> {
165154}
166155
167156/// Recursively build `node` (and its descendants) into `ast`, returning its id.
168- ///
169- /// This is a single traversal: each node's kind and field names are registered
170- /// in the schema on the fly, immediately before the node is created. Children
171- /// are built first so a parent's field lists reference existing ids. Any
172- /// comment/`unexpectedText` trivia carried by a token is harvested into
173- /// `trivia` during the same pass rather than embedded in the tree.
157+ /// Kinds and field names are registered in the schema on the fly. Comment /
158+ /// `unexpectedText` trivia is harvested into `trivia` in the same pass.
174159fn build ( node : & Value , ast : & mut Ast , trivia : & mut Vec < TriviaToken > ) -> Result < Id , String > {
175160 let info = classify ( node) ?;
176161 collect_trivia ( node, trivia) ;
@@ -200,9 +185,8 @@ fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<I
200185 ) )
201186}
202187
203- /// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already
204- /// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes
205- /// have no trivia keys, so this is a no-op for them.
188+ /// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (already filtered
189+ /// upstream) into `out`. No-op on non-token nodes.
206190fn collect_trivia ( node : & Value , out : & mut Vec < TriviaToken > ) {
207191 for key in [ "leadingTrivia" , "trailingTrivia" ] {
208192 let Some ( Value :: Array ( pieces) ) = node. get ( key) else {
@@ -231,11 +215,9 @@ fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
231215
232216/// Parse a node's `range` into a [`yeast::Range`].
233217///
234- /// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`,
235- /// a 1-based `line`, and a 1-based UTF-8 byte `column`. yeast (like tree-sitter)
236- /// uses byte offsets with 0-based rows/columns and an exclusive end, so the
237- /// line/column are shifted down by one. swift-syntax's end position is already
238- /// exclusive, so the byte offsets map across directly.
218+ /// JSON offsets are 0-based UTF-8 byte offsets; `line`/`column` are 1-based
219+ /// UTF-8. yeast uses byte offsets with 0-based rows/columns (exclusive end),
220+ /// so line/column are shifted down by one.
239221fn parse_range ( node : & Value ) -> Option < Range > {
240222 let range = node. get ( "range" ) ?;
241223 let point = |key : & str | -> Option < ( usize , Point ) > {
@@ -259,8 +241,7 @@ fn parse_range(node: &Value) -> Option<Range> {
259241}
260242
261243/// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`])
262- /// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested
263- /// from it. Both are produced in a single traversal.
244+ /// into a [`yeast::Ast`] plus harvested trivia, in a single traversal.
264245pub fn json_to_ast ( json : & str ) -> Result < AdaptedTree , String > {
265246 let root: Value = serde_json:: from_str ( json) . map_err ( |e| format ! ( "invalid JSON: {e}" ) ) ?;
266247
0 commit comments