1- //! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`].
1+ //! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the
2+ //! in-memory format the CodeQL desugaring rules operate on.
23//!
3- //! The JSON is produced out-of-process by `swift-syntax-rs`'s FFI shim, so
4- //! this module is pure Rust (no Swift toolchain needed).
4+ //! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
5+ //! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
6+ //! so the extractor consumes swift-syntax output without pulling in the Swift
7+ //! toolchain (the JSON is produced out-of-process).
58//!
6- //! The mapping mirrors tree-sitter's node model:
9+ //! The mapping mirrors tree-sitter's node model, which is what yeast (and the
10+ //! extractor's rewrite rules) expect:
711//!
8- //! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens**
9- //! (identifiers, literals, operators — text not determined by kind) become
10- //! **named** nodes, keyed by their kind name.
11- //! * **Fixed tokens** (keywords, punctuation) become **anonymous** nodes,
12- //! keyed by their text (like tree-sitter's anonymous tokens).
13- //! * Collection nodes are already elided to JSON arrays upstream.
12+ //! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens** (identifiers,
13+ //! literals, operators — the ones whose text is not determined by their kind)
14+ //! become **named** nodes, keyed by their kind name.
15+ //! * **Fixed tokens** (keywords and punctuation, whose text is fully determined
16+ //! by their kind) become **anonymous** nodes, keyed by their text — exactly
17+ //! how tree-sitter models anonymous tokens (e.g. `"func"`, `"->"`).
18+ //! * Collection nodes are already elided to JSON arrays upstream, so a
19+ //! list-valued field maps directly to that field holding several children.
1420//!
15- //! Kind/field names are preserved from swift-syntax; aligning them with the
16- //! tree-sitter-swift schema is done incrementally in the rewrite rules.
21+ //! Note: this preserves swift-syntax's own kind/field names. Aligning those
22+ //! names with the tree-sitter-swift schema (so the rewrite rules in
23+ //! [`super::swift`] fire) is done incrementally in the rules.
1724
1825use std:: collections:: BTreeMap ;
1926
@@ -22,29 +29,32 @@ use yeast::schema::Schema;
2229use yeast:: { Ast , Id , NodeContent , Point , Range } ;
2330
2431/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia.
25- /// Collected into a side channel rather than embedded in the [`yeast::Ast`],
26- /// mirroring how the extractor treats tree-sitter `extra` nodes.
32+ ///
33+ /// These are collected into a side channel rather than embedded in the
34+ /// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
35+ /// nodes: they carry a location and text but are not attached to a parent.
2736#[ derive( Debug , Clone , PartialEq , Eq ) ]
2837pub struct TriviaToken {
29- /// The trivia kind (e.g. `lineComment`, `docLineComment`, `unexpectedText`).
38+ /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`,
39+ /// `docBlockComment`, `unexpectedText`).
3040pub kind : String ,
31- /// The verbatim source text (e.g. `// comment`).
41+ /// The verbatim source text of the piece (e.g. `// comment`).
3242pub text : String ,
3343 /// The source range the piece occupies.
3444pub range : Range ,
3545}
3646
37- /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus
38- /// the comment/`unexpectedText` trivia harvested from it (in source order).
47+ /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
48+ /// comment/`unexpectedText` trivia harvested from it (in source order).
3949pub struct AdaptedTree {
4050 pub ast : Ast ,
4151 pub trivia : Vec < TriviaToken > ,
4252}
4353
4454/// swift-syntax `TokenKind` cases whose text is *not* determined by the kind
45- /// (i.e. `TokenKind.defaultText == nil`). Modelled as named leaf nodes; every
46- /// other token is a fixed keyword/punctuation, modelled as an anonymous token
47- /// keyed by its text.
55+ /// (i.e. `TokenKind.defaultText == nil`). These carry varying information and
56+ /// are modelled as named leaf nodes; every other token is a fixed
57+ /// keyword/punctuation token modelled as an anonymous token keyed by its text.
4858const VARYING_TOKEN_KINDS : & [ & str ] = & [
4959 "identifier" ,
5060 "integerLiteral" ,
@@ -106,7 +116,7 @@ fn classify(node: &Value) -> Result<KindInfo, String> {
106116 . unwrap_or ( "" )
107117 . to_string ( ) ;
108118
109- // Case name is the part before any `(payload)` in the debug rendering.
119+ // The case name is the part before any `(payload)` in the debug rendering.
110120 let case_name = token_kind. split ( '(' ) . next ( ) . unwrap_or ( token_kind) ;
111121
112122 if VARYING_TOKEN_KINDS . contains ( & case_name) {
@@ -132,7 +142,8 @@ fn classify(node: &Value) -> Result<KindInfo, String> {
132142 }
133143}
134144
135- /// Iterate over a node's structural (field, value) pairs, skipping metadata.
145+ /// Iterate over a node object's structural (field, value) pairs in a stable
146+ /// order, skipping metadata keys.
136147fn field_entries ( node : & Value ) -> Vec < ( & str , & Value ) > {
137148 node. as_object ( )
138149 . map ( |map| {
@@ -144,8 +155,8 @@ fn field_entries(node: &Value) -> Vec<(&str, &Value)> {
144155 . unwrap_or_default ( )
145156}
146157
147- /// Child nodes held by a field value: either a single node or an elided
148- /// collection (JSON array).
158+ /// The child node objects held by a field value, which is either a single node
159+ /// object or an array of them (an elided collection ).
149160fn children_of ( value : & Value ) -> Vec < & Value > {
150161 match value {
151162 Value :: Array ( items) => items. iter ( ) . collect ( ) ,
@@ -154,8 +165,12 @@ fn children_of(value: &Value) -> Vec<&Value> {
154165}
155166
156167/// Recursively build `node` (and its descendants) into `ast`, returning its id.
157- /// Kinds and field names are registered in the schema on the fly. Comment /
158- /// `unexpectedText` trivia is harvested into `trivia` in the same pass.
168+ ///
169+ /// This is a single traversal: each node's kind and field names are registered
170+ /// in the schema on the fly, immediately before the node is created. Children
171+ /// are built first so a parent's field lists reference existing ids. Any
172+ /// comment/`unexpectedText` trivia carried by a token is harvested into
173+ /// `trivia` during the same pass rather than embedded in the tree.
159174fn build ( node : & Value , ast : & mut Ast , trivia : & mut Vec < TriviaToken > ) -> Result < Id , String > {
160175 let info = classify ( node) ?;
161176 collect_trivia ( node, trivia) ;
@@ -185,8 +200,9 @@ fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<I
185200 ) )
186201}
187202
188- /// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (already filtered
189- /// upstream) into `out`. No-op on non-token nodes.
203+ /// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already
204+ /// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes
205+ /// have no trivia keys, so this is a no-op for them.
190206fn collect_trivia ( node : & Value , out : & mut Vec < TriviaToken > ) {
191207 for key in [ "leadingTrivia" , "trailingTrivia" ] {
192208 let Some ( Value :: Array ( pieces) ) = node. get ( key) else {
@@ -215,9 +231,11 @@ fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
215231
216232/// Parse a node's `range` into a [`yeast::Range`].
217233///
218- /// JSON offsets are 0-based UTF-8 byte offsets; `line`/`column` are 1-based
219- /// UTF-8. yeast uses byte offsets with 0-based rows/columns (exclusive end),
220- /// so line/column are shifted down by one.
234+ /// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`,
235+ /// a 1-based `line`, and a 1-based UTF-8 byte `column`. yeast (like tree-sitter)
236+ /// uses byte offsets with 0-based rows/columns and an exclusive end, so the
237+ /// line/column are shifted down by one. swift-syntax's end position is already
238+ /// exclusive, so the byte offsets map across directly.
221239fn parse_range ( node : & Value ) -> Option < Range > {
222240 let range = node. get ( "range" ) ?;
223241 let point = |key : & str | -> Option < ( usize , Point ) > {
@@ -241,7 +259,8 @@ fn parse_range(node: &Value) -> Option<Range> {
241259}
242260
243261/// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`])
244- /// into a [`yeast::Ast`] plus harvested trivia, in a single traversal.
262+ /// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested
263+ /// from it. Both are produced in a single traversal.
245264pub fn json_to_ast ( json : & str ) -> Result < AdaptedTree , String > {
246265 let root: Value = serde_json:: from_str ( json) . map_err ( |e| format ! ( "invalid JSON: {e}" ) ) ?;
247266
0 commit comments