diff --git a/Cargo.lock b/Cargo.lock index 799040c..3d33272 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,6 +274,12 @@ dependencies = [ "num-traits 0.1.43", ] +[[package]] +name = "countme" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" + [[package]] name = "cpufeatures" version = "0.3.0" @@ -1055,6 +1061,18 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "rowan" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417a3a9f582e349834051b8a10c8d71ca88da4211e4093528e36b9845f6b5f21" +dependencies = [ + "countme", + "hashbrown 0.14.5", + "rustc-hash", + "text-size", +] + [[package]] name = "rpds" version = "1.1.1" @@ -1070,6 +1088,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "1.0.7" @@ -1359,6 +1383,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "text-size" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233" + [[package]] name = "textwrap" version = "0.16.2" @@ -1684,6 +1714,7 @@ dependencies = [ "nom", "nom_locate", "rand", + "rowan", "rpds", "serde", "serde_json", diff --git a/crates/.gitignore b/crates/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/crates/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/crates/typr-core/Cargo.toml b/crates/typr-core/Cargo.toml index 132db88..1842b97 100644 --- a/crates/typr-core/Cargo.toml +++ b/crates/typr-core/Cargo.toml @@ -26,6 +26,7 @@ rand.workspace = true anyhow.workspace = true countmap.workspace = true bincode = "1.3" +rowan = "0.16.1" [dev-dependencies] insta = { version = "1", features = ["yaml"] } diff --git a/crates/typr-core/src/components/error_message/help_data.rs b/crates/typr-core/src/components/error_message/help_data.rs index 33f5589..c07e320 100644 --- a/crates/typr-core/src/components/error_message/help_data.rs +++ b/crates/typr-core/src/components/error_message/help_data.rs @@ -41,6 +41,10 @@ pub struct HelpData { } impl HelpData { + pub fn new(offset: usize, file_name: String) -> Self { + HelpData { offset, file_name } + } + pub fn get_offset(&self) -> usize { self.offset } diff --git a/crates/typr-core/src/components/language/mod.rs b/crates/typr-core/src/components/language/mod.rs index 6e92844..7c72c32 100644 --- a/crates/typr-core/src/components/language/mod.rs +++ b/crates/typr-core/src/components/language/mod.rs @@ -3,6 +3,7 @@ pub mod array_lang; pub mod function_lang; pub mod module_lang; pub mod operators; +pub mod syntax; pub mod use_lang; pub mod var; pub mod var_function; diff --git a/crates/typr-core/src/components/language/syntax/language.rs b/crates/typr-core/src/components/language/syntax/language.rs new file mode 100644 index 0000000..a821429 --- /dev/null +++ b/crates/typr-core/src/components/language/syntax/language.rs @@ -0,0 +1,23 @@ +use crate::components::language::syntax::syntax_kind::SyntaxKind; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum TyprLanguage {} + +impl rowan::Language for TyprLanguage { + type Kind = SyntaxKind; + + fn kind_from_raw(raw: rowan::SyntaxKind) -> Self::Kind { + // This is safe because rowan guarantees it only ever + // passes back the u16s it was given when building + // the tree. + unsafe { std::mem::transmute(raw.0) } + } + + fn kind_to_raw(kind: Self::Kind) -> rowan::SyntaxKind { + rowan::SyntaxKind(kind as u16) + } +} + +pub type SyntaxNode = rowan::SyntaxNode; +pub type SyntaxToken = rowan::SyntaxToken; +pub type SyntaxElement = rowan::SyntaxElement; diff --git a/crates/typr-core/src/components/language/syntax/mod.rs b/crates/typr-core/src/components/language/syntax/mod.rs new file mode 100644 index 0000000..6794396 --- /dev/null +++ b/crates/typr-core/src/components/language/syntax/mod.rs @@ -0,0 +1,2 @@ +pub mod language; +pub mod syntax_kind; diff --git a/crates/typr-core/src/components/language/syntax/syntax_kind.rs b/crates/typr-core/src/components/language/syntax/syntax_kind.rs new file mode 100644 index 0000000..29fedca --- /dev/null +++ b/crates/typr-core/src/components/language/syntax/syntax_kind.rs @@ -0,0 +1,210 @@ +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct Token { + pub kind: SyntaxKind, + pub len: usize, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Hash)] +#[repr(u16)] +#[allow(non_camel_case_types)] +pub enum SyntaxKind { + // Keywords + LET_KW, + FN_KW, + FUNCTION_KW, + TYPE_CONSTRUCTOR_KW, + RECURSIVE_KW, + TYPE_KW, + OPAQUE_KW, + MODULE_KW, + MOD_KW, + IMPORT_KW, + AS_KW, + USE_KW, + TEST_KW, + LIBRARY_KW, + EMBED_KW, + INTERFACE_KW, + CLASS_KW, + JS_KW, + RETURN_KW, + BREAK_KW, + NEXT_KW, + + // Record identifiers + RECORD_KW, + OBJECT_KW, + LIST_KW, + + ANY_KW, + SELF_KW, + EMPTY_KW, + + // Conditional + IF_KW, + ELSE_KW, + + // Types keywords + DATAFRAME_KW, + ARRAY_KW, + VEC_KW, + NUM_KW, + INT_KW, + BOOL_KW, + TUPLE_KW, + + // Dynamic tokens + IDENT, + NUMBER, + STRING, + + // Value keywords + TRUE_KW, + FALSE_KW, + NULL_KW, + NA_KW, + + // Loops + FOR_KW, + WHILE_KW, + LOOP_KW, + + // Annotations + AT_EXPORT, + AT_PUB, + AT_TESTABLE, + AT_IMPORT_FROM, + + // Operators + ADD, + ADD2, + MINUS, + MINUS2, + MUL, + MUL2, + DIV, + DIV2, + AT, + AT2, + MODULO, + MODULO2, + PIPE, + PIPE2, + DOLLAR, + DOLLAR2, + EQ, + EQ2, + DOT, + DOT2, + DOT3, + NOT_EQ, + LESSER_OR_EQUAL, + GREATER_OR_EQUAL, + LESSER_THAN, + GREATER_THAN, + IN_OP, + AND, + AND2, + OR, + OR2, + CUSTOM, + AS_EXCL, + L_ARROW, // <- + R_ARROW, // -> + + // Punctuation + L_CURLY, + R_CURLY, + L_BRACK, + R_BRACK, + L_PAREN, + R_PAREN, + L_VECTORIAL, + R_VECTORIAL, + SEMICOLON, + COLON, + COLON2, + COMMA, + STAR, // * + EXCLAMATION, + CARET, // ^ + QUESTION_MARK, + UNDERSCORE, + BACKSLASH, + + // Trivia + WHITESPACE, + COMMENT, + + // Special + ERROR, + EOF, + + // Nodes + SOURCE_FILE, + LITERAL_EXPR, + IDENT_EXPR, + BINARY_EXPR, + PREFIX_EXPR, + PAREN_EXPR, + CALL_EXPR, + ARRAY_EXPR, + LET_STMT, + EXPR_STMT, + BLOCK_EXPR, + INDEX_EXPR, +} + +impl SyntaxKind { + pub fn from_keyword(text: &str) -> Option { + match text { + "let" => Some(Self::LET_KW), + "fn" => Some(Self::FN_KW), + "function" => Some(Self::FUNCTION_KW), + "type_constructor" => Some(Self::TYPE_CONSTRUCTOR_KW), + "recursive" => Some(Self::RECURSIVE_KW), + "type" => Some(Self::TYPE_KW), + "opaque" => Some(Self::OPAQUE_KW), + "module" => Some(Self::MODULE_KW), + "mod" => Some(Self::MOD_KW), + "import" => Some(Self::IMPORT_KW), + "as" => Some(Self::AS_KW), + "use" => Some(Self::USE_KW), + "test" => Some(Self::TEST_KW), + "library" => Some(Self::LIBRARY_KW), + "embed" => Some(Self::EMBED_KW), + "interface" => Some(Self::INTERFACE_KW), + "class" => Some(Self::CLASS_KW), + "JS" => Some(Self::JS_KW), + "return" => Some(Self::RETURN_KW), + "break" => Some(Self::BREAK_KW), + "next" => Some(Self::NEXT_KW), + "record" => Some(Self::RECORD_KW), + "object" => Some(Self::OBJECT_KW), + "list" => Some(Self::LIST_KW), + "any" => Some(Self::ANY_KW), + "self" => Some(Self::SELF_KW), + "empty" => Some(Self::EMPTY_KW), + "if" => Some(Self::IF_KW), + "else" => Some(Self::ELSE_KW), + "dataframe" | "df" => Some(Self::DATAFRAME_KW), + "array" => Some(Self::ARRAY_KW), + "vec" => Some(Self::VEC_KW), + "num" => Some(Self::NUM_KW), + "int" => Some(Self::INT_KW), + "bool" => Some(Self::BOOL_KW), + "tuple" => Some(Self::TUPLE_KW), + "true" => Some(Self::TRUE_KW), + "false" => Some(Self::FALSE_KW), + "null" => Some(Self::NULL_KW), + "NA" => Some(Self::NA_KW), + "for" => Some(Self::FOR_KW), + "while" => Some(Self::WHILE_KW), + "loop" => Some(Self::LOOP_KW), + "and" => Some(Self::AND2), + "or" => Some(Self::OR2), + "in" => Some(Self::IN_OP), + _ => None, + } + } +} diff --git a/crates/typr-core/src/processes/lexing/input.rs b/crates/typr-core/src/processes/lexing/input.rs new file mode 100644 index 0000000..c058870 --- /dev/null +++ b/crates/typr-core/src/processes/lexing/input.rs @@ -0,0 +1,19 @@ +use crate::{ + components::language::syntax::syntax_kind::SyntaxKind, processes::lexing::lexed_str::LexedStr, +}; + +pub struct Input { + pub tokens: Vec, +} + +impl Input { + pub fn new(lexed_str: &LexedStr) -> Self { + let mut tokens = lexed_str.kinds.clone(); + tokens.retain(|kind| *kind != SyntaxKind::WHITESPACE && *kind != SyntaxKind::COMMENT); + Self { tokens } + } + + pub fn kind(&self, index: usize) -> SyntaxKind { + self.tokens.get(index).copied().unwrap_or(SyntaxKind::EOF) + } +} diff --git a/crates/typr-core/src/processes/lexing/lexed_str.rs b/crates/typr-core/src/processes/lexing/lexed_str.rs new file mode 100644 index 0000000..1a1965b --- /dev/null +++ b/crates/typr-core/src/processes/lexing/lexed_str.rs @@ -0,0 +1,63 @@ +use crate::{ + components::language::syntax::syntax_kind::SyntaxKind, processes::lexing::lexer::Lexer, +}; + +pub struct LexedStr<'src> { + pub text: &'src str, + pub kinds: Vec, + pub starts: Vec, +} + +impl<'src> LexedStr<'src> { + pub fn new(text: &'src str) -> Self { + let mut lexer = Lexer::new(text); + let mut kinds = Vec::new(); + let mut starts = Vec::new(); + let mut offset = 0; + + loop { + let token = lexer.next_token(); + starts.push(offset); + + let final_kind = if token.kind == SyntaxKind::IDENT { + let token_text = &text[offset..offset + token.len]; + SyntaxKind::from_keyword(token_text).unwrap_or(SyntaxKind::IDENT) + } else { + token.kind + }; + kinds.push(final_kind); + offset += token.len; + if final_kind == SyntaxKind::EOF { + starts.push(offset); + break; + } + } + + Self { + text, + kinds, + starts, + } + } + + pub fn kind(&self, index: usize) -> SyntaxKind { + self.kinds.get(index).copied().unwrap_or(SyntaxKind::EOF) + } + + pub fn text_for_token(&self, index: usize) -> &'src str { + // Safety: `starts` should have enough elements because + // we pushed an extra offset when the lexer hit the end of the file + let start = self.starts[index]; + let end = self.starts[index + 1]; + &self.text[start..end] + } + + pub fn len(&self) -> usize { + self.kinds.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} diff --git a/crates/typr-core/src/processes/lexing/lexer.rs b/crates/typr-core/src/processes/lexing/lexer.rs new file mode 100644 index 0000000..511e446 --- /dev/null +++ b/crates/typr-core/src/processes/lexing/lexer.rs @@ -0,0 +1,352 @@ +use std::str::Chars; + +use crate::components::language::syntax::syntax_kind::{SyntaxKind, Token}; + +pub const EOF_CHAR: char = '\0'; + +pub struct Lexer<'src> { + src: Chars<'src>, +} + +impl<'src> Lexer<'src> { + pub fn new(input: &'src str) -> Self { + Self { src: input.chars() } + } + + pub fn is_eof(&self) -> bool { + self.src.as_str().is_empty() + } + + fn first(&self) -> char { + self.src.clone().next().unwrap_or(EOF_CHAR) + } + + fn second(&self) -> char { + let mut it = self.src.clone(); + it.next(); + it.next().unwrap_or(EOF_CHAR) + } + + fn bump(&mut self) -> Option { + self.src.next() + } + + fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) { + while predicate(self.first()) && !self.is_eof() { + self.bump(); + } + } + + pub fn next_token(&mut self) -> Token { + let bytes_before = self.src.as_str().len(); + let kind = self.advance_token(); + let bytes_after = self.src.as_str().len(); + + // Fix: bytes_before is larger than bytes_after! + let len = bytes_before - bytes_after; + + Token { kind, len } + } + + fn advance_token(&mut self) -> SyntaxKind { + let Some(first_char) = self.bump() else { + return SyntaxKind::EOF; + }; + + match first_char { + // --- Specific Identifiers / Keywords with Lookahead (Must come BEFORE generic identifiers) --- + 'r' if self.first() == '#' && self.second() == '"' => { + self.bump(); // eat '#' + self.bump(); // eat '"' + while !self.is_eof() { + if self.first() == '"' && self.second() == '#' { + self.bump(); + self.bump(); + break; + } + self.bump(); + } + SyntaxKind::STRING + } + 'a' if self.first() == 's' && self.second() == '!' => { + self.bump(); // eat 's' + self.bump(); // eat '!' + SyntaxKind::AS_EXCL + } + + // --- Dynamic tokens --- + c if is_ident_start(c) => { + self.eat_while(is_ident_continue); + SyntaxKind::IDENT + } + '`' => { + self.eat_while(|c| c != '`'); + self.bump(); + SyntaxKind::IDENT + } + c if c.is_ascii_digit() => { + self.eat_while(|c| c.is_ascii_digit()); + + if self.first() == '.' && self.second().is_ascii_digit() { + self.bump(); // consume the '.' + self.eat_while(|c| c.is_ascii_digit()); // consume the trailing digits + } + + SyntaxKind::NUMBER + } + '"' | '\'' => { + let quote_type = first_char; + while !self.is_eof() { + let c = self.bump().unwrap(); + if c == '\\' { + self.bump(); + } else if c == quote_type { + break; + } + } + SyntaxKind::STRING + } + + // --- Operators --- + '+' if self.first() == '+' => { + self.bump(); + SyntaxKind::ADD2 + } + '+' => SyntaxKind::ADD, + '-' if self.first() == '-' => { + self.bump(); + SyntaxKind::MINUS2 + } + '-' if self.first() == '>' => { + self.bump(); + SyntaxKind::R_ARROW + } + '-' => SyntaxKind::MINUS, + '*' if self.first() == '*' => { + self.bump(); + SyntaxKind::MUL2 + } + '*' => SyntaxKind::MUL, + '/' if self.first() == '/' => { + self.bump(); + SyntaxKind::DIV2 + } + '/' => SyntaxKind::DIV, + '@' if self.first() == '@' => { + self.bump(); + SyntaxKind::AT2 + } + '@' if self.first() == '{' => { + self.bump(); + SyntaxKind::L_VECTORIAL + } + '@' => SyntaxKind::AT, + + // % Operator Logic + '%' if self.first() == '%' => { + self.bump(); + SyntaxKind::MODULO2 + } + '%' => { + let lookahead = self.src.clone(); + let mut found_closing_percent = false; + + for c in lookahead { + if c == '%' { + found_closing_percent = true; + } + if c == '%' || c.is_whitespace() { + break; + } + } + + if found_closing_percent { + self.eat_while(|c| c != '%'); + self.bump(); + SyntaxKind::CUSTOM + } else { + SyntaxKind::MODULO + } + } + + // | Operator Logic + '|' if self.first() == '>' && self.second() == '>' => { + self.bump(); + self.bump(); + SyntaxKind::PIPE2 + } + '|' if self.first() == '>' => { + self.bump(); + SyntaxKind::PIPE + } + '|' if self.first() == '|' => { + self.bump(); + SyntaxKind::OR2 + } + '|' => SyntaxKind::OR, + + '$' if self.first() == '$' => { + self.bump(); + SyntaxKind::DOLLAR2 + } + '$' => SyntaxKind::DOLLAR, + + '=' if self.first() == '=' => { + self.bump(); // BUG FIX: Don't forget to consume the second '='! + SyntaxKind::EQ2 + } + '=' => SyntaxKind::EQ, + + '.' if self.first() == '.' && self.second() == '.' => { + self.bump(); + self.bump(); + SyntaxKind::DOT3 + } + '.' if self.first() == '.' => { + self.bump(); + SyntaxKind::DOT2 + } + '.' => SyntaxKind::DOT, + + '!' if self.first() == '=' => { + self.bump(); + SyntaxKind::NOT_EQ + } + '!' => SyntaxKind::EXCLAMATION, + + '<' if self.first() == '=' => { + self.bump(); + SyntaxKind::LESSER_OR_EQUAL + } + '<' if self.first() == '-' => { + self.bump(); + SyntaxKind::L_ARROW + } + '<' => SyntaxKind::LESSER_THAN, + + '>' if self.first() == '=' => { + self.bump(); + SyntaxKind::GREATER_OR_EQUAL + } + '>' => SyntaxKind::GREATER_THAN, + + '&' if self.first() == '&' => { + self.bump(); + SyntaxKind::AND2 + } + '&' => SyntaxKind::AND, + + // --- Punctuation --- + '{' => SyntaxKind::L_CURLY, + '}' if self.first() == '@' => { + self.bump(); + SyntaxKind::R_VECTORIAL + } + '}' => SyntaxKind::R_CURLY, + '[' => SyntaxKind::L_BRACK, + ']' => SyntaxKind::R_BRACK, + '(' => SyntaxKind::L_PAREN, + ')' => SyntaxKind::R_PAREN, + ';' => SyntaxKind::SEMICOLON, + ':' if self.first() == ':' => { + self.bump(); + SyntaxKind::COLON2 + } + ':' => SyntaxKind::COLON, + ',' => SyntaxKind::COMMA, + '^' => SyntaxKind::CARET, + '?' => SyntaxKind::QUESTION_MARK, + '_' => SyntaxKind::UNDERSCORE, + '\\' => SyntaxKind::BACKSLASH, + + // --- Trivia --- + c if c.is_whitespace() => { + self.eat_while(char::is_whitespace); + SyntaxKind::WHITESPACE + } + '#' => { + self.eat_while(|c| c != '\n'); + SyntaxKind::COMMENT + } + + // --- Special --- + EOF_CHAR => SyntaxKind::EOF, + _ => SyntaxKind::ERROR, + } + } +} + +fn is_ident_start(c: char) -> bool { + c.is_ascii_alphabetic() || c == '_' +} + +fn is_ident_continue(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '_' +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lex_all(input: &str) -> Vec<(SyntaxKind, &str)> { + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + let mut offset = 0; + + loop { + let token = lexer.next_token(); + if token.kind == SyntaxKind::EOF { + break; + } + + let text = &input[offset..offset + token.len]; // len is usize here, so no cast needed + tokens.push((token.kind, text)); + offset += token.len; + } + tokens + } + + #[test] + fn test_custom_operators() { + // Normal modulo + assert_eq!(lex_all("5 % 2")[2], (SyntaxKind::MODULO, "%")); + // Modulo 2 + assert_eq!(lex_all("%%")[0], (SyntaxKind::MODULO2, "%%")); + // Custom Operator + assert_eq!(lex_all("%in%")[0], (SyntaxKind::CUSTOM, "%in%")); + } + + #[test] + fn test_pipe_operators() { + assert_eq!(lex_all("|>")[0], (SyntaxKind::PIPE, "|>")); + assert_eq!(lex_all("|>>")[0], (SyntaxKind::PIPE2, "|>>")); + } + + #[test] + fn test_numbers() { + assert_eq!(lex_all("42")[0], (SyntaxKind::NUMBER, "42")); + assert_eq!(lex_all("12.34")[0], (SyntaxKind::NUMBER, "12.34")); + + // Ensure method calls on numbers don't swallow the dot + let tokens = lex_all("1.to_string"); + assert_eq!(tokens[0], (SyntaxKind::NUMBER, "1")); + assert_eq!(tokens[1], (SyntaxKind::DOT, ".")); + assert_eq!(tokens[2], (SyntaxKind::IDENT, "to_string")); + } + + #[test] + fn test_strings() { + assert_eq!(lex_all("\"hello\"")[0], (SyntaxKind::STRING, "\"hello\"")); + assert_eq!(lex_all("'world'")[0], (SyntaxKind::STRING, "'world'")); + assert_eq!( + lex_all("r#\"hello world\"#")[0], + (SyntaxKind::STRING, "r#\"hello world\"#") + ); + } + + #[test] + fn test_as_excl() { + assert_eq!(lex_all("as!")[0], (SyntaxKind::AS_EXCL, "as!")); + assert_eq!(lex_all("astronaut")[0], (SyntaxKind::IDENT, "astronaut")); + } +} diff --git a/crates/typr-core/src/processes/lexing/mod.rs b/crates/typr-core/src/processes/lexing/mod.rs new file mode 100644 index 0000000..c19fd23 --- /dev/null +++ b/crates/typr-core/src/processes/lexing/mod.rs @@ -0,0 +1,3 @@ +pub mod input; +pub mod lexed_str; +pub mod lexer; diff --git a/crates/typr-core/src/processes/mod.rs b/crates/typr-core/src/processes/mod.rs index ee71998..71e7638 100644 --- a/crates/typr-core/src/processes/mod.rs +++ b/crates/typr-core/src/processes/mod.rs @@ -1,4 +1,6 @@ +pub mod lexing; pub mod parsing; +pub mod parsing_new; pub mod spg; pub mod transpiling; pub mod type_checking; diff --git a/crates/typr-core/src/processes/parsing_new/ast.rs b/crates/typr-core/src/processes/parsing_new/ast.rs new file mode 100644 index 0000000..bfa0757 --- /dev/null +++ b/crates/typr-core/src/processes/parsing_new/ast.rs @@ -0,0 +1,122 @@ +use crate::components::language::syntax::language::{SyntaxNode, SyntaxToken}; +use crate::components::language::syntax::syntax_kind::SyntaxKind; + +pub trait AstNode { + fn can_cast(kind: SyntaxKind) -> bool; + fn cast(node: SyntaxNode) -> Option + where + Self: Sized; + fn syntax(&self) -> &SyntaxNode; +} + +pub struct LiteralExpr(pub SyntaxNode); + +impl AstNode for LiteralExpr { + fn can_cast(kind: SyntaxKind) -> bool { + kind == SyntaxKind::LITERAL_EXPR + } + fn cast(node: SyntaxNode) -> Option { + if Self::can_cast(node.kind()) { + Some(Self(node)) + } else { + None + } + } + fn syntax(&self) -> &SyntaxNode { + &self.0 + } +} + +pub struct BinaryExpr(pub SyntaxNode); + +impl AstNode for BinaryExpr { + fn can_cast(kind: SyntaxKind) -> bool { + kind == SyntaxKind::BINARY_EXPR + } + fn cast(node: SyntaxNode) -> Option { + if Self::can_cast(node.kind()) { + Some(Self(node)) + } else { + None + } + } + fn syntax(&self) -> &SyntaxNode { + &self.0 + } +} + +impl BinaryExpr { + pub fn lhs(&self) -> Option { + self.syntax().children().find_map(Expr::cast) + } + + pub fn rhs(&self) -> Option { + self.syntax().children().filter_map(Expr::cast).nth(1) + } + + pub fn op_token(&self) -> Option { + self.syntax() + .children_with_tokens() + .filter_map(|it| it.into_token()) + .find(|it| { + matches!( + it.kind(), + SyntaxKind::ADD | SyntaxKind::MINUS | SyntaxKind::MUL | SyntaxKind::DIV + ) + }) + } +} + +pub struct ParenExpr(pub SyntaxNode); + +impl AstNode for ParenExpr { + fn can_cast(kind: SyntaxKind) -> bool { + kind == SyntaxKind::PAREN_EXPR + } + fn cast(node: SyntaxNode) -> Option { + if Self::can_cast(node.kind()) { + Some(Self(node)) + } else { + None + } + } + fn syntax(&self) -> &SyntaxNode { + &self.0 + } +} + +impl ParenExpr { + pub fn inner_expr(&self) -> Option { + self.syntax().children().find_map(Expr::cast) + } +} + +pub enum Expr { + Literal(LiteralExpr), + Binary(BinaryExpr), + Paren(ParenExpr), +} + +impl AstNode for Expr { + fn can_cast(kind: SyntaxKind) -> bool { + LiteralExpr::can_cast(kind) || BinaryExpr::can_cast(kind) || ParenExpr::can_cast(kind) + } + fn cast(node: SyntaxNode) -> Option { + if LiteralExpr::can_cast(node.kind()) { + Some(Expr::Literal(LiteralExpr(node))) + } else if BinaryExpr::can_cast(node.kind()) { + Some(Expr::Binary(BinaryExpr(node))) + } else if ParenExpr::can_cast(node.kind()) { + Some(Expr::Paren(ParenExpr(node))) + } else { + None + } + } + fn syntax(&self) -> &SyntaxNode { + match self { + Expr::Literal(it) => it.syntax(), + Expr::Binary(it) => it.syntax(), + Expr::Paren(it) => it.syntax(), + } + } +} diff --git a/crates/typr-core/src/processes/parsing_new/build_tree.rs b/crates/typr-core/src/processes/parsing_new/build_tree.rs new file mode 100644 index 0000000..9fc3a4f --- /dev/null +++ b/crates/typr-core/src/processes/parsing_new/build_tree.rs @@ -0,0 +1,49 @@ +use rowan::{GreenNode, GreenNodeBuilder}; + +use crate::{ + components::language::syntax::syntax_kind::SyntaxKind, + processes::{lexing::lexed_str::LexedStr, parsing_new::parser::Event}, +}; + +pub fn build_tree(lexed: &LexedStr, events: Vec) -> GreenNode { + let mut builder = GreenNodeBuilder::new(); + let mut lex_idx = 0; + + let eat_trivia = |builder: &mut GreenNodeBuilder, lex_idx: &mut usize| { + while *lex_idx < lexed.len() { + let kind = lexed.kind(*lex_idx); + if kind == SyntaxKind::WHITESPACE || kind == SyntaxKind::COMMENT { + let text = lexed.text_for_token(*lex_idx); + builder.token(rowan::SyntaxKind(kind as u16), text); + *lex_idx += 1; + } else { + break; + } + } + }; + + let mut events = events.into_iter().peekable(); + while let Some(event) = events.next() { + match event { + Event::Start { kind } => { + builder.start_node(rowan::SyntaxKind(kind as u16)); + } + Event::Token => { + eat_trivia(&mut builder, &mut lex_idx); + let kind = lexed.kind(lex_idx); + let text = lexed.text_for_token(lex_idx); + builder.token(rowan::SyntaxKind(kind as u16), text); + lex_idx += 1; + } + Event::Finish => { + if events.peek().is_none() { + eat_trivia(&mut builder, &mut lex_idx); + } + builder.finish_node(); + } + Event::Error(_) => todo!(), + } + } + + builder.finish() +} diff --git a/crates/typr-core/src/processes/parsing_new/grammar/expr.rs b/crates/typr-core/src/processes/parsing_new/grammar/expr.rs new file mode 100644 index 0000000..44a6b34 --- /dev/null +++ b/crates/typr-core/src/processes/parsing_new/grammar/expr.rs @@ -0,0 +1,143 @@ +use crate::{ + components::language::syntax::syntax_kind::SyntaxKind, + processes::parsing_new::parser::{CompletedMarker, Parser}, +}; + +/// Parses expressions using Pratt precedence climbing algorithm. +pub fn parse_expr(p: &mut Parser) { + expr_bp(p, 0); +} + +fn expr_bp(p: &mut Parser, min_bp: u8) { + let Some(mut lhs) = parse_lhs(p) else { + return; + }; + loop { + let Some((left_bp, right_bp)) = infix_binding_power(p.current()) else { + break; + }; + + // If the operator has lower binding power than the current context, stop parsing + if left_bp < min_bp { + break; + } + + // We wrap the LHS in the new node binary expression node + // which happens at the end of the loop, + // because we already know that LHS is part of + // a binary expression because of the previous lines. + // The binary expression node precedes the LHS node + let marker = lhs.precede(p); + + p.bump(); // Consume the operator + expr_bp(p, right_bp); + + lhs = marker.complete(p, SyntaxKind::BINARY_EXPR); + } +} + +/// Parses the base elements (Literals, Variables) +fn parse_lhs(p: &mut Parser) -> Option { + let mut lhs = match p.current() { + SyntaxKind::MINUS | SyntaxKind::EXCLAMATION => { + let m = p.start(); + p.bump(); + expr_bp(p, 255); + m.complete(p, SyntaxKind::PREFIX_EXPR) + } + SyntaxKind::L_PAREN => { + let m = p.start(); + p.bump(); + expr_bp(p, 0); + p.expect(SyntaxKind::R_PAREN); + m.complete(p, SyntaxKind::PAREN_EXPR) + } + SyntaxKind::NUMBER + | SyntaxKind::STRING + | SyntaxKind::TRUE_KW + | SyntaxKind::FALSE_KW + | SyntaxKind::NULL_KW + | SyntaxKind::NA_KW + | SyntaxKind::DOT3 => { + let m = p.start(); + p.bump(); + m.complete(p, SyntaxKind::LITERAL_EXPR) + } + SyntaxKind::IDENT => { + let m = p.start(); + p.bump(); + m.complete(p, SyntaxKind::IDENT_EXPR) + } + _ => { + p.error("Expected an expression"); + return None; + } + }; + + // The postfix loop - similar to rust analyzer + // Function calls and indexing are parsed here + loop { + match p.current() { + // function call: `foo(...)` + SyntaxKind::L_PAREN => { + let m = lhs.precede(p); + p.bump(); + while !p.at(SyntaxKind::R_PAREN) && !p.at(SyntaxKind::EOF) { + parse_expr(p); + if !p.eat(SyntaxKind::COMMA) { + break; + } + } + p.expect(SyntaxKind::R_PAREN); + lhs = m.complete(p, SyntaxKind::CALL_EXPR); + } + // Array indexing + SyntaxKind::L_BRACK => { + let m = lhs.precede(p); + p.bump(); + parse_expr(p); + p.expect(SyntaxKind::R_BRACK); + lhs = m.complete(p, SyntaxKind::INDEX_EXPR); + } + _ => break, + } + } + + Some(lhs) +} + +fn infix_binding_power(kind: SyntaxKind) -> Option<(u8, u8)> { + match kind { + SyntaxKind::ADD | SyntaxKind::MINUS => Some((1, 2)), + SyntaxKind::MUL | SyntaxKind::DIV => Some((3, 4)), + _ => None, + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::components::language::syntax::language::SyntaxNode; + use crate::processes::lexing::input::Input; + use crate::processes::lexing::lexed_str::LexedStr; + use crate::processes::parsing_new::build_tree::build_tree; + use crate::processes::parsing_new::grammar::expr::parse_expr; + + #[test] + fn test_pratt_parser() { + // We want to see if `*` binds tighter than `+` + let code = "1 + 2 * 3"; + let lexed = LexedStr::new(code); + let input = Input::new(&lexed); + let mut parser = Parser::new(&input); + + let root = parser.start(); + parse_expr(&mut parser); + root.complete(&mut parser, SyntaxKind::SOURCE_FILE); + let tree = build_tree(&lexed, parser.events); + let typed_node = SyntaxNode::new_root(tree); + + println!("{:#?}", typed_node); + } +} diff --git a/crates/typr-core/src/processes/parsing_new/grammar/mod.rs b/crates/typr-core/src/processes/parsing_new/grammar/mod.rs new file mode 100644 index 0000000..016a1ff --- /dev/null +++ b/crates/typr-core/src/processes/parsing_new/grammar/mod.rs @@ -0,0 +1 @@ +pub mod expr; diff --git a/crates/typr-core/src/processes/parsing_new/legacy_converter.rs b/crates/typr-core/src/processes/parsing_new/legacy_converter.rs new file mode 100644 index 0000000..5dd4943 --- /dev/null +++ b/crates/typr-core/src/processes/parsing_new/legacy_converter.rs @@ -0,0 +1,123 @@ +use crate::components::error_message::help_data::HelpData; +use crate::components::language::operators::Op; +use crate::components::language::syntax::syntax_kind::SyntaxKind; +use crate::components::language::Lang; +use crate::processes::parsing_new::ast::{AstNode, BinaryExpr, Expr, LiteralExpr, ParenExpr}; + +pub trait ToLegacy { + fn to_legacy(&self, file_name: &str) -> Lang; +} + +impl ToLegacy for Expr { + fn to_legacy(&self, file_name: &str) -> Lang { + match self { + Expr::Literal(lit) => lit.to_legacy(file_name), + Expr::Binary(bin) => bin.to_legacy(file_name), + Expr::Paren(paren) => paren.to_legacy(file_name), + } + } +} + +impl ToLegacy for ParenExpr { + fn to_legacy(&self, file_name: &str) -> Lang { + let inner = self + .inner_expr() + .expect("ParenExpr missing inner expression") + .to_legacy(file_name); + + // Find the offset of the inner expression, not the parenthesis itself (which matches old parser behavior) + let inner_offset = self + .inner_expr() + .unwrap() + .syntax() + .text_range() + .start() + .into(); + let help_data = HelpData::new(inner_offset, file_name.to_string()); + + Lang::Scope { + body: vec![inner], + help_data, + } + } +} + +impl ToLegacy for LiteralExpr { + fn to_legacy(&self, file_name: &str) -> Lang { + let token = self + .syntax() + .children_with_tokens() + .filter_map(|it| it.into_token()) + .find(|it| !matches!(it.kind(), SyntaxKind::WHITESPACE | SyntaxKind::COMMENT)) + .expect("LiteralExpr missing content token"); + let text = token.text().to_string(); + + let offset = token.text_range().start().into(); + let help_data = HelpData::new(offset, file_name.to_string()); + + match token.kind() { + SyntaxKind::TRUE_KW => Lang::Bool { + value: true, + help_data, + }, + SyntaxKind::FALSE_KW => Lang::Bool { + value: false, + help_data, + }, + SyntaxKind::STRING => Lang::Char { + value: text.replace("\"", ""), + help_data, + }, + SyntaxKind::NUMBER => { + if text.contains('.') { + Lang::Number { + value: text.parse().unwrap(), + help_data, + } + } else { + Lang::Integer { + value: text.parse().unwrap(), + help_data, + } + } + } + _ => unimplemented!("Literal not handled: {:?}", token.kind()), + } + } +} + +impl ToLegacy for BinaryExpr { + fn to_legacy(&self, file_name: &str) -> Lang { + let lhs = self + .lhs() + .expect("BinaryExpr missing lhs") + .to_legacy(file_name); + let rhs = self + .rhs() + .expect("BinaryExpr missing rhs") + .to_legacy(file_name); + let op_token = self.op_token().expect("BinaryExpr missing operator"); + + let offset = op_token.text_range().start().into(); + let help_data = HelpData::new(offset, file_name.to_string()); + + let operator = match op_token.kind() { + SyntaxKind::ADD => Op::Add(help_data.clone()), + SyntaxKind::MINUS => Op::Minus(help_data.clone()), + SyntaxKind::MUL => Op::Mul(help_data.clone()), + SyntaxKind::DIV => Op::Div(help_data.clone()), + _ => unimplemented!("Operator not handled: {:?}", op_token.kind()), + }; + + // Lang::Operator uses outer help_data, which usually matched the start of the expression in nom. + let expr_offset = self.syntax().text_range().start().into(); + let expr_help_data = HelpData::new(expr_offset, file_name.to_string()); + + Lang::Operator { + operator, + lhs: Box::new(rhs), // Legacy parser backwards naming: lhs holds the right side + rhs: Box::new(lhs), // rhs holds the left side + help_data: expr_help_data, + } + } +} diff --git a/crates/typr-core/src/processes/parsing_new/mod.rs b/crates/typr-core/src/processes/parsing_new/mod.rs new file mode 100644 index 0000000..eddd8da --- /dev/null +++ b/crates/typr-core/src/processes/parsing_new/mod.rs @@ -0,0 +1,5 @@ +pub mod ast; +pub mod build_tree; +pub mod grammar; +pub mod legacy_converter; +pub mod parser; diff --git a/crates/typr-core/src/processes/parsing_new/parser.rs b/crates/typr-core/src/processes/parsing_new/parser.rs new file mode 100644 index 0000000..f70de01 --- /dev/null +++ b/crates/typr-core/src/processes/parsing_new/parser.rs @@ -0,0 +1,111 @@ +use crate::{ + components::language::syntax::syntax_kind::SyntaxKind, processes::lexing::input::Input, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Event { + Start { kind: SyntaxKind }, + Token, + Finish, + Error(String), +} + +pub struct Parser<'src> { + input: &'src Input, + pos: usize, + pub events: Vec, +} + +impl<'src> Parser<'src> { + pub fn new(input: &'src Input) -> Self { + Self { + input, + pos: 0, + events: Vec::new(), + } + } + + pub fn current(&self) -> SyntaxKind { + self.input.kind(self.pos) + } + + pub fn nth(&self, n: usize) -> SyntaxKind { + self.input.kind(self.pos + n) + } + + pub fn at(&self, kind: SyntaxKind) -> bool { + self.current() == kind + } + + pub fn bump(&mut self) { + if self.current() == SyntaxKind::EOF { + return; + } + + self.events.push(Event::Token); + self.pos += 1; + } + + pub fn eat(&mut self, kind: SyntaxKind) -> bool { + if self.current() == kind { + self.bump(); + true + } else { + false + } + } + + pub fn error(&mut self, message: impl Into) { + self.events.push(Event::Error(message.into())); + } + + pub fn start(&mut self) -> Marker { + let event_pos = self.events.len(); + self.events.push(Event::Start { + kind: SyntaxKind::ERROR, + }); + Marker { pos: event_pos } + } + + pub fn expect(&mut self, kind: SyntaxKind) -> bool { + if self.eat(kind) { + true + } else { + self.error(format!("Expected {:?}", kind)); + false + } + } +} + +pub struct Marker { + pos: usize, +} + +impl Marker { + pub fn complete(self, parser: &mut Parser, kind: SyntaxKind) -> CompletedMarker { + match &mut parser.events[self.pos] { + Event::Start { kind: slot } => { + *slot = kind; + } + _ => unreachable!(), + } + parser.events.push(Event::Finish); + CompletedMarker { pos: self.pos } + } +} + +pub struct CompletedMarker { + pos: usize, +} + +impl CompletedMarker { + pub fn precede(&self, parser: &mut Parser) -> Marker { + parser.events.insert( + self.pos, + Event::Start { + kind: SyntaxKind::ERROR, + }, + ); + Marker { pos: self.pos } + } +} diff --git a/crates/typr-core/tests/parser_migration.rs b/crates/typr-core/tests/parser_migration.rs new file mode 100644 index 0000000..9ef76a0 --- /dev/null +++ b/crates/typr-core/tests/parser_migration.rs @@ -0,0 +1,60 @@ +use nom_locate::LocatedSpan; +use typr_core::components::language::syntax::language::SyntaxNode; +use typr_core::components::language::syntax::syntax_kind::SyntaxKind; +use typr_core::processes::lexing::input::Input; +use typr_core::processes::lexing::lexed_str::LexedStr; +use typr_core::processes::parsing::elements::parse_elements; +use typr_core::processes::parsing_new::ast::{AstNode, Expr}; +use typr_core::processes::parsing_new::build_tree::build_tree; +use typr_core::processes::parsing_new::grammar::expr::parse_expr; +use typr_core::processes::parsing_new::legacy_converter::ToLegacy; +use typr_core::processes::parsing_new::parser::Parser; + +fn verify_parser_parity(code: &str, file_name: &str) { + let span = LocatedSpan::new_extra(code, file_name.to_string()); + let (remaining, old_ast) = + parse_elements(span).expect("Old parser failed to parse the expression"); + assert!( + remaining.fragment().trim().is_empty(), + "Old parser did not consume the entire input! Remaining: {}", + remaining.fragment() + ); + + let lexed = LexedStr::new(code); + let input = Input::new(&lexed); + let mut parser = Parser::new(&input); + let root = parser.start(); + + parse_expr(&mut parser); + root.complete(&mut parser, SyntaxKind::SOURCE_FILE); + + let tree = build_tree(&lexed, parser.events.clone()); + let syntax_node = SyntaxNode::new_root(tree); + + let expr_node = syntax_node + .descendants() + .find_map(Expr::cast) + .expect("New parser did not produce a valid Expr"); + + let new_ast = expr_node.to_legacy(file_name); + + assert_eq!( + old_ast, new_ast, + "\n\n PARSER MISMATCH in file: {} \n\nOld AST:\n{:#?}\n\nNew AST:\n{:#?}\n\n", + file_name, old_ast, new_ast + ); +} + +macro_rules! migration_test { + ($test_name:ident, $file_path:expr) => { + #[test] + fn $test_name() { + let code = include_str!($file_path); + verify_parser_parity(code, $file_path); + } + }; +} + +migration_test!(test_expr_literal, "parser_migration_cases/expr_literal.ty"); +migration_test!(test_expr_binary, "parser_migration_cases/expr_binary.ty"); +migration_test!(test_expr_paren, "parser_migration_cases/expr_paren.ty"); diff --git a/crates/typr-core/tests/parser_migration_cases/expr_binary.ty b/crates/typr-core/tests/parser_migration_cases/expr_binary.ty new file mode 100644 index 0000000..cba549f --- /dev/null +++ b/crates/typr-core/tests/parser_migration_cases/expr_binary.ty @@ -0,0 +1 @@ +1 + 2 * 3 diff --git a/crates/typr-core/tests/parser_migration_cases/expr_literal.ty b/crates/typr-core/tests/parser_migration_cases/expr_literal.ty new file mode 100644 index 0000000..d81cc07 --- /dev/null +++ b/crates/typr-core/tests/parser_migration_cases/expr_literal.ty @@ -0,0 +1 @@ +42 diff --git a/crates/typr-core/tests/parser_migration_cases/expr_paren.ty b/crates/typr-core/tests/parser_migration_cases/expr_paren.ty new file mode 100644 index 0000000..5dfe572 --- /dev/null +++ b/crates/typr-core/tests/parser_migration_cases/expr_paren.ty @@ -0,0 +1 @@ +(1 + 2) * 3