diff --git a/handrolled_parser/parser.mbt b/handrolled_parser/parser.mbt index c7153ea74..39e647edc 100644 --- a/handrolled_parser/parser.mbt +++ b/handrolled_parser/parser.mbt @@ -293,6 +293,14 @@ fn State::parse_var(self : Self) -> @syntax.Var { { name, loc, } } +///| +fn State::parse_trait_constraint(self : Self) -> @syntax.TypeVarConstraint { + let spos = self.peek_spos() + let trait_ = self.parse_qual_type_ident(context="trait name") + let loc = self.loc_start_with(spos) + { trait_, loc, } +} + ///| fn State::parse_tvar_binder(self : Self) -> @syntax.TypeVarBinder { if self.mode is Panic(loc~, ..) { @@ -305,12 +313,7 @@ fn State::parse_tvar_binder(self : Self) -> @syntax.TypeVarBinder { [TK_COLON], fn(state) { state.expect_token(TK_COLON) - state.sepby1(delim=TK_PLUS, fn(state) { - let loc = state.peek_location() - let trait_ = state.parse_qual_type_ident(context="trait name") - { trait_, loc, } - }) - |> List + state.sepby1(delim=TK_PLUS, State::parse_trait_constraint) |> List }, ) let constraints = if constraints is Some(xs) { xs } else { @list.new() } @@ -571,12 +574,7 @@ fn State::parse_trait_decl( let supers = if self.peek_token() is COLON { self.skip() self.with_follow(follow_set=[TK_LBRACE], fn(state) { - state.sepby1(delim=TK_PLUS, fn(state) { - let loc = state.peek_location() - let trait_ = state.parse_qual_type_ident(context="trait name") - @syntax.TypeVarConstraint::{ trait_, loc, } - }) - |> @list.List + state.sepby1(delim=TK_PLUS, State::parse_trait_constraint) |> @list.List }) } else { @list.new() @@ -1147,7 +1145,7 @@ fn State::parse_top(self : Self) -> @syntax.Impl { FN => self.parse_top_func(attrs~, vis~, spos~, is_declare~) TRAIT => { let trait_decl = self.parse_trait_decl(attrs~, vis~, is_declare~) - let trait_decl = { ..trait_decl, vis, } + let trait_decl = { ..trait_decl, vis, loc: self.loc_start_with(spos), } TopTrait(trait_decl) } TEST => self.parse_test_block(attrs~, spos~, is_async=None) @@ -1983,21 +1981,22 @@ fn State::parse_simple_pattern(self : Self) -> @syntax.Pattern { Constant(c=@syntax.make_double(x), loc~) } MINUS => { + let sign_spos = self.peek_spos() self.expect_token(TK_MINUS) match self.peek_token() { INT(x) => { - let loc = self.peek_location() self.expect_token(TK_INT) + let loc = self.loc_start_with(sign_spos) Constant(c=@syntax.make_int("-\{x}"), loc~) } FLOAT(x) => { - let loc = self.peek_location() self.expect_token(TK_FLOAT) + let loc = self.loc_start_with(sign_spos) Constant(c=@syntax.make_float("-\{x}"), loc~) } DOUBLE(x) => { - let loc = self.peek_location() self.expect_token(TK_DOUBLE) + let loc = self.loc_start_with(sign_spos) Constant(c=@syntax.make_double("-\{x}"), loc~) } other => { @@ -3503,7 +3502,7 @@ fn State::parse_simple_try_expr( if self.mode is Panic(loc~, ..) { return Hole(loc~, kind=Synthesized) } - let spos = self.peek_spos() + let spos = self.peek_spos_with_parsed(parsed~) let e = self.parse_pipe_expr(parsed~) match self.peek_token() { CATCH => { @@ -5924,9 +5923,9 @@ fn State::parse_type_aux(self : Self) -> (Bool, @syntax.Type) { } AMPER => { self.skip() - let spos = self.peek_spos() + let name_spos = self.peek_spos() let lid = self.parse_qual_type_ident(context="object type") - let loc = self.loc_start_with(spos) + let loc = self.loc_start_with(name_spos) let ty = @syntax.Type::Object({ id: lid, loc, }) parse_question_mark(spos, ty) } diff --git a/test/manual_test/corpus_location_regression_test.mbt b/test/manual_test/corpus_location_regression_test.mbt new file mode 100644 index 000000000..8b5e5395b --- /dev/null +++ b/test/manual_test/corpus_location_regression_test.mbt @@ -0,0 +1,221 @@ +///| +priv struct CorpusLocationCollector { + spans : Array[(String, Int, Int)] +} + +///| +extend CorpusLocationCollector with IterVisitor::{base, visit_Impl} + +///| +fn CorpusLocationCollector::add( + self : Self, + kind : String, + loc : @basic.Location, +) -> Unit { + self.spans.push((kind, loc.start.cnum, loc.end.cnum)) +} + +///| +impl IterVisitor for CorpusLocationCollector with fn visit_TraitDecl(self, decl) { + self.add("trait", decl.loc) + IterVisitor::visit_TraitDecl(self.base(), decl) +} + +///| +impl IterVisitor for CorpusLocationCollector with fn visit_Pattern(self, pat) { + if pat is Constant(loc~, ..) { + self.add("pattern", loc) + } + IterVisitor::visit_Pattern(self.base(), pat) +} + +///| +impl IterVisitor for CorpusLocationCollector with fn visit_Type(self, ty) { + match ty { + Option(loc~, ..) => self.add("option", loc) + Object(constr) => self.add("object", constr.loc) + _ => () + } + IterVisitor::visit_Type(self.base(), ty) +} + +///| +impl IterVisitor for CorpusLocationCollector with fn visit_Expr(self, expr) { + match expr { + Try(loc~, ..) => self.add("try", loc) + Continue(loc~, ..) => self.add("continue", loc) + _ => () + } + IterVisitor::visit_Expr(self.base(), expr) +} + +///| +impl IterVisitor for CorpusLocationCollector with fn visit_TypeVarBinder( + self, + binder, +) { + self.add("binder", binder.name_loc) + IterVisitor::visit_TypeVarBinder(self.base(), binder) +} + +///| +impl IterVisitor for CorpusLocationCollector with fn visit_TypeVarConstraint( + self, + constraint, +) { + self.add("constraint", constraint.loc) +} + +///| +fn assert_corpus_locations( + source : String, + expected : Array[(String, Int, Int)], +) -> Unit raise Error { + let name = "corpus_locations.mbt" + let results : Array[(String, @syntax.Impls)] = [] + for + (parser_name, parser) in [ + ("Handrolled", @parser.Parser::Handrolled), + ("MoonYacc", MoonYacc), + ] { + let (impls, reports) = @parser.parse_string(source, name~, parser~) + if !reports.is_empty() { + fail("\{parser_name} rejected \{source}: \{Repr(reports)}") + } + results.push((parser_name, impls)) + } + let cst = @untyped_cst.parse_structure(source, name~) + @test.assert_eq(cst.diagnostics_view().length(), 0, msg=source) + let (impls, reports) = cst.to_impls() + @test.assert_eq(reports.length(), 0, msg=source) + results.push(("CST", impls)) + let mut reference : Json? = None + for (parser, impls) in results { + let collector : CorpusLocationCollector = { spans: [], } + impls.each(impl_ => collector.visit_Impl(impl_)) + @test.assert_eq(collector.spans, expected, msg="\{parser}: \{source}") + let json = @basic.show_loc.protect(String, fn() { + impls.map(impl_ => impl_.json_repr()).to_json() + }) + if reference is Some(expected_json) { + @test.assert_eq(json, expected_json, msg="\{parser}: \{source}") + } else { + reference = Some(json) + } + } +} + +///| +test "corpus locations include trait prefixes but not trailing comments" { + for + declaration in [ + "trait T {}", "pub(open) trait T {}", "#deprecated\npub trait T {}", "declare trait T {}", + "#deprecated\ndeclare pub(open) trait T {}", + ] { + assert_corpus_locations(declaration + " // trailing 🐇\n", [ + ("trait", 0, declaration.length()), + ]) + } +} + +///| +test "corpus locations include signs in negative patterns" { + let prefix = "fn f(x) { match x { " + for literal in ["-1", "- 2", "-3L", "-4.5F", "- 5.5F", "-6.5", "- 7.5"] { + assert_corpus_locations(prefix + literal + " => 0 } }", [ + ("pattern", prefix.length(), prefix.length() + literal.length()), + ]) + } + assert_corpus_locations(prefix + "Number(-32601) => 0 } }", [ + ("pattern", prefix.length() + 7, prefix.length() + 13), + ]) + for operator in ["..=", "..<"] { + assert_corpus_locations(prefix + "-10 " + operator + " -2 => 0 } }", [ + ("pattern", prefix.length(), prefix.length() + 3), + ("pattern", prefix.length() + 8, prefix.length() + 10), + ]) + } +} + +///| +test "corpus locations separate optional object types from their names" { + let prefix = "fn f(x : " + let start = prefix.length() + for name in ["T", "@pkg.Trait"] { + let end = start + 1 + name.length() + assert_corpus_locations(prefix + "&" + name + ") {}", [ + ("object", start + 1, end), + ]) + assert_corpus_locations(prefix + "&" + name + "?) {}", [ + ("option", start, end + 1), + ("object", start + 1, end), + ]) + assert_corpus_locations(prefix + "&" + name + "??) {}", [ + ("option", start, end + 2), + ("option", start, end + 1), + ("object", start + 1, end), + ]) + assert_corpus_locations(prefix + "(&" + name + ")?) {}", [ + ("option", start, end + 3), + ("object", start + 2, end + 1), + ]) + } +} + +///| +test "corpus locations start catch expressions at the pre-parsed body" { + let prefix = "fn f() { " + for + body in [ + "()", "(x)", "((f()))", "(x, y)", "(x : Int)", "(f)()", "(xs)[0]", "(x).method()", + "(x) |> f", + ] { + for suffix in [" catch { _ => 0 }", " catch { _ => 0 } noraise { x => x }"] { + let expr = body + suffix + assert_corpus_locations(prefix + expr + " }", [ + ("try", prefix.length(), prefix.length() + expr.length()), + ]) + } + } +} + +///| +test "corpus locations cover complete trait constraints and retain binder names" { + for name in ["Eq", "@json.FromJson"] { + assert_corpus_locations("fn[T : " + name + "] f(x : T) {}", [ + ("binder", 3, 4), + ("constraint", 7, 7 + name.length()), + ]) + } + assert_corpus_locations("fn[T : Eq + @json.FromJson] f(x : T) {}", [ + ("binder", 3, 4), + ("constraint", 7, 9), + ("constraint", 12, 26), + ]) + assert_corpus_locations("trait T : Eq + @json.FromJson {}", [ + ("trait", 0, 32), + ("constraint", 10, 12), + ("constraint", 15, 29), + ]) +} + +///| +test "corpus locations exclude comments after continue" { + for newline in ["\n", "\r\n"] { + let prefix = "fn f() {" + newline + " " + for + statement in [ + "continue", "continue label~", "continue label~~", "continue 1", "continue (1)", + "continue label~ 1, (2)", "continue label~~ (1), (2)", + ] { + assert_corpus_locations( + prefix + statement + " // trailing 🐇" + newline + "}", + [("continue", prefix.length(), prefix.length() + statement.length())], + ) + } + } + assert_corpus_locations( + "fn f() {\n continue // first\n continue // second\n}", + [("continue", 11, 19), ("continue", 31, 39)], + ) +} diff --git a/test/openseek_test/moon.pkg b/test/openseek_test/moon.pkg index af20eaf3b..0793f420b 100644 --- a/test/openseek_test/moon.pkg +++ b/test/openseek_test/moon.pkg @@ -1,5 +1,6 @@ import { "moonbitlang/parser", + "moonbitlang/parser/syntax", "moonbitlang/parser/untyped_cst", "moonbitlang/lexer/basic", "moonbitlang/x/fs", diff --git a/test/openseek_test/openseek_test.mbt b/test/openseek_test/openseek_test.mbt index 5c1ca2be9..cd887a2ba 100644 --- a/test/openseek_test/openseek_test.mbt +++ b/test/openseek_test/openseek_test.mbt @@ -57,9 +57,16 @@ fn record_diagnostics( } } +///| +fn impls_json(impls : @syntax.Impls) -> Json { + @basic.show_loc.protect(String, fn() { + impls.map(impl_ => impl_.json_repr()).to_json() + }) +} + ///| #skip("requires an external openseek checkout") -test "all OpenSeek MoonBit sources parse without diagnostics" { +test "all OpenSeek MoonBit sources parse to equivalent ASTs without diagnostics" { let root = @env.get_env_var("OPENSEEK_ROOT").unwrap_or_else(() => { fail("OPENSEEK_ROOT must point to an OpenSeek checkout") }) @@ -83,13 +90,13 @@ test "all OpenSeek MoonBit sources parse without diagnostics" { let source = @fs.read_file_to_string(path) catch { err => fail("failed to read \{path}: \{Repr(err)}") } - let (_, handrolled_reports) = @parser.parse_string( + let (handrolled_impls, handrolled_reports) = @parser.parse_string( source, name=path, parser=Handrolled, ) record_diagnostics(failures, path, "Handrolled", handrolled_reports) - let (_, moonyacc_reports) = @parser.parse_string( + let (moonyacc_impls, moonyacc_reports) = @parser.parse_string( source, name=path, parser=MoonYacc, @@ -102,14 +109,35 @@ test "all OpenSeek MoonBit sources parse without diagnostics" { "untyped CST", reports_from_view(cst.diagnostics_view()), ) - let (_, lowering_reports) = cst.to_impls() + let (cst_impls, lowering_reports) = cst.to_impls() record_diagnostics(failures, path, "untyped CST to_impls", lowering_reports) + if !(handrolled_reports.is_empty() && + moonyacc_reports.is_empty() && + cst.diagnostics_view().is_empty() && + lowering_reports.is_empty()) { + continue + } + let handrolled_json = impls_json(handrolled_impls) + let moonyacc_json = impls_json(moonyacc_impls) + let cst_json = impls_json(cst_impls) + if handrolled_json != moonyacc_json { + failures.push( + "\{path}\nAST mismatch: Handrolled vs MoonYacc\nHandrolled:\n\{handrolled_json.stringify(indent=2)}\nMoonYacc:\n\{moonyacc_json.stringify(indent=2)}", + ) + } + if handrolled_json != cst_json { + failures.push( + "\{path}\nAST mismatch: Handrolled vs untyped CST\nHandrolled:\n\{handrolled_json.stringify(indent=2)}\nuntyped CST:\n\{cst_json.stringify(indent=2)}", + ) + } } if !failures.is_empty() { fail( - "OpenSeek corpus parsing failed for \{failures.length()} parser/file checks:\n\n" + + "OpenSeek corpus validation failed for \{failures.length()} checks:\n\n" + failures.join("\n\n"), ) } - println("parsed \{files.length()} OpenSeek .mbt files with all three parsers") + println( + "parsed \{files.length()} OpenSeek .mbt files with equivalent ASTs from all three parsers", + ) } diff --git a/untyped_cst/continue_locations_test.mbt b/untyped_cst/continue_locations_test.mbt new file mode 100644 index 000000000..c07551e9a --- /dev/null +++ b/untyped_cst/continue_locations_test.mbt @@ -0,0 +1,46 @@ +///| +test "continue lowering preserves comments and complete CST source spans" { + for newline in ["\n", "\r\n"] { + for + statement in [ + "continue", "continue label~", "continue label~~", "continue (1)", "continue label~ 1, (2)", + "continue label~~ (1), (2)", + ] { + let prefix = "// 🐇" + newline + let source = prefix + statement + " // trailing 🦀" + newline + let parsed = @untyped_cst.parse_expression(source) + @test.assert_eq(parsed.diagnostics_view().length(), 0) + let root = parsed.root() + @test.assert_eq(root.source_span, (0, source.length())) + @test.assert_eq(root.text(source), source[:]) + guard api_find_node(root, node => node.kind is Expr_Continue) + is Some(continue_node) else { + fail("expected continue") + } + let span_before = continue_node.source_span + let text_before = continue_node.text(source).to_owned() + // Check the actual retained comment token, not just its source text. + guard api_find_node(root, node => { + node.kind is Syntax_Token(TK_COMMENT) && + node.text(source).has_prefix("// trailing") + }) + is Some(comment) else { + fail("expected retained trailing comment") + } + @test.assert_eq(comment.text(source), "// trailing 🦀"[:]) + let (expr, reports) = parsed.to_expr() + @test.assert_eq(reports.length(), 0) + guard expr is Some(Continue(loc~, ..)) else { + fail("expected lowered continue") + } + // AST offsets count characters, whereas source spans count UTF-16 units. + @test.assert_eq(loc.start.cnum, prefix.length() - 1) + @test.assert_eq(loc.end.cnum, prefix.length() - 1 + statement.length()) + @test.assert_eq(loc.start.lnum, 2) + @test.assert_eq(loc.end.lnum, 2) + @test.assert_eq(continue_node.source_span, span_before) + @test.assert_eq(continue_node.text(source).to_owned(), text_before) + @test.assert_eq(root.text(source), source[:]) + } + } +} diff --git a/untyped_cst/internal/lower/control.mbt b/untyped_cst/internal/lower/control.mbt index ececa7227..dabaaa93d 100644 --- a/untyped_cst/internal/lower/control.mbt +++ b/untyped_cst/internal/lower/control.mbt @@ -388,7 +388,6 @@ fn lower_try_expr(node : CstNode) -> @syntax.Expr { .unwrap_or(false) let catch_loc = lower_catch_loc(node) let else_loc = lower_else_loc(node) - let loc = lower_try_expr_loc(node) Try( body=node .first_child_with_label("body") @@ -400,7 +399,7 @@ fn lower_try_expr(node : CstNode) -> @syntax.Expr { try_loc~, catch_loc~, else_loc~, - loc~, + loc=node.loc, ) } diff --git a/untyped_cst/internal/lower/lower_locations.mbt b/untyped_cst/internal/lower/lower_locations.mbt index de902c15d..ab38cdfa9 100644 --- a/untyped_cst/internal/lower/lower_locations.mbt +++ b/untyped_cst/internal/lower/lower_locations.mbt @@ -55,11 +55,7 @@ fn lower_decl_loc_without_trailing_trivia(node : CstNode) -> Location { ///| fn lower_trait_decl_loc(node : CstNode) -> Location { - let loc = lower_decl_loc_without_trailing_trivia(node) - match node.first_child_with_label("kw") { - Some(keyword) => Location::{ start: keyword.loc.start, end: loc.end, } - None => loc - } + lower_decl_loc_without_trailing_trivia(node) } ///| @@ -185,52 +181,17 @@ fn lower_else_loc(node : CstNode) -> Location { } ///| -fn lower_leading_parenthesized_end(node : CstNode) -> Position? { - match node.kind { - Expr_Unit | Expr_Group | Expr_Tuple | Expr_Constraint => - return Some(node.loc.end) - _ => () - } - for (label, child) in node.children { - if label is Some("trivia") { - continue - } - return lower_leading_parenthesized_end(child) - } - None -} - -///| -fn lower_first_significant_token_at_or_after( - node : CstNode, - pos : Position, -) -> CstNode? { +fn lower_last_significant_token_end(node : CstNode) -> Position? { match node.kind.token_kind() { - Some(TK_COMMENT | TK_NEWLINE) => return None - Some(_) if node.loc.start.cnum >= pos.cnum => return Some(node) - _ => () + Some(TK_COMMENT | TK_NEWLINE | TK_EOF) => return None + Some(_) => return Some(node.loc.end) + None => () } - for (_, child) in node.children { - if lower_first_significant_token_at_or_after(child, pos) is Some(found) { + for i = node.children.length() - 1; i >= 0; i = i - 1 { + let (_, child) = node.children[i] + if lower_last_significant_token_end(child) is Some(found) { return Some(found) } } None } - -///| -fn lower_try_expr_loc(node : CstNode) -> Location { - if node.bool_marker_value(fn(kind) { kind is Flag_HasTry }) is Some(true) { - return node.loc - } - if node.first_child_with_label("body") is Some(body) && - lower_leading_parenthesized_end(body) is Some(paren_end) && - lower_token_loc(node, "kw", TK_CATCH) is Some(catch_loc) { - let start = lower_first_significant_token_at_or_after(body, paren_end) - .map(token => token.loc.start) - .unwrap_or(catch_loc.start) - Location::{ start, end: node.loc.end, } - } else { - node.loc - } -} diff --git a/untyped_cst/internal/lower/lower_locations_wbtest.mbt b/untyped_cst/internal/lower/lower_locations_wbtest.mbt index 4dcede3f0..8276b69e3 100644 --- a/untyped_cst/internal/lower/lower_locations_wbtest.mbt +++ b/untyped_cst/internal/lower/lower_locations_wbtest.mbt @@ -19,6 +19,59 @@ fn detached_token(kind : TokenKind, start : Int, end : Int) -> CstNode { ) } +///| +test "continue locations handle trivia-only lists and detached recovery nodes" { + let kw = detached_token(TK_CONTINUE, 2, 10) + let comment = detached_token(TK_COMMENT, 25, 35) + let trivia = CstNode::list_continue_argument_list( + loc=comment.loc, + source_span=comment.source_span, + entries=[Trivia(comment)], + ) + for label in [None, Some(detached_token(TK_POST_LABEL, 11, 17))] { + let node = CstNode::expr_continue( + loc=detached_loc(2, 35), + source_span=(3, 36), + kw~, + label?, + arg=trivia, + ) + guard lower_expr(node) is Continue(args~, loc~, ..) else { + fail("expected continue") + } + @test.assert_eq(args.length(), 0) + @test.assert_eq( + loc, + detached_loc(2, if label is Some(_) { 17 } else { 10 }), + ) + } + let missing = CstNode::missing(loc=detached_loc(20, 20), source_span=(21, 21)) + let args = CstNode::list_continue_argument_list( + loc=detached_loc(20, 35), + source_span=(21, 36), + entries=[Item(missing), Trivia(comment)], + ) + let recovered = CstNode::expr_continue( + loc=detached_loc(2, 35), + source_span=(3, 36), + kw~, + arg=args, + ) + guard lower_expr(recovered) is Continue(loc~, ..) else { + fail("expected recovered continue") + } + @test.assert_eq(loc, detached_loc(2, 20)) + let missing_keyword = CstNode::expr_continue( + loc=detached_loc(20, 20), + source_span=(21, 21), + kw=missing, + ) + guard lower_expr(missing_keyword) is Continue(loc~, ..) else { + fail("expected continue with missing keyword") + } + @test.assert_eq(loc, missing_keyword.loc) +} + ///| test "detached CST derives locations only from the owning syntax fields" { let nested = CstNode::expr_try_operator( diff --git a/untyped_cst/internal/lower/node_integration_wbtest.mbt b/untyped_cst/internal/lower/node_integration_wbtest.mbt index 2308fc4ab..0b8c0755d 100644 --- a/untyped_cst/internal/lower/node_integration_wbtest.mbt +++ b/untyped_cst/internal/lower/node_integration_wbtest.mbt @@ -1,43 +1,29 @@ ///| -test "significant token search skips comments and newlines by exact kind" { - let start = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 0, } - let token_start = Position::{ ..start, cnum: 2, } - let token_end = Position::{ ..start, cnum: 3, } - let root = CstNode::expression( - loc=mk_loc(start, token_end), - source_span=(0, 3), - entries=[ - Trivia( - CstNode::syntax_token( - token_kind=TK_COMMENT, - loc=mk_loc(start, token_start), - source_span=(0, 2), - entries=[], - ), +test "last significant token search skips trailing trivia by exact kind" { + let root = CstNode::expression(loc=detached_loc(2, 31), source_span=(3, 32), entries=[ + Expr( + CstNode::syntax_unexpected( + token_kind=TK_LIDENT, + loc=detached_loc(2, 3), + source_span=(3, 4), + entries=[Payload(String("x"))], ), - Trivia( - CstNode::syntax_unexpected( - token_kind=TK_NEWLINE, - loc=empty_loc(token_start), - source_span=(2, 2), - entries=[], - ), + ), + Trivia(detached_token(TK_COMMENT, 4, 30)), + Trivia( + CstNode::syntax_unexpected( + token_kind=TK_NEWLINE, + loc=detached_loc(30, 31), + source_span=(31, 32), + entries=[], ), - Expr( - CstNode::syntax_token( - token_kind=TK_LIDENT, - loc=mk_loc(token_start, token_end), - source_span=(2, 3), - entries=[Payload(String("x"))], - ), - ), - ], + ), + Trivia(detached_token(TK_EOF, 31, 31)), + ]) + @test.assert_eq( + lower_last_significant_token_end(root), + Some(detached_loc(2, 3).end), ) - guard lower_first_significant_token_at_or_after(root, start) is Some(found) else { - fail("expected significant token") - } - @test.assert_eq(found.kind.token_kind(), Some(TK_LIDENT)) - @test.assert_eq(found.source_span, (2, 3)) } ///| diff --git a/untyped_cst/internal/lower/pattern.mbt b/untyped_cst/internal/lower/pattern.mbt index f5cae3638..54270995c 100644 --- a/untyped_cst/internal/lower/pattern.mbt +++ b/untyped_cst/internal/lower/pattern.mbt @@ -10,17 +10,7 @@ fn lower_pattern(node : CstNode) -> @syntax.Pattern { Pattern_Any | Pattern_MissingPattern => Any(loc=node.loc) Pattern_Constant => if lower_constant_from_node(node) is Some(c) { - let loc = if node.has_child_kind(fn(kind) { - kind is Operator_UnaryMinus - }) { - node - .first_child_with_label("semantic_constant") - .map(constant => constant.loc) - .unwrap_or(node.loc) - } else { - node.loc - } - Constant(c~, loc~) + Constant(c~, loc=node.loc) } else { Any(loc=node.loc) } diff --git a/untyped_cst/internal/lower/statement.mbt b/untyped_cst/internal/lower/statement.mbt index a67d781d1..12e72468b 100644 --- a/untyped_cst/internal/lower/statement.mbt +++ b/untyped_cst/internal/lower/statement.mbt @@ -189,17 +189,27 @@ fn lower_break_expr(node : CstNode) -> @syntax.Expr { ///| fn lower_continue_expr(node : CstNode) -> @syntax.Expr { let args : Array[@syntax.Expr] = [] + let keyword_loc = lower_token_loc(node, "kw", TK_CONTINUE).unwrap_or(node.loc) + let start = keyword_loc.start + let mut end = node + .first_child_with_label("label") + .map(label => label.loc.end) + .unwrap_or(keyword_loc.end) if node.first_child_with_label("arg") is Some(list) { for item in lower_item_children(list) { args.push(lower_expr(item)) + // Use syntax tokens so grouping delimiters survive, but trailing trivia + // cannot extend the AST location. Detached recovery nodes may lack tokens. + end = lower_last_significant_token_end(item).unwrap_or(item.loc.end) } } else if node.first_child_with_label("expr") is Some(expr) { args.push(lower_expr(expr)) + end = lower_last_significant_token_end(expr).unwrap_or(expr.loc.end) } else { () } let label = lower_control_label(node) - Continue(args=lower_array_to_list(args), label~, loc=node.loc) + Continue(args=lower_array_to_list(args), label~, loc={ start, end, }) } ///| diff --git a/untyped_cst/internal/lower/type.mbt b/untyped_cst/internal/lower/type.mbt index 5694e12db..b4a2c5b93 100644 --- a/untyped_cst/internal/lower/type.mbt +++ b/untyped_cst/internal/lower/type.mbt @@ -86,25 +86,7 @@ fn lower_option_type(node : CstNode) -> @syntax.Type { .first_child_of_kind(NodeKind::is_operator) .map(fn(operator) { operator.loc }) .unwrap_or(node.loc) - let loc = match inner_node { - Some(ty) if option_type_wraps_object(ty) => - Location::{ start: inner.loc().start, end: node.loc.end, } - _ => node.loc - } - Option(ty=inner, loc~, question_loc~) -} - -///| -fn option_type_wraps_object(node : CstNode) -> Bool { - match node.kind { - Type_Object => true - Type_Option => - node - .first_child_with_label("type") - .map(option_type_wraps_object) - .unwrap_or(false) - _ => false - } + Option(ty=inner, loc=node.loc, question_loc~) } ///|