From 307bbd8683953d6f7b107ed9c7a8847d816f84a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Laferri=C3=A8re?= Date: Wed, 23 Sep 2026 22:06:34 -0400 Subject: [PATCH 1/3] lexer: split `Lexer::get_token` in two to enable refinements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intro `scan_token` as the service actually retrieving the next token from the source file, and keep `get_token` as the service returning the token to the clients. `get_token` can then act as a refinement point to filter or alter the token stream. Signed-off-by: Alexis Laferrière --- src/parser/lexer_work.nit | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/parser/lexer_work.nit b/src/parser/lexer_work.nit index c8b42e8a2e..7e7d61625d 100644 --- a/src/parser/lexer_work.nit +++ b/src/parser/lexer_work.nit @@ -141,8 +141,15 @@ class Lexer end # Primitive method to return a token, or return null if it is discarded - # Is used to implement `peek` and `next` - protected fun get_token: nullable Token + # + # Is used to implement `peek` and `next`. + # Refine it to filter or alter the token stream, see `scan_token` for the raw tokens. + protected fun get_token: nullable Token do return scan_token + + # Scan the next token from the source, or return null if it is discarded + # + # The raw tokens, before any refinement of `get_token`. + protected fun scan_token: nullable Token do var dfa_state = 0 From db1874ac03c3383b7cca1ac1a079baef3748b313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Laferri=C3=A8re?= Date: Wed, 23 Sep 2026 12:33:45 -0400 Subject: [PATCH 2/3] nitls: parse only the header of modules to list dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dependencies listed by `nitls -M` dropped any module with a parsing error. A broken file would then be missing from the dependencies of a Makefile, and make could see the binary as up to date. Without the rebuild the error is fully hidden from the user. Let's make this less likely by parsing only the part nitls actually needs to list dependencies. This way a syntax error after the import list doesn't affect nitls' behavior. Intro the module loader_header_only, imported by nitls only, which refines the Lexer to return EOF at the first token of the body. As a bonus, the reduced work speeds up nitls significantly, `nitls -M src/nitc.nit` is about 9x faster. Since it can run many time on every `make` invocation, it becomes noticeably more snappy in large projects. Signed-off-by: Alexis Laferrière --- src/loader_header_only.nit | 181 +++++++++++++++++++++++++ src/nitls.nit | 1 + tests/nitls.args | 4 + tests/nitls_error_body.nit | 18 +++ tests/nitls_header_annotations.nit | 29 ++++ tests/nitls_header_redef.nit | 23 ++++ tests/nitls_header_visibility.nit | 23 ++++ tests/sav/nitls_args10.res | 2 + tests/sav/nitls_args11.res | 2 + tests/sav/nitls_args12.res | 2 + tests/sav/nitls_args9.res | 3 + tests/sav/nitls_error_body.res | 1 + tests/sav/nitls_header_annotations.res | 6 + tests/sav/nitls_header_redef.res | 10 ++ tests/sav/nitls_header_visibility.res | 10 ++ 15 files changed, 315 insertions(+) create mode 100644 src/loader_header_only.nit create mode 100644 tests/nitls_error_body.nit create mode 100644 tests/nitls_header_annotations.nit create mode 100644 tests/nitls_header_redef.nit create mode 100644 tests/nitls_header_visibility.nit create mode 100644 tests/sav/nitls_args10.res create mode 100644 tests/sav/nitls_args11.res create mode 100644 tests/sav/nitls_args12.res create mode 100644 tests/sav/nitls_args9.res create mode 100644 tests/sav/nitls_error_body.res create mode 100644 tests/sav/nitls_header_annotations.res create mode 100644 tests/sav/nitls_header_redef.res create mode 100644 tests/sav/nitls_header_visibility.res diff --git a/src/loader_header_only.nit b/src/loader_header_only.nit new file mode 100644 index 0000000000..d8d1bb82e3 --- /dev/null +++ b/src/loader_header_only.nit @@ -0,0 +1,181 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Load only the header of the modules: module declaration and imports +# +# The AST of the loaded modules have only what's needed for the importation +# hierarchy and no classes, top-level functions, top-level code, or top-level +# foreign code bodies. This parsing is fast and robust to syntax errors +# in the body of the modules. +module loader_header_only + +import loader + +redef class Lexer + # Tokens already read ahead of the parser, to be returned next + private var ahead = new Array[Token] + + # Location of the first token of the body + private var header_end: nullable Location = null + + # Normal token stream up to the imports, then return EOF + # + # Look ahead for a definition or block opening as a way to detect that + # we're done with imports. The look ahead allows to skip over blank tokens + # and keywords before the definition markers like visibility and `redef`. + redef fun get_token + do + var header_end = header_end + if header_end == null then + if ahead.not_empty then return ahead.shift + + var t = super + if t == null then return null + if not is_body_start(t) then return t + + header_end = t.location + self.header_end = header_end + end + + return new EOF.init_tk(header_end) + end + + # Is `t` the first token of the body? Looking ahead as needed + private fun is_body_start(t: Token): Bool + do + if not t.is_prefix then return t.starts_body + + var n = scan_significant + ahead.add n + while n.is_blank or n.is_prefix do + n = scan_significant + ahead.add n + end + return not n isa TKwimport + end + + # Scan the next raw token that is not an ignored blank space + private fun scan_significant: Token + do + var t = scan_token + while t == null do t = scan_token + return t + end +end + +redef class Token + private fun is_blank: Bool do return false + private fun is_prefix: Bool do return false + private fun starts_body: Bool do return false +end + +# Blanks + +redef class TEol + redef fun is_blank do return true +end + +redef class TComment + redef fun is_blank do return true +end + +# Prefixes + +redef class TKwpublic + redef fun is_prefix do return true +end + +redef class TKwprotected + redef fun is_prefix do return true +end + +redef class TKwprivate + redef fun is_prefix do return true +end + +redef class TKwintrude + redef fun is_prefix do return true +end + +redef class TKwredef + redef fun is_prefix do return true +end + +# Definitions + +redef class TKwclass + redef fun starts_body do return true +end + +redef class TKwabstract + redef fun starts_body do return true +end + +redef class TKwinterface + redef fun starts_body do return true +end + +redef class TKwenum + redef fun starts_body do return true +end + +redef class TKwsubset + redef fun starts_body do return true +end + +redef class TKwextern + redef fun starts_body do return true +end + +redef class TKwmeth + redef fun starts_body do return true +end + +redef class TKwinit + redef fun starts_body do return true +end + +redef class TKwtype + redef fun starts_body do return true +end + +redef class TKwin + redef fun starts_body do return true +end + +# Statements opening a block + +redef class TKwif + redef fun starts_body do return true +end + +redef class TKwwhile + redef fun starts_body do return true +end + +redef class TKwfor + redef fun starts_body do return true +end + +redef class TKwloop + redef fun starts_body do return true +end + +redef class TKwdo + redef fun starts_body do return true +end + +redef class TKwwith + redef fun starts_body do return true +end diff --git a/src/nitls.nit b/src/nitls.nit index 62fc09f027..a3bd9c4e6f 100644 --- a/src/nitls.nit +++ b/src/nitls.nit @@ -18,6 +18,7 @@ module nitls import modelbuilder +import loader_header_only import ordered_tree import console diff --git a/tests/nitls.args b/tests/nitls.args index 5f5a31a9db..98fc858f77 100644 --- a/tests/nitls.args +++ b/tests/nitls.args @@ -6,3 +6,7 @@ base_simple3.nit project1 -td project1/module3.nit project1/subdir/subdir2/subdir3/submodule.nit test_prog --no-color test_prog/game/excluded.nit test_prog/game/excluded_dir/more.nit -t --no-color +-M nitls_error_body.nit +-M nitls_header_annotations.nit +-M nitls_header_visibility.nit +-M nitls_header_redef.nit diff --git a/tests/nitls_error_body.nit b/tests/nitls_error_body.nit new file mode 100644 index 0000000000..7e82650251 --- /dev/null +++ b/tests/nitls_error_body.nit @@ -0,0 +1,18 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The imports can be found, despite the syntax error below +import base_simple_import + +fun foo(label: Int) do end diff --git a/tests/nitls_header_annotations.nit b/tests/nitls_header_annotations.nit new file mode 100644 index 0000000000..b7ecd96f1b --- /dev/null +++ b/tests/nitls_header_annotations.nit @@ -0,0 +1,29 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module nitls_header_annotations is + ldflags("-lm", + "-lpthread") +end + +import end +import base_simple + +# The annotations of an import can span lines too +import base_simple_import is + conditional(base_simple) +end + +class NitlsHeaderAnnotations +end diff --git a/tests/nitls_header_redef.nit b/tests/nitls_header_redef.nit new file mode 100644 index 0000000000..4d746af97c --- /dev/null +++ b/tests/nitls_header_redef.nit @@ -0,0 +1,23 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module nitls_header_redef + +import end + +# The grammar accepts `redef` before an import, should it really? +redef import base_simple3 + +redef class A +end diff --git a/tests/nitls_header_visibility.nit b/tests/nitls_header_visibility.nit new file mode 100644 index 0000000000..c4004fcf0e --- /dev/null +++ b/tests/nitls_header_visibility.nit @@ -0,0 +1,23 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module nitls_header_visibility + +import end + +private +import base_simple3 + +# The body starts at the visibility keyword, not at `fun` +private fun nitls_header_foo do end diff --git a/tests/sav/nitls_args10.res b/tests/sav/nitls_args10.res new file mode 100644 index 0000000000..a5ab79683f --- /dev/null +++ b/tests/sav/nitls_args10.res @@ -0,0 +1,2 @@ +base_simple.nit +nitls_header_annotations.nit diff --git a/tests/sav/nitls_args11.res b/tests/sav/nitls_args11.res new file mode 100644 index 0000000000..d99cb8441b --- /dev/null +++ b/tests/sav/nitls_args11.res @@ -0,0 +1,2 @@ +base_simple3.nit +nitls_header_visibility.nit diff --git a/tests/sav/nitls_args12.res b/tests/sav/nitls_args12.res new file mode 100644 index 0000000000..b6062a8851 --- /dev/null +++ b/tests/sav/nitls_args12.res @@ -0,0 +1,2 @@ +base_simple3.nit +nitls_header_redef.nit diff --git a/tests/sav/nitls_args9.res b/tests/sav/nitls_args9.res new file mode 100644 index 0000000000..0a55748775 --- /dev/null +++ b/tests/sav/nitls_args9.res @@ -0,0 +1,3 @@ +base_simple.nit +base_simple_import.nit +nitls_error_body.nit diff --git a/tests/sav/nitls_error_body.res b/tests/sav/nitls_error_body.res new file mode 100644 index 0000000000..f005d3914d --- /dev/null +++ b/tests/sav/nitls_error_body.res @@ -0,0 +1 @@ +nitls_error_body.nit:18,9--13: Syntax Error: unexpected keyword 'label'. diff --git a/tests/sav/nitls_header_annotations.res b/tests/sav/nitls_header_annotations.res new file mode 100644 index 0000000000..e859615217 --- /dev/null +++ b/tests/sav/nitls_header_annotations.res @@ -0,0 +1,6 @@ +10 +20 +20 +30 +30 +30 diff --git a/tests/sav/nitls_header_redef.res b/tests/sav/nitls_header_redef.res new file mode 100644 index 0000000000..f00c965d83 --- /dev/null +++ b/tests/sav/nitls_header_redef.res @@ -0,0 +1,10 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 diff --git a/tests/sav/nitls_header_visibility.res b/tests/sav/nitls_header_visibility.res new file mode 100644 index 0000000000..f00c965d83 --- /dev/null +++ b/tests/sav/nitls_header_visibility.res @@ -0,0 +1,10 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 From d59a2344ec3e81c29b5d13bf607c0d6127ec9f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Laferri=C3=A8re?= Date: Wed, 23 Sep 2026 12:34:08 -0400 Subject: [PATCH 3/3] nitls: list the modules that failed parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With `nitls -M` a module that failed to parse was not listed in the output. This could have make think no rebuild was necessery which fully silenced the error. Let's list the files with the parse error to address this issue. To do so we keep a list of the parsing errors. An alternative would be to keep a list of the AModule parsed or the AError for the failed ones. Then `nitls` could list those in for the `-M` output instead of the type-checked MModules. Signed-off-by: Alexis Laferrière --- src/loader.nit | 4 ++++ src/nitls.nit | 9 +++++++++ tests/nitls.args | 2 ++ tests/nitls_error_header.nit | 18 ++++++++++++++++++ tests/sav/nitls_args13.res | 1 + tests/sav/nitls_args14.res | 1 + tests/sav/nitls_error_header.res | 1 + 7 files changed, 36 insertions(+) create mode 100644 tests/nitls_error_header.nit create mode 100644 tests/sav/nitls_args13.res create mode 100644 tests/sav/nitls_args14.res create mode 100644 tests/sav/nitls_error_header.res diff --git a/src/loader.nit b/src/loader.nit index 169d52e848..c9cefcbe5e 100644 --- a/src/loader.nit +++ b/src/loader.nit @@ -670,6 +670,9 @@ redef class ModelBuilder return path.realpath end + # Errors of the lexer or parser that prevented a module from loading + var parse_errors = new Array[AError] + # Try to load a module AST using a path. # Display an error if there is a problem (IO / lexer / parser) and return null # @@ -702,6 +705,7 @@ redef class ModelBuilder var neof = tree.n_eof assert neof isa AError error(neof, neof.message) + parse_errors.add neof return null end diff --git a/src/nitls.nit b/src/nitls.nit index a3bd9c4e6f..b716b02c4a 100644 --- a/src/nitls.nit +++ b/src/nitls.nit @@ -207,6 +207,15 @@ if opt_source.value then end end +if opt_make.value then + # The modules that could not be parsed didn't result in a mmodule that + # could be listed above. List them here from the collected errors. + for error in mb.parse_errors do + var file = error.location.file + if file != null then print file.filename.escape_to_mk + end +end + if opt_package.value then var mpackages = new Array[MPackage] for m in mmodules do diff --git a/tests/nitls.args b/tests/nitls.args index 98fc858f77..28b5957d38 100644 --- a/tests/nitls.args +++ b/tests/nitls.args @@ -10,3 +10,5 @@ test_prog/game/excluded.nit test_prog/game/excluded_dir/more.nit -t --no-color -M nitls_header_annotations.nit -M nitls_header_visibility.nit -M nitls_header_redef.nit +-M error_mod_unk.nit --no-color +-M nitls_error_header.nit diff --git a/tests/nitls_error_header.nit b/tests/nitls_error_header.nit new file mode 100644 index 0000000000..9a150346cd --- /dev/null +++ b/tests/nitls_error_header.nit @@ -0,0 +1,18 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base_simple_import + +# Syntax error in the header +import 5 diff --git a/tests/sav/nitls_args13.res b/tests/sav/nitls_args13.res new file mode 100644 index 0000000000..deb2c79c6c --- /dev/null +++ b/tests/sav/nitls_args13.res @@ -0,0 +1 @@ +error_mod_unk.nit diff --git a/tests/sav/nitls_args14.res b/tests/sav/nitls_args14.res new file mode 100644 index 0000000000..416275eca2 --- /dev/null +++ b/tests/sav/nitls_args14.res @@ -0,0 +1 @@ +nitls_error_header.nit diff --git a/tests/sav/nitls_error_header.res b/tests/sav/nitls_error_header.res new file mode 100644 index 0000000000..1ac1ce7e56 --- /dev/null +++ b/tests/sav/nitls_error_header.res @@ -0,0 +1 @@ +nitls_error_header.nit:18,8: Syntax Error: unexpected literal value '5'.