diff --git a/src/loader.nit b/src/loader.nit index 169d52e848..c9cefcbe5e 100644 --- a/src/loader.nit +++ b/src/loader.nit @@ -670,6 +670,9 @@ redef class ModelBuilder return path.realpath end + # Errors of the lexer or parser that prevented a module from loading + var parse_errors = new Array[AError] + # Try to load a module AST using a path. # Display an error if there is a problem (IO / lexer / parser) and return null # @@ -702,6 +705,7 @@ redef class ModelBuilder var neof = tree.n_eof assert neof isa AError error(neof, neof.message) + parse_errors.add neof return null end diff --git a/src/loader_header_only.nit b/src/loader_header_only.nit new file mode 100644 index 0000000000..d8d1bb82e3 --- /dev/null +++ b/src/loader_header_only.nit @@ -0,0 +1,181 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Load only the header of the modules: module declaration and imports +# +# The AST of the loaded modules have only what's needed for the importation +# hierarchy and no classes, top-level functions, top-level code, or top-level +# foreign code bodies. This parsing is fast and robust to syntax errors +# in the body of the modules. +module loader_header_only + +import loader + +redef class Lexer + # Tokens already read ahead of the parser, to be returned next + private var ahead = new Array[Token] + + # Location of the first token of the body + private var header_end: nullable Location = null + + # Normal token stream up to the imports, then return EOF + # + # Look ahead for a definition or block opening as a way to detect that + # we're done with imports. The look ahead allows to skip over blank tokens + # and keywords before the definition markers like visibility and `redef`. + redef fun get_token + do + var header_end = header_end + if header_end == null then + if ahead.not_empty then return ahead.shift + + var t = super + if t == null then return null + if not is_body_start(t) then return t + + header_end = t.location + self.header_end = header_end + end + + return new EOF.init_tk(header_end) + end + + # Is `t` the first token of the body? Looking ahead as needed + private fun is_body_start(t: Token): Bool + do + if not t.is_prefix then return t.starts_body + + var n = scan_significant + ahead.add n + while n.is_blank or n.is_prefix do + n = scan_significant + ahead.add n + end + return not n isa TKwimport + end + + # Scan the next raw token that is not an ignored blank space + private fun scan_significant: Token + do + var t = scan_token + while t == null do t = scan_token + return t + end +end + +redef class Token + private fun is_blank: Bool do return false + private fun is_prefix: Bool do return false + private fun starts_body: Bool do return false +end + +# Blanks + +redef class TEol + redef fun is_blank do return true +end + +redef class TComment + redef fun is_blank do return true +end + +# Prefixes + +redef class TKwpublic + redef fun is_prefix do return true +end + +redef class TKwprotected + redef fun is_prefix do return true +end + +redef class TKwprivate + redef fun is_prefix do return true +end + +redef class TKwintrude + redef fun is_prefix do return true +end + +redef class TKwredef + redef fun is_prefix do return true +end + +# Definitions + +redef class TKwclass + redef fun starts_body do return true +end + +redef class TKwabstract + redef fun starts_body do return true +end + +redef class TKwinterface + redef fun starts_body do return true +end + +redef class TKwenum + redef fun starts_body do return true +end + +redef class TKwsubset + redef fun starts_body do return true +end + +redef class TKwextern + redef fun starts_body do return true +end + +redef class TKwmeth + redef fun starts_body do return true +end + +redef class TKwinit + redef fun starts_body do return true +end + +redef class TKwtype + redef fun starts_body do return true +end + +redef class TKwin + redef fun starts_body do return true +end + +# Statements opening a block + +redef class TKwif + redef fun starts_body do return true +end + +redef class TKwwhile + redef fun starts_body do return true +end + +redef class TKwfor + redef fun starts_body do return true +end + +redef class TKwloop + redef fun starts_body do return true +end + +redef class TKwdo + redef fun starts_body do return true +end + +redef class TKwwith + redef fun starts_body do return true +end diff --git a/src/nitls.nit b/src/nitls.nit index 62fc09f027..b716b02c4a 100644 --- a/src/nitls.nit +++ b/src/nitls.nit @@ -18,6 +18,7 @@ module nitls import modelbuilder +import loader_header_only import ordered_tree import console @@ -206,6 +207,15 @@ if opt_source.value then end end +if opt_make.value then + # The modules that could not be parsed didn't result in a mmodule that + # could be listed above. List them here from the collected errors. + for error in mb.parse_errors do + var file = error.location.file + if file != null then print file.filename.escape_to_mk + end +end + if opt_package.value then var mpackages = new Array[MPackage] for m in mmodules do diff --git a/src/parser/lexer_work.nit b/src/parser/lexer_work.nit index c8b42e8a2e..7e7d61625d 100644 --- a/src/parser/lexer_work.nit +++ b/src/parser/lexer_work.nit @@ -141,8 +141,15 @@ class Lexer end # Primitive method to return a token, or return null if it is discarded - # Is used to implement `peek` and `next` - protected fun get_token: nullable Token + # + # Is used to implement `peek` and `next`. + # Refine it to filter or alter the token stream, see `scan_token` for the raw tokens. + protected fun get_token: nullable Token do return scan_token + + # Scan the next token from the source, or return null if it is discarded + # + # The raw tokens, before any refinement of `get_token`. + protected fun scan_token: nullable Token do var dfa_state = 0 diff --git a/tests/nitls.args b/tests/nitls.args index 5f5a31a9db..28b5957d38 100644 --- a/tests/nitls.args +++ b/tests/nitls.args @@ -6,3 +6,9 @@ base_simple3.nit project1 -td project1/module3.nit project1/subdir/subdir2/subdir3/submodule.nit test_prog --no-color test_prog/game/excluded.nit test_prog/game/excluded_dir/more.nit -t --no-color +-M nitls_error_body.nit +-M nitls_header_annotations.nit +-M nitls_header_visibility.nit +-M nitls_header_redef.nit +-M error_mod_unk.nit --no-color +-M nitls_error_header.nit diff --git a/tests/nitls_error_body.nit b/tests/nitls_error_body.nit new file mode 100644 index 0000000000..7e82650251 --- /dev/null +++ b/tests/nitls_error_body.nit @@ -0,0 +1,18 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The imports can be found, despite the syntax error below +import base_simple_import + +fun foo(label: Int) do end diff --git a/tests/nitls_error_header.nit b/tests/nitls_error_header.nit new file mode 100644 index 0000000000..9a150346cd --- /dev/null +++ b/tests/nitls_error_header.nit @@ -0,0 +1,18 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base_simple_import + +# Syntax error in the header +import 5 diff --git a/tests/nitls_header_annotations.nit b/tests/nitls_header_annotations.nit new file mode 100644 index 0000000000..b7ecd96f1b --- /dev/null +++ b/tests/nitls_header_annotations.nit @@ -0,0 +1,29 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module nitls_header_annotations is + ldflags("-lm", + "-lpthread") +end + +import end +import base_simple + +# The annotations of an import can span lines too +import base_simple_import is + conditional(base_simple) +end + +class NitlsHeaderAnnotations +end diff --git a/tests/nitls_header_redef.nit b/tests/nitls_header_redef.nit new file mode 100644 index 0000000000..4d746af97c --- /dev/null +++ b/tests/nitls_header_redef.nit @@ -0,0 +1,23 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module nitls_header_redef + +import end + +# The grammar accepts `redef` before an import, should it really? +redef import base_simple3 + +redef class A +end diff --git a/tests/nitls_header_visibility.nit b/tests/nitls_header_visibility.nit new file mode 100644 index 0000000000..c4004fcf0e --- /dev/null +++ b/tests/nitls_header_visibility.nit @@ -0,0 +1,23 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module nitls_header_visibility + +import end + +private +import base_simple3 + +# The body starts at the visibility keyword, not at `fun` +private fun nitls_header_foo do end diff --git a/tests/sav/nitls_args10.res b/tests/sav/nitls_args10.res new file mode 100644 index 0000000000..a5ab79683f --- /dev/null +++ b/tests/sav/nitls_args10.res @@ -0,0 +1,2 @@ +base_simple.nit +nitls_header_annotations.nit diff --git a/tests/sav/nitls_args11.res b/tests/sav/nitls_args11.res new file mode 100644 index 0000000000..d99cb8441b --- /dev/null +++ b/tests/sav/nitls_args11.res @@ -0,0 +1,2 @@ +base_simple3.nit +nitls_header_visibility.nit diff --git a/tests/sav/nitls_args12.res b/tests/sav/nitls_args12.res new file mode 100644 index 0000000000..b6062a8851 --- /dev/null +++ b/tests/sav/nitls_args12.res @@ -0,0 +1,2 @@ +base_simple3.nit +nitls_header_redef.nit diff --git a/tests/sav/nitls_args13.res b/tests/sav/nitls_args13.res new file mode 100644 index 0000000000..deb2c79c6c --- /dev/null +++ b/tests/sav/nitls_args13.res @@ -0,0 +1 @@ +error_mod_unk.nit diff --git a/tests/sav/nitls_args14.res b/tests/sav/nitls_args14.res new file mode 100644 index 0000000000..416275eca2 --- /dev/null +++ b/tests/sav/nitls_args14.res @@ -0,0 +1 @@ +nitls_error_header.nit diff --git a/tests/sav/nitls_args9.res b/tests/sav/nitls_args9.res new file mode 100644 index 0000000000..0a55748775 --- /dev/null +++ b/tests/sav/nitls_args9.res @@ -0,0 +1,3 @@ +base_simple.nit +base_simple_import.nit +nitls_error_body.nit diff --git a/tests/sav/nitls_error_body.res b/tests/sav/nitls_error_body.res new file mode 100644 index 0000000000..f005d3914d --- /dev/null +++ b/tests/sav/nitls_error_body.res @@ -0,0 +1 @@ +nitls_error_body.nit:18,9--13: Syntax Error: unexpected keyword 'label'. diff --git a/tests/sav/nitls_error_header.res b/tests/sav/nitls_error_header.res new file mode 100644 index 0000000000..1ac1ce7e56 --- /dev/null +++ b/tests/sav/nitls_error_header.res @@ -0,0 +1 @@ +nitls_error_header.nit:18,8: Syntax Error: unexpected literal value '5'. diff --git a/tests/sav/nitls_header_annotations.res b/tests/sav/nitls_header_annotations.res new file mode 100644 index 0000000000..e859615217 --- /dev/null +++ b/tests/sav/nitls_header_annotations.res @@ -0,0 +1,6 @@ +10 +20 +20 +30 +30 +30 diff --git a/tests/sav/nitls_header_redef.res b/tests/sav/nitls_header_redef.res new file mode 100644 index 0000000000..f00c965d83 --- /dev/null +++ b/tests/sav/nitls_header_redef.res @@ -0,0 +1,10 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 diff --git a/tests/sav/nitls_header_visibility.res b/tests/sav/nitls_header_visibility.res new file mode 100644 index 0000000000..f00c965d83 --- /dev/null +++ b/tests/sav/nitls_header_visibility.res @@ -0,0 +1,10 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10