nitls: Improve resiliency to parsing errors

src/loader.nit

@@ -670,6 +670,9 @@ redef class ModelBuilder
 		return path.realpath
 	end
 
+	# Errors of the lexer or parser that prevented a module from loading
+	var parse_errors = new Array[AError]
+
 	# Try to load a module AST using a path.
 	# Display an error if there is a problem (IO / lexer / parser) and return null
 	#
@@ -702,6 +705,7 @@ redef class ModelBuilder
 			var neof = tree.n_eof
 			assert neof isa AError
 			error(neof, neof.message)
+			parse_errors.add neof
 			return null
 		end
 

src/loader_header_only.nit

@@ -0,0 +1,181 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Load only the header of the modules: module declaration and imports
+#
+# The AST of the loaded modules have only what's needed for the importation
+# hierarchy and no classes, top-level functions, top-level code, or top-level
+# foreign code bodies. This parsing is fast and robust to syntax errors
+# in the body of the modules.
+module loader_header_only
+
+import loader
+
+redef class Lexer
+	# Tokens already read ahead of the parser, to be returned next
+	private var ahead = new Array[Token]
+
+	# Location of the first token of the body
+	private var header_end: nullable Location = null
+
+	# Normal token stream up to the imports, then return EOF
+	#
+	# Look ahead for a definition or block opening as a way to detect that
+	# we're done with imports. The look ahead allows to skip over blank tokens
+	# and keywords before the definition markers like visibility and `redef`.
+	redef fun get_token
+	do
+		var header_end = header_end
+		if header_end == null then
+			if ahead.not_empty then return ahead.shift
+
+			var t = super
+			if t == null then return null
+			if not is_body_start(t) then return t
+
+			header_end = t.location
+			self.header_end = header_end
+		end
+
+		return new EOF.init_tk(header_end)
+	end
+
+	# Is `t` the first token of the body? Looking ahead as needed
+	private fun is_body_start(t: Token): Bool
+	do
+		if not t.is_prefix then return t.starts_body
+
+		var n = scan_significant
+		ahead.add n
+		while n.is_blank or n.is_prefix do
+			n = scan_significant
+			ahead.add n
+		end
+		return not n isa TKwimport
+	end
+
+	# Scan the next raw token that is not an ignored blank space
+	private fun scan_significant: Token
+	do
+		var t = scan_token
+		while t == null do t = scan_token
+		return t
+	end
+end
+
+redef class Token
+	private fun is_blank: Bool do return false
+	private fun is_prefix: Bool do return false
+	private fun starts_body: Bool do return false
+end
+
+# Blanks
+
+redef class TEol
+	redef fun is_blank do return true
+end
+
+redef class TComment
+	redef fun is_blank do return true
+end
+
+# Prefixes
+
+redef class TKwpublic
+	redef fun is_prefix do return true
+end
+
+redef class TKwprotected
+	redef fun is_prefix do return true
+end
+
+redef class TKwprivate
+	redef fun is_prefix do return true
+end
+
+redef class TKwintrude
+	redef fun is_prefix do return true
+end
+
+redef class TKwredef
+	redef fun is_prefix do return true
+end
+
+# Definitions
+
+redef class TKwclass
+	redef fun starts_body do return true
+end
+
+redef class TKwabstract
+	redef fun starts_body do return true
+end
+
+redef class TKwinterface
+	redef fun starts_body do return true
+end
+
+redef class TKwenum
+	redef fun starts_body do return true
+end
+
+redef class TKwsubset
+	redef fun starts_body do return true
+end
+
+redef class TKwextern
+	redef fun starts_body do return true
+end
+
+redef class TKwmeth
+	redef fun starts_body do return true
+end
+
+redef class TKwinit
+	redef fun starts_body do return true
+end
+
+redef class TKwtype
+	redef fun starts_body do return true
+end
+
+redef class TKwin
+	redef fun starts_body do return true
+end
+
+# Statements opening a block
+
+redef class TKwif
+	redef fun starts_body do return true
+end
+
+redef class TKwwhile
+	redef fun starts_body do return true
+end
+
+redef class TKwfor
+	redef fun starts_body do return true
+end
+
+redef class TKwloop
+	redef fun starts_body do return true
+end
+
+redef class TKwdo
+	redef fun starts_body do return true
+end
+
+redef class TKwwith
+	redef fun starts_body do return true
+end

src/nitls.nit

@@ -18,6 +18,7 @@
 module nitls
 
 import modelbuilder
+import loader_header_only
 import ordered_tree
 import console
 
@@ -206,6 +207,15 @@ if opt_source.value then
 	end
 end
 
+if opt_make.value then
+	# The modules that could not be parsed didn't result in a mmodule that
+	# could be listed above. List them here from the collected errors.
+	for error in mb.parse_errors do
+		var file = error.location.file
+		if file != null then print file.filename.escape_to_mk
+	end
+end
+
 if opt_package.value then
 	var mpackages = new Array[MPackage]
 	for m in mmodules do

src/parser/lexer_work.nit

@@ -141,8 +141,15 @@ class Lexer
 	end
 
 	# Primitive method to return a token, or return null if it is discarded
-	# Is used to implement `peek` and `next`
-	protected fun get_token: nullable Token
+	#
+	# Is used to implement `peek` and `next`.
+	# Refine it to filter or alter the token stream, see `scan_token` for the raw tokens.
+	protected fun get_token: nullable Token do return scan_token
+
+	# Scan the next token from the source, or return null if it is discarded
+	#
+	# The raw tokens, before any refinement of `get_token`.
+	protected fun scan_token: nullable Token
 	do
 		var dfa_state = 0
 

tests/nitls.args

@@ -6,3 +6,9 @@ base_simple3.nit project1
 -td project1/module3.nit project1/subdir/subdir2/subdir3/submodule.nit
 test_prog --no-color
 test_prog/game/excluded.nit test_prog/game/excluded_dir/more.nit -t --no-color
+-M nitls_error_body.nit
+-M nitls_header_annotations.nit
+-M nitls_header_visibility.nit
+-M nitls_header_redef.nit
+-M error_mod_unk.nit --no-color
+-M nitls_error_header.nit

tests/nitls_error_body.nit

@@ -0,0 +1,18 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The imports can be found, despite the syntax error below
+import base_simple_import
+
+fun foo(label: Int) do end

tests/nitls_error_header.nit

@@ -0,0 +1,18 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base_simple_import
+
+# Syntax error in the header
+import 5

tests/nitls_header_annotations.nit

@@ -0,0 +1,29 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module nitls_header_annotations is
+	ldflags("-lm",
+		"-lpthread")
+end
+
+import end
+import base_simple
+
+# The annotations of an import can span lines too
+import base_simple_import is
+	conditional(base_simple)
+end
+
+class NitlsHeaderAnnotations
+end

tests/nitls_header_redef.nit

@@ -0,0 +1,23 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module nitls_header_redef
+
+import end
+
+# The grammar accepts `redef` before an import, should it really?
+redef import base_simple3
+
+redef class A
+end

tests/nitls_header_visibility.nit

@@ -0,0 +1,23 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module nitls_header_visibility
+
+import end
+
+private
+import base_simple3
+
+# The body starts at the visibility keyword, not at `fun`
+private fun nitls_header_foo do end

tests/sav/nitls_args10.res

@@ -0,0 +1,2 @@
+base_simple.nit
+nitls_header_annotations.nit

tests/sav/nitls_args11.res

@@ -0,0 +1,2 @@
+base_simple3.nit
+nitls_header_visibility.nit

tests/sav/nitls_args12.res

@@ -0,0 +1,2 @@
+base_simple3.nit
+nitls_header_redef.nit

tests/sav/nitls_args13.res

@@ -0,0 +1 @@
+error_mod_unk.nit

tests/sav/nitls_args14.res

@@ -0,0 +1 @@
+nitls_error_header.nit

tests/sav/nitls_args9.res

@@ -0,0 +1,3 @@
+base_simple.nit
+base_simple_import.nit
+nitls_error_body.nit

tests/sav/nitls_error_body.res

@@ -0,0 +1 @@
+nitls_error_body.nit:18,9--13: Syntax Error: unexpected keyword 'label'.

tests/sav/nitls_error_header.res

@@ -0,0 +1 @@
+nitls_error_header.nit:18,8: Syntax Error: unexpected literal value '5'.

tests/sav/nitls_header_annotations.res

@@ -0,0 +1,6 @@
+10
+20
+20
+30
+30
+30