Error on unicode surrogates in mypyc (#21936)

This PR is expected to fail until
mypyc/ast_serialize#82 is merged and released.

Author: ilevkivskyi

mypy/nativeparse.py

@@ -295,7 +295,7 @@ def parse_to_binary_ast(
         platform=options.platform,
         always_true=options.always_true,
         always_false=options.always_false,
-        cache_version=4,
+        cache_version=5,
     )
     return (
         ast_bytes,
@@ -1348,6 +1348,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression:
         return ce
     elif tag == nodes.STR_EXPR:
         se = StrExpr(read_str(data))
+        se.has_surrogates = read_bool(data)
         read_loc(data, se)
         expect_end_tag(data)
         return se
@@ -1437,7 +1438,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression:
                 s = StrExpr(read_str(data))
                 read_loc(data, s)
                 fitems.append(s)
-        expr = build_fstring_join(data, fitems)
+        expr = build_fstring_join(data, fitems, set_has_surrogates=True)
         expect_end_tag(data)
         return expr
     elif tag == nodes.LIST_COMPREHENSION:
@@ -1534,6 +1535,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression:
                 read_loc(data, s)
                 titems.append(s)
         expr = TemplateStrExpr(titems)
+        expr.has_surrogates = read_bool(data)
         read_loc(data, expr)
         state.check_min_version(
             "t-strings", (3, 14), expr.line, expr.column, enforce_in_stubs=True
@@ -1660,16 +1662,30 @@ def read_fstring_items(state: State, data: ReadBuffer) -> Expression:
     return build_fstring_join(data, items)
 
 
-def build_fstring_join(data: ReadBuffer, items: list[Expression]) -> Expression:
+def build_fstring_join(
+    data: ReadBuffer, items: list[Expression], set_has_surrogates: bool = False
+) -> Expression:
     items = collapse_consecutive_str_items(items)
     if len(items) == 1:
         expr = items[0]
+        if set_has_surrogates:
+            if isinstance(expr, StrExpr):
+                target = expr
+            else:
+                assert isinstance(expr, CallExpr) and isinstance(expr.callee, MemberExpr)
+                # It doesn't really matter where to set the surrogates flag,
+                # so we set it on the outermost format string.
+                target = expr.callee.expr
+                assert isinstance(target, StrExpr)
+            target.has_surrogates = read_bool(data)
         read_loc(data, expr)
         return expr
     args = ListExpr(items)
     str_expr = StrExpr("")
     member = MemberExpr(str_expr, "join")
     call = CallExpr(member, [args], [ARG_POS], [None])
+    if set_has_surrogates:
+        str_expr.has_surrogates = read_bool(data)
     read_loc(data, call)
     set_line_column(args, call)
     set_line_column(str_expr, call)

mypy/nodes.py

@@ -2275,7 +2275,7 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T:
 class StrExpr(Expression):
     """String literal"""
 
-    __slots__ = ("value", "as_type")
+    __slots__ = ("value", "as_type", "has_surrogates")
 
     __match_args__ = ("value",)
 
@@ -2284,11 +2284,16 @@ class StrExpr(Expression):
     # represents the type denoted by the type expression.
     # None means "is not a type expression".
     as_type: NotParsed | mypy.types.Type | None
+    # This indicates whether original string literal contained Unicode surrogate
+    # codepoints. Those are not supported by Ruff parser and are replaced by
+    # replacement characters. Thus, we can't support them in mypyc.
+    has_surrogates: bool
 
     def __init__(self, value: str) -> None:
         super().__init__()
         self.value = value
         self.as_type = NotParsed.VALUE
+        self.has_surrogates = False
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
         return visitor.visit_str_expr(self)
@@ -2937,7 +2942,7 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T:
 class TemplateStrExpr(Expression):
     """Template string expression t'...'."""
 
-    __slots__ = ("items",)
+    __slots__ = ("items", "has_surrogates")
     __match_args__ = ("items",)
 
     # Each item is either:
@@ -2946,12 +2951,14 @@ class TemplateStrExpr(Expression):
     #     where conversion is str | None ("r", "s", "a", or None)
     #     and format_spec_expr is Expression | None
     items: list[Expression | tuple[Expression, str, str | None, Expression | None]]
+    has_surrogates: bool
 
     def __init__(
         self, items: list[Expression | tuple[Expression, str, str | None, Expression | None]]
     ) -> None:
         super().__init__()
         self.items = items
+        self.has_surrogates = False
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
         return visitor.visit_template_str_expr(self)

mypy/test/test_nativeparse.py

@@ -247,6 +247,9 @@ def _assert_trivial_binary_data(self, b: bytes, /) -> None:
         def int_enc(n: int) -> int:
             return (n + 10) << 1
 
+        def bool_enc(b: bool) -> int:
+            return int(b)
+
         def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> list[int]:
             return [
                 LOCATION,
@@ -267,6 +270,7 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) ->
                 + [END_TAG, LIST_GEN, 22, nodes.STR_EXPR]
                 + [LITERAL_STR, int_enc(5)]
                 + list(b"hello")
+                + [bool_enc(False)]  # no unicode surrogates
                 + locs(1, 6, 1, 13)
                 + [END_TAG]
                 # arg_kinds: [ARG_POS]

mypyc/irbuild/prebuildvisitor.py

@@ -15,6 +15,7 @@
     MypyFile,
     NameExpr,
     Node,
+    StrExpr,
     SymbolNode,
     Var,
 )
@@ -259,6 +260,17 @@ def visit_name_expr(self, expr: NameExpr) -> None:
         if isinstance(expr.node, (Var, FuncDef)):
             self.visit_symbol_node(expr.node)
 
+    def visit_str_expr(self, o: StrExpr) -> None:
+        # Handle surrogates before main pass to avoid conflicts with various optimizations
+        # like replacing `ord("<some char>")` with its integer value statically, etc.
+        if o.has_surrogates:
+            self.errors.error(
+                "Surrogate codepoints in string literals not supported, use chr(...) instead",
+                self.current_file.path,
+                o.line,
+            )
+        super().visit_str_expr(o)
+
     def visit_var(self, var: Var) -> None:
         self.visit_symbol_node(var)
 
@@ -272,7 +284,7 @@ def visit_symbol_node(self, symbol: SymbolNode) -> None:
             orig_func = self.symbols_to_funcs[symbol]
             if self.is_parent(self.funcs[-1], orig_func):
                 # The function in which the symbol was previously seen is
-                # nested within the function currently being visited. Thus
+                # nested within the function currently being visited. Thus,
                 # the current function is a better candidate to contain the
                 # declaration.
                 self.symbols_to_funcs[symbol] = self.funcs[-1]

mypyc/irbuild/visitor.py

@@ -313,6 +313,11 @@ def visit_dict_expr(self, expr: DictExpr) -> Value:
         return transform_dict_expr(self.builder, expr)
 
     def visit_template_str_expr(self, expr: TemplateStrExpr) -> Value:
+        if expr.has_surrogates:
+            self.builder.error(
+                "Surrogate codepoints in string literals not supported, use chr(...) instead",
+                expr.line,
+            )
         self.bail("Template strings are not supported by mypyc", expr.line)
 
     def visit_set_expr(self, expr: SetExpr) -> Value:

mypyc/test-data/irbuild-str.test

@@ -1175,3 +1175,18 @@ L0:
     r1 = ''
     r2 = CPyStr_EqualLiteral(r0, r1, 0)
     return r2
+
+[case testUnicodeSurrogate]
+# flags: --native-parser
+
+def f() -> str:
+    return "\ud800"
+
+def test_surrogate() -> None:
+    assert ord(f()) == 0xd800
+    assert ord("\udfff") == 0xdfff
+    assert repr("foobar\x00\xab\ud912\U00012345") == r"'foobar\x00«\ud912𒍅'"
+[out]
+main:4: error: Surrogate codepoints in string literals not supported, use chr(...) instead
+main:8: error: Surrogate codepoints in string literals not supported, use chr(...) instead
+main:9: error: Surrogate codepoints in string literals not supported, use chr(...) instead

mypyc/test-data/run-strings.test

@@ -1099,15 +1099,6 @@ def test_encode() -> None:
     with assertRaises(UnicodeEncodeError):
         u.encode('latin1')
 
-[case testUnicodeSurrogate]
-def f() -> str:
-    return "\ud800"
-
-def test_surrogate() -> None:
-    assert ord(f()) == 0xd800
-    assert ord("\udfff") == 0xdfff
-    assert repr("foobar\x00\xab\ud912\U00012345") == r"'foobar\x00«\ud912𒍅'"
-
 [case testStrip]
 def test_all_strips_default() -> None:
     s = " a1\t"

mypyc/test/testutil.py

@@ -11,6 +11,7 @@
 
 from mypy import build
 from mypy.errors import CompileError
+from mypy.main import process_options
 from mypy.nodes import Expression, MypyFile
 from mypy.options import Options
 from mypy.test.config import test_temp_dir
@@ -102,10 +103,16 @@ def build_ir_for_single_file2(
 ) -> tuple[ModuleIR, MypyFile, dict[Expression, Type], Mapper]:
     program_text = "\n".join(input_lines)
 
-    # By default generate IR compatible with the earliest supported Python C API.
+    flags = re.search("# flags: (.*)$", program_text, flags=re.MULTILINE)
+
+    # By default, generate IR compatible with the earliest supported Python C API.
     # If a test needs more recent API features, this should be overridden.
     compiler_options = compiler_options or CompilerOptions(capi_version=(3, 10))
-    options = Options()
+    if flags:
+        flag_list = flags.group(1).split()
+        _, options = process_options(flag_list, require_targets=False)
+    else:
+        options = Options()
     options.show_traceback = True
     options.hide_error_codes = True
     options.use_builtins_fixtures = True
@@ -120,7 +127,7 @@ def build_ir_for_single_file2(
     options.per_module_options["__main__"] = {"mypyc": True}
 
     source = build.BuildSource("main", "__main__", program_text)
-    # Construct input as a single single.
+    # Construct input as a single source.
     # Parse and type check the input program.
     result = build.build(sources=[source], options=options, alt_lib_path=test_temp_dir)
     result.manager.metastore.close()