gh-153569: handle carriage returns after comments

Author: pablogsal

Lib/test/test_tokenize.py

@@ -2580,6 +2580,16 @@ def test_carriage_return_after_debug_comment(self):
                 tokens = self._get_tokens(f"{prefix}'''{{x=# comment\r}}'''")
                 self.assertEqual(tokens[4].string, "# comment\r}")
 
+                for extra_tokens in (False, True):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(
+                            f"{prefix}'{{#\r!", extra_tokens=extra_tokens
+                        )
+                    self.assertEqual(
+                        caught.exception.args,
+                        ("unexpected EOF in multi-line statement", (1, 7)),
+                    )
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(

Parser/lexer/lexer.c

@@ -338,7 +338,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
 
         if (current != NULL) {
             const char *comment_end = tok->cur;
-            if (c == '\n') {
+            if (c == '\n' || c == '\r') {
                 comment_end--;
             }
             if (_PyLexer_record_ftstring_comment(