4040import com .sun .tools .javac .util .Log ;
4141import com .sun .tools .javac .util .Log .DeferredDiagnosticHandler ;
4242import com .sun .tools .javac .util .Options ;
43+ import com .sun .tools .javac .util .Position ;
4344import java .io .IOException ;
4445import java .lang .reflect .Method ;
4546import java .net .URI ;
@@ -379,20 +380,20 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
379380 break ;
380381 }
381382 int charI0 = t .pos ();
382- // Get string, possibly with Unicode escapes.
383- String originalTokText = text .substring (charI0 , t .endPos ());
384- String tokText = t .kind () == TokenKind .STRINGLITERAL
385- ? checkNotNull (t .stringVal ()) // Unicode escapes removed.
386- : originalTokText ;
383+ String tokText = text .substring (charI0 , t .endPos ()); // The token's source text.
387384 char tokText0 = tokText .charAt (0 ); // The token's first character.
388385 final boolean isToken ; // Is this tok a token?
389386 final boolean isNumbered ; // Is this tok numbered? (tokens and comments)
390387 String extraNewline = null ; // Extra newline at end?
391388 List <String > strings = new ArrayList <>();
392389 if (Character .isWhitespace (tokText0 )) {
390+ int escape = tokText .indexOf ('\\' );
391+ if (escape != -1 ) {
392+ throw error (text , charI0 + escape , "Unicode escapes not allowed in whitespace" );
393+ }
393394 isToken = false ;
394395 isNumbered = false ;
395- Iterator <String > it = Newlines .lineIterator (originalTokText );
396+ Iterator <String > it = Newlines .lineIterator (tokText );
396397 while (it .hasNext ()) {
397398 String line = it .next ();
398399 String newline = Newlines .getLineEnding (line );
@@ -406,20 +407,41 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
406407 strings .add (line );
407408 }
408409 }
409- } else if (tokText .startsWith ("'" ) || tokText .startsWith ("\" " )) {
410+ } else if (t .kind () == TokenKind .CHARLITERAL || t .kind () == TokenKind .STRINGLITERAL ) {
411+ checkLiteral (text , charI0 , t .kind (), tokText );
410412 isToken = true ;
411413 isNumbered = true ;
412- strings .add (originalTokText );
413- } else if (tokText .startsWith ("//" ) || tokText .startsWith ("/*" )) {
414+ strings .add (tokText );
415+ } else if (tokText .startsWith ("//" ) || ( tokText .startsWith ("/*" ) && tokText . endsWith ( "*/" ) )) {
414416 // For compatibility with an earlier lexer, the newline after a // comment is its own tok.
415- if (tokText .startsWith ("//" ) && (originalTokText .endsWith ("\n " ) || originalTokText .endsWith ("\r " ))) {
416- extraNewline = checkNotNull (Newlines .getLineEnding (originalTokText ));
417+ if (tokText .startsWith ("//" ) && (tokText .endsWith ("\n " ) || tokText .endsWith ("\r " ))) {
418+ extraNewline = checkNotNull (Newlines .getLineEnding (tokText ));
417419 tokText = tokText .substring (0 , tokText .length () - extraNewline .length ());
418- originalTokText = originalTokText .substring (0 , originalTokText .length () - extraNewline .length ());
419420 }
420421 isToken = false ;
421422 isNumbered = true ;
422- strings .add (originalTokText );
423+ strings .add (tokText );
424+ } else if (t .kind () == null ) {
425+ // A non-token that is neither whitespace nor a comment, because a Unicode escape in a comment
426+ // delimiter or line terminator changed how javac lexed it. A block comment is only misidentified if
427+ // its closing delimiter is escaped.
428+ int escape = tokText .startsWith ("/*" ) ? tokText .lastIndexOf ('\\' ) : tokText .indexOf ('\\' );
429+ throw error (
430+ text ,
431+ charI0 + Math .max (escape , 0 ),
432+ "Unicode escapes not allowed in whitespace or comment delimiters" );
433+ } else if (tokText .indexOf ('\\' ) != -1 ) {
434+ throw error (
435+ text ,
436+ charI0 + tokText .indexOf ('\\' ),
437+ "Unicode escapes not allowed in identifiers, keywords, operators, or numeric literals" );
438+ } else if (indexOfIgnorable (tokText ) != -1 ) {
439+ // javac discards ignorable characters in identifiers and keywords, e.g. `in`, U+200B, `t` is lexed as
440+ // the keyword `int`.
441+ throw error (
442+ text ,
443+ charI0 + indexOfIgnorable (tokText ),
444+ "Ignorable characters not allowed in identifiers or keywords" );
423445 } else if (Character .isJavaIdentifierStart (tokText0 )
424446 || Character .isDigit (tokText0 )
425447 || (tokText0 == '.' && tokText .length () > 1 && Character .isDigit (tokText .charAt (1 )))) {
@@ -429,8 +451,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
429451 strings .add (tokText );
430452 } else {
431453 // Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>"
432- // cannot be lexed without syntactic knowledge. This implementation fails if the token
433- // contains Unicode escapes.
454+ // cannot be lexed without syntactic knowledge.
434455 isToken = true ;
435456 isNumbered = true ;
436457 for (int i = 0 ; i < tokText .length (); i ++) {
@@ -439,18 +460,12 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
439460 }
440461 }
441462 if (strings .size () == 1 ) {
442- if (!tokText .startsWith ("//" ) || !continueTrailingComment (toks , originalTokText , text )) {
443- toks .add (new Tok (
444- isNumbered ? kN ++ : -1 , originalTokText , tokText , charI , columnI , isToken , t .kind ()));
463+ if (!tokText .startsWith ("//" ) || !continueTrailingComment (toks , tokText , text )) {
464+ toks .add (new Tok (isNumbered ? kN ++ : -1 , tokText , tokText , charI , columnI , isToken , t .kind ()));
445465 }
446- charI += originalTokText .length ();
447- columnI = updateColumn (columnI , originalTokText );
448-
466+ charI += tokText .length ();
467+ columnI = updateColumn (columnI , tokText );
449468 } else {
450- if (strings .size () != 1 && !tokText .equals (originalTokText )) {
451- throw new FormatterException (
452- "Unicode escapes not allowed in whitespace or multi-character operators" );
453- }
454469 for (String str : strings ) {
455470 toks .add (new Tok (isNumbered ? kN ++ : -1 , str , str , charI , columnI , isToken , null ));
456471 charI += str .length ();
@@ -467,6 +482,92 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
467482 return List .copyOf (toks );
468483 }
469484
485+ /**
486+ * Rejects literals whose delimiters, or text block line terminators, are Unicode escapes. The formatter operates on
487+ * the source text of literals, which needs to agree with javac's lexing.
488+ */
489+ private static void checkLiteral (String text , int pos , TokenKind kind , String tokText ) throws FormatterException {
490+ int lineTerminator = indexOfUnicodeLineTerminatorEscape (tokText );
491+ String delimiter ;
492+ String description ;
493+ if (kind == TokenKind .CHARLITERAL ) {
494+ delimiter = "'" ;
495+ description = "character literal" ;
496+ } else if (Newlines .firstBreak (tokText ) != -1 || lineTerminator != -1 ) {
497+ // A Unicode line terminator will be rejected below. Here we identify it is in a text block.
498+ delimiter = "\" \" \" " ;
499+ description = "text block" ;
500+ } else {
501+ delimiter = "\" " ;
502+ description = "string literal" ;
503+ }
504+ if (!tokText .startsWith (delimiter )) {
505+ throw error (
506+ text , pos + tokText .indexOf ('\\' ), "Unicode escapes not allowed in " + description + " delimiters" );
507+ }
508+ if (!tokText .endsWith (delimiter )) {
509+ throw error (
510+ text ,
511+ pos + tokText .lastIndexOf ('\\' ),
512+ "Unicode escapes not allowed in " + description + " delimiters" );
513+ }
514+ if (lineTerminator != -1 ) {
515+ throw error (
516+ text , pos + lineTerminator , "Unicode escapes not allowed in " + description + " line terminators" );
517+ }
518+ }
519+
520+ /**
521+ * Returns the index of the first Unicode escape for a line terminator (LF U+000A or CR U+000D) in {@code s}, or
522+ * {@code -1}.
523+ */
524+ private static int indexOfUnicodeLineTerminatorEscape (String s ) {
525+ int start = s .indexOf ('\\' );
526+ while (start != -1 ) {
527+ int end = start ;
528+ while (end < s .length () && s .charAt (end ) == '\\' ) {
529+ end ++;
530+ }
531+ // A backslash is only eligible to begin a Unicode escape if it is preceded by an even number of contiguous
532+ // backslashes (JLS 3.3), and only the last backslash in a run can be followed by a `u`.
533+ int last = end - 1 ;
534+ if ((last - start ) % 2 == 0 && isUnicodeLineTerminatorEscape (s , last )) {
535+ return last ;
536+ }
537+ start = s .indexOf ('\\' , end );
538+ }
539+ return -1 ;
540+ }
541+
542+ /** Returns true if the backslash at {@code backslash} begins a Unicode escape for a line terminator. */
543+ private static boolean isUnicodeLineTerminatorEscape (String s , int backslash ) {
544+ int i = backslash + 1 ;
545+ while (i < s .length () && s .charAt (i ) == 'u' ) {
546+ i ++;
547+ }
548+ return i > backslash + 1
549+ && (s .regionMatches (/* ignoreCase= */ true , i , "000a" , 0 , 4 )
550+ || s .regionMatches (/* ignoreCase= */ true , i , "000d" , 0 , 4 ));
551+ }
552+
553+ /** Returns the index of the first identifier-ignorable character in {@code s}, or {@code -1}. */
554+ private static int indexOfIgnorable (String s ) {
555+ for (int i = 0 ; i < s .length (); ) {
556+ int codePoint = s .codePointAt (i );
557+ if (Character .isIdentifierIgnorable (codePoint )) {
558+ return i ;
559+ }
560+ i += Character .charCount (codePoint );
561+ }
562+ return -1 ;
563+ }
564+
565+ private static FormatterException error (String text , int inputPosition , String message ) {
566+ Position .LineMap lineMap = Position .makeLineMap (text .toCharArray (), text .length (), false );
567+ return new FormatterException (FormatterDiagnostic .create (
568+ lineMap .getLineNumber (inputPosition ), lineMap .getColumnNumber (inputPosition ), message ));
569+ }
570+
470571 /**
471572 * Gets diagnostics from a DeferredDiagnosticHandler using reflection.
472573 * This method handles the API change in JDK 25 where getDiagnostics()
0 commit comments