Skip to content

Commit 8eaeb50

Browse files
committed
Reject Unicode escapes outside literal and comment contents
javac translates Unicode escapes before it lexes, but the formatter works on the source text, so the two can disagree about where a token starts and ends (#244): class Foo { // \u000D } The escaped carriage return ends the comment for javac, and the escape is left in the gap before `}`, where JavaInput took it for a token: <stdin>:1:1: error: expected token: '\'; generated } instead An escape in a comment or literal delimiter, or in a line terminator of a text block, does the same, and javac also drops identifier-ignorable characters such as U+200B from names and keywords, so a zero-width space at the end of `getAndIncrement` failed with "expected token: 'getAndIncrement<U+200B>'; generated getAndIncrement instead". JavaInput.buildToks now rejects these inputs up front with a diagnostic at the line and column of the escape or character, as google/google-java-format#1527 does: <stdin>:2:6: error: Unicode escapes not allowed in whitespace or comment delimiters <stdin>:3:20: error: Ignorable characters not allowed in identifiers or keywords Escapes in the contents of comments, string and character literals and text blocks are still allowed and kept as they are. Every tok now holds its source text, so RawTok no longer carries javac's string value, and the old check for escapes in whitespace and multi-character operators, which reported no position, is gone. The cases in DiagnosticTest.unicodeEscapes and ignorableCharacters come from upstream; their exit code is 2, ours for an input that does not parse. FormatterTest.unicodeEscapesInLiteralAndCommentContents checks that escapes in contents format unchanged, in our style.
1 parent 9b11043 commit 8eaeb50

4 files changed

Lines changed: 306 additions & 42 deletions

File tree

‎open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java‎

Lines changed: 126 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import com.sun.tools.javac.util.Log;
4141
import com.sun.tools.javac.util.Log.DeferredDiagnosticHandler;
4242
import com.sun.tools.javac.util.Options;
43+
import com.sun.tools.javac.util.Position;
4344
import java.io.IOException;
4445
import java.lang.reflect.Method;
4546
import java.net.URI;
@@ -379,20 +380,20 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
379380
break;
380381
}
381382
int charI0 = t.pos();
382-
// Get string, possibly with Unicode escapes.
383-
String originalTokText = text.substring(charI0, t.endPos());
384-
String tokText = t.kind() == TokenKind.STRINGLITERAL
385-
? checkNotNull(t.stringVal()) // Unicode escapes removed.
386-
: originalTokText;
383+
String tokText = text.substring(charI0, t.endPos()); // The token's source text.
387384
char tokText0 = tokText.charAt(0); // The token's first character.
388385
final boolean isToken; // Is this tok a token?
389386
final boolean isNumbered; // Is this tok numbered? (tokens and comments)
390387
String extraNewline = null; // Extra newline at end?
391388
List<String> strings = new ArrayList<>();
392389
if (Character.isWhitespace(tokText0)) {
390+
int escape = tokText.indexOf('\\');
391+
if (escape != -1) {
392+
throw error(text, charI0 + escape, "Unicode escapes not allowed in whitespace");
393+
}
393394
isToken = false;
394395
isNumbered = false;
395-
Iterator<String> it = Newlines.lineIterator(originalTokText);
396+
Iterator<String> it = Newlines.lineIterator(tokText);
396397
while (it.hasNext()) {
397398
String line = it.next();
398399
String newline = Newlines.getLineEnding(line);
@@ -406,20 +407,41 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
406407
strings.add(line);
407408
}
408409
}
409-
} else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
410+
} else if (t.kind() == TokenKind.CHARLITERAL || t.kind() == TokenKind.STRINGLITERAL) {
411+
checkLiteral(text, charI0, t.kind(), tokText);
410412
isToken = true;
411413
isNumbered = true;
412-
strings.add(originalTokText);
413-
} else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
414+
strings.add(tokText);
415+
} else if (tokText.startsWith("//") || (tokText.startsWith("/*") && tokText.endsWith("*/"))) {
414416
// For compatibility with an earlier lexer, the newline after a // comment is its own tok.
415-
if (tokText.startsWith("//") && (originalTokText.endsWith("\n") || originalTokText.endsWith("\r"))) {
416-
extraNewline = checkNotNull(Newlines.getLineEnding(originalTokText));
417+
if (tokText.startsWith("//") && (tokText.endsWith("\n") || tokText.endsWith("\r"))) {
418+
extraNewline = checkNotNull(Newlines.getLineEnding(tokText));
417419
tokText = tokText.substring(0, tokText.length() - extraNewline.length());
418-
originalTokText = originalTokText.substring(0, originalTokText.length() - extraNewline.length());
419420
}
420421
isToken = false;
421422
isNumbered = true;
422-
strings.add(originalTokText);
423+
strings.add(tokText);
424+
} else if (t.kind() == null) {
425+
// A non-token that is neither whitespace nor a comment, because a Unicode escape in a comment
426+
// delimiter or line terminator changed how javac lexed it. A block comment is only misidentified if
427+
// its closing delimiter is escaped.
428+
int escape = tokText.startsWith("/*") ? tokText.lastIndexOf('\\') : tokText.indexOf('\\');
429+
throw error(
430+
text,
431+
charI0 + Math.max(escape, 0),
432+
"Unicode escapes not allowed in whitespace or comment delimiters");
433+
} else if (tokText.indexOf('\\') != -1) {
434+
throw error(
435+
text,
436+
charI0 + tokText.indexOf('\\'),
437+
"Unicode escapes not allowed in identifiers, keywords, operators, or numeric literals");
438+
} else if (indexOfIgnorable(tokText) != -1) {
439+
// javac discards ignorable characters in identifiers and keywords, e.g. `in`, U+200B, `t` is lexed as
440+
// the keyword `int`.
441+
throw error(
442+
text,
443+
charI0 + indexOfIgnorable(tokText),
444+
"Ignorable characters not allowed in identifiers or keywords");
423445
} else if (Character.isJavaIdentifierStart(tokText0)
424446
|| Character.isDigit(tokText0)
425447
|| (tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1)))) {
@@ -429,8 +451,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
429451
strings.add(tokText);
430452
} else {
431453
// Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>"
432-
// cannot be lexed without syntactic knowledge. This implementation fails if the token
433-
// contains Unicode escapes.
454+
// cannot be lexed without syntactic knowledge.
434455
isToken = true;
435456
isNumbered = true;
436457
for (int i = 0; i < tokText.length(); i++) {
@@ -439,18 +460,12 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
439460
}
440461
}
441462
if (strings.size() == 1) {
442-
if (!tokText.startsWith("//") || !continueTrailingComment(toks, originalTokText, text)) {
443-
toks.add(new Tok(
444-
isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken, t.kind()));
463+
if (!tokText.startsWith("//") || !continueTrailingComment(toks, tokText, text)) {
464+
toks.add(new Tok(isNumbered ? kN++ : -1, tokText, tokText, charI, columnI, isToken, t.kind()));
445465
}
446-
charI += originalTokText.length();
447-
columnI = updateColumn(columnI, originalTokText);
448-
466+
charI += tokText.length();
467+
columnI = updateColumn(columnI, tokText);
449468
} else {
450-
if (strings.size() != 1 && !tokText.equals(originalTokText)) {
451-
throw new FormatterException(
452-
"Unicode escapes not allowed in whitespace or multi-character operators");
453-
}
454469
for (String str : strings) {
455470
toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null));
456471
charI += str.length();
@@ -467,6 +482,92 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
467482
return List.copyOf(toks);
468483
}
469484

485+
/**
486+
* Rejects literals whose delimiters, or text block line terminators, are Unicode escapes. The formatter operates on
487+
* the source text of literals, which needs to agree with javac's lexing.
488+
*/
489+
private static void checkLiteral(String text, int pos, TokenKind kind, String tokText) throws FormatterException {
490+
int lineTerminator = indexOfUnicodeLineTerminatorEscape(tokText);
491+
String delimiter;
492+
String description;
493+
if (kind == TokenKind.CHARLITERAL) {
494+
delimiter = "'";
495+
description = "character literal";
496+
} else if (Newlines.firstBreak(tokText) != -1 || lineTerminator != -1) {
497+
// A Unicode line terminator will be rejected below. Here we identify it is in a text block.
498+
delimiter = "\"\"\"";
499+
description = "text block";
500+
} else {
501+
delimiter = "\"";
502+
description = "string literal";
503+
}
504+
if (!tokText.startsWith(delimiter)) {
505+
throw error(
506+
text, pos + tokText.indexOf('\\'), "Unicode escapes not allowed in " + description + " delimiters");
507+
}
508+
if (!tokText.endsWith(delimiter)) {
509+
throw error(
510+
text,
511+
pos + tokText.lastIndexOf('\\'),
512+
"Unicode escapes not allowed in " + description + " delimiters");
513+
}
514+
if (lineTerminator != -1) {
515+
throw error(
516+
text, pos + lineTerminator, "Unicode escapes not allowed in " + description + " line terminators");
517+
}
518+
}
519+
520+
/**
521+
* Returns the index of the first Unicode escape for a line terminator (LF U+000A or CR U+000D) in {@code s}, or
522+
* {@code -1}.
523+
*/
524+
private static int indexOfUnicodeLineTerminatorEscape(String s) {
525+
int start = s.indexOf('\\');
526+
while (start != -1) {
527+
int end = start;
528+
while (end < s.length() && s.charAt(end) == '\\') {
529+
end++;
530+
}
531+
// A backslash is only eligible to begin a Unicode escape if it is preceded by an even number of contiguous
532+
// backslashes (JLS 3.3), and only the last backslash in a run can be followed by a `u`.
533+
int last = end - 1;
534+
if ((last - start) % 2 == 0 && isUnicodeLineTerminatorEscape(s, last)) {
535+
return last;
536+
}
537+
start = s.indexOf('\\', end);
538+
}
539+
return -1;
540+
}
541+
542+
/** Returns true if the backslash at {@code backslash} begins a Unicode escape for a line terminator. */
543+
private static boolean isUnicodeLineTerminatorEscape(String s, int backslash) {
544+
int i = backslash + 1;
545+
while (i < s.length() && s.charAt(i) == 'u') {
546+
i++;
547+
}
548+
return i > backslash + 1
549+
&& (s.regionMatches(/* ignoreCase= */ true, i, "000a", 0, 4)
550+
|| s.regionMatches(/* ignoreCase= */ true, i, "000d", 0, 4));
551+
}
552+
553+
/** Returns the index of the first identifier-ignorable character in {@code s}, or {@code -1}. */
554+
private static int indexOfIgnorable(String s) {
555+
for (int i = 0; i < s.length(); ) {
556+
int codePoint = s.codePointAt(i);
557+
if (Character.isIdentifierIgnorable(codePoint)) {
558+
return i;
559+
}
560+
i += Character.charCount(codePoint);
561+
}
562+
return -1;
563+
}
564+
565+
private static FormatterException error(String text, int inputPosition, String message) {
566+
Position.LineMap lineMap = Position.makeLineMap(text.toCharArray(), text.length(), false);
567+
return new FormatterException(FormatterDiagnostic.create(
568+
lineMap.getLineNumber(inputPosition), lineMap.getColumnNumber(inputPosition), message));
569+
}
570+
470571
/**
471572
* Gets diagnostics from a DeferredDiagnosticHandler using reflection.
472573
* This method handles the API change in JDK 25 where getDiagnostics()

‎open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,13 @@ class JavacTokens {
4040

4141
/** An unprocessed input token, including whitespace and comments. */
4242
static class RawTok {
43-
@Nullable
44-
private final String stringVal;
45-
4643
@Nullable
4744
private final TokenKind kind;
4845

4946
private final int pos;
5047
private final int endPos;
5148

52-
RawTok(@Nullable String stringVal, @Nullable TokenKind kind, int pos, int endPos) {
53-
this.stringVal = stringVal;
49+
RawTok(@Nullable TokenKind kind, int pos, int endPos) {
5450
this.kind = kind;
5551
this.pos = pos;
5652
this.endPos = endPos;
@@ -71,12 +67,6 @@ public int pos() {
7167
public int endPos() {
7268
return endPos;
7369
}
74-
75-
/** The escaped string value of a literal, or {@code null} for other tokens. */
76-
@Nullable
77-
public String stringVal() {
78-
return stringVal;
79-
}
8070
}
8171

8272
/** Lex the input and return a list of {@link RawTok}s. */
@@ -96,10 +86,9 @@ public static List<RawTok> getTokens(String source, Context context, Set<TokenKi
9686
if (t.comments != null) {
9787
for (Comment c : Lists.reverse(t.comments)) {
9888
if (last < c.getSourcePos(0)) {
99-
tokens.add(new RawTok(null, null, last, c.getSourcePos(0)));
89+
tokens.add(new RawTok(null, last, c.getSourcePos(0)));
10090
}
10191
tokens.add(new RawTok(
102-
null,
10392
null,
10493
c.getSourcePos(0),
10594
c.getSourcePos(0) + c.getText().length()));
@@ -113,14 +102,13 @@ public static List<RawTok> getTokens(String source, Context context, Set<TokenKi
113102
break;
114103
}
115104
if (last < t.pos) {
116-
tokens.add(new RawTok(null, null, last, t.pos));
105+
tokens.add(new RawTok(null, last, t.pos));
117106
}
118-
tokens.add(new RawTok(
119-
t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null, t.kind, t.pos, t.endPos));
107+
tokens.add(new RawTok(t.kind, t.pos, t.endPos));
120108
last = t.endPos;
121109
} while (scanner.token().kind != TokenKind.EOF);
122110
if (last < end) {
123-
tokens.add(new RawTok(null, null, last, end));
111+
tokens.add(new RawTok(null, last, end));
124112
}
125113
return List.copyOf(tokens);
126114
}

0 commit comments

Comments
 (0)