Skip to content

Commit b3c6c9b

Browse files
authored
Merge branch '21.X.X' into abashev/javadoc-drop-an-empty-trailing-p-tag
2 parents 4a371f5 + 442f75b commit b3c6c9b

7 files changed

Lines changed: 341 additions & 45 deletions

File tree

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ functionaljava = { module = "org.functionaljava:functionaljava", version = "5.0"
3030
guava = { module = "com.google.guava:guava", version = "33.8.0-jre" }
3131
guava-testlib = { module = "com.google.guava:guava-testlib", version = "33.8.0-jre" }
3232
immutables-value = { module = "org.immutables:value", version = "2.12.3" }
33-
jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version = "2.22" }
33+
jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version = "2.23" }
3434
jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version = "2.23.0" }
3535
jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version = "2.22.4" }
3636
jackson-dataformat-xml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-xml", version = "2.23.0" }

‎open-java-format/src/main/java/com/palantir/javaformat/java/ImportOrderer.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,13 @@ private String reorderImports() throws FormatterException {
7676
ImportsAndIndex imports = scanImports(firstImportStart);
7777
int afterLastImport = imports.index;
7878

79-
// Make sure there are no more imports before the next class (etc) definition.
79+
// Another import before the next class (etc) definition means something other than whitespace and comments
80+
// separates the imports, such as a stray `;`, which javac rejects from Java 21 on. Leave the imports as they
81+
// are rather than fail the whole file (google/google-java-format#1528): the formatter reports what is wrong
82+
// with the file, or formats it if nothing is.
8083
Optional<Integer> maybeLaterImport = findIdentifier(afterLastImport, IMPORT_OR_CLASS_START);
8184
if (maybeLaterImport.isPresent() && tokenAt(maybeLaterImport.get()).equals("import")) {
82-
throw new FormatterException("Imports not contiguous (perhaps a comment separates them?)");
85+
return text;
8386
}
8487

8588
StringBuilder result = new StringBuilder();

‎open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java‎

Lines changed: 126 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import com.sun.tools.javac.util.Log;
4141
import com.sun.tools.javac.util.Log.DeferredDiagnosticHandler;
4242
import com.sun.tools.javac.util.Options;
43+
import com.sun.tools.javac.util.Position;
4344
import java.io.IOException;
4445
import java.lang.reflect.Method;
4546
import java.net.URI;
@@ -379,20 +380,20 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
379380
break;
380381
}
381382
int charI0 = t.pos();
382-
// Get string, possibly with Unicode escapes.
383-
String originalTokText = text.substring(charI0, t.endPos());
384-
String tokText = t.kind() == TokenKind.STRINGLITERAL
385-
? checkNotNull(t.stringVal()) // Unicode escapes removed.
386-
: originalTokText;
383+
String tokText = text.substring(charI0, t.endPos()); // The token's source text.
387384
char tokText0 = tokText.charAt(0); // The token's first character.
388385
final boolean isToken; // Is this tok a token?
389386
final boolean isNumbered; // Is this tok numbered? (tokens and comments)
390387
String extraNewline = null; // Extra newline at end?
391388
List<String> strings = new ArrayList<>();
392389
if (Character.isWhitespace(tokText0)) {
390+
int escape = tokText.indexOf('\\');
391+
if (escape != -1) {
392+
throw error(text, charI0 + escape, "Unicode escapes not allowed in whitespace");
393+
}
393394
isToken = false;
394395
isNumbered = false;
395-
Iterator<String> it = Newlines.lineIterator(originalTokText);
396+
Iterator<String> it = Newlines.lineIterator(tokText);
396397
while (it.hasNext()) {
397398
String line = it.next();
398399
String newline = Newlines.getLineEnding(line);
@@ -406,20 +407,41 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
406407
strings.add(line);
407408
}
408409
}
409-
} else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
410+
} else if (t.kind() == TokenKind.CHARLITERAL || t.kind() == TokenKind.STRINGLITERAL) {
411+
checkLiteral(text, charI0, t.kind(), tokText);
410412
isToken = true;
411413
isNumbered = true;
412-
strings.add(originalTokText);
413-
} else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
414+
strings.add(tokText);
415+
} else if (tokText.startsWith("//") || (tokText.startsWith("/*") && tokText.endsWith("*/"))) {
414416
// For compatibility with an earlier lexer, the newline after a // comment is its own tok.
415-
if (tokText.startsWith("//") && (originalTokText.endsWith("\n") || originalTokText.endsWith("\r"))) {
416-
extraNewline = checkNotNull(Newlines.getLineEnding(originalTokText));
417+
if (tokText.startsWith("//") && (tokText.endsWith("\n") || tokText.endsWith("\r"))) {
418+
extraNewline = checkNotNull(Newlines.getLineEnding(tokText));
417419
tokText = tokText.substring(0, tokText.length() - extraNewline.length());
418-
originalTokText = originalTokText.substring(0, originalTokText.length() - extraNewline.length());
419420
}
420421
isToken = false;
421422
isNumbered = true;
422-
strings.add(originalTokText);
423+
strings.add(tokText);
424+
} else if (t.kind() == null) {
425+
// A non-token that is neither whitespace nor a comment, because a Unicode escape in a comment
426+
// delimiter or line terminator changed how javac lexed it. A block comment is only misidentified if
427+
// its closing delimiter is escaped.
428+
int escape = tokText.startsWith("/*") ? tokText.lastIndexOf('\\') : tokText.indexOf('\\');
429+
throw error(
430+
text,
431+
charI0 + Math.max(escape, 0),
432+
"Unicode escapes not allowed in whitespace or comment delimiters");
433+
} else if (tokText.indexOf('\\') != -1) {
434+
throw error(
435+
text,
436+
charI0 + tokText.indexOf('\\'),
437+
"Unicode escapes not allowed in identifiers, keywords, operators, or numeric literals");
438+
} else if (indexOfIgnorable(tokText) != -1) {
439+
// javac discards ignorable characters in identifiers and keywords, e.g. `in`, U+200B, `t` is lexed as
440+
// the keyword `int`.
441+
throw error(
442+
text,
443+
charI0 + indexOfIgnorable(tokText),
444+
"Ignorable characters not allowed in identifiers or keywords");
423445
} else if (Character.isJavaIdentifierStart(tokText0)
424446
|| Character.isDigit(tokText0)
425447
|| (tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1)))) {
@@ -429,8 +451,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
429451
strings.add(tokText);
430452
} else {
431453
// Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>"
432-
// cannot be lexed without syntactic knowledge. This implementation fails if the token
433-
// contains Unicode escapes.
454+
// cannot be lexed without syntactic knowledge.
434455
isToken = true;
435456
isNumbered = true;
436457
for (int i = 0; i < tokText.length(); i++) {
@@ -439,18 +460,12 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
439460
}
440461
}
441462
if (strings.size() == 1) {
442-
if (!tokText.startsWith("//") || !continueTrailingComment(toks, originalTokText, text)) {
443-
toks.add(new Tok(
444-
isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken, t.kind()));
463+
if (!tokText.startsWith("//") || !continueTrailingComment(toks, tokText, text)) {
464+
toks.add(new Tok(isNumbered ? kN++ : -1, tokText, tokText, charI, columnI, isToken, t.kind()));
445465
}
446-
charI += originalTokText.length();
447-
columnI = updateColumn(columnI, originalTokText);
448-
466+
charI += tokText.length();
467+
columnI = updateColumn(columnI, tokText);
449468
} else {
450-
if (strings.size() != 1 && !tokText.equals(originalTokText)) {
451-
throw new FormatterException(
452-
"Unicode escapes not allowed in whitespace or multi-character operators");
453-
}
454469
for (String str : strings) {
455470
toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null));
456471
charI += str.length();
@@ -467,6 +482,92 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
467482
return List.copyOf(toks);
468483
}
469484

485+
/**
486+
* Rejects literals whose delimiters, or text block line terminators, are Unicode escapes. The formatter operates on
487+
* the source text of literals, which needs to agree with javac's lexing.
488+
*/
489+
private static void checkLiteral(String text, int pos, TokenKind kind, String tokText) throws FormatterException {
490+
int lineTerminator = indexOfUnicodeLineTerminatorEscape(tokText);
491+
String delimiter;
492+
String description;
493+
if (kind == TokenKind.CHARLITERAL) {
494+
delimiter = "'";
495+
description = "character literal";
496+
} else if (Newlines.firstBreak(tokText) != -1 || lineTerminator != -1) {
497+
// A Unicode line terminator will be rejected below. Here we identify it is in a text block.
498+
delimiter = "\"\"\"";
499+
description = "text block";
500+
} else {
501+
delimiter = "\"";
502+
description = "string literal";
503+
}
504+
if (!tokText.startsWith(delimiter)) {
505+
throw error(
506+
text, pos + tokText.indexOf('\\'), "Unicode escapes not allowed in " + description + " delimiters");
507+
}
508+
if (!tokText.endsWith(delimiter)) {
509+
throw error(
510+
text,
511+
pos + tokText.lastIndexOf('\\'),
512+
"Unicode escapes not allowed in " + description + " delimiters");
513+
}
514+
if (lineTerminator != -1) {
515+
throw error(
516+
text, pos + lineTerminator, "Unicode escapes not allowed in " + description + " line terminators");
517+
}
518+
}
519+
520+
/**
521+
* Returns the index of the first Unicode escape for a line terminator (LF U+000A or CR U+000D) in {@code s}, or
522+
* {@code -1}.
523+
*/
524+
private static int indexOfUnicodeLineTerminatorEscape(String s) {
525+
int start = s.indexOf('\\');
526+
while (start != -1) {
527+
int end = start;
528+
while (end < s.length() && s.charAt(end) == '\\') {
529+
end++;
530+
}
531+
// A backslash is only eligible to begin a Unicode escape if it is preceded by an even number of contiguous
532+
// backslashes (JLS 3.3), and only the last backslash in a run can be followed by a `u`.
533+
int last = end - 1;
534+
if ((last - start) % 2 == 0 && isUnicodeLineTerminatorEscape(s, last)) {
535+
return last;
536+
}
537+
start = s.indexOf('\\', end);
538+
}
539+
return -1;
540+
}
541+
542+
/** Returns true if the backslash at {@code backslash} begins a Unicode escape for a line terminator. */
543+
private static boolean isUnicodeLineTerminatorEscape(String s, int backslash) {
544+
int i = backslash + 1;
545+
while (i < s.length() && s.charAt(i) == 'u') {
546+
i++;
547+
}
548+
return i > backslash + 1
549+
&& (s.regionMatches(/* ignoreCase= */ true, i, "000a", 0, 4)
550+
|| s.regionMatches(/* ignoreCase= */ true, i, "000d", 0, 4));
551+
}
552+
553+
/** Returns the index of the first identifier-ignorable character in {@code s}, or {@code -1}. */
554+
private static int indexOfIgnorable(String s) {
555+
for (int i = 0; i < s.length(); ) {
556+
int codePoint = s.codePointAt(i);
557+
if (Character.isIdentifierIgnorable(codePoint)) {
558+
return i;
559+
}
560+
i += Character.charCount(codePoint);
561+
}
562+
return -1;
563+
}
564+
565+
private static FormatterException error(String text, int inputPosition, String message) {
566+
Position.LineMap lineMap = Position.makeLineMap(text.toCharArray(), text.length(), false);
567+
return new FormatterException(FormatterDiagnostic.create(
568+
lineMap.getLineNumber(inputPosition), lineMap.getColumnNumber(inputPosition), message));
569+
}
570+
470571
/**
471572
* Gets diagnostics from a DeferredDiagnosticHandler using reflection.
472573
* This method handles the API change in JDK 25 where getDiagnostics()

‎open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,13 @@ class JavacTokens {
4040

4141
/** An unprocessed input token, including whitespace and comments. */
4242
static class RawTok {
43-
@Nullable
44-
private final String stringVal;
45-
4643
@Nullable
4744
private final TokenKind kind;
4845

4946
private final int pos;
5047
private final int endPos;
5148

52-
RawTok(@Nullable String stringVal, @Nullable TokenKind kind, int pos, int endPos) {
53-
this.stringVal = stringVal;
49+
RawTok(@Nullable TokenKind kind, int pos, int endPos) {
5450
this.kind = kind;
5551
this.pos = pos;
5652
this.endPos = endPos;
@@ -71,12 +67,6 @@ public int pos() {
7167
public int endPos() {
7268
return endPos;
7369
}
74-
75-
/** The escaped string value of a literal, or {@code null} for other tokens. */
76-
@Nullable
77-
public String stringVal() {
78-
return stringVal;
79-
}
8070
}
8171

8272
/** Lex the input and return a list of {@link RawTok}s. */
@@ -96,10 +86,9 @@ public static List<RawTok> getTokens(String source, Context context, Set<TokenKi
9686
if (t.comments != null) {
9787
for (Comment c : Lists.reverse(t.comments)) {
9888
if (last < c.getSourcePos(0)) {
99-
tokens.add(new RawTok(null, null, last, c.getSourcePos(0)));
89+
tokens.add(new RawTok(null, last, c.getSourcePos(0)));
10090
}
10191
tokens.add(new RawTok(
102-
null,
10392
null,
10493
c.getSourcePos(0),
10594
c.getSourcePos(0) + c.getText().length()));
@@ -113,14 +102,13 @@ public static List<RawTok> getTokens(String source, Context context, Set<TokenKi
113102
break;
114103
}
115104
if (last < t.pos) {
116-
tokens.add(new RawTok(null, null, last, t.pos));
105+
tokens.add(new RawTok(null, last, t.pos));
117106
}
118-
tokens.add(new RawTok(
119-
t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null, t.kind, t.pos, t.endPos));
107+
tokens.add(new RawTok(t.kind, t.pos, t.endPos));
120108
last = t.endPos;
121109
} while (scanner.token().kind != TokenKind.EOF);
122110
if (last < end) {
123-
tokens.add(new RawTok(null, null, last, end));
111+
tokens.add(new RawTok(null, last, end));
124112
}
125113
return List.copyOf(tokens);
126114
}

0 commit comments

Comments
 (0)