Skip to content

Commit 67a4c8b

Browse files
lowassergoogle-java-format Team
authored andcommitted
Cache JavaInput in Cleanups and fuse AST cleanup passes
Eliminates redundant full-file lexing and parsing across `Cleanups`, `CleanupPass`, and `Formatter.getFormatReplacements`: - Retain `ImmutableList<Tok>` on `JavaInput` (`getToks()`), cache `JavaInput` in `Cleanups` (invalidated alongside `parsed` in `applyReplacements`), and return it in `Cleanups.Result` so `CleanupScanner`, `ImportOrderer` (when `javaInput` is already cached), `ModifierOrderer`, and `Formatter.getFormatReplacements` share a single full-file lex instead of 3 full-file lexes + 1 prefix lex on clean files. - Fuse `REMOVE_UNUSED_IMPORTS` and `AST_CLEANUPS` onto a single shared `JCCompilationUnit` parse (`applyFusedAstPasses`), passing `preExistingDeletions` into `CleanupScanner` so adjacent deletions (such as `import java.util.List;;`) remain identical. - Order `SORT_IMPORTS` before `REORDER_MODIFIERS` in `CleanupPass` so when imports are sorted (invalidating `javaInput`), `SORT_IMPORTS` only does a prefix lex (`CLASS_START`) and `REORDER_MODIFIERS` then builds `JavaInput` once for `Formatter` to reuse. - Update `CleanupsTest` to cover fused AST pass behavior, `javaInput` reuse, and adjacent unused-import + redundant-semicolon deletion. Benchmarks (warm in-JVM `ThreadMXBean` CPU time): - `JavaInputAstVisitor.java` (140 KB, clean): med 178.1 -> 146.1 ms (1.22x), min 147.4 -> 105.2 ms (1.40x). PiperOrigin-RevId: 1025905856
1 parent f10be21 commit 67a4c8b

7 files changed

Lines changed: 168 additions & 31 deletions

File tree

‎core/src/main/java/com/google/googlejavaformat/java/CleanupPass.java‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package com.google.googlejavaformat.java;
1616

1717
import com.google.common.collect.ImmutableList;
18+
import com.google.common.collect.ImmutableRangeSet;
1819

1920
/// Standard cleanup passes applied before formatting.
2021
enum CleanupPass {
@@ -30,22 +31,31 @@ ImmutableList<Replacement> findReplacements(Cleanups cleanups) throws FormatterE
3031
AST_CLEANUPS {
3132
@Override
3233
ImmutableList<Replacement> findReplacements(Cleanups cleanups) throws FormatterException {
33-
return CleanupScanner.scan(cleanups.unit(), cleanups.contents(), cleanups.ranges());
34+
return CleanupScanner.scan(
35+
cleanups.unit(),
36+
cleanups.contents(),
37+
cleanups.javaInput().getToks(),
38+
cleanups.ranges(),
39+
ImmutableRangeSet.of());
3440
}
3541
},
36-
/// Reorders modifiers into JLS order.
37-
REORDER_MODIFIERS {
42+
/// Reorders imports into canonical style order.
43+
SORT_IMPORTS {
3844
@Override
3945
ImmutableList<Replacement> findReplacements(Cleanups cleanups) throws FormatterException {
40-
JavaInput javaInput = new JavaInput(cleanups.contents());
41-
return ModifierOrderer.findModifierReplacements(javaInput, cleanups.ranges());
46+
JavaInput cached = cleanups.cachedJavaInput();
47+
if (cached != null) {
48+
return ImportOrderer.findReplacements(
49+
cleanups.contents(), cached.getToks(), cleanups.style());
50+
}
51+
return ImportOrderer.findReplacements(cleanups.contents(), cleanups.style());
4252
}
4353
},
44-
/// Reorders imports into canonical style order.
45-
SORT_IMPORTS {
54+
/// Reorders modifiers into JLS order.
55+
REORDER_MODIFIERS {
4656
@Override
4757
ImmutableList<Replacement> findReplacements(Cleanups cleanups) throws FormatterException {
48-
return ImportOrderer.findReplacements(cleanups.contents(), cleanups.style());
58+
return ModifierOrderer.findModifierReplacements(cleanups.javaInput(), cleanups.ranges());
4959
}
5060
};
5161

‎core/src/main/java/com/google/googlejavaformat/java/CleanupScanner.java‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import static java.lang.Math.min;
2424

2525
import com.google.common.collect.ImmutableList;
26+
import com.google.common.collect.ImmutableRangeSet;
2627
import com.google.common.collect.ImmutableSet;
2728
import com.google.common.collect.Range;
2829
import com.google.common.collect.RangeSet;
@@ -59,10 +60,25 @@ final class CleanupScanner extends TreePathScanner<@Nullable Void, @Nullable Voi
5960
static ImmutableList<Replacement> scan(
6061
JCCompilationUnit unit, String contents, Collection<Range<Integer>> ranges)
6162
throws FormatterException {
63+
return scan(
64+
unit,
65+
contents,
66+
JavaInput.buildToks(contents, ImmutableSet.of()),
67+
ranges,
68+
ImmutableRangeSet.of());
69+
}
70+
71+
static ImmutableList<Replacement> scan(
72+
JCCompilationUnit unit,
73+
String contents,
74+
ImmutableList<Tok> toks,
75+
Collection<Range<Integer>> ranges,
76+
RangeSet<Integer> preExistingDeletions)
77+
throws FormatterException {
6278
RangeSet<Integer> targetRanges = TreeRangeSet.create(ranges);
63-
ImmutableList<Tok> toks = JavaInput.buildToks(contents, ImmutableSet.of());
6479
RangeSet<Integer> activeDeletions = TreeRangeSet.create();
65-
new CleanupScanner(contents, unit, toks, targetRanges, activeDeletions).scan(unit, null);
80+
new CleanupScanner(contents, unit, toks, targetRanges, activeDeletions, preExistingDeletions)
81+
.scan(unit, null);
6682
if (activeDeletions.isEmpty()) {
6783
return ImmutableList.of();
6884
}
@@ -122,18 +138,21 @@ private static void verifyCommentsPreserved(
122138
private final ImmutableList<Tok> toks;
123139
private final RangeSet<Integer> targetRanges;
124140
private final RangeSet<Integer> activeDeletions;
141+
private final RangeSet<Integer> preExistingDeletions;
125142

126143
CleanupScanner(
127144
String contents,
128145
JCCompilationUnit unit,
129146
ImmutableList<Tok> toks,
130147
RangeSet<Integer> targetRanges,
131-
RangeSet<Integer> activeDeletions) {
148+
RangeSet<Integer> activeDeletions,
149+
RangeSet<Integer> preExistingDeletions) {
132150
this.contents = contents;
133151
this.unit = unit;
134152
this.toks = toks;
135153
this.targetRanges = targetRanges;
136154
this.activeDeletions = activeDeletions;
155+
this.preExistingDeletions = preExistingDeletions;
137156
}
138157

139158
private boolean inRange(int startPos, int endPos) {
@@ -142,7 +161,9 @@ private boolean inRange(int startPos, int endPos) {
142161

143162
private void deleteRedundantSemicolon(int semiPos) {
144163
int start = semiPos;
145-
while (start > 0 && isHorizontalWhitespace(contents.charAt(start - 1))) {
164+
while (start > 0
165+
&& isHorizontalWhitespace(contents.charAt(start - 1))
166+
&& !preExistingDeletions.contains(start - 1)) {
146167
start--;
147168
}
148169
int lineBegin = start;
@@ -153,8 +174,7 @@ private void deleteRedundantSemicolon(int semiPos) {
153174
while (end < contents.length() && isHorizontalWhitespace(contents.charAt(end))) {
154175
end++;
155176
}
156-
boolean nothingBeforeOnLine =
157-
start == lineBegin || activeDeletions.encloses(Range.closedOpen(lineBegin, start));
177+
boolean nothingBeforeOnLine = start == lineBegin || isDeletedRange(lineBegin, start);
158178
if (nothingBeforeOnLine && end < contents.length()) {
159179
if (contents.charAt(end) == '\r') {
160180
end++;
@@ -168,6 +188,19 @@ private void deleteRedundantSemicolon(int semiPos) {
168188
activeDeletions.add(Range.closedOpen(start, end));
169189
}
170190

191+
private boolean isDeletedRange(int start, int end) {
192+
Range<Integer> range = Range.closedOpen(start, end);
193+
if (activeDeletions.encloses(range)) {
194+
return true;
195+
}
196+
if (preExistingDeletions.isEmpty()) {
197+
return false;
198+
}
199+
RangeSet<Integer> combined = TreeRangeSet.create(activeDeletions.subRangeSet(range));
200+
combined.addAll(preExistingDeletions.subRangeSet(range));
201+
return combined.encloses(range);
202+
}
203+
171204
private static boolean isHorizontalWhitespace(char c) {
172205
return c == ' ' || c == '\t';
173206
}

‎core/src/main/java/com/google/googlejavaformat/java/Cleanups.java‎

Lines changed: 64 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import static java.lang.Math.max;
1818
import static java.lang.Math.min;
19+
import static java.util.Comparator.comparingInt;
1920

2021
import com.google.common.base.CharMatcher;
2122
import com.google.common.collect.ImmutableList;
@@ -43,11 +44,13 @@
4344
final class Cleanups {
4445

4546
/// The cleaned contents, the requested character ranges remapped onto them (plus any modified
46-
/// regions), and the javac unit for `contents` if it can be reused for formatting.
47+
/// regions), and the javac unit and lexed [JavaInput] for `contents` if they can be reused for
48+
/// formatting.
4749
record Result(
4850
String contents,
4951
Collection<Range<Integer>> characterRanges,
50-
@Nullable JCCompilationUnit unit) {}
52+
@Nullable JCCompilationUnit unit,
53+
@Nullable JavaInput javaInput) {}
5154

5255
private record Parsed(Context context, JCCompilationUnit unit) {}
5356

@@ -78,7 +81,7 @@ static Result apply(
7881
throws FormatterException {
7982
JavaInput.validateRanges(characterRanges, contents.length());
8083
if (passes.isEmpty()) {
81-
return new Result(contents, characterRanges, null);
84+
return new Result(contents, characterRanges, null, null);
8285
}
8386
RangeSet<Integer> normalized = TreeRangeSet.create();
8487
for (Range<Integer> range : characterRanges) {
@@ -88,8 +91,17 @@ static Result apply(
8891
: range);
8992
}
9093
Cleanups cleanups = new Cleanups(contents, ImmutableRangeSet.copyOf(normalized), style);
94+
boolean fuseAstPasses =
95+
passes.contains(CleanupPass.REMOVE_UNUSED_IMPORTS)
96+
&& passes.contains(CleanupPass.AST_CLEANUPS);
9197
for (CleanupPass pass : passes) {
92-
cleanups.applyReplacements(pass.findReplacements(cleanups));
98+
if (fuseAstPasses && pass == CleanupPass.REMOVE_UNUSED_IMPORTS) {
99+
cleanups.applyFusedAstPasses();
100+
} else if (fuseAstPasses && pass == CleanupPass.AST_CLEANUPS) {
101+
// Already applied together with REMOVE_UNUSED_IMPORTS on the same parsed AST.
102+
} else {
103+
cleanups.applyReplacements(pass.findReplacements(cleanups));
104+
}
93105
}
94106
// An empty range can't be represented in a RangeSet, so restore it if the passes removed all
95107
// contents (or the input was empty).
@@ -98,12 +110,16 @@ static Result apply(
98110
? ImmutableList.of(Range.closedOpen(0, 0))
99111
: cleanups.ranges.asRanges();
100112
return new Result(
101-
cleanups.contents, resultRanges, cleanups.parsed == null ? null : cleanups.parsed.unit());
113+
cleanups.contents,
114+
resultRanges,
115+
cleanups.parsed == null ? null : cleanups.parsed.unit(),
116+
cleanups.javaInput);
102117
}
103118

104119
private String contents;
105120
private ImmutableRangeSet<Integer> ranges;
106121
private @Nullable Parsed parsed;
122+
private @Nullable JavaInput javaInput;
107123
private final Style style;
108124

109125
private Cleanups(String contents, ImmutableRangeSet<Integer> ranges, Style style) {
@@ -148,6 +164,48 @@ Context context() throws FormatterException {
148164
return parsed().context();
149165
}
150166

167+
JavaInput javaInput() throws FormatterException {
168+
if (javaInput == null) {
169+
javaInput = new JavaInput(contents);
170+
}
171+
return javaInput;
172+
}
173+
174+
@Nullable JavaInput cachedJavaInput() {
175+
return javaInput;
176+
}
177+
178+
/// Runs `REMOVE_UNUSED_IMPORTS` and `AST_CLEANUPS` against a single parsed [JCCompilationUnit] so
179+
/// unused-import edits do not force an extra full-file parse before AST cleanups.
180+
private void applyFusedAstPasses() throws FormatterException {
181+
ImmutableList<Replacement> unusedImports =
182+
RemoveUnusedImports.findUnusedImports(context(), unit(), contents);
183+
RangeSet<Integer> preExistingDeletions;
184+
if (unusedImports.isEmpty()) {
185+
preExistingDeletions = ImmutableRangeSet.of();
186+
} else {
187+
RangeSet<Integer> deletions = TreeRangeSet.create();
188+
for (Replacement r : unusedImports) {
189+
deletions.add(r.replaceRange());
190+
}
191+
preExistingDeletions = deletions;
192+
}
193+
ImmutableList<Replacement> astCleanups =
194+
CleanupScanner.scan(
195+
unit(), contents, javaInput().getToks(), ranges(), preExistingDeletions);
196+
if (unusedImports.isEmpty()) {
197+
applyReplacements(astCleanups);
198+
} else if (astCleanups.isEmpty()) {
199+
applyReplacements(unusedImports);
200+
} else {
201+
List<Replacement> merged = new ArrayList<>(unusedImports.size() + astCleanups.size());
202+
merged.addAll(unusedImports);
203+
merged.addAll(astCleanups);
204+
merged.sort(comparingInt(r -> r.replaceRange().lowerEndpoint()));
205+
applyReplacements(merged);
206+
}
207+
}
208+
151209
/// Applies a pass's replacements to the current contents, remapping requested ranges and adding
152210
/// modified regions.
153211
private void applyReplacements(List<Replacement> replacements) {
@@ -166,6 +224,7 @@ private void applyReplacements(List<Replacement> replacements) {
166224
this.contents = edit.contents();
167225
this.ranges = ImmutableRangeSet.copyOf(newRanges);
168226
this.parsed = null;
227+
this.javaInput = null;
169228
}
170229

171230
/// Returns the range to reformat for the given span.

‎core/src/main/java/com/google/googlejavaformat/java/Formatter.java‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,11 @@ String formatSource(
193193
Edit.apply(
194194
cleaned.contents(),
195195
getFormatReplacements(
196-
cleaned.contents(), cleaned.characterRanges(), options.cleanup(), cleaned.unit()));
196+
cleaned.contents(),
197+
cleaned.characterRanges(),
198+
options.cleanup(),
199+
cleaned.unit(),
200+
cleaned.javaInput()));
197201
if (reflowLongStrings) {
198202
// Only reflow long strings in the requested ranges, mapped onto the formatted output.
199203
return StringWrapper.wrap(
@@ -265,7 +269,8 @@ public ImmutableList<Replacement> getFormatReplacements(
265269
ImmutableList<Replacement> getFormatReplacements(
266270
String input, Collection<Range<Integer>> characterRanges, boolean reindent)
267271
throws FormatterException {
268-
return getFormatReplacements(input, characterRanges, reindent, /* unit= */ null);
272+
return getFormatReplacements(
273+
input, characterRanges, reindent, /* unit= */ null, /* cachedJavaInput= */ null);
269274
}
270275

271276
ImmutableList<Replacement> getFormatReplacements(
@@ -274,7 +279,18 @@ ImmutableList<Replacement> getFormatReplacements(
274279
boolean reindent,
275280
@Nullable JCCompilationUnit unit)
276281
throws FormatterException {
277-
JavaInput javaInput = new JavaInput(input);
282+
return getFormatReplacements(
283+
input, characterRanges, reindent, unit, /* cachedJavaInput= */ null);
284+
}
285+
286+
ImmutableList<Replacement> getFormatReplacements(
287+
String input,
288+
Collection<Range<Integer>> characterRanges,
289+
boolean reindent,
290+
@Nullable JCCompilationUnit unit,
291+
@Nullable JavaInput cachedJavaInput)
292+
throws FormatterException {
293+
JavaInput javaInput = cachedJavaInput != null ? cachedJavaInput : new JavaInput(input);
278294
if (unit != null) {
279295
javaInput.setCompilationUnit(unit);
280296
}

‎core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ public static String reorderImports(String text) throws FormatterException {
7676
static ImmutableList<Replacement> findReplacements(String text, Style style)
7777
throws FormatterException {
7878
ImmutableList<Tok> toks = JavaInput.buildToks(text, CLASS_START);
79+
return findReplacements(text, toks, style);
80+
}
81+
82+
/// Variant of [#findReplacements(String, Style)] that reuses an already-lexed token stream.
83+
static ImmutableList<Replacement> findReplacements(
84+
String text, ImmutableList<Tok> toks, Style style) throws FormatterException {
7985
return new ImportOrderer(text, toks, style).findReplacements();
8086
}
8187

‎core/src/main/java/com/google/googlejavaformat/java/JavaInput.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,7 @@ public String toString() {
251251
// `/*` comments can lie on multiple lines. These data structures (along with equivalent ones for
252252
// the formatted output) let us compute correspondences between the input and output.
253253

254+
private final ImmutableList<Tok> toks; // All Toks (including whitespace and comments).
254255
private final ImmutableMap<Integer, Integer> positionToColumnMap; // Map Tok position to column.
255256
private final ImmutableList<Token> tokens; // The Tokens for this input.
256257
private final ImmutableRangeMap<Integer, Token> positionTokenMap; // Map position to Token.
@@ -265,7 +266,7 @@ public String toString() {
265266
JavaInput(String text) throws FormatterException {
266267
this.text = checkNotNull(text);
267268
setLines(ImmutableList.copyOf(Newlines.lineIterator(text)));
268-
ImmutableList<Tok> toks = buildToks(text);
269+
this.toks = buildToks(text);
269270
positionToColumnMap = makePositionToColumnMap(toks);
270271
tokens = buildTokens(toks);
271272
ImmutableRangeMap.Builder<Integer, Token> tokenLocations = ImmutableRangeMap.builder();
@@ -751,6 +752,11 @@ public Token getToken(int k) {
751752
return kToToken[k];
752753
}
753754

755+
/// Get all input toks (tokens, comments, whitespace, and EOF).
756+
ImmutableList<Tok> getToks() {
757+
return toks;
758+
}
759+
754760
/// Get the input tokens.
755761
///
756762
/// @return the input tokens

0 commit comments

Comments
 (0)