From 67a4c8b1325fbf3dbf182ffed32c819dd98f3a31 Mon Sep 17 00:00:00 2001 From: lowasser CA Date: Thu, 19 Nov 2026 14:28:57 -0800 Subject: [PATCH] Cache JavaInput in Cleanups and fuse AST cleanup passes Eliminates redundant full-file lexing and parsing across `Cleanups`, `CleanupPass`, and `Formatter.getFormatReplacements`: - Retain `ImmutableList` on `JavaInput` (`getToks()`), cache `JavaInput` in `Cleanups` (invalidated alongside `parsed` in `applyReplacements`), and return it in `Cleanups.Result` so `CleanupScanner`, `ImportOrderer` (when `javaInput` is already cached), `ModifierOrderer`, and `Formatter.getFormatReplacements` share a single full-file lex instead of 3 full-file lexes + 1 prefix lex on clean files. - Fuse `REMOVE_UNUSED_IMPORTS` and `AST_CLEANUPS` onto a single shared `JCCompilationUnit` parse (`applyFusedAstPasses`), passing `preExistingDeletions` into `CleanupScanner` so adjacent deletions (such as `import java.util.List;;`) remain identical. - Order `SORT_IMPORTS` before `REORDER_MODIFIERS` in `CleanupPass` so when imports are sorted (invalidating `javaInput`), `SORT_IMPORTS` only does a prefix lex (`CLASS_START`) and `REORDER_MODIFIERS` then builds `JavaInput` once for `Formatter` to reuse. - Update `CleanupsTest` to cover fused AST pass behavior, `javaInput` reuse, and adjacent unused-import + redundant-semicolon deletion. Benchmarks (warm in-JVM `ThreadMXBean` CPU time): - `JavaInputAstVisitor.java` (140 KB, clean): med 178.1 -> 146.1 ms (1.22x), min 147.4 -> 105.2 ms (1.40x). PiperOrigin-RevId: 1025905856 --- .../googlejavaformat/java/CleanupPass.java | 26 ++++--- .../googlejavaformat/java/CleanupScanner.java | 45 ++++++++++-- .../googlejavaformat/java/Cleanups.java | 69 +++++++++++++++++-- .../googlejavaformat/java/Formatter.java | 22 +++++- .../googlejavaformat/java/ImportOrderer.java | 6 ++ .../googlejavaformat/java/JavaInput.java | 8 ++- .../googlejavaformat/java/CleanupsTest.java | 23 ++++--- 7 files changed, 168 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/com/google/googlejavaformat/java/CleanupPass.java b/core/src/main/java/com/google/googlejavaformat/java/CleanupPass.java index e92ed4d32..25e9639de 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/CleanupPass.java +++ b/core/src/main/java/com/google/googlejavaformat/java/CleanupPass.java @@ -15,6 +15,7 @@ package com.google.googlejavaformat.java; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableRangeSet; /// Standard cleanup passes applied before formatting. enum CleanupPass { @@ -30,22 +31,31 @@ ImmutableList findReplacements(Cleanups cleanups) throws FormatterE AST_CLEANUPS { @Override ImmutableList findReplacements(Cleanups cleanups) throws FormatterException { - return CleanupScanner.scan(cleanups.unit(), cleanups.contents(), cleanups.ranges()); + return CleanupScanner.scan( + cleanups.unit(), + cleanups.contents(), + cleanups.javaInput().getToks(), + cleanups.ranges(), + ImmutableRangeSet.of()); } }, - /// Reorders modifiers into JLS order. - REORDER_MODIFIERS { + /// Reorders imports into canonical style order. + SORT_IMPORTS { @Override ImmutableList findReplacements(Cleanups cleanups) throws FormatterException { - JavaInput javaInput = new JavaInput(cleanups.contents()); - return ModifierOrderer.findModifierReplacements(javaInput, cleanups.ranges()); + JavaInput cached = cleanups.cachedJavaInput(); + if (cached != null) { + return ImportOrderer.findReplacements( + cleanups.contents(), cached.getToks(), cleanups.style()); + } + return ImportOrderer.findReplacements(cleanups.contents(), cleanups.style()); } }, - /// Reorders imports into canonical style order. - SORT_IMPORTS { + /// Reorders modifiers into JLS order. + REORDER_MODIFIERS { @Override ImmutableList findReplacements(Cleanups cleanups) throws FormatterException { - return ImportOrderer.findReplacements(cleanups.contents(), cleanups.style()); + return ModifierOrderer.findModifierReplacements(cleanups.javaInput(), cleanups.ranges()); } }; diff --git a/core/src/main/java/com/google/googlejavaformat/java/CleanupScanner.java b/core/src/main/java/com/google/googlejavaformat/java/CleanupScanner.java index b36167630..49b4e6ac5 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/CleanupScanner.java +++ b/core/src/main/java/com/google/googlejavaformat/java/CleanupScanner.java @@ -23,6 +23,7 @@ import static java.lang.Math.min; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableRangeSet; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; @@ -59,10 +60,25 @@ final class CleanupScanner extends TreePathScanner<@Nullable Void, @Nullable Voi static ImmutableList scan( JCCompilationUnit unit, String contents, Collection> ranges) throws FormatterException { + return scan( + unit, + contents, + JavaInput.buildToks(contents, ImmutableSet.of()), + ranges, + ImmutableRangeSet.of()); + } + + static ImmutableList scan( + JCCompilationUnit unit, + String contents, + ImmutableList toks, + Collection> ranges, + RangeSet preExistingDeletions) + throws FormatterException { RangeSet targetRanges = TreeRangeSet.create(ranges); - ImmutableList toks = JavaInput.buildToks(contents, ImmutableSet.of()); RangeSet activeDeletions = TreeRangeSet.create(); - new CleanupScanner(contents, unit, toks, targetRanges, activeDeletions).scan(unit, null); + new CleanupScanner(contents, unit, toks, targetRanges, activeDeletions, preExistingDeletions) + .scan(unit, null); if (activeDeletions.isEmpty()) { return ImmutableList.of(); } @@ -122,18 +138,21 @@ private static void verifyCommentsPreserved( private final ImmutableList toks; private final RangeSet targetRanges; private final RangeSet activeDeletions; + private final RangeSet preExistingDeletions; CleanupScanner( String contents, JCCompilationUnit unit, ImmutableList toks, RangeSet targetRanges, - RangeSet activeDeletions) { + RangeSet activeDeletions, + RangeSet preExistingDeletions) { this.contents = contents; this.unit = unit; this.toks = toks; this.targetRanges = targetRanges; this.activeDeletions = activeDeletions; + this.preExistingDeletions = preExistingDeletions; } private boolean inRange(int startPos, int endPos) { @@ -142,7 +161,9 @@ private boolean inRange(int startPos, int endPos) { private void deleteRedundantSemicolon(int semiPos) { int start = semiPos; - while (start > 0 && isHorizontalWhitespace(contents.charAt(start - 1))) { + while (start > 0 + && isHorizontalWhitespace(contents.charAt(start - 1)) + && !preExistingDeletions.contains(start - 1)) { start--; } int lineBegin = start; @@ -153,8 +174,7 @@ private void deleteRedundantSemicolon(int semiPos) { while (end < contents.length() && isHorizontalWhitespace(contents.charAt(end))) { end++; } - boolean nothingBeforeOnLine = - start == lineBegin || activeDeletions.encloses(Range.closedOpen(lineBegin, start)); + boolean nothingBeforeOnLine = start == lineBegin || isDeletedRange(lineBegin, start); if (nothingBeforeOnLine && end < contents.length()) { if (contents.charAt(end) == '\r') { end++; @@ -168,6 +188,19 @@ private void deleteRedundantSemicolon(int semiPos) { activeDeletions.add(Range.closedOpen(start, end)); } + private boolean isDeletedRange(int start, int end) { + Range range = Range.closedOpen(start, end); + if (activeDeletions.encloses(range)) { + return true; + } + if (preExistingDeletions.isEmpty()) { + return false; + } + RangeSet combined = TreeRangeSet.create(activeDeletions.subRangeSet(range)); + combined.addAll(preExistingDeletions.subRangeSet(range)); + return combined.encloses(range); + } + private static boolean isHorizontalWhitespace(char c) { return c == ' ' || c == '\t'; } diff --git a/core/src/main/java/com/google/googlejavaformat/java/Cleanups.java b/core/src/main/java/com/google/googlejavaformat/java/Cleanups.java index baaa45d3e..be636818b 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/Cleanups.java +++ b/core/src/main/java/com/google/googlejavaformat/java/Cleanups.java @@ -16,6 +16,7 @@ import static java.lang.Math.max; import static java.lang.Math.min; +import static java.util.Comparator.comparingInt; import com.google.common.base.CharMatcher; import com.google.common.collect.ImmutableList; @@ -43,11 +44,13 @@ final class Cleanups { /// The cleaned contents, the requested character ranges remapped onto them (plus any modified - /// regions), and the javac unit for `contents` if it can be reused for formatting. + /// regions), and the javac unit and lexed [JavaInput] for `contents` if they can be reused for + /// formatting. record Result( String contents, Collection> characterRanges, - @Nullable JCCompilationUnit unit) {} + @Nullable JCCompilationUnit unit, + @Nullable JavaInput javaInput) {} private record Parsed(Context context, JCCompilationUnit unit) {} @@ -78,7 +81,7 @@ static Result apply( throws FormatterException { JavaInput.validateRanges(characterRanges, contents.length()); if (passes.isEmpty()) { - return new Result(contents, characterRanges, null); + return new Result(contents, characterRanges, null, null); } RangeSet normalized = TreeRangeSet.create(); for (Range range : characterRanges) { @@ -88,8 +91,17 @@ static Result apply( : range); } Cleanups cleanups = new Cleanups(contents, ImmutableRangeSet.copyOf(normalized), style); + boolean fuseAstPasses = + passes.contains(CleanupPass.REMOVE_UNUSED_IMPORTS) + && passes.contains(CleanupPass.AST_CLEANUPS); for (CleanupPass pass : passes) { - cleanups.applyReplacements(pass.findReplacements(cleanups)); + if (fuseAstPasses && pass == CleanupPass.REMOVE_UNUSED_IMPORTS) { + cleanups.applyFusedAstPasses(); + } else if (fuseAstPasses && pass == CleanupPass.AST_CLEANUPS) { + // Already applied together with REMOVE_UNUSED_IMPORTS on the same parsed AST. + } else { + cleanups.applyReplacements(pass.findReplacements(cleanups)); + } } // An empty range can't be represented in a RangeSet, so restore it if the passes removed all // contents (or the input was empty). @@ -98,12 +110,16 @@ static Result apply( ? ImmutableList.of(Range.closedOpen(0, 0)) : cleanups.ranges.asRanges(); return new Result( - cleanups.contents, resultRanges, cleanups.parsed == null ? null : cleanups.parsed.unit()); + cleanups.contents, + resultRanges, + cleanups.parsed == null ? null : cleanups.parsed.unit(), + cleanups.javaInput); } private String contents; private ImmutableRangeSet ranges; private @Nullable Parsed parsed; + private @Nullable JavaInput javaInput; private final Style style; private Cleanups(String contents, ImmutableRangeSet ranges, Style style) { @@ -148,6 +164,48 @@ Context context() throws FormatterException { return parsed().context(); } + JavaInput javaInput() throws FormatterException { + if (javaInput == null) { + javaInput = new JavaInput(contents); + } + return javaInput; + } + + @Nullable JavaInput cachedJavaInput() { + return javaInput; + } + + /// Runs `REMOVE_UNUSED_IMPORTS` and `AST_CLEANUPS` against a single parsed [JCCompilationUnit] so + /// unused-import edits do not force an extra full-file parse before AST cleanups. + private void applyFusedAstPasses() throws FormatterException { + ImmutableList unusedImports = + RemoveUnusedImports.findUnusedImports(context(), unit(), contents); + RangeSet preExistingDeletions; + if (unusedImports.isEmpty()) { + preExistingDeletions = ImmutableRangeSet.of(); + } else { + RangeSet deletions = TreeRangeSet.create(); + for (Replacement r : unusedImports) { + deletions.add(r.replaceRange()); + } + preExistingDeletions = deletions; + } + ImmutableList astCleanups = + CleanupScanner.scan( + unit(), contents, javaInput().getToks(), ranges(), preExistingDeletions); + if (unusedImports.isEmpty()) { + applyReplacements(astCleanups); + } else if (astCleanups.isEmpty()) { + applyReplacements(unusedImports); + } else { + List merged = new ArrayList<>(unusedImports.size() + astCleanups.size()); + merged.addAll(unusedImports); + merged.addAll(astCleanups); + merged.sort(comparingInt(r -> r.replaceRange().lowerEndpoint())); + applyReplacements(merged); + } + } + /// Applies a pass's replacements to the current contents, remapping requested ranges and adding /// modified regions. private void applyReplacements(List replacements) { @@ -166,6 +224,7 @@ private void applyReplacements(List replacements) { this.contents = edit.contents(); this.ranges = ImmutableRangeSet.copyOf(newRanges); this.parsed = null; + this.javaInput = null; } /// Returns the range to reformat for the given span. diff --git a/core/src/main/java/com/google/googlejavaformat/java/Formatter.java b/core/src/main/java/com/google/googlejavaformat/java/Formatter.java index 2b763c1fa..507ee59f1 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/Formatter.java +++ b/core/src/main/java/com/google/googlejavaformat/java/Formatter.java @@ -193,7 +193,11 @@ String formatSource( Edit.apply( cleaned.contents(), getFormatReplacements( - cleaned.contents(), cleaned.characterRanges(), options.cleanup(), cleaned.unit())); + cleaned.contents(), + cleaned.characterRanges(), + options.cleanup(), + cleaned.unit(), + cleaned.javaInput())); if (reflowLongStrings) { // Only reflow long strings in the requested ranges, mapped onto the formatted output. return StringWrapper.wrap( @@ -265,7 +269,8 @@ public ImmutableList getFormatReplacements( ImmutableList getFormatReplacements( String input, Collection> characterRanges, boolean reindent) throws FormatterException { - return getFormatReplacements(input, characterRanges, reindent, /* unit= */ null); + return getFormatReplacements( + input, characterRanges, reindent, /* unit= */ null, /* cachedJavaInput= */ null); } ImmutableList getFormatReplacements( @@ -274,7 +279,18 @@ ImmutableList getFormatReplacements( boolean reindent, @Nullable JCCompilationUnit unit) throws FormatterException { - JavaInput javaInput = new JavaInput(input); + return getFormatReplacements( + input, characterRanges, reindent, unit, /* cachedJavaInput= */ null); + } + + ImmutableList getFormatReplacements( + String input, + Collection> characterRanges, + boolean reindent, + @Nullable JCCompilationUnit unit, + @Nullable JavaInput cachedJavaInput) + throws FormatterException { + JavaInput javaInput = cachedJavaInput != null ? cachedJavaInput : new JavaInput(input); if (unit != null) { javaInput.setCompilationUnit(unit); } diff --git a/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java b/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java index 27d2fde68..ce84dabef 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java +++ b/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java @@ -76,6 +76,12 @@ public static String reorderImports(String text) throws FormatterException { static ImmutableList findReplacements(String text, Style style) throws FormatterException { ImmutableList toks = JavaInput.buildToks(text, CLASS_START); + return findReplacements(text, toks, style); + } + + /// Variant of [#findReplacements(String, Style)] that reuses an already-lexed token stream. + static ImmutableList findReplacements( + String text, ImmutableList toks, Style style) throws FormatterException { return new ImportOrderer(text, toks, style).findReplacements(); } diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java b/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java index edfb2ab10..55b839bf5 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java +++ b/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java @@ -251,6 +251,7 @@ public String toString() { // `/*` comments can lie on multiple lines. These data structures (along with equivalent ones for // the formatted output) let us compute correspondences between the input and output. + private final ImmutableList toks; // All Toks (including whitespace and comments). private final ImmutableMap positionToColumnMap; // Map Tok position to column. private final ImmutableList tokens; // The Tokens for this input. private final ImmutableRangeMap positionTokenMap; // Map position to Token. @@ -265,7 +266,7 @@ public String toString() { JavaInput(String text) throws FormatterException { this.text = checkNotNull(text); setLines(ImmutableList.copyOf(Newlines.lineIterator(text))); - ImmutableList toks = buildToks(text); + this.toks = buildToks(text); positionToColumnMap = makePositionToColumnMap(toks); tokens = buildTokens(toks); ImmutableRangeMap.Builder tokenLocations = ImmutableRangeMap.builder(); @@ -751,6 +752,11 @@ public Token getToken(int k) { return kToToken[k]; } + /// Get all input toks (tokens, comments, whitespace, and EOF). + ImmutableList getToks() { + return toks; + } + /// Get the input tokens. /// /// @return the input tokens diff --git a/core/src/test/java/com/google/googlejavaformat/java/CleanupsTest.java b/core/src/test/java/com/google/googlejavaformat/java/CleanupsTest.java index 3e685ed8a..2fc8fd3b6 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/CleanupsTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/CleanupsTest.java @@ -607,11 +607,12 @@ class Test { ImmutableList.of(Range.closedOpen(0, input.length())), Style.GOOGLE, Cleanups.enabledPasses( - /* cleanup= */ false, + /* cleanup= */ true, /* removeUnusedImports= */ true, - /* reorderModifiers= */ false, - /* sortImports= */ false)); + /* reorderModifiers= */ true, + /* sortImports= */ true)); assertThat(result.unit()).isNotNull(); + assertThat(result.javaInput()).isNotNull(); assertThat(result.contents()).isEqualTo(input); } @@ -634,6 +635,7 @@ class Test { /* reorderModifiers= */ true, /* sortImports= */ false)); assertThat(result.unit()).isNull(); + assertThat(result.javaInput()).isNull(); assertThat(result.contents()).contains("public static int x;"); } @@ -671,12 +673,12 @@ class Test { } @Test - public void parseReusedWhenLaterPassParsesAfterEarlierEdits() throws Exception { + public void fusedAstPassesInvalidateUnitOnEditAndReuseJavaInputInLaterPass() throws Exception { String input = """ package test; - import java.util.ArrayList; + import java.util.ArrayList;; class Test { void f(boolean a) { @@ -694,10 +696,15 @@ void f(boolean a) { Cleanups.enabledPasses( /* cleanup= */ true, /* removeUnusedImports= */ true, - /* reorderModifiers= */ false, - /* sortImports= */ false)); - assertThat(result.unit()).isNotNull(); + /* reorderModifiers= */ true, + /* sortImports= */ true)); + // REMOVE_UNUSED_IMPORTS and AST_CLEANUPS ran on a single shared parse and removed both the + // unused import and its redundant semicolon, invalidating the parsed unit; REORDER_MODIFIERS + // then lexed once with no edits so javaInput is cached for Formatter. + assertThat(result.unit()).isNull(); + assertThat(result.javaInput()).isNotNull(); assertThat(result.contents()).doesNotContain("ArrayList"); + assertThat(result.contents()).doesNotContain(";;"); assertThat(result.contents()).contains("if (a) {"); }