From 8eaeb501ce49bdde45d0e2ef49237d2b4b2ae38d Mon Sep 17 00:00:00 2001 From: Alex Abashev Date: Thu, 19 Nov 2026 14:00:08 +0300 Subject: [PATCH] Reject Unicode escapes outside literal and comment contents javac translates Unicode escapes before it lexes, but the formatter works on the source text, so the two can disagree about where a token starts and ends (#244): class Foo { // \u000D } The escaped carriage return ends the comment for javac, and the escape is left in the gap before `}`, where JavaInput took it for a token: :1:1: error: expected token: '\'; generated } instead An escape in a comment or literal delimiter, or in a line terminator of a text block, does the same, and javac also drops identifier-ignorable characters such as U+200B from names and keywords, so a zero-width space at the end of `getAndIncrement` failed with "expected token: 'getAndIncrement'; generated getAndIncrement instead". JavaInput.buildToks now rejects these inputs up front with a diagnostic at the line and column of the escape or character, as google/google-java-format#1527 does: :2:6: error: Unicode escapes not allowed in whitespace or comment delimiters :3:20: error: Ignorable characters not allowed in identifiers or keywords Escapes in the contents of comments, string and character literals and text blocks are still allowed and kept as they are. Every tok now holds its source text, so RawTok no longer carries javac's string value, and the old check for escapes in whitespace and multi-character operators, which reported no position, is gone. The cases in DiagnosticTest.unicodeEscapes and ignorableCharacters come from upstream; their exit code is 2, ours for an input that does not parse. FormatterTest.unicodeEscapesInLiteralAndCommentContents checks that escapes in contents format unchanged, in our style. --- .../palantir/javaformat/java/JavaInput.java | 151 ++++++++++++++--- .../palantir/javaformat/java/JavacTokens.java | 22 +-- .../javaformat/java/DiagnosticTest.java | 157 ++++++++++++++++++ .../javaformat/java/FormatterTest.java | 18 ++ 4 files changed, 306 insertions(+), 42 deletions(-) diff --git a/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java b/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java index cb946f05d..89fa74668 100644 --- a/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java +++ b/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java @@ -40,6 +40,7 @@ import com.sun.tools.javac.util.Log; import com.sun.tools.javac.util.Log.DeferredDiagnosticHandler; import com.sun.tools.javac.util.Options; +import com.sun.tools.javac.util.Position; import java.io.IOException; import java.lang.reflect.Method; import java.net.URI; @@ -379,20 +380,20 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept break; } int charI0 = t.pos(); - // Get string, possibly with Unicode escapes. - String originalTokText = text.substring(charI0, t.endPos()); - String tokText = t.kind() == TokenKind.STRINGLITERAL - ? checkNotNull(t.stringVal()) // Unicode escapes removed. - : originalTokText; + String tokText = text.substring(charI0, t.endPos()); // The token's source text. char tokText0 = tokText.charAt(0); // The token's first character. final boolean isToken; // Is this tok a token? final boolean isNumbered; // Is this tok numbered? (tokens and comments) String extraNewline = null; // Extra newline at end? List strings = new ArrayList<>(); if (Character.isWhitespace(tokText0)) { + int escape = tokText.indexOf('\\'); + if (escape != -1) { + throw error(text, charI0 + escape, "Unicode escapes not allowed in whitespace"); + } isToken = false; isNumbered = false; - Iterator it = Newlines.lineIterator(originalTokText); + Iterator it = Newlines.lineIterator(tokText); while (it.hasNext()) { String line = it.next(); String newline = Newlines.getLineEnding(line); @@ -406,20 +407,41 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept strings.add(line); } } - } else if (tokText.startsWith("'") || tokText.startsWith("\"")) { + } else if (t.kind() == TokenKind.CHARLITERAL || t.kind() == TokenKind.STRINGLITERAL) { + checkLiteral(text, charI0, t.kind(), tokText); isToken = true; isNumbered = true; - strings.add(originalTokText); - } else if (tokText.startsWith("//") || tokText.startsWith("/*")) { + strings.add(tokText); + } else if (tokText.startsWith("//") || (tokText.startsWith("/*") && tokText.endsWith("*/"))) { // For compatibility with an earlier lexer, the newline after a // comment is its own tok. - if (tokText.startsWith("//") && (originalTokText.endsWith("\n") || originalTokText.endsWith("\r"))) { - extraNewline = checkNotNull(Newlines.getLineEnding(originalTokText)); + if (tokText.startsWith("//") && (tokText.endsWith("\n") || tokText.endsWith("\r"))) { + extraNewline = checkNotNull(Newlines.getLineEnding(tokText)); tokText = tokText.substring(0, tokText.length() - extraNewline.length()); - originalTokText = originalTokText.substring(0, originalTokText.length() - extraNewline.length()); } isToken = false; isNumbered = true; - strings.add(originalTokText); + strings.add(tokText); + } else if (t.kind() == null) { + // A non-token that is neither whitespace nor a comment, because a Unicode escape in a comment + // delimiter or line terminator changed how javac lexed it. A block comment is only misidentified if + // its closing delimiter is escaped. + int escape = tokText.startsWith("/*") ? tokText.lastIndexOf('\\') : tokText.indexOf('\\'); + throw error( + text, + charI0 + Math.max(escape, 0), + "Unicode escapes not allowed in whitespace or comment delimiters"); + } else if (tokText.indexOf('\\') != -1) { + throw error( + text, + charI0 + tokText.indexOf('\\'), + "Unicode escapes not allowed in identifiers, keywords, operators, or numeric literals"); + } else if (indexOfIgnorable(tokText) != -1) { + // javac discards ignorable characters in identifiers and keywords, e.g. `in`, U+200B, `t` is lexed as + // the keyword `int`. + throw error( + text, + charI0 + indexOfIgnorable(tokText), + "Ignorable characters not allowed in identifiers or keywords"); } else if (Character.isJavaIdentifierStart(tokText0) || Character.isDigit(tokText0) || (tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1)))) { @@ -429,8 +451,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept strings.add(tokText); } else { // Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>" - // cannot be lexed without syntactic knowledge. This implementation fails if the token - // contains Unicode escapes. + // cannot be lexed without syntactic knowledge. isToken = true; isNumbered = true; for (int i = 0; i < tokText.length(); i++) { @@ -439,18 +460,12 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept } } if (strings.size() == 1) { - if (!tokText.startsWith("//") || !continueTrailingComment(toks, originalTokText, text)) { - toks.add(new Tok( - isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken, t.kind())); + if (!tokText.startsWith("//") || !continueTrailingComment(toks, tokText, text)) { + toks.add(new Tok(isNumbered ? kN++ : -1, tokText, tokText, charI, columnI, isToken, t.kind())); } - charI += originalTokText.length(); - columnI = updateColumn(columnI, originalTokText); - + charI += tokText.length(); + columnI = updateColumn(columnI, tokText); } else { - if (strings.size() != 1 && !tokText.equals(originalTokText)) { - throw new FormatterException( - "Unicode escapes not allowed in whitespace or multi-character operators"); - } for (String str : strings) { toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null)); charI += str.length(); @@ -467,6 +482,92 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept return List.copyOf(toks); } + /** + * Rejects literals whose delimiters, or text block line terminators, are Unicode escapes. The formatter operates on + * the source text of literals, which needs to agree with javac's lexing. + */ + private static void checkLiteral(String text, int pos, TokenKind kind, String tokText) throws FormatterException { + int lineTerminator = indexOfUnicodeLineTerminatorEscape(tokText); + String delimiter; + String description; + if (kind == TokenKind.CHARLITERAL) { + delimiter = "'"; + description = "character literal"; + } else if (Newlines.firstBreak(tokText) != -1 || lineTerminator != -1) { + // A Unicode line terminator will be rejected below. Here we identify it is in a text block. + delimiter = "\"\"\""; + description = "text block"; + } else { + delimiter = "\""; + description = "string literal"; + } + if (!tokText.startsWith(delimiter)) { + throw error( + text, pos + tokText.indexOf('\\'), "Unicode escapes not allowed in " + description + " delimiters"); + } + if (!tokText.endsWith(delimiter)) { + throw error( + text, + pos + tokText.lastIndexOf('\\'), + "Unicode escapes not allowed in " + description + " delimiters"); + } + if (lineTerminator != -1) { + throw error( + text, pos + lineTerminator, "Unicode escapes not allowed in " + description + " line terminators"); + } + } + + /** + * Returns the index of the first Unicode escape for a line terminator (LF U+000A or CR U+000D) in {@code s}, or + * {@code -1}. + */ + private static int indexOfUnicodeLineTerminatorEscape(String s) { + int start = s.indexOf('\\'); + while (start != -1) { + int end = start; + while (end < s.length() && s.charAt(end) == '\\') { + end++; + } + // A backslash is only eligible to begin a Unicode escape if it is preceded by an even number of contiguous + // backslashes (JLS 3.3), and only the last backslash in a run can be followed by a `u`. + int last = end - 1; + if ((last - start) % 2 == 0 && isUnicodeLineTerminatorEscape(s, last)) { + return last; + } + start = s.indexOf('\\', end); + } + return -1; + } + + /** Returns true if the backslash at {@code backslash} begins a Unicode escape for a line terminator. */ + private static boolean isUnicodeLineTerminatorEscape(String s, int backslash) { + int i = backslash + 1; + while (i < s.length() && s.charAt(i) == 'u') { + i++; + } + return i > backslash + 1 + && (s.regionMatches(/* ignoreCase= */ true, i, "000a", 0, 4) + || s.regionMatches(/* ignoreCase= */ true, i, "000d", 0, 4)); + } + + /** Returns the index of the first identifier-ignorable character in {@code s}, or {@code -1}. */ + private static int indexOfIgnorable(String s) { + for (int i = 0; i < s.length(); ) { + int codePoint = s.codePointAt(i); + if (Character.isIdentifierIgnorable(codePoint)) { + return i; + } + i += Character.charCount(codePoint); + } + return -1; + } + + private static FormatterException error(String text, int inputPosition, String message) { + Position.LineMap lineMap = Position.makeLineMap(text.toCharArray(), text.length(), false); + return new FormatterException(FormatterDiagnostic.create( + lineMap.getLineNumber(inputPosition), lineMap.getColumnNumber(inputPosition), message)); + } + /** * Gets diagnostics from a DeferredDiagnosticHandler using reflection. * This method handles the API change in JDK 25 where getDiagnostics() diff --git a/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java b/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java index 55748cc76..55071ce4b 100644 --- a/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java +++ b/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java @@ -40,17 +40,13 @@ class JavacTokens { /** An unprocessed input token, including whitespace and comments. */ static class RawTok { - @Nullable - private final String stringVal; - @Nullable private final TokenKind kind; private final int pos; private final int endPos; - RawTok(@Nullable String stringVal, @Nullable TokenKind kind, int pos, int endPos) { - this.stringVal = stringVal; + RawTok(@Nullable TokenKind kind, int pos, int endPos) { this.kind = kind; this.pos = pos; this.endPos = endPos; @@ -71,12 +67,6 @@ public int pos() { public int endPos() { return endPos; } - - /** The escaped string value of a literal, or {@code null} for other tokens. */ - @Nullable - public String stringVal() { - return stringVal; - } } /** Lex the input and return a list of {@link RawTok}s. */ @@ -96,10 +86,9 @@ public static List getTokens(String source, Context context, Set getTokens(String source, Context context, Set:2:4: error: unclosed character literal"); } + + @Test + void unicodeEscapes() throws Exception { + assertStdinError(""" + class Foo { + // \\u000D + } + """, ":2:6: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError(""" + class Foo {\\u0020} + """, ":1:12: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError(""" + class Foo { \\u0020 } + """, ":1:13: error: Unicode escapes not allowed in whitespace"); + assertStdinError(""" + class Foo { + \\u002f\\u002f comment + } + """, ":2:3: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError(""" + class Foo { + /* \\u221e *\\u002f + } + """, ":2:14: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError( + """ + class Foo { + int x = 1 \\u002b 2; + } + """, + ":2:13: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + int x = 1\\u0030; + } + """, + ":2:12: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + void f() { + getAndIncrement\\u200b("foo"); + } + } + """, + ":3:20: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + void f() { + f\\u006fo("bar"); + } + } + """, + ":3:6: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + \\u0069nt x = 1; + } + """, + ":2:3: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError(""" + class Foo { + String s = \\u0022hello\\u0022; + } + """, ":2:14: error: Unicode escapes not allowed in string literal delimiters"); + assertStdinError(""" + class Foo { + String s = "hello\\u0022; + } + """, ":2:20: error: Unicode escapes not allowed in string literal delimiters"); + assertStdinError(""" + class Foo { + char c = 'a\\u0027; + } + """, ":2:14: error: Unicode escapes not allowed in character literal delimiters"); + assertStdinError(""" + class Foo { + String s = \\u0022"" + hello + \"""; + } + """, ":2:14: error: Unicode escapes not allowed in text block delimiters"); + assertStdinError(""" + class Foo { + String s = \""" + hello + ""\\u0022; + } + """, ":4:3: error: Unicode escapes not allowed in text block delimiters"); + assertStdinError(""" + class Foo { + String s = \"""\\u000ahello\"""; + } + """, ":2:17: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\u000Aworld + \"""; + } + """, ":3:8: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\uuu000dworld + \"""; + } + """, ":3:8: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\\\\\u000aworld + \"""; + } + """, ":3:10: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\u005c\\u000aworld + \"""; + } + """, ":3:14: error: Unicode escapes not allowed in text block line terminators"); + } + + @Test + void ignorableCharacters() throws Exception { + assertStdinError(""" + class Foo { + void f() { + getAndIncrement\u200b("foo"); + } + } + """, ":3:20: error: Ignorable characters not allowed in identifiers or keywords"); + assertStdinError(""" + class Foo { + in\u200bt x = 1; + } + """, ":2:5: error: Ignorable characters not allowed in identifiers or keywords"); + } + + private static void assertStdinError(String input, String expected) throws Exception { + InputStream inStream = new ByteArrayInputStream(input.getBytes(UTF_8)); + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + Main main = new Main(new PrintWriter(out, true), new PrintWriter(err, true), inStream); + int exitCode = main.format("-"); + assertThat(exitCode).isEqualTo(2); + assertThat(err.toString()).startsWith(expected + System.lineSeparator()); + } } diff --git a/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java b/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java index 5ffb10a78..a8ae1a738 100644 --- a/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java +++ b/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java @@ -379,4 +379,22 @@ public List items() { assertThat(ImportOrderer.reorderImports(input, palantirStyle)) .isEqualTo(ImportOrderer.reorderImports(input, Style.OJF)); } + + @Test + void unicodeEscapesInLiteralAndCommentContents() throws FormatterException { + // Unicode escapes in the contents of comments and literals are preserved. The input contains Unicode escapes, + // so they need to be escaped in the string literal in this test. + String input = """ + class T { + // \\u221e + /* \\u221e */ + String s = "\\u0041\\u005c\\u005c"; + char c = '\\u0041'; + String t = \""" + \\\\u000a \\u0041 + \"""; + } + """; + assertThat(FORMATTER.formatSource(input)).isEqualTo(input); + } }