diff --git a/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java b/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java index cb946f05d..89fa74668 100644 --- a/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java +++ b/open-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java @@ -40,6 +40,7 @@ import com.sun.tools.javac.util.Log; import com.sun.tools.javac.util.Log.DeferredDiagnosticHandler; import com.sun.tools.javac.util.Options; +import com.sun.tools.javac.util.Position; import java.io.IOException; import java.lang.reflect.Method; import java.net.URI; @@ -379,20 +380,20 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept break; } int charI0 = t.pos(); - // Get string, possibly with Unicode escapes. - String originalTokText = text.substring(charI0, t.endPos()); - String tokText = t.kind() == TokenKind.STRINGLITERAL - ? checkNotNull(t.stringVal()) // Unicode escapes removed. - : originalTokText; + String tokText = text.substring(charI0, t.endPos()); // The token's source text. char tokText0 = tokText.charAt(0); // The token's first character. final boolean isToken; // Is this tok a token? final boolean isNumbered; // Is this tok numbered? (tokens and comments) String extraNewline = null; // Extra newline at end? List strings = new ArrayList<>(); if (Character.isWhitespace(tokText0)) { + int escape = tokText.indexOf('\\'); + if (escape != -1) { + throw error(text, charI0 + escape, "Unicode escapes not allowed in whitespace"); + } isToken = false; isNumbered = false; - Iterator it = Newlines.lineIterator(originalTokText); + Iterator it = Newlines.lineIterator(tokText); while (it.hasNext()) { String line = it.next(); String newline = Newlines.getLineEnding(line); @@ -406,20 +407,41 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept strings.add(line); } } - } else if (tokText.startsWith("'") || tokText.startsWith("\"")) { + } else if (t.kind() == TokenKind.CHARLITERAL || t.kind() == TokenKind.STRINGLITERAL) { + checkLiteral(text, charI0, t.kind(), tokText); isToken = true; isNumbered = true; - strings.add(originalTokText); - } else if (tokText.startsWith("//") || tokText.startsWith("/*")) { + strings.add(tokText); + } else if (tokText.startsWith("//") || (tokText.startsWith("/*") && tokText.endsWith("*/"))) { // For compatibility with an earlier lexer, the newline after a // comment is its own tok. - if (tokText.startsWith("//") && (originalTokText.endsWith("\n") || originalTokText.endsWith("\r"))) { - extraNewline = checkNotNull(Newlines.getLineEnding(originalTokText)); + if (tokText.startsWith("//") && (tokText.endsWith("\n") || tokText.endsWith("\r"))) { + extraNewline = checkNotNull(Newlines.getLineEnding(tokText)); tokText = tokText.substring(0, tokText.length() - extraNewline.length()); - originalTokText = originalTokText.substring(0, originalTokText.length() - extraNewline.length()); } isToken = false; isNumbered = true; - strings.add(originalTokText); + strings.add(tokText); + } else if (t.kind() == null) { + // A non-token that is neither whitespace nor a comment, because a Unicode escape in a comment + // delimiter or line terminator changed how javac lexed it. A block comment is only misidentified if + // its closing delimiter is escaped. + int escape = tokText.startsWith("/*") ? tokText.lastIndexOf('\\') : tokText.indexOf('\\'); + throw error( + text, + charI0 + Math.max(escape, 0), + "Unicode escapes not allowed in whitespace or comment delimiters"); + } else if (tokText.indexOf('\\') != -1) { + throw error( + text, + charI0 + tokText.indexOf('\\'), + "Unicode escapes not allowed in identifiers, keywords, operators, or numeric literals"); + } else if (indexOfIgnorable(tokText) != -1) { + // javac discards ignorable characters in identifiers and keywords, e.g. `in`, U+200B, `t` is lexed as + // the keyword `int`. + throw error( + text, + charI0 + indexOfIgnorable(tokText), + "Ignorable characters not allowed in identifiers or keywords"); } else if (Character.isJavaIdentifierStart(tokText0) || Character.isDigit(tokText0) || (tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1)))) { @@ -429,8 +451,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept strings.add(tokText); } else { // Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>" - // cannot be lexed without syntactic knowledge. This implementation fails if the token - // contains Unicode escapes. + // cannot be lexed without syntactic knowledge. isToken = true; isNumbered = true; for (int i = 0; i < tokText.length(); i++) { @@ -439,18 +460,12 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept } } if (strings.size() == 1) { - if (!tokText.startsWith("//") || !continueTrailingComment(toks, originalTokText, text)) { - toks.add(new Tok( - isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken, t.kind())); + if (!tokText.startsWith("//") || !continueTrailingComment(toks, tokText, text)) { + toks.add(new Tok(isNumbered ? kN++ : -1, tokText, tokText, charI, columnI, isToken, t.kind())); } - charI += originalTokText.length(); - columnI = updateColumn(columnI, originalTokText); - + charI += tokText.length(); + columnI = updateColumn(columnI, tokText); } else { - if (strings.size() != 1 && !tokText.equals(originalTokText)) { - throw new FormatterException( - "Unicode escapes not allowed in whitespace or multi-character operators"); - } for (String str : strings) { toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null)); charI += str.length(); @@ -467,6 +482,92 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept return List.copyOf(toks); } + /** + * Rejects literals whose delimiters, or text block line terminators, are Unicode escapes. The formatter operates on + * the source text of literals, which needs to agree with javac's lexing. + */ + private static void checkLiteral(String text, int pos, TokenKind kind, String tokText) throws FormatterException { + int lineTerminator = indexOfUnicodeLineTerminatorEscape(tokText); + String delimiter; + String description; + if (kind == TokenKind.CHARLITERAL) { + delimiter = "'"; + description = "character literal"; + } else if (Newlines.firstBreak(tokText) != -1 || lineTerminator != -1) { + // A Unicode line terminator will be rejected below. Here we identify it is in a text block. + delimiter = "\"\"\""; + description = "text block"; + } else { + delimiter = "\""; + description = "string literal"; + } + if (!tokText.startsWith(delimiter)) { + throw error( + text, pos + tokText.indexOf('\\'), "Unicode escapes not allowed in " + description + " delimiters"); + } + if (!tokText.endsWith(delimiter)) { + throw error( + text, + pos + tokText.lastIndexOf('\\'), + "Unicode escapes not allowed in " + description + " delimiters"); + } + if (lineTerminator != -1) { + throw error( + text, pos + lineTerminator, "Unicode escapes not allowed in " + description + " line terminators"); + } + } + + /** + * Returns the index of the first Unicode escape for a line terminator (LF U+000A or CR U+000D) in {@code s}, or + * {@code -1}. + */ + private static int indexOfUnicodeLineTerminatorEscape(String s) { + int start = s.indexOf('\\'); + while (start != -1) { + int end = start; + while (end < s.length() && s.charAt(end) == '\\') { + end++; + } + // A backslash is only eligible to begin a Unicode escape if it is preceded by an even number of contiguous + // backslashes (JLS 3.3), and only the last backslash in a run can be followed by a `u`. + int last = end - 1; + if ((last - start) % 2 == 0 && isUnicodeLineTerminatorEscape(s, last)) { + return last; + } + start = s.indexOf('\\', end); + } + return -1; + } + + /** Returns true if the backslash at {@code backslash} begins a Unicode escape for a line terminator. */ + private static boolean isUnicodeLineTerminatorEscape(String s, int backslash) { + int i = backslash + 1; + while (i < s.length() && s.charAt(i) == 'u') { + i++; + } + return i > backslash + 1 + && (s.regionMatches(/* ignoreCase= */ true, i, "000a", 0, 4) + || s.regionMatches(/* ignoreCase= */ true, i, "000d", 0, 4)); + } + + /** Returns the index of the first identifier-ignorable character in {@code s}, or {@code -1}. */ + private static int indexOfIgnorable(String s) { + for (int i = 0; i < s.length(); ) { + int codePoint = s.codePointAt(i); + if (Character.isIdentifierIgnorable(codePoint)) { + return i; + } + i += Character.charCount(codePoint); + } + return -1; + } + + private static FormatterException error(String text, int inputPosition, String message) { + Position.LineMap lineMap = Position.makeLineMap(text.toCharArray(), text.length(), false); + return new FormatterException(FormatterDiagnostic.create( + lineMap.getLineNumber(inputPosition), lineMap.getColumnNumber(inputPosition), message)); + } + /** * Gets diagnostics from a DeferredDiagnosticHandler using reflection. * This method handles the API change in JDK 25 where getDiagnostics() diff --git a/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java b/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java index 55748cc76..55071ce4b 100644 --- a/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java +++ b/open-java-format/src/main/java/com/palantir/javaformat/java/JavacTokens.java @@ -40,17 +40,13 @@ class JavacTokens { /** An unprocessed input token, including whitespace and comments. */ static class RawTok { - @Nullable - private final String stringVal; - @Nullable private final TokenKind kind; private final int pos; private final int endPos; - RawTok(@Nullable String stringVal, @Nullable TokenKind kind, int pos, int endPos) { - this.stringVal = stringVal; + RawTok(@Nullable TokenKind kind, int pos, int endPos) { this.kind = kind; this.pos = pos; this.endPos = endPos; @@ -71,12 +67,6 @@ public int pos() { public int endPos() { return endPos; } - - /** The escaped string value of a literal, or {@code null} for other tokens. */ - @Nullable - public String stringVal() { - return stringVal; - } } /** Lex the input and return a list of {@link RawTok}s. */ @@ -96,10 +86,9 @@ public static List getTokens(String source, Context context, Set getTokens(String source, Context context, Set:2:4: error: unclosed character literal"); } + + @Test + void unicodeEscapes() throws Exception { + assertStdinError(""" + class Foo { + // \\u000D + } + """, ":2:6: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError(""" + class Foo {\\u0020} + """, ":1:12: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError(""" + class Foo { \\u0020 } + """, ":1:13: error: Unicode escapes not allowed in whitespace"); + assertStdinError(""" + class Foo { + \\u002f\\u002f comment + } + """, ":2:3: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError(""" + class Foo { + /* \\u221e *\\u002f + } + """, ":2:14: error: Unicode escapes not allowed in whitespace or comment delimiters"); + assertStdinError( + """ + class Foo { + int x = 1 \\u002b 2; + } + """, + ":2:13: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + int x = 1\\u0030; + } + """, + ":2:12: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + void f() { + getAndIncrement\\u200b("foo"); + } + } + """, + ":3:20: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + void f() { + f\\u006fo("bar"); + } + } + """, + ":3:6: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError( + """ + class Foo { + \\u0069nt x = 1; + } + """, + ":2:3: error: Unicode escapes not allowed in identifiers, keywords, operators, or" + + " numeric literals"); + assertStdinError(""" + class Foo { + String s = \\u0022hello\\u0022; + } + """, ":2:14: error: Unicode escapes not allowed in string literal delimiters"); + assertStdinError(""" + class Foo { + String s = "hello\\u0022; + } + """, ":2:20: error: Unicode escapes not allowed in string literal delimiters"); + assertStdinError(""" + class Foo { + char c = 'a\\u0027; + } + """, ":2:14: error: Unicode escapes not allowed in character literal delimiters"); + assertStdinError(""" + class Foo { + String s = \\u0022"" + hello + \"""; + } + """, ":2:14: error: Unicode escapes not allowed in text block delimiters"); + assertStdinError(""" + class Foo { + String s = \""" + hello + ""\\u0022; + } + """, ":4:3: error: Unicode escapes not allowed in text block delimiters"); + assertStdinError(""" + class Foo { + String s = \"""\\u000ahello\"""; + } + """, ":2:17: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\u000Aworld + \"""; + } + """, ":3:8: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\uuu000dworld + \"""; + } + """, ":3:8: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\\\\\u000aworld + \"""; + } + """, ":3:10: error: Unicode escapes not allowed in text block line terminators"); + assertStdinError(""" + class Foo { + String s = \""" + hello\\u005c\\u000aworld + \"""; + } + """, ":3:14: error: Unicode escapes not allowed in text block line terminators"); + } + + @Test + void ignorableCharacters() throws Exception { + assertStdinError(""" + class Foo { + void f() { + getAndIncrement\u200b("foo"); + } + } + """, ":3:20: error: Ignorable characters not allowed in identifiers or keywords"); + assertStdinError(""" + class Foo { + in\u200bt x = 1; + } + """, ":2:5: error: Ignorable characters not allowed in identifiers or keywords"); + } + + private static void assertStdinError(String input, String expected) throws Exception { + InputStream inStream = new ByteArrayInputStream(input.getBytes(UTF_8)); + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + Main main = new Main(new PrintWriter(out, true), new PrintWriter(err, true), inStream); + int exitCode = main.format("-"); + assertThat(exitCode).isEqualTo(2); + assertThat(err.toString()).startsWith(expected + System.lineSeparator()); + } } diff --git a/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java b/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java index 5ffb10a78..a8ae1a738 100644 --- a/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java +++ b/open-java-format/src/test/java/com/palantir/javaformat/java/FormatterTest.java @@ -379,4 +379,22 @@ public List items() { assertThat(ImportOrderer.reorderImports(input, palantirStyle)) .isEqualTo(ImportOrderer.reorderImports(input, Style.OJF)); } + + @Test + void unicodeEscapesInLiteralAndCommentContents() throws FormatterException { + // Unicode escapes in the contents of comments and literals are preserved. The input contains Unicode escapes, + // so they need to be escaped in the string literal in this test. + String input = """ + class T { + // \\u221e + /* \\u221e */ + String s = "\\u0041\\u005c\\u005c"; + char c = '\\u0041'; + String t = \""" + \\\\u000a \\u0041 + \"""; + } + """; + assertThat(FORMATTER.formatSource(input)).isEqualTo(input); + } }