Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import com.sun.tools.javac.util.Log;
import com.sun.tools.javac.util.Log.DeferredDiagnosticHandler;
import com.sun.tools.javac.util.Options;
import com.sun.tools.javac.util.Position;
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URI;
Expand Down Expand Up @@ -379,20 +380,20 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
break;
}
int charI0 = t.pos();
// Get string, possibly with Unicode escapes.
String originalTokText = text.substring(charI0, t.endPos());
String tokText = t.kind() == TokenKind.STRINGLITERAL
? checkNotNull(t.stringVal()) // Unicode escapes removed.
: originalTokText;
String tokText = text.substring(charI0, t.endPos()); // The token's source text.
char tokText0 = tokText.charAt(0); // The token's first character.
final boolean isToken; // Is this tok a token?
final boolean isNumbered; // Is this tok numbered? (tokens and comments)
String extraNewline = null; // Extra newline at end?
List<String> strings = new ArrayList<>();
if (Character.isWhitespace(tokText0)) {
int escape = tokText.indexOf('\\');
if (escape != -1) {
throw error(text, charI0 + escape, "Unicode escapes not allowed in whitespace");
}
isToken = false;
isNumbered = false;
Iterator<String> it = Newlines.lineIterator(originalTokText);
Iterator<String> it = Newlines.lineIterator(tokText);
while (it.hasNext()) {
String line = it.next();
String newline = Newlines.getLineEnding(line);
Expand All @@ -406,20 +407,41 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
strings.add(line);
}
}
} else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
} else if (t.kind() == TokenKind.CHARLITERAL || t.kind() == TokenKind.STRINGLITERAL) {
checkLiteral(text, charI0, t.kind(), tokText);
isToken = true;
isNumbered = true;
strings.add(originalTokText);
} else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
strings.add(tokText);
} else if (tokText.startsWith("//") || (tokText.startsWith("/*") && tokText.endsWith("*/"))) {
// For compatibility with an earlier lexer, the newline after a // comment is its own tok.
if (tokText.startsWith("//") && (originalTokText.endsWith("\n") || originalTokText.endsWith("\r"))) {
extraNewline = checkNotNull(Newlines.getLineEnding(originalTokText));
if (tokText.startsWith("//") && (tokText.endsWith("\n") || tokText.endsWith("\r"))) {
extraNewline = checkNotNull(Newlines.getLineEnding(tokText));
tokText = tokText.substring(0, tokText.length() - extraNewline.length());
originalTokText = originalTokText.substring(0, originalTokText.length() - extraNewline.length());
}
isToken = false;
isNumbered = true;
strings.add(originalTokText);
strings.add(tokText);
} else if (t.kind() == null) {
// A non-token that is neither whitespace nor a comment, because a Unicode escape in a comment
// delimiter or line terminator changed how javac lexed it. A block comment is only misidentified if
// its closing delimiter is escaped.
int escape = tokText.startsWith("/*") ? tokText.lastIndexOf('\\') : tokText.indexOf('\\');
throw error(
text,
charI0 + Math.max(escape, 0),
"Unicode escapes not allowed in whitespace or comment delimiters");
} else if (tokText.indexOf('\\') != -1) {
throw error(
text,
charI0 + tokText.indexOf('\\'),
"Unicode escapes not allowed in identifiers, keywords, operators, or numeric literals");
} else if (indexOfIgnorable(tokText) != -1) {
// javac discards ignorable characters in identifiers and keywords, e.g. `in`, U+200B, `t` is lexed as
// the keyword `int`.
throw error(
text,
charI0 + indexOfIgnorable(tokText),
"Ignorable characters not allowed in identifiers or keywords");
} else if (Character.isJavaIdentifierStart(tokText0)
|| Character.isDigit(tokText0)
|| (tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1)))) {
Expand All @@ -429,8 +451,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
strings.add(tokText);
} else {
// Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>"
// cannot be lexed without syntactic knowledge. This implementation fails if the token
// contains Unicode escapes.
// cannot be lexed without syntactic knowledge.
isToken = true;
isNumbered = true;
for (int i = 0; i < tokText.length(); i++) {
Expand All @@ -439,18 +460,12 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
}
}
if (strings.size() == 1) {
if (!tokText.startsWith("//") || !continueTrailingComment(toks, originalTokText, text)) {
toks.add(new Tok(
isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken, t.kind()));
if (!tokText.startsWith("//") || !continueTrailingComment(toks, tokText, text)) {
toks.add(new Tok(isNumbered ? kN++ : -1, tokText, tokText, charI, columnI, isToken, t.kind()));
}
charI += originalTokText.length();
columnI = updateColumn(columnI, originalTokText);

charI += tokText.length();
columnI = updateColumn(columnI, tokText);
} else {
if (strings.size() != 1 && !tokText.equals(originalTokText)) {
throw new FormatterException(
"Unicode escapes not allowed in whitespace or multi-character operators");
}
for (String str : strings) {
toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null));
charI += str.length();
Expand All @@ -467,6 +482,92 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
return List.copyOf(toks);
}

/**
* Rejects literals whose delimiters, or text block line terminators, are Unicode escapes. The formatter operates on
* the source text of literals, which needs to agree with javac's lexing.
*/
private static void checkLiteral(String text, int pos, TokenKind kind, String tokText) throws FormatterException {
int lineTerminator = indexOfUnicodeLineTerminatorEscape(tokText);
String delimiter;
String description;
if (kind == TokenKind.CHARLITERAL) {
delimiter = "'";
description = "character literal";
} else if (Newlines.firstBreak(tokText) != -1 || lineTerminator != -1) {
// A Unicode line terminator will be rejected below. Here we identify it is in a text block.
delimiter = "\"\"\"";
description = "text block";
} else {
delimiter = "\"";
description = "string literal";
}
if (!tokText.startsWith(delimiter)) {
throw error(
text, pos + tokText.indexOf('\\'), "Unicode escapes not allowed in " + description + " delimiters");
}
if (!tokText.endsWith(delimiter)) {
throw error(
text,
pos + tokText.lastIndexOf('\\'),
"Unicode escapes not allowed in " + description + " delimiters");
}
if (lineTerminator != -1) {
throw error(
text, pos + lineTerminator, "Unicode escapes not allowed in " + description + " line terminators");
}
}

/**
* Returns the index of the first Unicode escape for a line terminator (LF U+000A or CR U+000D) in {@code s}, or
* {@code -1}.
*/
private static int indexOfUnicodeLineTerminatorEscape(String s) {
int start = s.indexOf('\\');
while (start != -1) {
int end = start;
while (end < s.length() && s.charAt(end) == '\\') {
end++;
}
// A backslash is only eligible to begin a Unicode escape if it is preceded by an even number of contiguous
// backslashes (JLS 3.3), and only the last backslash in a run can be followed by a `u`.
int last = end - 1;
if ((last - start) % 2 == 0 && isUnicodeLineTerminatorEscape(s, last)) {
return last;
}
start = s.indexOf('\\', end);
}
return -1;
}

/** Returns true if the backslash at {@code backslash} begins a Unicode escape for a line terminator. */
private static boolean isUnicodeLineTerminatorEscape(String s, int backslash) {
int i = backslash + 1;
while (i < s.length() && s.charAt(i) == 'u') {
i++;
}
return i > backslash + 1
&& (s.regionMatches(/* ignoreCase= */ true, i, "000a", 0, 4)
|| s.regionMatches(/* ignoreCase= */ true, i, "000d", 0, 4));
}

/** Returns the index of the first identifier-ignorable character in {@code s}, or {@code -1}. */
private static int indexOfIgnorable(String s) {
for (int i = 0; i < s.length(); ) {
int codePoint = s.codePointAt(i);
if (Character.isIdentifierIgnorable(codePoint)) {
return i;
}
i += Character.charCount(codePoint);
}
return -1;
}

private static FormatterException error(String text, int inputPosition, String message) {
Position.LineMap lineMap = Position.makeLineMap(text.toCharArray(), text.length(), false);
return new FormatterException(FormatterDiagnostic.create(
lineMap.getLineNumber(inputPosition), lineMap.getColumnNumber(inputPosition), message));
}

/**
* Gets diagnostics from a DeferredDiagnosticHandler using reflection.
* This method handles the API change in JDK 25 where getDiagnostics()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,13 @@ class JavacTokens {

/** An unprocessed input token, including whitespace and comments. */
static class RawTok {
@Nullable
private final String stringVal;

@Nullable
private final TokenKind kind;

private final int pos;
private final int endPos;

RawTok(@Nullable String stringVal, @Nullable TokenKind kind, int pos, int endPos) {
this.stringVal = stringVal;
RawTok(@Nullable TokenKind kind, int pos, int endPos) {
this.kind = kind;
this.pos = pos;
this.endPos = endPos;
Expand All @@ -71,12 +67,6 @@ public int pos() {
public int endPos() {
return endPos;
}

/** The escaped string value of a literal, or {@code null} for other tokens. */
@Nullable
public String stringVal() {
return stringVal;
}
}

/** Lex the input and return a list of {@link RawTok}s. */
Expand All @@ -96,10 +86,9 @@ public static List<RawTok> getTokens(String source, Context context, Set<TokenKi
if (t.comments != null) {
for (Comment c : Lists.reverse(t.comments)) {
if (last < c.getSourcePos(0)) {
tokens.add(new RawTok(null, null, last, c.getSourcePos(0)));
tokens.add(new RawTok(null, last, c.getSourcePos(0)));
}
tokens.add(new RawTok(
null,
null,
c.getSourcePos(0),
c.getSourcePos(0) + c.getText().length()));
Expand All @@ -113,14 +102,13 @@ public static List<RawTok> getTokens(String source, Context context, Set<TokenKi
break;
}
if (last < t.pos) {
tokens.add(new RawTok(null, null, last, t.pos));
tokens.add(new RawTok(null, last, t.pos));
}
tokens.add(new RawTok(
t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null, t.kind, t.pos, t.endPos));
tokens.add(new RawTok(t.kind, t.pos, t.endPos));
last = t.endPos;
} while (scanner.token().kind != TokenKind.EOF);
if (last < end) {
tokens.add(new RawTok(null, null, last, end));
tokens.add(new RawTok(null, last, end));
}
return List.copyOf(tokens);
}
Expand Down
Loading
Loading