Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,7 @@ cython_debug/
/.idea
.acli/
settings.xml

# Qwen tokenizer vocab: fetched from ModelScope at runtime, never commit the
# 2.4MB file (a local copy may be placed here for offline development/tests).
src/main/resources/qwen.tiktoken
19 changes: 18 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
<name>DashScope Java SDK</name>
<groupId>com.alibaba</groupId>
<artifactId>dashscope-sdk-java</artifactId>
<version>2.23.6</version>
<version>2.23.7</version>

<properties>
<maven.compiler.source>8</maven.compiler.source>
Expand Down Expand Up @@ -169,6 +169,18 @@
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<excludes>
<!-- vocab is resolved at runtime by QwenTokenizer; keep any local
offline dev copy out of distribution jars -->
<exclude>qwen.tiktoken</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
Expand All @@ -192,6 +204,11 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.0.1</version>
<configuration>
<excludes>
<exclude>**/qwen.tiktoken</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>attach-sources</id>
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/alibaba/dashscope/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@
package com.alibaba.dashscope;

public final class Version {
public static final String version = "2.23.6";
public static final String version = "2.23.7";
}
132 changes: 123 additions & 9 deletions src/main/java/com/alibaba/dashscope/tokenizers/QwenTokenizer.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,17 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
Expand All @@ -33,7 +43,16 @@ public class QwenTokenizer implements Tokenizer {
"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";
private static final int SPECIAL_START_ID = 151643;
private static final String TOKEN_RANK_SEPARATOR = " ";
private static final String vocabularyBpeFile = "qwen.tiktoken";
private static final String VOCAB_NAME = "qwen.tiktoken";
// The vocab is no longer tracked in git; it is downloaded from ModelScope on
// first use and cached locally (a manual classpath copy is still honored).
private static final String VOCAB_URL =
"https://www.modelscope.cn/models/Qwen/Qwen-7B-Chat/resolve/master/" + VOCAB_NAME;
private static final String VOCAB_SHA256 =
"b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186";
private static final String VOCAB_ENV = "DASHSCOPE_QWEN_VOCAB";
private static final Path VOCAB_CACHE_DIR =
Paths.get(System.getProperty("user.home"), ".cache", "dashscope", "tokenizers");
private static final Map<EncodeBytesEntity, Integer> mergeableRanks;
private static final Map<String, Integer> specialTokens;
private static final byte[][] decodeMap;
Expand All @@ -54,17 +73,14 @@ public class QwenTokenizer implements Tokenizer {
static {
// ref: https://github.com/openai/tiktoken/blob/main/tiktoken/load.py#L143
mergeableRanks = new LinkedHashMap<>();
ClassLoader classLoader = QwenTokenizer.class.getClassLoader();
try {
InputStream inputStream = classLoader.getResourceAsStream(vocabularyBpeFile);

BufferedReader reader =
new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
try (InputStream inputStream = resolveVocab();
BufferedReader reader =
new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
// 8J+Vkw== 149934 split
String[] splits = line.split(TOKEN_RANK_SEPARATOR);
assert splits.length == 2 : "Invalid line in " + vocabularyBpeFile + ": " + line;
assert splits.length == 2 : "Invalid line in " + VOCAB_NAME + ": " + line;

byte[] token = Base64.getDecoder().decode(splits[0].getBytes(StandardCharsets.UTF_8));
int rank = Integer.valueOf(splits[1]);
Expand All @@ -82,7 +98,105 @@ public class QwenTokenizer implements Tokenizer {
decodeMap[entry.getValue()] = Arrays.copyOf(b, b.length);
}
} catch (IOException e) {
throw new RuntimeException("Could not load " + vocabularyBpeFile + " from resources", e);
throw new RuntimeException("Could not load " + VOCAB_NAME, e);
}
}

/**
* Resolves the vocab from, in order: the {@code DASHSCOPE_QWEN_VOCAB} environment variable, the
* classpath (optional local copy), a SHA256-verified local cache, or a ModelScope download into
* {@code ~/.cache/dashscope/tokenizers}.
*/
static InputStream resolveVocab() throws IOException {
String override = System.getenv(VOCAB_ENV);
if (override != null && !override.isEmpty()) {
return Files.newInputStream(expandHome(override));
}
InputStream bundled = QwenTokenizer.class.getClassLoader().getResourceAsStream(VOCAB_NAME);
if (bundled != null) {
return bundled;
}
Path cached = VOCAB_CACHE_DIR.resolve(VOCAB_NAME);
if (Files.isRegularFile(cached) && VOCAB_SHA256.equals(sha256(cached))) {
return Files.newInputStream(cached);
}
downloadVocab(cached);
return Files.newInputStream(cached);
}

static Path expandHome(String path) {
String home = System.getProperty("user.home");
if (path.equals("~")) {
return Paths.get(home);
}
if (path.startsWith("~/")) {
return Paths.get(home, path.substring(2));
}
return Paths.get(path);
}

static String sha256(Path path) throws IOException {
MessageDigest digest;
try {
digest = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException(e);
}
try (InputStream in = Files.newInputStream(path)) {
byte[] buffer = new byte[1 << 20];
int read;
while ((read = in.read(buffer)) != -1) {
digest.update(buffer, 0, read);
}
}
StringBuilder sb = new StringBuilder();
for (byte b : digest.digest()) {
sb.append(String.format("%02x", b));
}
return sb.toString();
}

private static void downloadVocab(Path dest) throws IOException {
Files.createDirectories(dest.getParent());
Path tmp = Files.createTempFile(dest.getParent(), VOCAB_NAME + ".", ".download");
try {
HttpURLConnection connection =
(HttpURLConnection) URI.create(VOCAB_URL).toURL().openConnection();
connection.setConnectTimeout(60_000);
connection.setReadTimeout(60_000);
try (InputStream in = connection.getInputStream();
OutputStream out = Files.newOutputStream(tmp)) {
byte[] buffer = new byte[1 << 20];
int read;
while ((read = in.read(buffer)) != -1) {
out.write(buffer, 0, read);
}
} finally {
connection.disconnect();
}
if (!VOCAB_SHA256.equals(sha256(tmp))) {
throw new IOException("SHA256 mismatch for vocab downloaded from " + VOCAB_URL);
}
try {
Files.move(tmp, dest, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(tmp, dest, StandardCopyOption.REPLACE_EXISTING);
}
} catch (IOException | RuntimeException e) {
throw new RuntimeException(
"Failed to obtain the Qwen tokenizer vocab ("
+ VOCAB_NAME
+ "). Download it manually from "
+ VOCAB_URL
+ " and either place it at "
+ dest
+ " or point "
+ VOCAB_ENV
+ " at the file (required for offline environments). Cause: "
+ e,
e);
} finally {
Files.deleteIfExists(tmp);
}
}

Expand Down
Loading
Loading