From 3cef3df7a4eb410a12271e1f6cce7f20a8682a4c Mon Sep 17 00:00:00 2001 From: "zhansheng.lzs" Date: Fri, 20 Nov 2026 14:08:31 +0800 Subject: [PATCH] ci: pre-download Qwen tokenizer vocab before tests The vocab is fetched from ModelScope on first use, which is flaky from overseas runners. Pre-download it into the SHA256-verified local cache (with retries) in both the test and release workflows so tokenizer tests never hit the network during mvn test. --- .github/workflows/build-and-test.yml | 19 +++++++++++++++++++ .github/workflows/release.yml | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e45bec81..aa639557 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -50,6 +50,25 @@ jobs: distribution: ${{ env.JAVA_DISTRIBUTION }} cache: 'maven' + - name: Pre-download Qwen tokenizer vocab + run: | + vocab_dir="$HOME/.cache/dashscope/tokenizers" + vocab="$vocab_dir/qwen.tiktoken" + sha256="b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186" + url="https://www.modelscope.cn/models/Qwen/Qwen-7B-Chat/resolve/master/qwen.tiktoken" + mkdir -p "$vocab_dir" + for attempt in 1 2 3; do + if curl -fsSL --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 5 \ + -o "$vocab" "$url" && [ "$(sha256sum "$vocab" | cut -d' ' -f1)" = "$sha256" ]; then + echo "Vocab cached at $vocab" + exit 0 + fi + echo "Vocab download attempt $attempt failed, retrying..." + sleep 10 + done + echo "::error::Failed to download qwen.tiktoken from ModelScope after 3 attempts" + exit 1 + - name: Run tests run: mvn clean test env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 977c9e72..a13ef22c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,25 @@ jobs: V=$(mvn -q help:evaluate -Dexpression=project.version -DforceStdout) test "v$V" = "$GITHUB_REF_NAME" + - name: Pre-download Qwen tokenizer vocab + run: | + vocab_dir="$HOME/.cache/dashscope/tokenizers" + vocab="$vocab_dir/qwen.tiktoken" + sha256="b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186" + url="https://www.modelscope.cn/models/Qwen/Qwen-7B-Chat/resolve/master/qwen.tiktoken" + mkdir -p "$vocab_dir" + for attempt in 1 2 3; do + if curl -fsSL --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 5 \ + -o "$vocab" "$url" && [ "$(sha256sum "$vocab" | cut -d' ' -f1)" = "$sha256" ]; then + echo "Vocab cached at $vocab" + exit 0 + fi + echo "Vocab download attempt $attempt failed, retrying..." + sleep 10 + done + echo "::error::Failed to download qwen.tiktoken from ModelScope after 3 attempts" + exit 1 + - name: Unit tests run: mvn -B test