From 360b6396386be7292716adc100dc48c97ac96bd8 Mon Sep 17 00:00:00 2001 From: "zhansheng.lzs" Date: Fri, 20 Nov 2026 14:07:12 +0800 Subject: [PATCH] fix(ci): pre-fetch Qwen tokenizer vocab with retries to deflake tests The vocab download from ModelScope times out intermittently on GitHub runners, failing test_root_get_tokenizer_encodes_locally and the legacy tokenizer tests. Cache the vocab across runs and pre-download it with retries in both test jobs, reusing the SDK's URL/SHA256/cache logic. --- .github/workflows/unit_test.yml | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 8ec5ac8f..73f42bdf 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -65,6 +65,30 @@ jobs: run: | python -m pip list --format=columns + - name: Cache Qwen tokenizer vocab + uses: actions/cache@v4 + with: + path: ~/.cache/dashscope/tokenizers + key: qwen-tokenizer-vocab-v1 + + # ModelScope downloads from GitHub runners time out intermittently; + # pre-fetch with retries so network blips don't fail the test matrix. + - name: Pre-download Qwen tokenizer vocab + run: | + python - <<'EOF' + import time + from dashscope.legacy_1x.tokenizers import tokenizer as t + for attempt in range(1, 6): + try: + print("vocab ready:", t._resolve_vocab_path()) + break + except RuntimeError as exc: + print(f"attempt {attempt}/5 failed: {exc}") + if attempt == 5: + raise SystemExit(1) + time.sleep(5 * attempt) + EOF + - name: Run current tests with coverage run: | coverage run -m pytest tests --ignore=tests/legacy_1x @@ -113,6 +137,30 @@ jobs: pip install -r requirements.txt pip install -r requirements-test.txt + - name: Cache Qwen tokenizer vocab + uses: actions/cache@v4 + with: + path: ~/.cache/dashscope/tokenizers + key: qwen-tokenizer-vocab-v1 + + # ModelScope downloads from GitHub runners time out intermittently; + # pre-fetch with retries so network blips don't fail the test matrix. + - name: Pre-download Qwen tokenizer vocab + run: | + python - <<'EOF' + import time + from dashscope.legacy_1x.tokenizers import tokenizer as t + for attempt in range(1, 6): + try: + print("vocab ready:", t._resolve_vocab_path()) + break + except RuntimeError as exc: + print(f"attempt {attempt}/5 failed: {exc}") + if attempt == 5: + raise SystemExit(1) + time.sleep(5 * attempt) + EOF + - name: Run archived legacy tests run: | python -m pytest tests/legacy_1x