From ab5bf756266cbac2920628b3cbfae665c3a6ef20 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 13 Nov 2026 09:25:18 +0000 Subject: [PATCH] stress_test: add --synthetic cases for models missing from the set export The Context engineering set has no smollm2_infer response area, so the script could not load-test it. --synthetic MODEL builds cases from built-in student-style prompts crossed with --word-counts (default 16,64,256). Each length is reported as its own row, since generation length drives latency. Co-Authored-By: Claude Opus 5.5 --- scripts/stress_test.py | 49 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/scripts/stress_test.py b/scripts/stress_test.py index 902ba33..b6637e4 100644 --- a/scripts/stress_test.py +++ b/scripts/stress_test.py @@ -88,6 +88,43 @@ def load_cases(set_dir: str, models: list[str] | None) -> list[Case]: return cases +SYNTHETIC_PROMPTS = { + "smollm2_infer": [ + "What is a language model?", + "Explain attention in transformers in one sentence.", + "Write a haiku about tokens.", + "Why do large language models sometimes make things up?", + "What is the difference between a bigram and a trigram?", + ], +} + + +def synthetic_cases(models: list[str], word_counts: list[int]) -> list[Case]: + """Cases for models with no response area in the set export, shaped like + {"response": prompt, "params": {"model": ..., "word_count": N}}. Case.model + carries the length so report() breaks latency down per word_count. + Cases interleave lengths so every stage sees a mix.""" + cases = [] + for model in models: + prompts = SYNTHETIC_PROMPTS.get(model) + if not prompts: + raise SystemExit(f"no synthetic prompts for {model!r}; known: {', '.join(SYNTHETIC_PROMPTS)}") + short = model.removesuffix("_infer") + for i, prompt in enumerate(prompts): + for wc in word_counts: + body = { + "submission": {"type": "OTHER", "format": "", "content": {"value": prompt}}, + "task": {"title": f"synthetic {model}", "referenceSolution": {"value": ""}}, + "configuration": {"params": {"model": model, "word_count": wc}}, + } + cases.append(Case( + label=f"{short}.w{wc}.p{i}", + model=f"{model}/w{wc}", + body=json.dumps(body).encode(), + )) + return cases + + def is_success_body(data) -> bool: """The endpoint returns a JSON list of feedback items on success, e.g. [{"awardedPoints": 1, "message": "...", ...}]. Also accept the toolkit's @@ -227,13 +264,23 @@ def main() -> int: ap.add_argument("--duration", type=float, help="run each stage for N seconds instead of --requests") ap.add_argument("--ramp", help="comma-separated concurrency stages, e.g. 1,5,10,25,50") ap.add_argument("--model", action="append", help="only use cases for this model (repeatable)") + ap.add_argument("--synthetic", action="append", + help="add built-in prompt cases for a model not in the set export, e.g. smollm2_infer " + "(repeatable; set-export cases are then only included if --model is given)") + ap.add_argument("--word-counts", default="16,64,256", + help="word_count values for --synthetic cases (default: 16,64,256)") ap.add_argument("--timeout", type=float, default=60, help="per-request timeout in seconds") ap.add_argument("--no-warmup", action="store_true", help="skip the sequential warm-up pass") ap.add_argument("--dry-run", action="store_true", help="print request bodies and exit") ap.add_argument("--out", help="write per-request results to this CSV") args = ap.parse_args() - cases = load_cases(args.set_dir, args.model) + cases = [] + if args.model or not args.synthetic: + cases += load_cases(args.set_dir, args.model) + if args.synthetic: + word_counts = [int(x) for x in args.word_counts.split(",")] + cases += synthetic_cases(args.synthetic, word_counts) if not cases: print(f"No langModels cases found in {args.set_dir}", file=sys.stderr) return 1