#!/usr/bin/env python3
"""Repro: llama-server cross-request response mix-up under -np 4 --kv-unified.
Each round: (B) short "repeat exactly: <nonce>" request, (C) large-prompt
(~10k tokens) request whose system prompt requires echoing a per-request nonce,
(D) burst of 4 concurrent short nonce requests (2 streamed / 2 plain).
Every response must contain its OWN nonce and no nonce of an EARLIER request.
Usage: python3 repro-crossover.py --url http://127.0.0.1:8080 [--rounds 8]
Requires: pip install httpx
"""
import argparse, asyncio, json, random, string, sys, time
import httpx
FILLER = [
"Quantized MoE models reach higher aggregate throughput under continuous batching.",
"The KV cache grows linearly with context length and dominates memory for long prompts.",
"Speculative decoding only pays off when the draft acceptance rate is high enough.",
"A reranker only improves retrieval when the candidate set is broad enough.",
]
def big_prompt(seed, chars=30000):
rnd = random.Random(seed)
lines = [f"## Research block {seed}"]
while sum(len(l) for l in lines) < chars:
tag = "".join(rnd.choices(string.ascii_lowercase + string.digits, k=8))
lines.append(f"- [{tag}] {rnd.choice(FILLER)}")
lines.append("\nSummarize the key points in 5 sentences.")
return "\n".join(lines)
seen, violations, checks = [], [], 0
def check(phase, rnd, nonce, text):
global checks
checks += 1
stem = nonce.rsplit("-", 1)[0]
problems = []
if nonce not in text and not (phase == "C" and stem in text):
problems.append("own_nonce_missing")
problems += [f"REPLAY_of:{o}" for o in seen if o != nonce and o in text]
seen.append(nonce)
if problems:
violations.append((rnd, phase, problems, text[:120]))
print(f" VIOLATION R{rnd}/{phase}: {problems} got={text[:80]!r}", flush=True)
async def echo_req(c, url, model, nonce, stream=False):
p = {"model": model, "max_tokens": 40, "temperature": 0,
"messages": [{"role": "user", "content": f"Repeat back exactly this code and nothing else: {nonce}"}]}
if not stream:
r = await c.post(f"{url}/v1/chat/completions", json=p)
return r.json()["choices"][0]["message"]["content"]
p["stream"] = True
out = []
async with c.stream("POST", f"{url}/v1/chat/completions", json=p) as r:
async for line in r.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
try: out.append(json.loads(line[6:])["choices"][0]["delta"].get("content") or "")
except Exception: pass
return "".join(out)
async def synth_req(c, url, model, nonce, prompt):
sysmsg = f"The FIRST line of your reply must be exactly `<!-- run:{nonce} -->`."
r = await c.post(f"{url}/v1/chat/completions", json={
"model": model, "max_tokens": 600,
"messages": [{"role": "system", "content": sysmsg},
{"role": "user", "content": sysmsg + "\n\n" + prompt}]})
return r.json()["choices"][0]["message"]["content"]
async def main():
ap = argparse.ArgumentParser()
ap.add_argument("--url", default="http://127.0.0.1:8080")
ap.add_argument("--model", default="default")
ap.add_argument("--rounds", type=int, default=8)
a = ap.parse_args()
ts = time.strftime("%H%M%S")
async with httpx.AsyncClient(timeout=600.0) as c:
for rnd in range(1, a.rounds + 1):
bp = big_prompt("cachehit" if rnd % 2 == 0 else f"uniq-{ts}-{rnd}")
nb = f"pb-{ts}-{rnd}-b"
check("B", rnd, nb, await echo_req(c, a.url, a.model, nb))
nc = f"pc-{ts}-{rnd}-c"
check("C", rnd, nc, await synth_req(c, a.url, a.model, nc, bp))
nd = [f"pd-{ts}-{rnd}-d{i}" for i in range(4)]
res = await asyncio.gather(*[echo_req(c, a.url, a.model, n, stream=(i % 2 == 0))
for i, n in enumerate(nd)], return_exceptions=True)
for n, r in zip(nd, res):
if not isinstance(r, Exception):
check("D", rnd, n, r)
print(f"round {rnd} done — violations so far: {len(violations)}/{checks}", flush=True)
print(f"\nRESULT: {len(violations)} violations / {checks} validated responses")
return 1 if violations else 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
Summary
Under parallel mixed load (
-np 4 --kv-unified),llama-serveron an integrated HIP GPU (Strix Halo / gfx1151, 128 GB UMA) returns complete responses that verbatim belong to a different, earlier request. A large-prompt request receives, as its entire response, the literal answer text of a short request from an earlier round. We also observed a chimeric response fusing tokens of two different requests' expected outputs.Bisected to a single commit: c7d8722 "ggml-cuda : restore prop.integrated on HIP builds" (#24233).
The same harness against CPU builds of both window endpoints shows no build-dependent difference (model-compliance noise only) — consistent with the culprit being the HIP integrated-memory path. This presumably explains why it went unnoticed: discrete GPUs are unaffected; it needs an integrated AMD GPU (
prop.integrated == 1), multiple slots and concurrent mixed traffic.Environment
cmake -DGGML_HIP=ON -DAMDGPU_TARGETS=gfx1151 -DGPU_TARGETS=gfx1151 -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++(Release)llama-server -m <model> -ngl 999 -c 65536 -np 4 --kv-unified --jinja --chat-template-kwargs '{"enable_thinking":false}'llama-serverrules the wrapper out.Reproduction
Load pattern per round (~30 s, script below): (B) short "repeat exactly this code: " request → (C) large-prompt (~10k tokens) request whose system prompt requires echoing a per-request nonce in the first line → (D) burst of 4 concurrent short nonce requests (2 streamed / 2 plain). Every response is validated to contain its own nonce and no nonce of an earlier request.
On affected builds the round-N large-prompt request receives a literal foreign nonce as its complete response, typically from a round-(N−1) burst request. Verbatim from run logs at e8f19cc:
In production we additionally observed: a poisoned slot re-serving its stored answer to unrelated requests 20+ minutes later (the victim's
usagethen reports the foreign request's token counts), and occasional degeneration into repeated single characters (/////...).repro-crossover.py (needs httpx)
Happy to run additional experiments on this hardware (it reproduces deterministically within 2–4 rounds).