Submission · qwen3-8b
b9b6f9c6034a4949
by Red-team engine: skips 2 of 36 layers (reference) · test split · runner kubernetes
redteam-skip-2-layers Qwen3-8B-FP8-Dynamic
W8A8KV16
Pipeline
- build done
- contract done
- quality done
- efficiency done
- score done
Result
0.000LowBScore
contract: KL from BF16 of 0.20 nats per token, above 0.15: not a quantization of the task's model
| W·A·KV | W8A8KV16 declared · the service reads W8A8KV16 |
| Drift from BF16 | KL 0.202 nats per token |
| Speed gain | — BF16 in the same pod (gate: at least 0.95× less its error) |
| Memory gain | — BF16's peak · — GiB against BF16's — GiB on the first workload |
| Per-domain results on the hidden split are not released. | |
Engine: 8.5 GiB of weights held, 8.8 GiB stored.
0.79 GPU-hours
Raw record
{
"agent": "zoo-engine-redteam-skip",
"created_at": "2026-10-11T20:06:33Z",
"error": null,
"id": "b9b6f9c6034a4949",
"participant": "Red-team engine: skips 2 of 36 layers",
"partition": "reference",
"result": {
"admitted": false,
"anchor_efficiency": {
"interactive": 246.25517401299143,
"long_context": 733.6856563342271,
"serve": 1937.6236306833698
},
"anchor_peak_memory": {
"interactive": {
"device_peak_gib": 163.132,
"kv_bytes_per_token": 147464,
"kv_gib": 0.176,
"kv_usage": 0.0012,
"other_gib": 2.145,
"peak_gib": 17.591,
"weights_gib": 15.27
},
"long_context": {
"device_peak_gib": 163.132,
"kv_bytes_per_token": 146929,
"kv_gib": 142.364,
"kv_usage": 0.9904,
"other_gib": 2.145,
"peak_gib": 159.78,
"weights_gib": 15.27
},
"serve": {
"device_peak_gib": 163.132,
"kv_bytes_per_token": 147295,
"kv_gib": 38.208,
"kv_usage": 0.2658,
"other_gib": 2.145,
"peak_gib": 55.624,
"weights_gib": 15.27
}
},
"efficiency": {
"interactive": 323.91633699773575,
"long_context": 910.1095460118087,
"serve": 2386.1128321594756
},
"engine": {
"audit": {
"interactive": {
"bf16_mean": -0.13162121712621935,
"bf16_top1": 0.989544863597613,
"empty": 1,
"empty_alone": 0,
"empty_reference": 1,
"engine_mean": -0.2165623430278814,
"engine_top1": 0.9529551630434783,
"gap": 0.08494112590166204,
"gap_max": 0.5762329556075024,
"gap_median": 0.07069960891711469,
"length_ratio": 1.0,
"length_ratio_min": 0.0,
"picked": 24,
"run": "interactive-r0.json",
"samples": 23,
"status": "audited",
"top1_gap": 0.036589700554134685
},
"long_context": {
"bf16_mean": -0.32080665352398724,
"bf16_top1": 0.9861887845103142,
"empty": 1,
"empty_alone": 1,
"empty_reference": 0,
"engine_mean": -0.24724186092560851,
"engine_top1": 0.9432548874592771,
"gap": -0.07356479259837874,
"gap_max": 0.7224158946139976,
"gap_median": 0.018138986246698685,
"length_ratio": 1.0,
"length_ratio_min": 0.0,
"picked": 24,
"run": "long_context-r0.json",
"samples": 23,
"status": "audited",
"top1_gap": 0.04293389705103712
},
"memory_capped": {
"status": "missing"
},
"offline": {
"status": "missing"
},
"online_slo": {
"status": "missing"
},
"serve": {
"bf16_mean": -0.06717945055129734,
"bf16_top1": 0.9948675221132671,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.128159510677224,
"engine_top1": 0.9647770746405758,
"gap": 0.06098006012592666,
"gap_max": 0.6326763987717743,
"gap_median": 0.01106339192317396,
"length_ratio": 1.0,
"length_ratio_min": 0.50390625,
"picked": 24,
"run": "serve-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.030090447472691295
}
},
"budget": {
"interactive@1.02": {
"budget_gib": 10.364,
"factor": 1.02,
"full_running": 1.0,
"healthy": true,
"running": 1.0,
"value": 347.5832864849371
},
"long_context@1.02": {
"budget_gib": 147.306,
"factor": 1.02,
"full_running": 64.0,
"healthy": true,
"running": 64.0,
"value": 911.7542711595847
},
"serve@1.02": {
"budget_gib": 46.905,
"factor": 1.02,
"full_running": 32.0,
"healthy": true,
"running": 32.0,
"value": 2390.743485824174
}
},
"startup": {
"cuda_graph_gib": 0.41,
"engine_init_seconds": 10.84,
"kv_bytes_per_token": 139268.46719806697,
"kv_cache_dtype": "auto",
"kv_cache_gib": 151.16,
"kv_cache_tokens": 1165424.0,
"load_seconds": 3.009146,
"max_concurrency": {
"sequences": 35.57,
"tokens_per_request": 32768
},
"speculative_config": "None",
"vllm_version": "0.1.dev18498+g7a33ba95f",
"weights_gib": 8.48
},
"stored_gib": 8.79
},
"gpu_hours": 0.787,
"kl": {
"identity": {
"kl": 0.2034248430534346,
"kl_median": 0.19235552083315616,
"kl_per_token": 0.20007630082143477,
"kl_se": 0.00624518203751105,
"sequences": 256,
"tokens": 61987
},
"kl": 0.20241031911857107,
"kl_median": 0.1988011965054395,
"kl_per_token": 0.19945719832503958,
"kl_se": 0.006118412745463271,
"ppl": {
"hidden": {
"dnll": 0.06941989935493911,
"dnll_se": 0.003012534367826989,
"ppl": 16.74152321889948,
"ppl_bf16": 15.618750604937217,
"tokens": 262016,
"windows": 128
},
"wikitext2": {
"dnll": 0.05429154071381028,
"dnll_se": 0.0021520764752293946,
"ppl": 10.264314782792098,
"ppl_bf16": 9.721906631657593,
"tokens": 298862,
"windows": 146
}
},
"sequences": 256,
"tokens": 62197
},
"metric": "lowbit_score",
"peak_memory": {
"interactive": {
"device_peak_gib": 163.497,
"kv_bytes_per_token": 139270,
"kv_gib": 0.166,
"kv_usage": 0.0011,
"other_gib": 1.515,
"peak_gib": 10.161,
"weights_gib": 8.48
},
"long_context": {
"device_peak_gib": 163.497,
"kv_bytes_per_token": 138733,
"kv_gib": 134.423,
"kv_usage": 0.8893,
"other_gib": 1.515,
"peak_gib": 144.418,
"weights_gib": 8.48
},
"serve": {
"device_peak_gib": 163.497,
"kv_bytes_per_token": 138742,
"kv_gib": 35.99,
"kv_usage": 0.2381,
"other_gib": 1.515,
"peak_gib": 45.985,
"weights_gib": 8.48
}
},
"precision": {
"activations": 8,
"kv": 16,
"label": "W8A8KV16",
"weights": 8,
"weights_average": 8.0
},
"qualified": false,
"reason": "contract: KL from BF16 of 0.20 nats per token, above 0.15: not a quantization of the task\u0027s model",
"score": 0.0,
"task_id": "qwen3-8b",
"timed_audit": {
"interactive": {
"bf16_mean": -0.13162121712621935,
"bf16_top1": 0.989544863597613,
"empty": 1,
"empty_alone": 0,
"empty_reference": 1,
"engine_mean": -0.2165623430278814,
"engine_top1": 0.9529551630434783,
"gap": 0.08494112590166204,
"gap_max": 0.5762329556075024,
"gap_median": 0.07069960891711469,
"length_ratio": 1.0,
"length_ratio_min": 0.0,
"picked": 24,
"run": "interactive-r0.json",
"samples": 23,
"status": "audited",
"top1_gap": 0.036589700554134685
},
"long_context": {
"bf16_mean": -0.32080665352398724,
"bf16_top1": 0.9861887845103142,
"empty": 1,
"empty_alone": 1,
"empty_reference": 0,
"engine_mean": -0.24724186092560851,
"engine_top1": 0.9432548874592771,
"gap": -0.07356479259837874,
"gap_max": 0.7224158946139976,
"gap_median": 0.018138986246698685,
"length_ratio": 1.0,
"length_ratio_min": 0.0,
"picked": 24,
"run": "long_context-r0.json",
"samples": 23,
"status": "audited",
"top1_gap": 0.04293389705103712
},
"memory_capped": {
"status": "missing"
},
"offline": {
"status": "missing"
},
"online_slo": {
"status": "missing"
},
"serve": {
"bf16_mean": -0.06717945055129734,
"bf16_top1": 0.9948675221132671,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.128159510677224,
"engine_top1": 0.9647770746405758,
"gap": 0.06098006012592666,
"gap_max": 0.6326763987717743,
"gap_median": 0.01106339192317396,
"length_ratio": 1.0,
"length_ratio_min": 0.50390625,
"picked": 24,
"run": "serve-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.030090447472691295
}
}
},
"runner": "kubernetes",
"solution": {
"checkpoint": "Qwen3-8B-FP8-Dynamic",
"declared_precision": true,
"detected_precision": "W8A8KV16",
"method": "redteam-skip-2-layers",
"precision": "W8A8KV16"
},
"split": "test",
"stages": [
{
"name": "build",
"status": "done"
},
{
"name": "contract",
"status": "done"
},
{
"name": "quality",
"status": "done"
},
{
"name": "efficiency",
"status": "done"
},
{
"name": "score",
"status": "done"
}
],
"status": "done",
"task_id": "qwen3-8b",
"track": "engine",
"updated_at": "2026-10-11T21:00:54Z"
}