Submission · qwen35-27b
275d38f89a794f8b
by trial2-27b (participant) · test split · runner kubernetes
NVFP4+FP8 head+INT4 emb+fp16 GDN state+cap-safe serving-trial2-27b lowbarena-trial-27b-v1
W4A4KV8
Pipeline
- build done
- contract done
- quality done
- efficiency done
- score done
Result
0.000LowBScore
contract: limited to a hard cap of its interactive peak plus 2% (22.531 GiB with room for its CUDA context), the engine neither held its requests at once nor kept 90% of its rate; limited to a hard cap of its serve peak plus 2% (33.576 GiB with room for its CUDA context), the engine neither held its requests at once nor kept 90% of its rate
| W·A·KV | W4A4KV8 declared · the service reads W16A16KV8 |
| Drift from BF16 | KL 0.027 nats per token |
| Speed gain | — BF16 in the same pod (gate: at least 0.95× less its error) |
| Memory gain | — BF16's peak · — GiB against BF16's — GiB on the first workload |
| Per-domain results on the hidden split are not released. | |
Engine: 15.1 GiB of weights held, 16.3 GiB stored.
3.16 GPU-hours
Raw record
{
"agent": "trial2-27b",
"created_at": "2026-10-13T16:47:05Z",
"error": null,
"id": "275d38f89a794f8b",
"participant": "trial2-27b",
"partition": "participant",
"result": {
"admitted": false,
"anchor_efficiency": {
"interactive": 93.8947337219063,
"long_context": 324.2982599954284,
"serve": 864.9791839542864
},
"anchor_peak_memory": {
"interactive": {
"device_peak_gib": 161.798,
"kv_bytes_per_token": 200806,
"kv_gib": 0.239,
"kv_usage": 0.0023,
"other_gib": 4.555,
"peak_gib": 55.015,
"weights_gib": 50.22
},
"long_context": {
"device_peak_gib": 161.8,
"kv_bytes_per_token": 75895,
"kv_gib": 73.537,
"kv_usage": 0.6913,
"other_gib": 4.555,
"peak_gib": 128.313,
"weights_gib": 50.22
},
"serve": {
"device_peak_gib": 161.798,
"kv_bytes_per_token": 88591,
"kv_gib": 22.98,
"kv_usage": 0.216,
"other_gib": 4.555,
"peak_gib": 77.756,
"weights_gib": 50.22
}
},
"efficiency": {
"interactive": 142.24113462794793,
"long_context": 692.8238915400858,
"serve": 1710.1517371777927
},
"engine": {
"audit": {
"interactive": {
"bf16_mean": -0.16084521190991544,
"bf16_top1": 0.9899944453130497,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.490333145200283,
"engine_top1": 0.8777954005480206,
"gap": 0.3294879332903675,
"gap_max": 1.8505635789404375,
"gap_median": 0.1823480744366915,
"length_ratio": 1.0,
"length_ratio_min": 0.515625,
"picked": 24,
"run": "interactive-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.11219904476502907
},
"long_context": {
"bf16_mean": -0.1567404825290429,
"bf16_top1": 0.9938340233073568,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.3320344422672976,
"engine_top1": 0.9283703330616352,
"gap": 0.1752939597382547,
"gap_max": 0.7798232334974955,
"gap_median": 0.0954482725686816,
"length_ratio": 1.0,
"length_ratio_min": 0.98046875,
"picked": 24,
"run": "long_context-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.06546369024572163
},
"memory_capped": {
"status": "missing"
},
"offline": {
"status": "missing"
},
"online_slo": {
"status": "missing"
},
"serve": {
"bf16_mean": -0.20096103407553478,
"bf16_top1": 0.9845974282328177,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.225946009616939,
"engine_top1": 0.9446232380762856,
"gap": 0.024984975541404213,
"gap_max": 0.2857234773704345,
"gap_median": 0.0340169900407119,
"length_ratio": 1.0,
"length_ratio_min": 0.986328125,
"picked": 24,
"run": "serve-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.03997419015653214
}
},
"budget": {
"interactive@1.02": {
"budget_gib": 18.531,
"cap_gib": 22.531,
"factor": 1.02,
"full_running": 1.0,
"healthy": true,
"running": 0,
"value": 125.61338090883974
},
"long_context@1.02": {
"budget_gib": 56.701,
"cap_gib": 60.701,
"factor": 1.02,
"full_running": 64.0,
"healthy": true,
"running": 0,
"value": 661.3282093562074
},
"serve@1.02": {
"budget_gib": 29.576,
"cap_gib": 33.576,
"factor": 1.02,
"full_running": 32.0,
"healthy": true,
"running": 0,
"value": 1518.4211883888443
}
},
"hard_cap": true,
"startup": {
"cuda_graph_gib": 0.29,
"engine_init_seconds": 100.07,
"kv_bytes_per_token": 35204.72255668127,
"kv_cache_dtype": "fp8",
"kv_cache_gib": 143.11,
"kv_cache_tokens": 4364846.0,
"max_concurrency": {
"sequences": 133.2,
"tokens_per_request": 32768
},
"speculative_config": "None",
"vllm_version": "0.1.dev18498+g7a33ba95f",
"weights_gib": 15.05
},
"stored_gib": 16.293
},
"gpu_hours": 3.164,
"kl": {
"kl": 0.026833965750123666,
"kl_median": 0.022971409568917217,
"kl_per_token": 0.026815486365564845,
"kl_se": 0.0011877508403167513,
"ppl_error": "RuntimeError: HTTP 500: {\"error\":{\"message\":\"EngineCore encountered an issue. See stack trace (above) for the root cause.\",\"type\":\"InternalServerError\",\"param\":null,\"code\":500}}",
"sequences": 256,
"tokens": 63668
},
"metric": "lowbit_score",
"peak_memory": {
"interactive": {
"device_peak_gib": 162.939,
"kv_bytes_per_token": 102519,
"kv_gib": 0.122,
"kv_usage": 0.0009,
"other_gib": 2.995,
"peak_gib": 18.168,
"weights_gib": 15.05
},
"long_context": {
"device_peak_gib": 162.939,
"kv_bytes_per_token": 38747,
"kv_gib": 37.543,
"kv_usage": 0.2623,
"other_gib": 2.995,
"peak_gib": 55.589,
"weights_gib": 15.05
},
"serve": {
"device_peak_gib": 162.939,
"kv_bytes_per_token": 42214,
"kv_gib": 10.95,
"kv_usage": 0.0765,
"other_gib": 2.995,
"peak_gib": 28.996,
"weights_gib": 15.05
}
},
"precision": {
"activations": 16,
"kv": 8,
"label": "W16A16KV8",
"weights": 16,
"weights_average": 4.91
},
"qualified": false,
"reason": "contract: limited to a hard cap of its interactive peak plus 2% (22.531 GiB with room for its CUDA context), the engine neither held its requests at once nor kept 90% of its rate; limited to a hard cap of its serve peak plus 2% (33.576 GiB with room for its CUDA context), the engine neither held its requests at once nor kept 90% of its rate",
"score": 0.0,
"task_id": "qwen35-27b",
"timed_audit": {
"interactive": {
"bf16_mean": -0.16084521190991544,
"bf16_top1": 0.9899944453130497,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.490333145200283,
"engine_top1": 0.8777954005480206,
"gap": 0.3294879332903675,
"gap_max": 1.8505635789404375,
"gap_median": 0.1823480744366915,
"length_ratio": 1.0,
"length_ratio_min": 0.515625,
"picked": 24,
"run": "interactive-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.11219904476502907
},
"long_context": {
"bf16_mean": -0.1567404825290429,
"bf16_top1": 0.9938340233073568,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.3320344422672976,
"engine_top1": 0.9283703330616352,
"gap": 0.1752939597382547,
"gap_max": 0.7798232334974955,
"gap_median": 0.0954482725686816,
"length_ratio": 1.0,
"length_ratio_min": 0.98046875,
"picked": 24,
"run": "long_context-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.06546369024572163
},
"memory_capped": {
"status": "missing"
},
"offline": {
"status": "missing"
},
"online_slo": {
"status": "missing"
},
"serve": {
"bf16_mean": -0.20096103407553478,
"bf16_top1": 0.9845974282328177,
"empty": 0,
"empty_alone": 0,
"empty_reference": 0,
"engine_mean": -0.225946009616939,
"engine_top1": 0.9446232380762856,
"gap": 0.024984975541404213,
"gap_max": 0.2857234773704345,
"gap_median": 0.0340169900407119,
"length_ratio": 1.0,
"length_ratio_min": 0.986328125,
"picked": 24,
"run": "serve-r0.json",
"samples": 24,
"status": "audited",
"top1_gap": 0.03997419015653214
}
}
},
"runner": "kubernetes",
"solution": {
"checkpoint": "lowbarena-trial-27b-v1",
"declared_precision": true,
"detected_precision": "W16A16KV8",
"method": "NVFP4+FP8 head+INT4 emb+fp16 GDN state+cap-safe serving-trial2-27b",
"precision": "W4A4KV8"
},
"split": "test",
"stages": [
{
"name": "build",
"status": "done"
},
{
"name": "contract",
"status": "done"
},
{
"name": "quality",
"status": "done"
},
{
"name": "efficiency",
"status": "done"
},
{
"name": "score",
"status": "done"
}
],
"status": "done",
"task_id": "qwen35-27b",
"track": "engine",
"updated_at": "2026-10-13T20:01:56Z"
}