Submission · qwen35-27b
4632bca02a3846d5
by trial2-27b (participant) · dev split · runner kubernetes
NVFP4+FP8 head+INT4 emb+fp16 GDN state+seqs64-trial2-27b lowbarena-trial-27b-v1
W4A4KV8
Pipeline
- build done
- contract done
- quality done
- efficiency done
- score skipped
Result
0.000LowBScore on the development split (never ranked)
contract: limited to a hard cap of its interactive peak plus 2% (22.653 GiB with room for its CUDA context), the engine neither held its requests at once nor kept 90% of its rate
| W·A·KV | W4A4KV8 declared · the service reads W16A16KV8 |
| Accuracy | tier A · mean drop — points · worst domain knowledge (upper bound 2.9) |
| Speed gain | — BF16 in the same pod |
| Memory gain | — BF16's peak · — GiB against — GiB on the first workload |
| Contract | passed, 4.91 bits per weight stored |
| GPU-hours | 1.75 |
Raw record
{
"agent": "trial2-27b",
"created_at": "2026-10-13T07:18:52Z",
"error": null,
"id": "4632bca02a3846d5",
"participant": "trial2-27b",
"partition": "participant",
"result": {
"anchor_efficiency": {
"interactive": 94.00846698278934,
"long_context": 318.0804228920214,
"serve": 850.589748810954
},
"anchor_peak_memory": {
"interactive": {
"device_peak_gib": 161.798,
"kv_bytes_per_token": 200806,
"kv_gib": 0.239,
"kv_usage": 0.0023,
"other_gib": 4.555,
"peak_gib": 55.015,
"weights_gib": 50.22
},
"long_context": {
"device_peak_gib": 161.8,
"kv_bytes_per_token": 75895,
"kv_gib": 73.537,
"kv_usage": 0.6913,
"other_gib": 4.555,
"peak_gib": 128.313,
"weights_gib": 50.22
},
"serve": {
"device_peak_gib": 161.798,
"kv_bytes_per_token": 88591,
"kv_gib": 22.98,
"kv_usage": 0.216,
"other_gib": 4.555,
"peak_gib": 77.756,
"weights_gib": 50.22
}
},
"contract": {
"activation_bits": 16,
"bits_per_weight": 4.914912481581198,
"foreign_tensors": {
"bytes": 0,
"count": 0,
"largest": [],
"share": 0.0
},
"format": "bf16",
"kv_bits": null,
"model_bits_per_weight": 5.062876869446282,
"ok": true,
"quantization": {
"activations": null,
"format": "bf16",
"ignore": [
"model.language_model.layers.0.linear_attn.conv1d",
"model.language_model.layers.0.linear_attn.in_proj_a",
"model.language_model.layers.0.linear_attn.in_proj_b",
"model.language_model.layers.1.linear_attn.conv1d",
"model.language_model.layers.1.linear_attn.in_proj_a",
"model.language_model.layers.1.linear_attn.in_proj_b",
"model.language_model.layers.10.linear_attn.conv1d",
"model.language_model.layers.10.linear_attn.in_proj_a",
"model.language_model.layers.10.linear_attn.in_proj_b",
"model.language_model.layers.12.linear_attn.conv1d",
"model.language_model.layers.12.linear_attn.in_proj_a",
"model.language_model.layers.12.linear_attn.in_proj_b",
"model.language_model.layers.13.linear_attn.conv1d",
"model.language_model.layers.13.linear_attn.in_proj_a",
"model.language_model.layers.13.linear_attn.in_proj_b",
"model.language_model.layers.14.linear_attn.conv1d",
"model.language_model.layers.14.linear_attn.in_proj_a",
"model.language_model.layers.14.linear_attn.in_proj_b",
"model.language_model.layers.16.linear_attn.conv1d",
"model.language_model.layers.16.linear_attn.in_proj_a",
"model.language_model.layers.16.linear_attn.in_proj_b",
"model.language_model.layers.17.linear_attn.conv1d",
"model.language_model.layers.17.linear_attn.in_proj_a",
"model.language_model.layers.17.linear_attn.in_proj_b",
"model.language_model.layers.18.linear_attn.conv1d",
"model.language_model.layers.18.linear_attn.in_proj_a",
"model.language_model.layers.18.linear_attn.in_proj_b",
"model.language_model.layers.2.linear_attn.conv1d",
"model.language_model.layers.2.linear_attn.in_proj_a",
"model.language_model.layers.2.linear_attn.in_proj_b",
"model.language_model.layers.20.linear_attn.conv1d",
"model.language_model.layers.20.linear_attn.in_proj_a",
"model.language_model.layers.20.linear_attn.in_proj_b",
"model.language_model.layers.21.linear_attn.conv1d",
"model.language_model.layers.21.linear_attn.in_proj_a",
"model.language_model.layers.21.linear_attn.in_proj_b",
"model.language_model.layers.22.linear_attn.conv1d",
"model.language_model.layers.22.linear_attn.in_proj_a",
"model.language_model.layers.22.linear_attn.in_proj_b",
"model.language_model.layers.24.linear_attn.conv1d",
"model.language_model.layers.24.linear_attn.in_proj_a",
"model.language_model.layers.24.linear_attn.in_proj_b",
"model.language_model.layers.25.linear_attn.conv1d",
"model.language_model.layers.25.linear_attn.in_proj_a",
"model.language_model.layers.25.linear_attn.in_proj_b",
"model.language_model.layers.26.linear_attn.conv1d",
"model.language_model.layers.26.linear_attn.in_proj_a",
"model.language_model.layers.26.linear_attn.in_proj_b",
"model.language_model.layers.28.linear_attn.conv1d",
"model.language_model.layers.28.linear_attn.in_proj_a",
"model.language_model.layers.28.linear_attn.in_proj_b",
"model.language_model.layers.29.linear_attn.conv1d",
"model.language_model.layers.29.linear_attn.in_proj_a",
"model.language_model.layers.29.linear_attn.in_proj_b",
"model.language_model.layers.30.linear_attn.conv1d",
"model.language_model.layers.30.linear_attn.in_proj_a",
"model.language_model.layers.30.linear_attn.in_proj_b",
"model.language_model.layers.32.linear_attn.conv1d",
"model.language_model.layers.32.linear_attn.in_proj_a",
"model.language_model.layers.32.linear_attn.in_proj_b",
"model.language_model.layers.33.linear_attn.conv1d",
"model.language_model.layers.33.linear_attn.in_proj_a",
"model.language_model.layers.33.linear_attn.in_proj_b",
"model.language_model.layers.34.linear_attn.conv1d",
"model.language_model.layers.34.linear_attn.in_proj_a",
"model.language_model.layers.34.linear_attn.in_proj_b",
"model.language_model.layers.36.linear_attn.conv1d",
"model.language_model.layers.36.linear_attn.in_proj_a",
"model.language_model.layers.36.linear_attn.in_proj_b",
"model.language_model.layers.37.linear_attn.conv1d",
"model.language_model.layers.37.linear_attn.in_proj_a",
"model.language_model.layers.37.linear_attn.in_proj_b",
"model.language_model.layers.38.linear_attn.conv1d",
"model.language_model.layers.38.linear_attn.in_proj_a",
"model.language_model.layers.38.linear_attn.in_proj_b",
"model.language_model.layers.4.linear_attn.conv1d",
"model.language_model.layers.4.linear_attn.in_proj_a",
"model.language_model.layers.4.linear_attn.in_proj_b",
"model.language_model.layers.40.linear_attn.conv1d",
"model.language_model.layers.40.linear_attn.in_proj_a",
"model.language_model.layers.40.linear_attn.in_proj_b",
"model.language_model.layers.41.linear_attn.conv1d",
"model.language_model.layers.41.linear_attn.in_proj_a",
"model.language_model.layers.41.linear_attn.in_proj_b",
"model.language_model.layers.42.linear_attn.conv1d",
"model.language_model.layers.42.linear_attn.in_proj_a",
"model.language_model.layers.42.linear_attn.in_proj_b",
"model.language_model.layers.44.linear_attn.conv1d",
"model.language_model.layers.44.linear_attn.in_proj_a",
"model.language_model.layers.44.linear_attn.in_proj_b",
"model.language_model.layers.45.linear_attn.conv1d",
"model.language_model.layers.45.linear_attn.in_proj_a",
"model.language_model.layers.45.linear_attn.in_proj_b",
"model.language_model.layers.46.linear_attn.conv1d",
"model.language_model.layers.46.linear_attn.in_proj_a",
"model.language_model.layers.46.linear_attn.in_proj_b",
"model.language_model.layers.48.linear_attn.conv1d",
"model.language_model.layers.48.linear_attn.in_proj_a",
"model.language_model.layers.48.linear_attn.in_proj_b",
"model.language_model.layers.49.linear_attn.conv1d",
"model.language_model.layers.49.linear_attn.in_proj_a",
"model.language_model.layers.49.linear_attn.in_proj_b",
"model.language_model.layers.5.linear_attn.conv1d",
"model.language_model.layers.5.linear_attn.in_proj_a",
"model.language_model.layers.5.linear_attn.in_proj_b",
"model.language_model.layers.50.linear_attn.conv1d",
"model.language_model.layers.50.linear_attn.in_proj_a",
"model.language_model.layers.50.linear_attn.in_proj_b",
"model.language_model.layers.52.linear_attn.conv1d",
"model.language_model.layers.52.linear_attn.in_proj_a",
"model.language_model.layers.52.linear_attn.in_proj_b",
"model.language_model.layers.53.linear_attn.conv1d",
"model.language_model.layers.53.linear_attn.in_proj_a",
"model.language_model.layers.53.linear_attn.in_proj_b",
"model.language_model.layers.54.linear_attn.conv1d",
"model.language_model.layers.54.linear_attn.in_proj_a",
"model.language_model.layers.54.linear_attn.in_proj_b",
"model.language_model.layers.56.linear_attn.conv1d",
"model.language_model.layers.56.linear_attn.in_proj_a",
"model.language_model.layers.56.linear_attn.in_proj_b",
"model.language_model.layers.57.linear_attn.conv1d",
"model.language_model.layers.57.linear_attn.in_proj_a",
"model.language_model.layers.57.linear_attn.in_proj_b",
"model.language_model.layers.58.linear_attn.conv1d",
"model.language_model.layers.58.linear_attn.in_proj_a",
"model.language_model.layers.58.linear_attn.in_proj_b",
"model.language_model.layers.6.linear_attn.conv1d",
"model.language_model.layers.6.linear_attn.in_proj_a",
"model.language_model.layers.6.linear_attn.in_proj_b",
"model.language_model.layers.60.linear_attn.conv1d",
"model.language_model.layers.60.linear_attn.in_proj_a",
"model.language_model.layers.60.linear_attn.in_proj_b",
"model.language_model.layers.61.linear_attn.conv1d",
"model.language_model.layers.61.linear_attn.in_proj_a",
"model.language_model.layers.61.linear_attn.in_proj_b",
"model.language_model.layers.62.linear_attn.conv1d",
"model.language_model.layers.62.linear_attn.in_proj_a",
"model.language_model.layers.62.linear_attn.in_proj_b",
"model.language_model.layers.8.linear_attn.conv1d",
"model.language_model.layers.8.linear_attn.in_proj_a",
"model.language_model.layers.8.linear_attn.in_proj_b",
"model.language_model.layers.9.linear_attn.conv1d",
"model.language_model.layers.9.linear_attn.in_proj_a",
"model.language_model.layers.9.linear_attn.in_proj_b",
"model.visual*",
"mtp*"
],
"kv_quantized": false,
"sparse": false,
"weights": {}
},
"rope_artifact": {
"rope_theta": 10000000,
"rope_type": "default"
},
"rope_source": {
"rope_theta": 10000000,
"rope_type": "default"
},
"safetensor_bytes": 17494111736,
"sparse": false,
"violations": [],
"weight_bits": 16
},
"dev_quality": {
"domains": {
"code": {
"bf16": 0.3053977272727273,
"bf16_se": 0.04707248778770569,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.002840909090909091,
"headroom": 0.3053977272727273,
"items": 88,
"quantized": 0.3025568181818182,
"recovery": 0.9906976744186046,
"recovery_se": 0.035611232255291926,
"se": 0.010875589396147392,
"status": "valid",
"upper": 0.020731253647571554
},
"commonsense": {
"bf16": 0.93,
"bf16_se": 0.01256199786707855,
"candidate_lost": 0.0,
"chance": 0.24983333333333332,
"drop": 0.006666666666666667,
"format_contract": {
"bf16": 1.0,
"quantized": 1.0
},
"headroom": 0.6801666666666667,
"items": 300,
"quantized": 0.9233333333333333,
"recovery": 0.9901984807645184,
"recovery_se": 0.0079331799781169,
"se": 0.005395884581782512,
"status": "valid",
"upper": 0.0155428968036989
},
"expert": {
"bf16": 0.5375,
"bf16_se": 0.07095155786322377,
"candidate_lost": 0.0,
"chance": 0.25,
"drop": 0.029166666666666667,
"format_contract": {
"bf16": 1.0,
"quantized": 1.0
},
"headroom": 0.2875,
"items": 30,
"quantized": 0.5083333333333333,
"recovery": 0.8985507246376812,
"recovery_se": 0.09928439024089819,
"se": 0.02854426219425823,
"status": "R5_diagnostic",
"upper": 0.07612197797622144
},
"knowledge": {
"bf16": 0.64,
"bf16_se": 0.02515171357287956,
"candidate_lost": 0.0,
"chance": 0.10940343915343916,
"drop": 0.010833333333333334,
"format_contract": {
"bf16": 1.0,
"quantized": 0.9975
},
"headroom": 0.5305965608465608,
"items": 300,
"quantized": 0.6291666666666667,
"recovery": 0.9795827298314017,
"recovery_se": 0.021248849509686706,
"se": 0.011274566471785897,
"status": "valid",
"upper": 0.029379995179421133
},
"long_context": {
"bf16": 0.91575,
"bf16_se": 0.021385410644641716,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.0015000000000000002,
"headroom": 0.91575,
"items": 100,
"quantized": 0.91425,
"recovery": 0.9983619983619985,
"recovery_se": 0.006729980956596471,
"se": 0.006162980061003218,
"status": "valid",
"upper": 0.011638102200350295
},
"math": {
"bf16": 0.8125,
"bf16_se": 0.026330786192355474,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.00375,
"headroom": 0.8125,
"items": 200,
"quantized": 0.80875,
"recovery": 0.9953846153846153,
"recovery_se": 0.009870259436780287,
"se": 0.008019585792383982,
"status": "valid",
"upper": 0.01694221862847165
}
},
"evaluations": 1,
"first_quality": null,
"length": {
"length_generations": 4544,
"length_shift": 0.012287862015106343,
"length_shift_se": 0.014053879155640665,
"length_tokens": 172.8661971830986,
"length_tokens_bf16": 170.76782570422534
},
"recovery": 0.9908450997522277,
"recovery_se": 0.00877570163079022,
"reference_rows": "/work/results/qwen35-27b/bf16",
"tier": "A",
"valid_domains": 5,
"worst_domain": "knowledge",
"worst_drop": 0.010833333333333334,
"worst_upper": 0.029379995179421133
},
"dev_score": {
"bf16_peak_gib": null,
"drift": null,
"efficiency_gain": null,
"mean_drop": null,
"memories": null,
"memory": null,
"memory_saved": null,
"peak_gib": null,
"precision": {
"activations": 16,
"kv": 8,
"label": "W16A16KV8",
"weights": 16,
"weights_average": 4.91
},
"qualified": false,
"reason": "contract: limited to a hard cap of its interactive peak plus 2% (22.653 GiB with room for its CUDA context), the engine neither held its requests at once nor kept 90% of its rate",
"score": 0.0,
"speed": null,
"speeds": null
},
"efficiency": {
"interactive": 159.7520544853387,
"long_context": 688.5787543599912,
"serve": 1761.9888088549617
},
"engine": {
"audit": null,
"budget": {
"interactive@1.02": {
"budget_gib": 18.653,
"cap_gib": 22.653,
"factor": 1.02,
"full_running": 1.0,
"healthy": false,
"running": 0,
"value": null
},
"long_context@1.02": {
"budget_gib": 56.791,
"cap_gib": 60.791,
"factor": 1.02,
"full_running": 64.0,
"healthy": true,
"running": 64.0,
"value": 689.4051414879292
},
"serve@1.02": {
"budget_gib": 29.688,
"cap_gib": 33.688,
"factor": 1.02,
"full_running": 32.0,
"healthy": true,
"running": 32.0,
"value": 1774.04493525793
}
},
"hard_cap": true,
"startup": {
"cuda_graph_gib": 0.29,
"engine_init_seconds": 993.38,
"kv_bytes_per_token": 35205.23925122021,
"kv_cache_dtype": "fp8",
"kv_cache_gib": 142.99,
"kv_cache_tokens": 4361122.0,
"max_concurrency": {
"sequences": 133.09,
"tokens_per_request": 32768
},
"speculative_config": "None",
"vllm_version": "0.1.dev18498+g7a33ba95f",
"weights_gib": 15.05
},
"stored_gib": 16.293
},
"gpu_hours": 1.751,
"kl": {
"kl": 0.033644429275141136,
"kl_median": 0.034312587795444716,
"kl_per_token": 0.033695340864255865,
"kl_se": 0.0022542627556029916,
"sequences": 64,
"tokens": 16088
},
"peak_memory": {
"interactive": {
"device_peak_gib": 162.939,
"kv_bytes_per_token": 102433,
"kv_gib": 0.122,
"kv_usage": 0.0009,
"other_gib": 3.115,
"peak_gib": 18.287,
"weights_gib": 15.05
},
"long_context": {
"device_peak_gib": 162.939,
"kv_bytes_per_token": 38715,
"kv_gib": 37.512,
"kv_usage": 0.2623,
"other_gib": 3.115,
"peak_gib": 55.677,
"weights_gib": 15.05
},
"serve": {
"device_peak_gib": 162.939,
"kv_bytes_per_token": 42178,
"kv_gib": 10.941,
"kv_usage": 0.0765,
"other_gib": 3.115,
"peak_gib": 29.106,
"weights_gib": 15.05
}
},
"split": "dev",
"workload": {
"bf16": {
"peak_gib": 77.756,
"rate": 850.589748810954
},
"submission": {
"peak_gib": 29.106,
"rate": 1761.9888088549617
}
}
},
"runner": "kubernetes",
"solution": {
"checkpoint": "lowbarena-trial-27b-v1",
"declared_precision": true,
"detected_precision": "W16A16KV8",
"method": "NVFP4+FP8 head+INT4 emb+fp16 GDN state+seqs64-trial2-27b",
"precision": "W4A4KV8"
},
"split": "dev",
"stages": [
{
"name": "build",
"status": "done"
},
{
"name": "contract",
"status": "done"
},
{
"name": "quality",
"status": "done"
},
{
"name": "efficiency",
"status": "done"
},
{
"name": "score",
"status": "skipped"
}
],
"status": "done",
"task_id": "qwen35-27b",
"track": "engine",
"updated_at": "2026-10-13T09:07:37Z"
}