Submission · qwen35-27b
820740bcfc5049af
by trial2-27b (participant) · dev split · runner kubernetes
NVFP4-GPTQ+FP8 head+INT4 emb+fp16 GDN state+capfix-trial2-27b lowbarena-trial-27b-v2
W4A4KV8
Pipeline
- build done
- contract done
- quality done
- efficiency done
- score skipped
Result
0.000LowBScore on the development split (never ranked)
contract: the engine returned no log-probabilities for the KL probe
| W·A·KV | W4A4KV8 declared · the service reads W16A16KV8 |
| Accuracy | tier B · mean drop — points · worst domain code (upper bound 4.6) |
| Speed gain | — BF16 in the same pod |
| Memory gain | — BF16's peak · — GiB against — GiB on the first workload |
| Contract | passed, 4.91 bits per weight stored |
| GPU-hours | 3.40 |
Raw record
{
"agent": "trial2-27b",
"created_at": "2026-10-13T10:57:59Z",
"error": null,
"id": "820740bcfc5049af",
"participant": "trial2-27b",
"partition": "participant",
"result": {
"anchor_efficiency": {
"interactive": 93.90008238303317,
"long_context": 321.8997581477662,
"serve": 854.772576603853
},
"anchor_peak_memory": {
"interactive": {
"device_peak_gib": 161.798,
"kv_bytes_per_token": 200806,
"kv_gib": 0.239,
"kv_usage": 0.0023,
"other_gib": 4.555,
"peak_gib": 55.015,
"weights_gib": 50.22
},
"long_context": {
"device_peak_gib": 161.8,
"kv_bytes_per_token": 75895,
"kv_gib": 73.537,
"kv_usage": 0.6913,
"other_gib": 4.555,
"peak_gib": 128.313,
"weights_gib": 50.22
},
"serve": {
"device_peak_gib": 161.798,
"kv_bytes_per_token": 88591,
"kv_gib": 22.98,
"kv_usage": 0.216,
"other_gib": 4.555,
"peak_gib": 77.756,
"weights_gib": 50.22
}
},
"contract": {
"activation_bits": 16,
"bits_per_weight": 4.914912481581198,
"foreign_tensors": {
"bytes": 0,
"count": 0,
"largest": [],
"share": 0.0
},
"format": "bf16",
"kv_bits": null,
"model_bits_per_weight": 5.062876869446282,
"ok": true,
"quantization": {
"activations": null,
"format": "bf16",
"ignore": [
"model.language_model.layers.0.linear_attn.conv1d",
"model.language_model.layers.0.linear_attn.in_proj_a",
"model.language_model.layers.0.linear_attn.in_proj_b",
"model.language_model.layers.1.linear_attn.conv1d",
"model.language_model.layers.1.linear_attn.in_proj_a",
"model.language_model.layers.1.linear_attn.in_proj_b",
"model.language_model.layers.10.linear_attn.conv1d",
"model.language_model.layers.10.linear_attn.in_proj_a",
"model.language_model.layers.10.linear_attn.in_proj_b",
"model.language_model.layers.12.linear_attn.conv1d",
"model.language_model.layers.12.linear_attn.in_proj_a",
"model.language_model.layers.12.linear_attn.in_proj_b",
"model.language_model.layers.13.linear_attn.conv1d",
"model.language_model.layers.13.linear_attn.in_proj_a",
"model.language_model.layers.13.linear_attn.in_proj_b",
"model.language_model.layers.14.linear_attn.conv1d",
"model.language_model.layers.14.linear_attn.in_proj_a",
"model.language_model.layers.14.linear_attn.in_proj_b",
"model.language_model.layers.16.linear_attn.conv1d",
"model.language_model.layers.16.linear_attn.in_proj_a",
"model.language_model.layers.16.linear_attn.in_proj_b",
"model.language_model.layers.17.linear_attn.conv1d",
"model.language_model.layers.17.linear_attn.in_proj_a",
"model.language_model.layers.17.linear_attn.in_proj_b",
"model.language_model.layers.18.linear_attn.conv1d",
"model.language_model.layers.18.linear_attn.in_proj_a",
"model.language_model.layers.18.linear_attn.in_proj_b",
"model.language_model.layers.2.linear_attn.conv1d",
"model.language_model.layers.2.linear_attn.in_proj_a",
"model.language_model.layers.2.linear_attn.in_proj_b",
"model.language_model.layers.20.linear_attn.conv1d",
"model.language_model.layers.20.linear_attn.in_proj_a",
"model.language_model.layers.20.linear_attn.in_proj_b",
"model.language_model.layers.21.linear_attn.conv1d",
"model.language_model.layers.21.linear_attn.in_proj_a",
"model.language_model.layers.21.linear_attn.in_proj_b",
"model.language_model.layers.22.linear_attn.conv1d",
"model.language_model.layers.22.linear_attn.in_proj_a",
"model.language_model.layers.22.linear_attn.in_proj_b",
"model.language_model.layers.24.linear_attn.conv1d",
"model.language_model.layers.24.linear_attn.in_proj_a",
"model.language_model.layers.24.linear_attn.in_proj_b",
"model.language_model.layers.25.linear_attn.conv1d",
"model.language_model.layers.25.linear_attn.in_proj_a",
"model.language_model.layers.25.linear_attn.in_proj_b",
"model.language_model.layers.26.linear_attn.conv1d",
"model.language_model.layers.26.linear_attn.in_proj_a",
"model.language_model.layers.26.linear_attn.in_proj_b",
"model.language_model.layers.28.linear_attn.conv1d",
"model.language_model.layers.28.linear_attn.in_proj_a",
"model.language_model.layers.28.linear_attn.in_proj_b",
"model.language_model.layers.29.linear_attn.conv1d",
"model.language_model.layers.29.linear_attn.in_proj_a",
"model.language_model.layers.29.linear_attn.in_proj_b",
"model.language_model.layers.30.linear_attn.conv1d",
"model.language_model.layers.30.linear_attn.in_proj_a",
"model.language_model.layers.30.linear_attn.in_proj_b",
"model.language_model.layers.32.linear_attn.conv1d",
"model.language_model.layers.32.linear_attn.in_proj_a",
"model.language_model.layers.32.linear_attn.in_proj_b",
"model.language_model.layers.33.linear_attn.conv1d",
"model.language_model.layers.33.linear_attn.in_proj_a",
"model.language_model.layers.33.linear_attn.in_proj_b",
"model.language_model.layers.34.linear_attn.conv1d",
"model.language_model.layers.34.linear_attn.in_proj_a",
"model.language_model.layers.34.linear_attn.in_proj_b",
"model.language_model.layers.36.linear_attn.conv1d",
"model.language_model.layers.36.linear_attn.in_proj_a",
"model.language_model.layers.36.linear_attn.in_proj_b",
"model.language_model.layers.37.linear_attn.conv1d",
"model.language_model.layers.37.linear_attn.in_proj_a",
"model.language_model.layers.37.linear_attn.in_proj_b",
"model.language_model.layers.38.linear_attn.conv1d",
"model.language_model.layers.38.linear_attn.in_proj_a",
"model.language_model.layers.38.linear_attn.in_proj_b",
"model.language_model.layers.4.linear_attn.conv1d",
"model.language_model.layers.4.linear_attn.in_proj_a",
"model.language_model.layers.4.linear_attn.in_proj_b",
"model.language_model.layers.40.linear_attn.conv1d",
"model.language_model.layers.40.linear_attn.in_proj_a",
"model.language_model.layers.40.linear_attn.in_proj_b",
"model.language_model.layers.41.linear_attn.conv1d",
"model.language_model.layers.41.linear_attn.in_proj_a",
"model.language_model.layers.41.linear_attn.in_proj_b",
"model.language_model.layers.42.linear_attn.conv1d",
"model.language_model.layers.42.linear_attn.in_proj_a",
"model.language_model.layers.42.linear_attn.in_proj_b",
"model.language_model.layers.44.linear_attn.conv1d",
"model.language_model.layers.44.linear_attn.in_proj_a",
"model.language_model.layers.44.linear_attn.in_proj_b",
"model.language_model.layers.45.linear_attn.conv1d",
"model.language_model.layers.45.linear_attn.in_proj_a",
"model.language_model.layers.45.linear_attn.in_proj_b",
"model.language_model.layers.46.linear_attn.conv1d",
"model.language_model.layers.46.linear_attn.in_proj_a",
"model.language_model.layers.46.linear_attn.in_proj_b",
"model.language_model.layers.48.linear_attn.conv1d",
"model.language_model.layers.48.linear_attn.in_proj_a",
"model.language_model.layers.48.linear_attn.in_proj_b",
"model.language_model.layers.49.linear_attn.conv1d",
"model.language_model.layers.49.linear_attn.in_proj_a",
"model.language_model.layers.49.linear_attn.in_proj_b",
"model.language_model.layers.5.linear_attn.conv1d",
"model.language_model.layers.5.linear_attn.in_proj_a",
"model.language_model.layers.5.linear_attn.in_proj_b",
"model.language_model.layers.50.linear_attn.conv1d",
"model.language_model.layers.50.linear_attn.in_proj_a",
"model.language_model.layers.50.linear_attn.in_proj_b",
"model.language_model.layers.52.linear_attn.conv1d",
"model.language_model.layers.52.linear_attn.in_proj_a",
"model.language_model.layers.52.linear_attn.in_proj_b",
"model.language_model.layers.53.linear_attn.conv1d",
"model.language_model.layers.53.linear_attn.in_proj_a",
"model.language_model.layers.53.linear_attn.in_proj_b",
"model.language_model.layers.54.linear_attn.conv1d",
"model.language_model.layers.54.linear_attn.in_proj_a",
"model.language_model.layers.54.linear_attn.in_proj_b",
"model.language_model.layers.56.linear_attn.conv1d",
"model.language_model.layers.56.linear_attn.in_proj_a",
"model.language_model.layers.56.linear_attn.in_proj_b",
"model.language_model.layers.57.linear_attn.conv1d",
"model.language_model.layers.57.linear_attn.in_proj_a",
"model.language_model.layers.57.linear_attn.in_proj_b",
"model.language_model.layers.58.linear_attn.conv1d",
"model.language_model.layers.58.linear_attn.in_proj_a",
"model.language_model.layers.58.linear_attn.in_proj_b",
"model.language_model.layers.6.linear_attn.conv1d",
"model.language_model.layers.6.linear_attn.in_proj_a",
"model.language_model.layers.6.linear_attn.in_proj_b",
"model.language_model.layers.60.linear_attn.conv1d",
"model.language_model.layers.60.linear_attn.in_proj_a",
"model.language_model.layers.60.linear_attn.in_proj_b",
"model.language_model.layers.61.linear_attn.conv1d",
"model.language_model.layers.61.linear_attn.in_proj_a",
"model.language_model.layers.61.linear_attn.in_proj_b",
"model.language_model.layers.62.linear_attn.conv1d",
"model.language_model.layers.62.linear_attn.in_proj_a",
"model.language_model.layers.62.linear_attn.in_proj_b",
"model.language_model.layers.8.linear_attn.conv1d",
"model.language_model.layers.8.linear_attn.in_proj_a",
"model.language_model.layers.8.linear_attn.in_proj_b",
"model.language_model.layers.9.linear_attn.conv1d",
"model.language_model.layers.9.linear_attn.in_proj_a",
"model.language_model.layers.9.linear_attn.in_proj_b",
"model.visual*",
"mtp*"
],
"kv_quantized": false,
"sparse": false,
"weights": {}
},
"rope_artifact": {
"rope_theta": 10000000,
"rope_type": "default"
},
"rope_source": {
"rope_theta": 10000000,
"rope_type": "default"
},
"safetensor_bytes": 17494111736,
"sparse": false,
"violations": [],
"weight_bits": 16
},
"dev_quality": {
"domains": {
"code": {
"bf16": 0.3053977272727273,
"bf16_se": 0.04707248778770569,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.02130681818181818,
"headroom": 0.3053977272727273,
"items": 88,
"quantized": 0.2840909090909091,
"recovery": 0.9302325581395349,
"recovery_se": 0.04871294382506705,
"se": 0.014876822332939512,
"status": "valid",
"upper": 0.04577919091950368
},
"commonsense": {
"bf16": 0.93,
"bf16_se": 0.01256199786707855,
"candidate_lost": 0.0,
"chance": 0.24983333333333332,
"drop": 0.015833333333333335,
"format_contract": {
"bf16": 1.0,
"quantized": 1.0
},
"headroom": 0.6801666666666667,
"items": 300,
"quantized": 0.9141666666666667,
"recovery": 0.9767213918157314,
"recovery_se": 0.006470305733135467,
"se": 0.004400886282820974,
"status": "valid",
"upper": 0.023072791268573836
},
"expert": {
"bf16": 0.5375,
"bf16_se": 0.07095155786322377,
"candidate_lost": 0.0,
"chance": 0.25,
"drop": 0.03333333333333333,
"format_contract": {
"bf16": 1.0,
"quantized": 1.0
},
"headroom": 0.2875,
"items": 30,
"quantized": 0.5041666666666667,
"recovery": 0.8840579710144928,
"recovery_se": 0.06570146298668585,
"se": 0.01888917060867218,
"status": "R5_diagnostic",
"upper": 0.06440601898459906
},
"knowledge": {
"bf16": 0.64,
"bf16_se": 0.02515171357287956,
"candidate_lost": 0.0,
"chance": 0.10940343915343916,
"drop": 0.02,
"format_contract": {
"bf16": 1.0,
"quantized": 0.9983333333333333
},
"headroom": 0.5305965608465608,
"items": 300,
"quantized": 0.62,
"recovery": 0.9623065781502801,
"recovery_se": 0.019779614364464397,
"se": 0.010494995356656042,
"status": "valid",
"upper": 0.03726426736169919
},
"long_context": {
"bf16": 0.91575,
"bf16_se": 0.021385410644641716,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.006708333333333335,
"headroom": 0.91575,
"items": 100,
"quantized": 0.9090416666666666,
"recovery": 0.9926744926744927,
"recovery_se": 0.008150821938506893,
"se": 0.007464115190187687,
"status": "valid",
"upper": 0.01898680282119208
},
"math": {
"bf16": 0.8125,
"bf16_se": 0.026330786192355474,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": -0.005,
"headroom": 0.8125,
"items": 200,
"quantized": 0.8175,
"recovery": 1.0061538461538462,
"recovery_se": 0.008436375418290067,
"se": 0.00685455502736068,
"status": "valid",
"upper": 0.006275743020008317
}
},
"evaluations": 1,
"first_quality": null,
"length": {
"length_generations": 4544,
"length_shift": 0.006629130802905836,
"length_shift_se": 0.008972790691940834,
"length_tokens": 171.89986795774647,
"length_tokens_bf16": 170.76782570422534
},
"recovery": 0.973617773386777,
"recovery_se": 0.010851096212388653,
"reference_rows": "/work/results/qwen35-27b/bf16",
"tier": "B",
"valid_domains": 5,
"worst_domain": "code",
"worst_drop": 0.02130681818181818,
"worst_upper": 0.04577919091950368
},
"dev_score": {
"bf16_peak_gib": null,
"drift": null,
"efficiency_gain": null,
"mean_drop": null,
"memories": null,
"memory": null,
"memory_saved": null,
"peak_gib": null,
"precision": {
"activations": 16,
"kv": 8,
"label": "W16A16KV8",
"weights": 16,
"weights_average": 4.91
},
"qualified": false,
"reason": "contract: the engine returned no log-probabilities for the KL probe",
"score": 0.0,
"speed": null,
"speeds": null
},
"efficiency": {
"interactive": 160.06509765739276,
"long_context": 713.192365835199,
"serve": 1824.1243024896796
},
"engine": {
"audit": null,
"budget": {
"interactive@1.02": {
"budget_gib": 18.173,
"cap_gib": 22.173,
"factor": 1.02,
"full_running": 1.0,
"healthy": true,
"running": 1.0,
"value": 159.47031256396795
},
"long_context@1.02": {
"budget_gib": 56.307,
"cap_gib": 60.307,
"factor": 1.02,
"full_running": 64.0,
"healthy": true,
"running": 64.0,
"value": 700.4312487547961
},
"serve@1.02": {
"budget_gib": 29.208,
"cap_gib": 33.208,
"factor": 1.02,
"full_running": 32.0,
"healthy": true,
"running": 32.0,
"value": 1705.902318208448
}
},
"hard_cap": true,
"startup": {
"cuda_graph_gib": 0.18,
"engine_init_seconds": 86.92,
"kv_bytes_per_token": 35200.732097485,
"kv_cache_dtype": "fp8",
"kv_cache_gib": 143.46,
"kv_cache_tokens": 4376017.0,
"max_concurrency": {
"sequences": 133.55,
"tokens_per_request": 32768
},
"speculative_config": "None",
"vllm_version": "0.1.dev18498+g7a33ba95f",
"weights_gib": 15.05
},
"stored_gib": 16.293
},
"gpu_hours": 3.403,
"peak_memory": {
"interactive": {
"device_peak_gib": 163.65,
"kv_bytes_per_token": 102420,
"kv_gib": 0.122,
"kv_usage": 0.0009,
"other_gib": 2.645,
"peak_gib": 17.817,
"weights_gib": 15.05
},
"long_context": {
"device_peak_gib": 163.65,
"kv_bytes_per_token": 38710,
"kv_gib": 37.507,
"kv_usage": 0.2614,
"other_gib": 2.645,
"peak_gib": 55.203,
"weights_gib": 15.05
},
"serve": {
"device_peak_gib": 163.65,
"kv_bytes_per_token": 42173,
"kv_gib": 10.94,
"kv_usage": 0.0763,
"other_gib": 2.645,
"peak_gib": 28.635,
"weights_gib": 15.05
}
},
"split": "dev",
"workload": {
"bf16": {
"peak_gib": 77.756,
"rate": 854.772576603853
},
"submission": {
"peak_gib": 28.635,
"rate": 1824.1243024896796
}
}
},
"runner": "kubernetes",
"solution": {
"checkpoint": "lowbarena-trial-27b-v2",
"declared_precision": true,
"detected_precision": "W16A16KV8",
"method": "NVFP4-GPTQ+FP8 head+INT4 emb+fp16 GDN state+capfix-trial2-27b",
"precision": "W4A4KV8"
},
"split": "dev",
"stages": [
{
"name": "build",
"status": "done"
},
{
"name": "contract",
"status": "done"
},
{
"name": "quality",
"status": "done"
},
{
"name": "efficiency",
"status": "done"
},
{
"name": "score",
"status": "skipped"
}
],
"status": "done",
"task_id": "qwen35-27b",
"track": "engine",
"updated_at": "2026-10-13T14:27:55Z"
}