Submission · qwen3-8b
d55250a215c34184
by trial2-8b (participant) · dev split · runner kubernetes
FP8-W8A8+FP8head+INT4emb+KV8+seqs12+bt4096-trial2-8b lowbarena-trial-8b2-fp8-fp8head-int4emb
W8A8KV8
Pipeline
- build done
- contract done
- quality done
- efficiency done
- score skipped
Result
1.792LowBScore on the development split (never ranked)
| W·A·KV | W8A8KV8 declared · the service reads W8A8KV8 |
| Accuracy | tier B · mean drop 0.96 points · worst domain commonsense (upper bound 3.8) |
| Speed gain | 1.02× BF16 in the same pod (serve 0.76×, interactive 1.27×, long context 1.11×) |
| Memory gain | 3.71× BF16's peak (serve 3.51×, interactive 2.01×, long context 7.24×) · 15.8 GiB against 55.6 GiB on the first workload |
| E · D | efficiency gain 1.95 · drift 0.84 points |
| Contract | passed, 8.00 bits per weight stored |
| GPU-hours | 0.70 |
Raw record
{
"agent": "trial2-8b",
"created_at": "2026-10-13T07:13:14Z",
"error": null,
"id": "d55250a215c34184",
"participant": "trial2-8b",
"partition": "participant",
"result": {
"contract": {
"activation_bits": 8,
"bits_per_weight": 8.003226902173912,
"foreign_tensors": {
"bytes": 0,
"count": 0,
"largest": [],
"share": 0.0
},
"format": "fp8-w8a8",
"kv_bits": null,
"model_bits_per_weight": 8.003282765413925,
"ok": true,
"quantization": {
"activations": {
"actorder": null,
"block_structure": null,
"dynamic": true,
"group_size": null,
"num_bits": 8,
"observer": null,
"observer_kwargs": {},
"strategy": "token",
"symmetric": true,
"type": "float"
},
"format": "fp8-w8a8",
"ignore": [],
"kv_quantized": false,
"sparse": false,
"weights": {
"actorder": null,
"block_structure": null,
"dynamic": false,
"group_size": null,
"num_bits": 8,
"observer": "mse",
"observer_kwargs": {},
"strategy": "channel",
"symmetric": true,
"type": "float"
}
},
"rope_artifact": {
"rope_theta": 1000000,
"rope_type": "default"
},
"rope_source": {
"rope_theta": 1000000,
"rope_type": "default"
},
"safetensor_bytes": 7925060048,
"sparse": false,
"violations": [],
"weight_bits": 8
},
"dev_quality": {
"domains": {
"code": {
"bf16": 0.3380681818181818,
"bf16_se": 0.047431027842082016,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.004261363636363636,
"headroom": 0.3380681818181818,
"items": 88,
"quantized": 0.3338068181818182,
"recovery": 0.9873949579831932,
"recovery_se": 0.022716247757476812,
"se": 0.007679640577101536,
"status": "valid",
"upper": 0.016894372385695664
},
"commonsense": {
"bf16": 0.7466666666666667,
"bf16_se": 0.024134209165109787,
"candidate_lost": 0.0,
"chance": 0.24983333333333332,
"drop": 0.015833333333333335,
"format_contract": {
"bf16": 1.0,
"quantized": 1.0
},
"headroom": 0.49683333333333335,
"items": 300,
"quantized": 0.7308333333333333,
"recovery": 0.9681314994968131,
"recovery_se": 0.02655378171889604,
"se": 0.01319280388400485,
"status": "valid",
"upper": 0.03753549572252131
},
"expert": {
"bf16": 0.44166666666666665,
"bf16_se": 0.08491318532563476,
"candidate_lost": 0.0,
"chance": 0.25,
"drop": -0.1125,
"format_contract": {
"bf16": 0.9916666666666667,
"quantized": 0.9916666666666667
},
"headroom": 0.19166666666666665,
"items": 30,
"quantized": 0.5541666666666667,
"recovery": 1.5869565217391308,
"recovery_se": 0.23052528561356556,
"se": 0.04418401307593339,
"status": "R5_diagnostic",
"upper": -0.03981729849008957
},
"knowledge": {
"bf16": 0.44666666666666666,
"bf16_se": 0.027939588170592746,
"candidate_lost": 0.0,
"chance": 0.10940343915343916,
"drop": 0.015833333333333335,
"format_contract": {
"bf16": 0.92,
"quantized": 0.9283333333333333
},
"headroom": 0.3372632275132275,
"items": 300,
"quantized": 0.43083333333333335,
"recovery": 0.9530534845139251,
"recovery_se": 0.03225071708506634,
"se": 0.010876980933725463,
"status": "valid",
"upper": 0.033725966969311724
},
"long_context": {
"bf16": 0.8561666666666666,
"bf16_se": 0.02867970594529983,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.0020833333333333337,
"headroom": 0.8561666666666666,
"items": 100,
"quantized": 0.8540833333333333,
"recovery": 0.9975666731555383,
"recovery_se": 0.012682665850035529,
"se": 0.010858475745272085,
"status": "valid",
"upper": 0.019945525934305913
},
"math": {
"bf16": 0.89,
"bf16_se": 0.019057569487578137,
"candidate_lost": 0.0,
"chance": 0.0,
"drop": 0.01,
"headroom": 0.89,
"items": 200,
"quantized": 0.88,
"recovery": 0.9887640449438202,
"recovery_se": 0.009924285730611847,
"se": 0.008832614300244544,
"status": "valid",
"upper": 0.024529650523902276
}
},
"evaluations": 1,
"first_quality": null,
"length": {
"length_generations": 4544,
"length_shift": -0.012300378816014312,
"length_shift_se": 0.005468344699969847,
"length_tokens": 146.31822183098592,
"length_tokens_bf16": 148.14040492957747
},
"recovery": 0.978982132018658,
"recovery_se": 0.010041078524902235,
"reference_rows": "/work/results/qwen-qwen3-8b/bf16-lb",
"tier": "B",
"valid_domains": 5,
"worst_domain": "commonsense",
"worst_drop": 0.015833333333333335,
"worst_upper": 0.03753549572252131
},
"dev_score": {
"bf16_peak_gib": 55.615,
"drift": 0.8403,
"efficiency_gain": 1.9489,
"mean_drop": 0.0096,
"memories": {
"interactive": 2.0074,
"long_context": 7.2432,
"serve": 3.5108
},
"memory": 3.7096,
"memory_saved": 0.7152,
"peak_gib": 15.841,
"precision": {
"activations": 8,
"kv": 8,
"label": "W8A8KV8",
"weights": 8,
"weights_average": 8.0
},
"qualified": true,
"reason": null,
"score": 1.7918,
"speed": 1.0239,
"speeds": {
"interactive": 1.2744,
"long_context": 1.1085,
"serve": 0.7598
}
},
"efficiency": {
"interactive": 331.5895082163212,
"long_context": 815.8783272087899,
"serve": 1464.539631706047
},
"engine": {
"audit": null,
"budget": {
"interactive@1.02": {
"budget_gib": 8.938,
"cap_gib": 12.938,
"factor": 1.02,
"full_running": 1.0,
"healthy": true,
"running": 1.0,
"value": 337.8146476362128
},
"long_context@1.02": {
"budget_gib": 22.498,
"cap_gib": 26.498,
"factor": 1.02,
"full_running": 12.0,
"healthy": true,
"running": 12.0,
"value": 808.8421235454565
},
"serve@1.02": {
"budget_gib": 16.158,
"cap_gib": 20.158,
"factor": 1.02,
"full_running": 12.0,
"healthy": true,
"running": 12.0,
"value": 1479.2302514680964
}
},
"hard_cap": true,
"startup": {
"cuda_graph_gib": 0.06,
"engine_init_seconds": 64.13,
"kv_bytes_per_token": 73729.4260506672,
"kv_cache_dtype": "fp8",
"kv_cache_gib": 152.48,
"kv_cache_tokens": 2220608.0,
"load_seconds": 5.160326,
"max_concurrency": {
"sequences": 67.77,
"tokens_per_request": 32768
},
"speculative_config": "None",
"vllm_version": "0.1.dev18498+g7a33ba95f",
"weights_gib": 7.42
},
"stored_gib": 7.381
},
"gpu_hours": 0.698,
"kl": {
"kl": 0.009092044286058707,
"kl_median": 0.009791557695287487,
"kl_per_token": 0.008995099241270968,
"kl_se": 0.001290113780927231,
"sequences": 64,
"tokens": 15418
},
"split": "dev",
"workload": {
"bf16": {
"peak_gib": 55.615,
"rate": 1927.5479023209139
},
"submission": {
"peak_gib": 15.841,
"rate": 1464.539631706047
}
}
},
"runner": "kubernetes",
"solution": {
"checkpoint": "lowbarena-trial-8b2-fp8-fp8head-int4emb",
"declared_precision": true,
"detected_precision": "W8A8KV8",
"method": "FP8-W8A8+FP8head+INT4emb+KV8+seqs12+bt4096-trial2-8b",
"precision": "W8A8KV8"
},
"split": "dev",
"stages": [
{
"name": "build",
"status": "done"
},
{
"name": "contract",
"status": "done"
},
{
"name": "quality",
"status": "done"
},
{
"name": "efficiency",
"status": "done"
},
{
"name": "score",
"status": "skipped"
}
],
"status": "done",
"task_id": "qwen3-8b",
"track": "engine",
"updated_at": "2026-10-13T07:56:49Z"
}