{"version":"23","rules":{"version":"23","scale":10.0,"w_acc":0.79,"w_kl":0.21,"kl_points":42.8,"fail_share":0.1,"speed_floor":0.95,"speed_se":0.05,"length_limit":0.1,"kl_ceiling":0.15,"workloads":3,"cost_scale":100.0},"vllm":{"release":"0.25.1","package":"0.1.dev18498+g7a33ba95f"},"voided_tasks":{},"workload":{"description":"64 requests of 8,192 prompt tokens, 32 in flight, 512 generated tokens each","input_len":8192,"output_len":512,"num_prompts":64,"max_concurrency":32,"request_rate":"inf","metric":"output_throughput","better":"higher"},"workloads":{"serve":{"description":"64 requests of 8,192 prompt tokens, 32 in flight, 512 generated tokens each","input_len":8192,"output_len":512,"num_prompts":64,"max_concurrency":32,"request_rate":"inf","metric":"output_throughput","better":"higher"},"interactive":{"description":"single user, short prompt, decode-bound","input_len":1024,"output_len":256,"num_prompts":32,"max_concurrency":1,"request_rate":"inf","metric":"median_tpot_ms","better":"lower"},"long_context":{"description":"16K-token prompts, KV-bound","input_len":16000,"output_len":256,"num_prompts":128,"max_concurrency":64,"request_rate":"inf","metric":"output_throughput","better":"higher"}},"engine_flags":{"max_model_len":32768,"gpu_memory_utilization":0.9,"max_num_seqs":256,"max_num_batched_tokens":8192,"enable_prefix_caching":false,"seed":0},"gate":{"quality":{"margins":{"A":0.03,"B":0.05},"required_tier":"B","evaluations_pooled":2},"speed_floor":"the geometric mean over serve, interactive, long_context of the throughput ratio to BF16 in the same pod, less 1.645 * 0.05 / sqrt(3), at least the task's speed_floor (0.95 unless the task says otherwise)","length":"generated tokens within 10% of BF16's"},"lowbit_score":"LowBScore = E * exp(-max(D, 0) / 10) if the gates pass, 0.1 of it if one fails, 0 if the contract or the engine rules are broken; efficiency E = sqrt(memory_gain * speed_gain / max(1, 1 + length_shift)), memory_gain and speed_gain the geometric means over serve, interactive, long_context of BF16 peak / peak and throughput / BF16 throughput in the same pod, length_shift the change in generated tokens; drift D = 0.79 * mean accuracy drop in points + 0.21 * 42.8 * KL(BF16 || solution) in nats per token on hidden prompts, the weights fitted to the references; BF16 scores 1","perplexity":"the excess loss over BF16 on hidden C4 text and on WikiText-2, shown beside the score but not part of it: on the references it moves with KL (correlation 0.95), so it adds nothing to the drift once KL is in","engine_track":{"checkpoint":"a Hugging Face repository at an exact commit, downloaded by the service and sealed before any of the engine's code runs, held to the contract like any checkpoint","serving":"launch.sh serve may run any server and any kernels and compute as it chooses, judged like any solution by the gates, the KL ceiling, and the drift, plus the engine rules below","order":"the engine is timed first; its accuracy runs and KL probe are then served under a hard cap of its largest checked peak, so it cannot hold more while it is scored than while it is timed"},"contract":{"configuration":"the source's architecture fields and RoPE parameters","foreign_tensor_share":0.01,"foreign_tensors":"tensors that are neither a source weight nor a quantization format's companion to one may hold at most this share of the source's bytes","kl_ceiling_nats":0.15,"kl_rule":"a KL from BF16 above the ceiling, or an engine that returns no log-probabilities for the probe, breaks the contract (score 0)"},"cost":{"scale_gpu_hours":100.0,"rule":"a participant's score on a task, the one the board ranks, is its LowBScore times exp(-gpu_hours / 100), gpu_hours being what the arena's GPUs ran for all of its evaluations of the task, development and scored, failed ones included; references are measured, not searched for, and carry no cost"},"precision":"W{weight bits}A{activation bits}KV{KV-cache bits}, any bit width the engine serves","submission":{"scored_per_task":{"qwen3-8b":3,"qwen35-27b":3},"counts":"the last finished scored submission, not the best; every one is public"},"engine":{"memory":"each workload's peak in the engine's own accounts where it keeps vLLM's, and the device's count where it keeps none","budget_factor":1.02,"context_gib":4.0,"budget_rate":0.9,"budget_rule":"served again under a hard cap of its peak times budget_factor plus context_gib, a filler holding the rest of the GPU, it holds as many requests at once as with the whole device, or keeps budget_rate of its rate","offload":"weights or KV blocks in host memory, as its start-up log reports them, break the engine rules","timed_output_audit":{"samples_per_scenario":24,"median_gap_limit_nats":{"qwen3-8b":0.034,"qwen35-27b":0.133},"min_length_ratio":0.9,"max_empty_share":0.25}}}