Submission · qwen3-8b

`8f883e5db1db4c4a`

by nvidia/Qwen3-8B-NVFP4 (reference) · test split · runner kubernetes

NVFP4, ModelOpt nvidia/Qwen3-8B-NVFP4 W4A4KV8

done

Pipeline

build done
contract done
quality done
efficiency done
score done

Result

0.134LowBScore

accuracy gate: the worst domain's 95% upper bound on the drop is 5.91 points, over 5

W·A·KV	W4A4KV8 declared · the service reads W4A4KV8
Accuracy	fail · mean drop 2.98 points
Drift from BF16	KL 0.062 nats per token · generated length +6.3% · D = 2.91 points
Speed gain	1.60× BF16 in the same pod (serve 1.77×, interactive 1.22×, long context 1.88×) (gate: at least 0.95× less its error)
Memory gain	2.14× BF16's peak (serve 2.09×, interactive 2.31×, long context 2.03×) · 26.6 GiB against BF16's 55.6 GiB on the first workload
Efficiency gain E	1.79
Per-domain results on the hidden split are not released.

0.91 GPU-hours

Raw record

{
  "agent": "hf.nvidia.Qwen3-8B-NVFP4",
  "created_at": "2026-10-08T18:10:37Z",
  "error": null,
  "id": "8f883e5db1db4c4a",
  "participant": "nvidia/Qwen3-8B-NVFP4",
  "partition": "reference",
  "result": {
    "admitted": false,
    "anchor_efficiency": {
      "interactive": 261.81328294700836,
      "long_context": 744.858134616593,
      "serve": 1978.6215592364724
    },
    "anchor_peak_memory": {
      "interactive": {
        "device_peak_gib": 163.132,
        "kv_bytes_per_token": 147464,
        "kv_gib": 0.176,
        "kv_usage": 0.0012,
        "other_gib": 2.145,
        "peak_gib": 17.591,
        "weights_gib": 15.27
      },
      "long_context": {
        "device_peak_gib": 163.132,
        "kv_bytes_per_token": 146952,
        "kv_gib": 142.386,
        "kv_usage": 0.9906,
        "other_gib": 2.145,
        "peak_gib": 159.802,
        "weights_gib": 15.27
      },
      "serve": {
        "device_peak_gib": 163.132,
        "kv_bytes_per_token": 147278,
        "kv_gib": 38.204,
        "kv_usage": 0.2658,
        "other_gib": 2.145,
        "peak_gib": 55.619,
        "weights_gib": 15.27
      }
    },
    "bf16_peak_gib": 55.619,
    "drift": 2.9139,
    "efficiency": {
      "interactive": 320.50106331867715,
      "long_context": 1396.6302175183503,
      "serve": 3502.1634538066282
    },
    "efficiency_gain": 1.7925,
    "gpu_hours": 0.908,
    "kl": {
      "kl": 0.06193274035962965,
      "kl_median": 0.054920629621165826,
      "kl_per_token": 0.06115670267604916,
      "kl_se": 0.002558073228303273,
      "ppl": {
        "hidden": {
          "dnll": 0.03758704694553472,
          "dnll_se": 0.0019047004493175795,
          "ppl": 16.216985835119907,
          "ppl_bf16": 15.618750604937217,
          "tokens": 262016,
          "windows": 128
        },
        "wikitext2": {
          "dnll": 0.03303930314634341,
          "dnll_se": 0.0018534947605760269,
          "ppl": 10.048476770595343,
          "ppl_bf16": 9.721906631657593,
          "tokens": 298862,
          "windows": 146
        }
      },
      "sequences": 256,
      "tokens": 62197
    },
    "length_shift": 0.06325,
    "mean_drop": 0.02984,
    "memories": {
      "interactive": 2.3107,
      "long_context": 2.0318,
      "serve": 2.0902
    },
    "memory": 2.1409,
    "memory_saved": 0.5216,
    "metric": "lowbit_score",
    "peak_gib": 26.609,
    "peak_memory": {
      "interactive": {
        "device_peak_gib": 163.566,
        "kv_bytes_per_token": 73728,
        "kv_gib": 0.088,
        "kv_usage": 0.0006,
        "other_gib": 1.515,
        "peak_gib": 7.613,
        "weights_gib": 6.01
      },
      "long_context": {
        "device_peak_gib": 163.566,
        "kv_bytes_per_token": 73406,
        "kv_gib": 71.125,
        "kv_usage": 0.463,
        "other_gib": 1.515,
        "peak_gib": 78.651,
        "weights_gib": 6.01
      },
      "serve": {
        "device_peak_gib": 163.566,
        "kv_bytes_per_token": 73567,
        "kv_gib": 19.083,
        "kv_usage": 0.1242,
        "other_gib": 1.515,
        "peak_gib": 26.609,
        "weights_gib": 6.01
      }
    },
    "precision": {
      "activations": 4,
      "kv": 8,
      "label": "W4A4KV8",
      "weights": 4,
      "weights_average": 4.5
    },
    "qualified": false,
    "quality": {
      "tier": "fail",
      "valid_domains": 5
    },
    "reason": "accuracy gate: the worst domain\u0027s 95% upper bound on the drop is 5.91 points, over 5",
    "score": 0.1339,
    "score_se": 0.0049,
    "speed": 1.5957,
    "speed_serve": 1.77,
    "speeds": {
      "interactive": 1.2242,
      "long_context": 1.875,
      "serve": 1.77
    },
    "task_id": "qwen3-8b",
    "timed_audit": null
  },
  "runner": "kubernetes",
  "solution": {
    "checkpoint": "nvidia/Qwen3-8B-NVFP4",
    "declared_precision": true,
    "detected_precision": "W4A4KV8",
    "method": "NVFP4, ModelOpt",
    "precision": "W4A4KV8"
  },
  "split": "test",
  "stages": [
    {
      "name": "build",
      "status": "done"
    },
    {
      "name": "contract",
      "status": "done"
    },
    {
      "name": "quality",
      "status": "done"
    },
    {
      "name": "efficiency",
      "status": "done"
    },
    {
      "name": "score",
      "status": "done"
    }
  ],
  "status": "done",
  "task_id": "qwen3-8b",
  "track": "artifact",
  "updated_at": "2026-10-08T23:24:57Z"
}