Submission · qwen35-27b

`d968488677754f2f`

by Qwen/Qwen3.5-27B-GPTQ-Int4 (reference) · test split · runner kubernetes

GPTQ Int4, Qwen Qwen/Qwen3.5-27B-GPTQ-Int4 W4A16KV16

done

Pipeline

build done
contract done
quality done
efficiency done
score done

Result

0.099LowBScore

0.68x BF16's speed (lower bound 0.63) is not shown to be within the floor of 0.95x

W·A·KV	W4A16KV16 declared · the service reads W4A16KV16
Accuracy	tier A · mean drop -0.19 points
Drift from BF16	KL 0.010 nats per token · generated length -1.1% · D = -0.06 points
Speed gain	0.68× BF16 in the same pod (serve 0.55×, interactive 1.25×, long context 0.45×) (gate: at least 0.95× less its error)
Memory gain	1.45× BF16's peak (serve 1.43×, interactive 1.74×, long context 1.22×) · 54.3 GiB against BF16's 77.8 GiB on the first workload
Efficiency gain E	0.99
Per-domain results on the hidden split are not released.

2.05 GPU-hours

Raw record

{
  "agent": "hf.Qwen.Qwen3.5-27B-GPTQ-Int4",
  "created_at": "2026-10-08T18:10:37Z",
  "error": null,
  "id": "d968488677754f2f",
  "participant": "Qwen/Qwen3.5-27B-GPTQ-Int4",
  "partition": "reference",
  "result": {
    "admitted": true,
    "anchor_efficiency": {
      "interactive": 93.40355166771842,
      "long_context": 317.65315538979405,
      "serve": 852.7006454861819
    },
    "anchor_peak_memory": {
      "interactive": {
        "device_peak_gib": 161.798,
        "kv_bytes_per_token": 200806,
        "kv_gib": 0.239,
        "kv_usage": 0.0023,
        "other_gib": 4.555,
        "peak_gib": 55.015,
        "weights_gib": 50.22
      },
      "long_context": {
        "device_peak_gib": 161.8,
        "kv_bytes_per_token": 75895,
        "kv_gib": 73.537,
        "kv_usage": 0.6913,
        "other_gib": 4.555,
        "peak_gib": 128.313,
        "weights_gib": 50.22
      },
      "serve": {
        "device_peak_gib": 161.798,
        "kv_bytes_per_token": 88591,
        "kv_gib": 22.98,
        "kv_usage": 0.216,
        "other_gib": 4.555,
        "peak_gib": 77.756,
        "weights_gib": 50.22
      }
    },
    "bf16_peak_gib": 77.756,
    "drift": -0.0612,
    "efficiency": {
      "interactive": 117.18574630714163,
      "long_context": 141.9638967192057,
      "serve": 470.14447093650324
    },
    "efficiency_gain": 0.9903,
    "gpu_hours": 2.05,
    "kl": {
      "kl": 0.010316117481757599,
      "kl_median": 0.009385561842827947,
      "kl_per_token": 0.010271231137583816,
      "kl_se": 0.0007150044789762303,
      "ppl": {
        "hidden": {
          "dnll": 0.0149986749374399,
          "dnll_se": 0.0005421817021672299,
          "ppl": 10.831363653117604,
          "ppl_bf16": 10.670119795435681,
          "tokens": 262016,
          "windows": 128
        },
        "wikitext2": {
          "dnll": 0.013958866873998577,
          "dnll_se": 0.0011838443245906585,
          "ppl": 6.996342675652046,
          "ppl_bf16": 6.8993601176944415,
          "tokens": 296815,
          "windows": 145
        }
      },
      "sequences": 256,
      "tokens": 63668
    },
    "length_shift": -0.01105,
    "mean_drop": -0.00195,
    "memories": {
      "interactive": 1.7424,
      "long_context": 1.2233,
      "serve": 1.4314
    },
    "memory": 1.4504,
    "memory_saved": 0.3014,
    "metric": "lowbit_score",
    "peak_gib": 54.321,
    "peak_memory": {
      "interactive": {
        "device_peak_gib": 161.755,
        "kv_bytes_per_token": 200850,
        "kv_gib": 0.239,
        "kv_usage": 0.0018,
        "other_gib": 4.605,
        "peak_gib": 31.575,
        "weights_gib": 26.73
      },
      "long_context": {
        "device_peak_gib": 161.755,
        "kv_bytes_per_token": 75912,
        "kv_gib": 73.553,
        "kv_usage": 0.5666,
        "other_gib": 4.605,
        "peak_gib": 104.889,
        "weights_gib": 26.73
      },
      "serve": {
        "device_peak_gib": 161.755,
        "kv_bytes_per_token": 88610,
        "kv_gib": 22.985,
        "kv_usage": 0.1771,
        "other_gib": 4.605,
        "peak_gib": 54.321,
        "weights_gib": 26.73
      }
    },
    "precision": {
      "activations": 16,
      "kv": 16,
      "label": "W4A16KV16",
      "weights": 4,
      "weights_average": 7.97
    },
    "qualified": false,
    "quality": {
      "tier": "A",
      "valid_domains": 5
    },
    "reason": "0.68x BF16\u0027s speed (lower bound 0.63) is not shown to be within the floor of 0.95x",
    "score": 0.099,
    "score_se": 0.0014,
    "speed": 0.6762,
    "speed_serve": 0.5514,
    "speeds": {
      "interactive": 1.2546,
      "long_context": 0.4469,
      "serve": 0.5514
    },
    "task_id": "qwen35-27b",
    "timed_audit": null
  },
  "runner": "kubernetes",
  "solution": {
    "checkpoint": "Qwen/Qwen3.5-27B-GPTQ-Int4",
    "declared_precision": true,
    "detected_precision": "W4A16KV16",
    "method": "GPTQ Int4, Qwen",
    "precision": "W4A16KV16"
  },
  "split": "test",
  "stages": [
    {
      "name": "build",
      "status": "done"
    },
    {
      "name": "contract",
      "status": "done"
    },
    {
      "name": "quality",
      "status": "done"
    },
    {
      "name": "efficiency",
      "status": "done"
    },
    {
      "name": "score",
      "status": "done"
    }
  ],
  "status": "done",
  "task_id": "qwen35-27b",
  "track": "artifact",
  "updated_at": "2026-10-09T05:12:44Z"
}