{
 "hosted_sample": "145 cases: 25 banking77, 20 btzsc, 25 abcd, 25 agentrewardbench, 25 guardrails, 25 beer_alignment",
 "partial": false,
 "collection_status": "every hosted request recorded or declared",
 "acceptance": {
  "local_agreement": 0.95,
  "hosted_valid_share": 0.95,
  "hosted_agreement": 0.9
 },
 "hosted_collection": {
  "0": {
   "requests": 3700,
   "recorded": 3700,
   "uncertain": 0,
   "unattempted": 0,
   "not_collected": 0,
   "uncertain_keys": [],
   "early_terminated": {},
   "sessions": [
    {
     "session": "only session (02:06 to 02:42Z)",
     "started": "2026-10-03T02:06:18Z",
     "requests": 3700,
     "first_request": "2026-10-03T02:06:23Z",
     "last_request": "2026-10-03T02:41:57Z",
     "end": "end",
     "ended": "2026-10-03T02:41:57Z",
     "why": null,
     "resume_of": null,
     "resumed_by": null
    }
   ]
  },
  "1": {
   "requests": 3700,
   "recorded": 3509,
   "uncertain": 1,
   "unattempted": 190,
   "not_collected": 0,
   "uncertain_keys": [
    [
     "Nex-N2.5-mini",
     "btzsc",
     "biasframes_sex/3574",
     "biasframes_sex"
    ]
   ],
   "early_terminated": {
    "Nex-N2.5-mini": "Provider cannot meet the predeclared 95% valid-case gate; repeated ~300s empty responses."
   },
   "sessions": [
    {
     "session": "first session (04:45 to 06:26Z)",
     "started": "2026-10-03T04:45:26Z",
     "requests": 1794,
     "first_request": "2026-10-03T04:45:31Z",
     "last_request": "2026-10-03T06:26:24Z",
     "end": null,
     "ended": null,
     "why": null,
     "resume_of": null,
     "resumed_by": "second session (16:26 to 16:45Z)"
    },
    {
     "session": "second session (16:26 to 16:45Z)",
     "started": "2026-10-03T16:25:57Z",
     "requests": 1715,
     "first_request": "2026-10-03T16:26:02Z",
     "last_request": "2026-10-03T16:44:57Z",
     "end": "end",
     "ended": "2026-10-03T16:44:57Z",
     "why": null,
     "resume_of": "first session (04:45 to 06:26Z)",
     "resumed_by": null
    }
   ]
  }
 },
 "drift": "the interval covers the sampling of cases, conditional on the sessions measured; pass_ms is each pass's own point, and fragment_ms each session's own point within a pass that spans two sessions (passes [1])",
 "prices": "Nothing plotted uses a price: the y axis is the overall score, and the batch throughput rows are our measured decisions per second, both from the same frozen results as the cost figure (its price snapshot of 2026-09-29). The listing prices recorded with the hosted requests are each session's live listing, kept as recorded and used for nothing here. cost_usd is the API spend billed for the requests.",
 "jev": "Jev's point is its stored calls, made on other days than every other system's; we made no new Jev calls, so Jev was not timed in the same sessions as the others",
 "contention_control": {
  "alone_ms": 73.7910270690918,
  "beside_other_runs_ms": 73.44989478588104,
  "change": -0.004622950740221343,
  "limit": 0.1
 },
 "classes_never_substituted": [
  "local interactive",
  "hosted end-to-end",
  "batch throughput"
 ],
 "retime": {
  "timing_date": "2026-10-04",
  "collection": {
   "0": {
    "requests": 740,
    "recorded": 55,
    "uncertain": 0,
    "unattempted": 685,
    "not_collected": 0,
    "uncertain_keys": [],
    "early_terminated": {
     "gemma-4-31B-it": "rate-limited again at the slower pace, at case 1 of 145 of pass 0; no request sent after that",
     "Qwen3.5-9B": "rate-limited again at the slower pace, at case 21 of 145 of pass 0; no request sent after that"
    },
    "sessions": [
     {
      "session": "only session (09:34 to 09:44Z)",
      "started": "2026-10-04T09:33:49Z",
      "requests": 55,
      "first_request": "2026-10-04T09:33:54Z",
      "last_request": "2026-10-04T09:44:13Z",
      "end": "end",
      "ended": "2026-10-04T09:44:13Z",
      "why": null,
      "resume_of": null,
      "resumed_by": null
     }
    ]
   },
   "1": {
    "requests": 740,
    "recorded": 0,
    "uncertain": 0,
    "unattempted": 740,
    "not_collected": 0,
    "uncertain_keys": [],
    "early_terminated": {
     "gemma-4-31B-it": "stopped in pass 0 and not sent again in this version",
     "Qwen3.5-9B": "stopped in pass 0 and not sent again in this version"
    },
    "sessions": []
   }
  },
  "targets": [
   "gemma-4-31B-it",
   "Qwen3.5-9B"
  ],
  "pause_s": {
   "gemma-4-31B-it": 30.0,
   "Qwen3.5-9B": 30.0
  },
  "pause": "a host gets its next case only after it has been idle this long since its previous case ended: idle time between cases, outside the case clock, so no request takes longer",
  "pause_basis": {
   "gemma-4-31B-it": "Crusoe's published limit, per project and model in fixed one-minute windows: 30 requests and 500,000 tokens a minute in its lower tier (OpenRouter's tier is not published); a case sends at most 5 requests of at most 3 tries, so at most 2 cases a minute: 30 s",
   "Qwen3.5-9B": "DeepInfra publishes only 200 concurrent requests per model, which one case in flight (5 requests at most) cannot reach; its 429s on 2026-10-03 came at every pace, one even after 308 s idle, so the more conservative choice: Crusoe's 30 s"
  },
  "stop": "a target with a try answered HTTP 429, or with any error response that names a rate limit or carries the code 429 (an HTTP 200 whose body holds an error included), finishes that case with its registered retries and then sends nothing more in this version",
  "task_cap_usd": 1.15,
  "pass_gap_s": 7200
 },
 "points": [
  {
   "system": "Jev",
   "class": "hosted end-to-end (Jev's API, stored); reference in the local panel",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the system compared against: its stored hosted calls",
   "score": 0.7874129122812762,
   "score_ci95": [
    0.7760932335526933,
    0.796742189257802
   ],
   "ms": 121.86713190749288,
   "ci95_ms": [
    118.20954829454422,
    126.1389022693038
   ],
   "p95_ms": 211.1117891035974,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 1,
   "pass_ms": {
    "0": 121.86713190749288
   },
   "cases_from": "the local sample (600 cases)",
   "note": "Jev's stored calls on the overall score's questions: one case per call, serial, the SDK's retries inside the timed span, warm keep-alive; other days than every other row; its Classification calls include a slow window. jev_spread_passes: three passes with one call in flight on another sample, as per-call medians: its session-to-session spread, not an interval. Jev's point is its stored calls, made on other days than every other system's; we made no new Jev calls, so Jev was not timed in the same sessions as the others",
   "jev_spread_passes": [
    {
     "calls": 500,
     "median_ms_per_call": 205.91438957490027
    },
    {
     "calls": 500,
     "median_ms_per_call": 113.82105736993253
    },
    {
     "calls": 50,
     "median_ms_per_call": 92.28184376843274
    }
   ]
  },
  {
   "system": "Jev",
   "class": "hosted end-to-end (Jev's API, stored)",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the system compared against: its stored hosted calls",
   "score": 0.7874129122812762,
   "score_ci95": [
    0.7760932335526933,
    0.796742189257802
   ],
   "ms": 119.78238681331277,
   "ci95_ms": [
    115.72577524930239,
    122.9751049540937
   ],
   "p95_ms": 220.77097930014133,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 1,
   "pass_ms": {
    "0": 119.78238681331277
   },
   "cases_from": "the hosted sample (145 cases: 25 banking77, 20 btzsc, 25 abcd, 25 agentrewardbench, 25 guardrails, 25 beer_alignment)",
   "note": "Jev's stored calls on the overall score's questions: one case per call, serial, the SDK's retries inside the timed span, warm keep-alive; other days than every other row; its Classification calls include a slow window. jev_spread_passes: three passes with one call in flight on another sample, as per-call medians: its session-to-session spread, not an interval. Jev's point is its stored calls, made on other days than every other system's; we made no new Jev calls, so Jev was not timed in the same sessions as the others",
   "jev_spread_passes": [
    {
     "calls": 500,
     "median_ms_per_call": 205.91438957490027
    },
    {
     "calls": 500,
     "median_ms_per_call": 113.82105736993253
    },
    {
     "calls": 50,
     "median_ms_per_call": 92.28184376843274
    }
   ]
  },
  {
   "system": "gemma-4-31B-it",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev with the highest score; on the cost figure's frontier",
   "score": 0.7936289416033507,
   "score_ci95": [
    0.7811502598091409,
    0.8037610457550619
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 4.675225937944792,
     "cards": 2
    },
    "a6000": {
     "decisions_per_s": 2.803450386356455,
     "cards": 2
    }
   }
  },
  {
   "system": "gemma-4-31B-it",
   "class": "local interactive",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev with the highest score; on the cost figure's frontier",
   "score": 0.7936289416033507,
   "score_ci95": [
    0.7811502598091409,
    0.8037610457550619
   ],
   "ms": 176.88438622280955,
   "ci95_ms": [
    175.33520981669426,
    178.21595584973693
   ],
   "p95_ms": 3235.7194093056023,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 176.90544109791517,
    "1": 177.09515104070306,
    "2": 176.2476651929319
   },
   "ratio_to_jev": {
    "ratio": 1.365567524777519,
    "ci95": [
     1.3083892493955402,
     1.456900098982584
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "2 of 2 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9985207100591716,
    "agree": 675,
    "n": 676,
    "undetermined": 4
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 2,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T01:33:05Z",
   "ended": "2026-10-03T02:01:42Z",
   "load_s": 638.8317438736558
  },
  {
   "system": "gemma-4-31B-it",
   "class": "hosted end-to-end",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev with the highest score; on the cost figure's frontier",
   "score": 0.7936289416033507,
   "score_ci95": [
    0.7811502598091409,
    0.8037610457550619
   ],
   "ms": 834.4226609915495,
   "ci95_ms": [
    717.8303198888898,
    1211.1484268680215
   ],
   "p95_ms": 4116.1312740296125,
   "cases": 117,
   "units": 117,
   "decisions": 133,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 24,
   "passes": 2,
   "pass_ms": {
    "0": 1589.2991572618484,
    "1": 717.95470174402
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 812.5763521529734,
      "cases": 48
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 600.1054961234331,
      "cases": 53
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 8.666506265489108,
    "ci95": [
     7.704921830407018,
     10.687363726074276
    ],
    "cases": 117
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 6.051428211999003,
     "ci95": [
      6.051428211999003,
      6.051428211999003
     ],
     "cases": 44
    },
    "workflows": {
     "ratio": 8.966200587036878,
     "ci95": [
      8.005615211923669,
      14.59546480006805
     ],
     "cases": 42
    },
    "typesafe": {
     "ratio": 10.995695151307501,
     "ci95": [
      5.778780512158242,
      17.967171304775203
     ],
     "cases": 31
    }
   },
   "accepted": false,
   "rejected_for": [
    "valid cases below 0.95 in pass 0: 63 of 145",
    "valid cases below 0.95 in pass 1: 101 of 145"
   ],
   "valid_share": {
    "0": 0.43448275862068964,
    "1": 0.696551724137931
   },
   "agreement_with_scored": {
    "share": 0.9955357142857143,
    "agree": 223,
    "n": 224,
    "undetermined": 1
   },
   "host": "crusoe/bf16",
   "provider": "Crusoe",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 63,
     "failed_cases": 82,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 194,
     "no_label": 0,
     "off_host": 0,
     "empty": 1,
     "retried": 294
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 101,
     "failed_cases": 44,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 91,
     "no_label": 0,
     "off_host": 0,
     "empty": 2,
     "retried": 182
    }
   },
   "early_terminated": {},
   "cost_usd": 0.10417750000000003
  },
  {
   "system": "gemma-4-31B-it",
   "class": "hosted end-to-end, slower protocol (timed 2026-10-04)",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev with the highest score; on the cost figure's frontier",
   "score": 0.7936289416033507,
   "score_ci95": [
    0.7811502598091409,
    0.8037610457550619
   ],
   "ms": 4054.858762770891,
   "ci95_ms": [
    4054.858762770891,
    4054.858762770891
   ],
   "p95_ms": 4054.858762770891,
   "cases": 1,
   "units": 1,
   "decisions": 1,
   "parts": [
    "workflows"
   ],
   "blocks": 1,
   "passes": 1,
   "pass_ms": {
    "0": 4054.858762770891
   },
   "fragment_ms": {},
   "ratio_to_jev": {
    "ratio": 48.19964519408336,
    "ci95": [
     48.19964519408336,
     48.19964519408336
    ],
    "cases": 1
   },
   "ratio_to_jev_by_part": {
    "classification": null,
    "workflows": {
     "ratio": 48.19964519408336,
     "ci95": [
      48.19964519408336,
      48.19964519408336
     ],
     "cases": 1
    },
    "typesafe": null
   },
   "accepted": false,
   "rejected_for": [
    "retried under the slower protocol on 2026-10-04; rate-limited again at the slower pace, at case 1 of 145 of pass 0; no request sent after that; its first 429 came from OpenRouter's upstream shared pool at Crusoe (limit_source upstream_provider_shared_pool) at the first case of the re-time, and this study's previous request to Crusoe was on 2026-10-03 at 16:44Z, so the stop reflects the host's shared limit, not a limit on this study's own request rate; still 1 of 145 valid in pass 0 and 0 of 145 valid in pass 1",
    "pass 0 early-terminated (rate-limited again at the slower pace, at case 1 of 145 of pass 0; no request sent after that): 369 of 370 requests never sent and 0 uncertain, so not a complete latency endpoint",
    "valid cases below 0.95 in pass 0: 1 of 145 (1 of the 1 recorded in full: 1.00)",
    "pass 1 early-terminated (stopped in pass 0 and not sent again in this version): 370 of 370 requests never sent and 0 uncertain, so not a complete latency endpoint",
    "valid cases below 0.95 in pass 1: 0 of 145"
   ],
   "valid_share": {
    "0": 0.006896551724137931,
    "1": 0.0
   },
   "agreement_with_scored": {
    "share": 1.0,
    "agree": 1,
    "n": 1,
    "undetermined": 0
   },
   "host": "crusoe/bf16",
   "provider": "Crusoe",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 1,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 144,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 1,
     "uncertain_requests": 0,
     "unattempted_requests": 369,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 1
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 0,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 145,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 0,
     "uncertain_requests": 0,
     "unattempted_requests": 370,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    }
   },
   "early_terminated": {
    "0": "rate-limited again at the slower pace, at case 1 of 145 of pass 0; no request sent after that",
    "1": "stopped in pass 0 and not sent again in this version"
   },
   "cost_usd": 0.00011184,
   "protocol_variant": "slower sender: 30 s pause between cases at the host, and a stop at the first rate limit; otherwise the registered hosted protocol",
   "pause_s": 30.0,
   "timing_date": "2026-10-04",
   "note": "timed under the slower protocol on 2026-10-04, with a 30 s pause between cases: a host gets its next case only after it has been idle this long since its previous case ended: idle time between cases, outside the case clock, so no request takes longer; a target with a try answered HTTP 429, or with any error response that names a rate limit or carries the code 429 (an HTTP 200 whose body holds an error included), finishes that case with its registered retries and then sends nothing more in this version. A later date and other host load than the first timing, so the two are not pooled. Why 30 s: Crusoe's published limit, per project and model in fixed one-minute windows: 30 requests and 500,000 tokens a minute in its lower tier (OpenRouter's tier is not published); a case sends at most 5 requests of at most 3 tries, so at most 2 cases a minute: 30 s"
  },
  {
   "system": "MiMo-V2.6-Pro-MOPD",
   "class": "local interactive",
   "subset": "omitted",
   "role": null,
   "subset_reason": "matches Jev, but 1,024B: held, as are all models of 300B or more (its scored run was offloaded to system RAM); no OpenRouter listing",
   "score": 0.7921800559564733,
   "score_ci95": [
    0.7800503695307753,
    0.8015973260105869
   ],
   "gap": "held: models of 300B or more; its scored run was offloaded to system RAM"
  },
  {
   "system": "MiMo-V2.6-Pro-MOPD",
   "class": "hosted end-to-end",
   "subset": "omitted",
   "role": null,
   "subset_reason": "matches Jev, but 1,024B: held, as are all models of 300B or more (its scored run was offloaded to system RAM); no OpenRouter listing",
   "score": 0.7921800559564733,
   "score_ci95": [
    0.7800503695307753,
    0.8015973260105869
   ],
   "gap": "held: models of 300B or more; its scored run was offloaded to system RAM; not listed"
  },
  {
   "system": "Qwen3.8-Flash-Next-FP8",
   "class": "local interactive",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev; local only: no listing at the precision we scored",
   "score": 0.7879258097010293,
   "score_ci95": [
    0.7751836343552823,
    0.798723802971052
   ],
   "ms": 492.96933598816395,
   "ci95_ms": [
    487.3178889974952,
    498.8967375829816
   ],
   "p95_ms": 3877.853823825717,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 494.9440900236368,
    "1": 491.84796307235956,
    "2": 494.1001031547785
   },
   "ratio_to_jev": {
    "ratio": 3.8649811641771876,
    "ci95": [
     3.663519820840462,
     4.025566665869204
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "4 of 4 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9852941176470589,
    "agree": 670,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 4,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S",
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T06:07:13Z",
   "ended": "2026-10-03T06:44:52Z",
   "load_s": 459.63066773163155
  },
  {
   "system": "Qwen3.8-Flash-Next-FP8",
   "class": "hosted end-to-end",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev; local only: no listing at the precision we scored",
   "score": 0.7879258097010293,
   "score_ci95": [
    0.7751836343552823,
    0.798723802971052
   ],
   "gap": "no OpenRouter listing at the precision we scored"
  },
  {
   "system": "Qwen3.8-27B",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev; the cheapest matching listing (Wafer) and on the cost figure's frontier",
   "score": 0.7873852327328256,
   "score_ci95": [
    0.7736974816505293,
    0.7978583903667467
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 6.769679903517999,
     "cards": 2
    },
    "a6000": {
     "decisions_per_s": 4.377873316561891,
     "cards": 2
    }
   }
  },
  {
   "system": "Qwen3.8-27B",
   "class": "local interactive",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev; the cheapest matching listing (Wafer) and on the cost figure's frontier",
   "score": 0.7873852327328256,
   "score_ci95": [
    0.7736974816505293,
    0.7978583903667467
   ],
   "ms": 208.17103004083037,
   "ci95_ms": [
    206.90205367282033,
    210.2210628800094
   ],
   "p95_ms": 1609.4826818443835,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 207.20919826999307,
    "1": 209.89456214010715,
    "2": 208.27621780335903
   },
   "ratio_to_jev": {
    "ratio": 1.6259816671232659,
    "ci95": [
     1.5777706752460792,
     1.6593574487234393
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "2 of 2 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9940828402366864,
    "agree": 672,
    "n": 676,
    "undetermined": 4
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 2,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T01:18:45Z",
   "ended": "2026-10-03T01:32:55Z",
   "load_s": 100.54210917372257
  },
  {
   "system": "Qwen3.8-27B",
   "class": "hosted end-to-end",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev; the cheapest matching listing (Wafer) and on the cost figure's frontier",
   "score": 0.7873852327328256,
   "score_ci95": [
    0.7736974816505293,
    0.7978583903667467
   ],
   "ms": 559.9270183593035,
   "ci95_ms": [
    551.3598658144474,
    569.9094026349485
   ],
   "p95_ms": 1577.2487833164632,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 552.7491248212755,
    "1": 566.927385982126
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 619.4791621528566,
      "cases": 70
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 540.2353387326002,
      "cases": 75
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 5.121473989383342,
    "ci95": [
     4.868238231407477,
     5.228317565549754
    ],
    "cases": 145
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 4.172624612507134,
     "ci95": [
      4.172624612507134,
      4.172624612507134
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 6.473843766078555,
     "ci95": [
      6.054186662971417,
      7.834056528888902
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 5.121473989383342,
     "ci95": [
      4.766459238039714,
      5.543057335920615
     ],
     "cases": 50
    }
   },
   "accepted": true,
   "rejected_for": [],
   "valid_share": {
    "0": 1.0,
    "1": 1.0
   },
   "agreement_with_scored": {
    "share": 0.9610778443113772,
    "agree": 321,
    "n": 334,
    "undetermined": 2
   },
   "host": "wafer",
   "provider": "Wafer",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    }
   },
   "early_terminated": {},
   "cost_usd": 0.0326595896
  },
  {
   "system": "Qwen3.8-27B",
   "class": "hosted end-to-end (second host, check only)",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev; the cheapest matching listing (Wafer) and on the cost figure's frontier",
   "score": 0.7873852327328256,
   "score_ci95": [
    0.7736974816505293,
    0.7978583903667467
   ],
   "ms": 400.0086379237473,
   "ci95_ms": [
    391.54945155605674,
    416.6310792788863
   ],
   "p95_ms": 3707.9711039550602,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 385.6450701132417,
    "1": 420.7611051388085
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 608.3491020835936,
      "cases": 70
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 391.9737609103322,
      "cases": 75
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 3.6506761262753558,
    "ci95": [
     3.50264883918442,
     4.344174021822715
    ],
    "cases": 145
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 3.154441380538302,
     "ci95": [
      3.154441380538302,
      3.154441380538302
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 10.617135226084846,
     "ci95": [
      4.789703742197593,
      12.753243049651974
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 4.660362537539447,
     "ci95": [
      3.6506761262753558,
      5.860793088856921
     ],
     "cases": 50
    }
   },
   "accepted": true,
   "rejected_for": [],
   "valid_share": {
    "0": 1.0,
    "1": 1.0
   },
   "agreement_with_scored": {
    "share": 0.9550898203592815,
    "agree": 319,
    "n": 334,
    "undetermined": 2
   },
   "host": "ionstream/fp8",
   "provider": "Ionstream",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    }
   },
   "early_terminated": {},
   "cost_usd": 0.1052271535999999
  },
  {
   "system": "gemma-4-26B-A4B-it",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev",
   "score": 0.7838858450796923,
   "score_ci95": [
    0.771462762961876,
    0.7944463798197942
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 18.901368108107683,
     "cards": 2
    },
    "a6000": {
     "decisions_per_s": 12.777055554191197,
     "cards": 2
    }
   }
  },
  {
   "system": "gemma-4-26B-A4B-it",
   "class": "local interactive",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev",
   "score": 0.7838858450796923,
   "score_ci95": [
    0.771462762961876,
    0.7944463798197942
   ],
   "ms": 80.79485781490803,
   "ci95_ms": [
    79.67262109741569,
    81.6810168325901
   ],
   "p95_ms": 813.7559397146106,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 80.98091324791312,
    "1": 81.18696603924036,
    "2": 79.70468420535326
   },
   "ratio_to_jev": {
    "ratio": 0.6838274523555269,
    "ci95": [
     0.6664224523787361,
     0.7038360013058845
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "2 of 2 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9852941176470589,
    "agree": 670,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 2,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T02:03:10Z",
   "ended": "2026-10-03T02:10:27Z",
   "load_s": 98.21617882605642
  },
  {
   "system": "gemma-4-26B-A4B-it",
   "class": "hosted end-to-end",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev",
   "score": 0.7838858450796923,
   "score_ci95": [
    0.771462762961876,
    0.7944463798197942
   ],
   "ms": 944.7718700394034,
   "ci95_ms": [
    866.9338868930936,
    989.2960540018976
   ],
   "p95_ms": 3817.5795702263713,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 868.0321429856122,
    "1": 1090.339204762131
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 989.3573760055006,
      "cases": 71
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 1101.9108328036964,
      "cases": 74
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 8.609145842251912,
    "ci95": [
     7.985673071137378,
     9.493761201111937
    ],
    "cases": 145
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 6.404999657842701,
     "ci95": [
      6.404999657842701,
      6.404999657842701
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 8.912015738436937,
     "ci95": [
      7.805941546729516,
      10.480920009865123
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 11.842920216826812,
     "ci95": [
      9.918070027370405,
      14.03739526501542
     ],
     "cases": 50
    }
   },
   "accepted": true,
   "rejected_for": [],
   "valid_share": {
    "0": 1.0,
    "1": 1.0
   },
   "agreement_with_scored": {
    "share": 0.9578313253012049,
    "agree": 318,
    "n": 332,
    "undetermined": 4
   },
   "host": "dekallm/bf16",
   "provider": "DekaLLM",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 9
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 29
    }
   },
   "early_terminated": {},
   "cost_usd": 0.07119187999999994
  },
  {
   "system": "gpt-6-luna",
   "class": "hosted end-to-end",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a closed API whose interval holds Jev's score (hosted only)",
   "score": 0.7817287546109833,
   "score_ci95": [
    0.7688965271915621,
    0.7922078332076429
   ],
   "ms": 839.4500249996781,
   "ci95_ms": [
    811.8731300346553,
    865.567578934133
   ],
   "p95_ms": 1489.1562000848353,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 803.8713028654456,
    "1": 874.5578280650079
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 917.2008689492941,
      "cases": 70
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 848.5773890279233,
      "cases": 75
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 7.277339870310855,
    "ci95": [
     6.967292503627069,
     7.559027530580599
    ],
    "cases": 145
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 6.967292503627069,
     "ci95": [
      6.967292503627069,
      6.967292503627069
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 7.357460234152219,
     "ci95": [
      6.560555402847657,
      8.096344715058793
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 7.471292137043677,
     "ci95": [
      6.754907808301085,
      8.200181908441085
     ],
     "cases": 50
    }
   },
   "accepted": true,
   "rejected_for": [],
   "valid_share": {
    "0": 1.0,
    "1": 1.0
   },
   "agreement_with_scored": {
    "share": 0.9788519637462235,
    "agree": 324,
    "n": 331,
    "undetermined": 5
   },
   "host": "its lab's API",
   "provider": "gpt-6-luna",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    }
   },
   "early_terminated": {},
   "cost_usd": 0.10622979999999996
  },
  {
   "system": "gpt-6-luna",
   "class": "hosted end-to-end, 16 workers (check only)",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a closed API whose interval holds Jev's score (hosted only)",
   "score": 0.7817287546109833,
   "score_ci95": [
    0.7688965271915621,
    0.7922078332076429
   ],
   "ms": 791.7297361418605,
   "ci95_ms": [
    711.6755107417703,
    826.9575443118811
   ],
   "p95_ms": 1826.6665791161358,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 1,
   "pass_ms": {
    "0": 791.7297361418605
   },
   "note": "every graded decision of the case, 16 requests in flight across cases; slowest of the case"
  },
  {
   "system": "MiMo-V2.6-Flash-RL",
   "class": "local interactive",
   "subset": "omitted",
   "role": null,
   "subset_reason": "matches Jev, but 310.8B: held, as are all models of 300B or more (its scored run was offloaded); its DeepInfra listing is outside the hosted study",
   "score": 0.7801495461609519,
   "score_ci95": [
    0.7678583123422807,
    0.7902372975875347
   ],
   "gap": "held: models of 300B or more; its scored run was offloaded to system RAM"
  },
  {
   "system": "MiMo-V2.6-Flash-RL",
   "class": "hosted end-to-end",
   "subset": "omitted",
   "role": null,
   "subset_reason": "matches Jev, but 310.8B: held, as are all models of 300B or more (its scored run was offloaded); its DeepInfra listing is outside the hosted study",
   "score": 0.7801495461609519,
   "score_ci95": [
    0.7678583123422807,
    0.7902372975875347
   ],
   "gap": "held: models of 300B or more; its scored run was offloaded to system RAM"
  },
  {
   "system": "Qwen3.6-35B-A3B-FP8",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev",
   "score": 0.7787387046226982,
   "score_ci95": [
    0.7656265422123629,
    0.7896189002292264
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 26.24435356001495,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 13.292106756443436,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.6-35B-A3B-FP8",
   "class": "local interactive",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev",
   "score": 0.7787387046226982,
   "score_ci95": [
    0.7656265422123629,
    0.7896189002292264
   ],
   "ms": 153.5617560148239,
   "ci95_ms": [
    152.64879819005728,
    154.69944337382913
   ],
   "p95_ms": 480.37539422512054,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 181.14914372563362,
    "1": 154.0210791863501,
    "2": 149.68390902504325
   },
   "ratio_to_jev": {
    "ratio": 1.4196098235538437,
    "ci95": [
     1.3472819451500528,
     1.4613090195107115
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "2 of 2 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9808823529411764,
    "agree": 667,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 2,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T02:10:35Z",
   "ended": "2026-10-03T02:19:08Z",
   "load_s": 128.27855804376304
  },
  {
   "system": "Qwen3.6-35B-A3B-FP8",
   "class": "hosted end-to-end",
   "subset": "chosen: match",
   "role": "match",
   "subset_reason": "matches Jev",
   "score": 0.7787387046226982,
   "score_ci95": [
    0.7656265422123629,
    0.7896189002292264
   ],
   "ms": 383.0989859998226,
   "ci95_ms": [
    355.13739800080657,
    411.62490379065275
   ],
   "p95_ms": 1919.8653441853821,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 352.1846877411008,
    "1": 437.9256730899215
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 427.4548818357289,
      "cases": 67
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 440.10099302977324,
      "cases": 73
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 3.614961426454513,
    "ci95": [
     3.426475516721625,
     3.9948817229840325
    ],
    "cases": 145
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 2.590133651610023,
     "ci95": [
      2.590133651610023,
      2.590133651610023
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 4.108200708358556,
     "ci95": [
      3.489768075582145,
      5.3185173097173
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 3.9948817229840325,
     "ci95": [
      3.5715678131448008,
      5.839558526111113
     ],
     "cases": 50
    }
   },
   "accepted": true,
   "rejected_for": [],
   "valid_share": {
    "0": 1.0,
    "1": 0.9655172413793104
   },
   "agreement_with_scored": {
    "share": 0.9577039274924471,
    "agree": 317,
    "n": 331,
    "undetermined": 2
   },
   "host": "akashml/fp8",
   "provider": "AkashML",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 145,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 10
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 140,
     "failed_cases": 5,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 5,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 55
    }
   },
   "early_terminated": {},
   "cost_usd": 0.1087806
  },
  {
   "system": "Qwen3.5-9B",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the borderline competitor (paired lower end -0.029 against -0.03)",
   "score": 0.7690839236104026,
   "score_ci95": [
    0.7559820604722134,
    0.779165454271013
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 16.561979873372767,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 9.951782301311377,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.5-9B",
   "class": "local interactive",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the borderline competitor (paired lower end -0.029 against -0.03)",
   "score": 0.7690839236104026,
   "score_ci95": [
    0.7559820604722134,
    0.779165454271013
   ],
   "ms": 105.82308704033494,
   "ci95_ms": [
    105.00481491908431,
    107.10977530106902
   ],
   "p95_ms": 693.4913112781942,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 102.6390278711915,
    "1": 104.60926219820976,
    "2": 111.60799488425255
   },
   "ratio_to_jev": {
    "ratio": 0.8464773067872303,
    "ci95": [
     0.812738808074668,
     0.8927460961580554
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "1 of 1 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9955882352941177,
    "agree": 677,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 1,
   "cards": [
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T03:14:02Z",
   "ended": "2026-10-03T03:20:51Z",
   "load_s": 59.53396737901494
  },
  {
   "system": "Qwen3.5-9B",
   "class": "hosted end-to-end",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the borderline competitor (paired lower end -0.029 against -0.03)",
   "score": 0.7690839236104026,
   "score_ci95": [
    0.7559820604722134,
    0.779165454271013
   ],
   "ms": 649.6674818918109,
   "ci95_ms": [
    630.1273149438202,
    678.6258281208575
   ],
   "p95_ms": 1776.9359690137208,
   "cases": 144,
   "units": 144,
   "decisions": 167,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 640.0432279333472,
    "1": 676.1276791803539
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 629.0451749227941,
      "cases": 70
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 829.8088889569044,
      "cases": 67
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 5.863636141898248,
    "ci95": [
     5.564127802733684,
     6.282857982964044
    ],
    "cases": 144
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 4.2937599605738335,
     "ci95": [
      4.2937599605738335,
      4.335098150430844
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 6.236951798493348,
     "ci95": [
      5.406357873848737,
      6.613914794902794
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 7.278308126366241,
     "ci95": [
      6.022475763223608,
      8.650013243907104
     ],
     "cases": 49
    }
   },
   "accepted": false,
   "rejected_for": [
    "valid cases below 0.95 in pass 1: 137 of 145"
   ],
   "valid_share": {
    "0": 0.9862068965517241,
    "1": 0.9448275862068966
   },
   "agreement_with_scored": {
    "share": 0.9628482972136223,
    "agree": 311,
    "n": 323,
    "undetermined": 6
   },
   "host": "deepinfra/bf16",
   "provider": "DeepInfra",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 143,
     "failed_cases": 2,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 1,
     "no_label": 1,
     "off_host": 0,
     "empty": 0,
     "retried": 3
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 137,
     "failed_cases": 8,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 21,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 67
    }
   },
   "early_terminated": {},
   "cost_usd": 0.11651530000000007
  },
  {
   "system": "Qwen3.5-9B",
   "class": "hosted end-to-end, slower protocol (timed 2026-10-04)",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the borderline competitor (paired lower end -0.029 against -0.03)",
   "score": 0.7690839236104026,
   "score_ci95": [
    0.7559820604722134,
    0.779165454271013
   ],
   "ms": 699.2628909647465,
   "ci95_ms": [
    654.0193306282163,
    799.6599180623889
   ],
   "p95_ms": 1738.2055018097162,
   "cases": 20,
   "units": 20,
   "decisions": 24,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 6,
   "passes": 1,
   "pass_ms": {
    "0": 699.2628909647465
   },
   "fragment_ms": {},
   "ratio_to_jev": {
    "ratio": 4.4810869400677955,
    "ci95": [
     4.290118554217435,
     6.70020421410461
    ],
    "cases": 20
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 3.207191801113031,
     "ci95": [
      2.340644604350769,
      3.207191801113031
     ],
     "cases": 4
    },
    "workflows": {
     "ratio": 4.441080655087711,
     "ci95": [
      3.304480447531645,
      12.266727727925653
     ],
     "cases": 9
    },
    "typesafe": {
     "ratio": 4.4810869400677955,
     "ci95": [
      4.290118554217435,
      7.7514576777547735
     ],
     "cases": 7
    }
   },
   "accepted": false,
   "rejected_for": [
    "retried under the slower protocol on 2026-10-04; rate-limited again at the slower pace, at case 21 of 145 of pass 0; no request sent after that; the 429 came from OpenRouter's upstream shared pool at DeepInfra (limit_source upstream_provider_shared_pool, provider error engine_overloaded) at case 21, sent after the 30 s pause like every case since its first; this study's previous request to DeepInfra before the re-time was on 2026-10-03 at 16:44Z, so the stop reflects the host's shared limit, not a limit on this study's own request rate; still 20 of 145 valid in pass 0 and 0 of 145 valid in pass 1",
    "pass 0 early-terminated (rate-limited again at the slower pace, at case 21 of 145 of pass 0; no request sent after that): 316 of 370 requests never sent and 0 uncertain, so not a complete latency endpoint",
    "valid cases below 0.95 in pass 0: 20 of 145 (20 of the 21 recorded in full: 0.95)",
    "pass 1 early-terminated (stopped in pass 0 and not sent again in this version): 370 of 370 requests never sent and 0 uncertain, so not a complete latency endpoint",
    "valid cases below 0.95 in pass 1: 0 of 145"
   ],
   "valid_share": {
    "0": 0.13793103448275862,
    "1": 0.0
   },
   "agreement_with_scored": {
    "share": 0.9583333333333334,
    "agree": 23,
    "n": 24,
    "undetermined": 2
   },
   "host": "deepinfra/bf16",
   "provider": "DeepInfra",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 20,
     "failed_cases": 1,
     "uncertain_cases": 0,
     "unattempted_cases": 124,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 54,
     "uncertain_requests": 0,
     "unattempted_requests": 316,
     "not_collected_requests": 0,
     "http_failed": 1,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 1
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 0,
     "failed_cases": 0,
     "uncertain_cases": 0,
     "unattempted_cases": 145,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 0,
     "uncertain_requests": 0,
     "unattempted_requests": 370,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 0,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    }
   },
   "early_terminated": {
    "0": "rate-limited again at the slower pace, at case 21 of 145 of pass 0; no request sent after that",
    "1": "stopped in pass 0 and not sent again in this version"
   },
   "cost_usd": 0.00919765,
   "protocol_variant": "slower sender: 30 s pause between cases at the host, and a stop at the first rate limit; otherwise the registered hosted protocol",
   "pause_s": 30.0,
   "timing_date": "2026-10-04",
   "note": "timed under the slower protocol on 2026-10-04, with a 30 s pause between cases: a host gets its next case only after it has been idle this long since its previous case ended: idle time between cases, outside the case clock, so no request takes longer; a target with a try answered HTTP 429, or with any error response that names a rate limit or carries the code 429 (an HTTP 200 whose body holds an error included), finishes that case with its registered retries and then sends nothing more in this version. A later date and other host load than the first timing, so the two are not pooled. Why 30 s: DeepInfra publishes only 200 concurrent requests per model, which one case in flight (5 requests at most) cannot reach; its 429s on 2026-10-03 came at every pace, one even after 308 s idle, so the more conservative choice: Crusoe's 30 s"
  },
  {
   "system": "Qwen3-Coder-Next-FP8",
   "class": "local interactive",
   "subset": "omitted",
   "role": null,
   "subset_reason": "does not match Jev and is on no cost frontier; four cards",
   "score": 0.764729257052419,
   "score_ci95": [
    0.7538129875415372,
    0.7737747153828779
   ],
   "gap": "does not match Jev and is on no cost frontier; four cards"
  },
  {
   "system": "Qwen3-Coder-Next-FP8",
   "class": "hosted end-to-end",
   "subset": "omitted",
   "role": null,
   "subset_reason": "does not match Jev and is on no cost frontier; four cards",
   "score": 0.764729257052419,
   "score_ci95": [
    0.7538129875415372,
    0.7737747153828779
   ],
   "gap": "outside the hosted subset (not called in the full passes)"
  },
  {
   "system": "Nex-N2.5-mini",
   "class": "local interactive",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "on the cost figure's frontier",
   "score": 0.7633952406284953,
   "score_ci95": [
    0.7508385508248363,
    0.7738876950795762
   ],
   "ms": 147.4212142638862,
   "ci95_ms": [
    146.34504215791821,
    149.7181747108698
   ],
   "p95_ms": 488.3969370275736,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 147.09973009303212,
    "1": 149.7181747108698,
    "2": 146.16133086383343
   },
   "ratio_to_jev": {
    "ratio": 1.159682777444958,
    "ci95": [
     1.1285552242395203,
     1.1915067323730206
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "2 of 2 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9705882352941176,
    "agree": 660,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 2,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T02:19:18Z",
   "ended": "2026-10-03T02:28:02Z",
   "load_s": 173.27265942189842
  },
  {
   "system": "Nex-N2.5-mini",
   "class": "hosted end-to-end",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "on the cost figure's frontier",
   "score": 0.7633952406284953,
   "score_ci95": [
    0.7508385508248363,
    0.7738876950795762
   ],
   "ms": 669.5672259666026,
   "ci95_ms": [
    631.2099481001496,
    741.948549170047
   ],
   "p95_ms": 3329.4979091733694,
   "cases": 144,
   "units": 144,
   "decisions": 167,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 631.2099481001496,
    "1": 1366.8159702792764
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 1366.8159702792764,
      "cases": 54
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 5.881090474971254,
    "ci95": [
     5.64043273841923,
     6.623367030047594
    ],
    "cases": 144
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 4.0966557443093805,
     "ci95": [
      4.0966557443093805,
      4.0966557443093805
     ],
     "cases": 44
    },
    "workflows": {
     "ratio": 6.407735205659087,
     "ci95": [
      5.616637889568106,
      8.023855716092578
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 7.259507124102191,
     "ci95": [
      6.325501121135051,
      8.960872385433152
     ],
     "cases": 50
    }
   },
   "accepted": false,
   "rejected_for": [
    "pass 1 early-terminated (Provider cannot meet the predeclared 95% valid-case gate; repeated ~300s empty responses): 190 of 370 requests never sent and 1 uncertain, so not a complete latency endpoint",
    "valid cases below 0.95 in pass 1: 54 of 145 (54 of the 70 recorded in full: 0.77)"
   ],
   "valid_share": {
    "0": 0.993103448275862,
    "1": 0.3724137931034483
   },
   "agreement_with_scored": {
    "share": 0.9567099567099567,
    "agree": 221,
    "n": 231,
    "undetermined": 3
   },
   "host": "nex-agi/bf16",
   "provider": "Nex AGI",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 144,
     "failed_cases": 1,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 1,
     "off_host": 0,
     "empty": 1,
     "retried": 1
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 54,
     "failed_cases": 16,
     "uncertain_cases": 1,
     "unattempted_cases": 74,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 179,
     "uncertain_requests": 1,
     "unattempted_requests": 190,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 15,
     "off_host": 0,
     "empty": 25,
     "retried": 0
    }
   },
   "early_terminated": {
    "1": "Provider cannot meet the predeclared 95% valid-case gate; repeated ~300s empty responses."
   },
   "cost_usd": 0.02482159
  },
  {
   "system": "Mistral-Medium-3.5-128B",
   "class": "local interactive",
   "subset": "omitted",
   "role": null,
   "subset_reason": "does not match Jev; unpriced; four cards",
   "score": 0.7474445055131964,
   "score_ci95": [
    0.7334491736004284,
    0.7588756816697096
   ],
   "gap": "does not match Jev; unpriced; four cards"
  },
  {
   "system": "Mistral-Medium-3.5-128B",
   "class": "hosted end-to-end",
   "subset": "omitted",
   "role": null,
   "subset_reason": "does not match Jev; unpriced; four cards",
   "score": 0.7474445055131964,
   "score_ci95": [
    0.7334491736004284,
    0.7588756816697096
   ],
   "gap": "no OpenRouter listing at the precision we scored (first-party only)"
  },
  {
   "system": "gemini-2.5-flash-lite",
   "class": "hosted end-to-end",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a cheap closed API (hosted only)",
   "score": 0.7353249504019783,
   "score_ci95": [
    0.7206292975102984,
    0.7470186420956758
   ],
   "ms": 325.1515696756542,
   "ci95_ms": [
    320.08681911975145,
    330.3411300294101
   ],
   "p95_ms": 459.8979209549725,
   "cases": 144,
   "units": 144,
   "decisions": 167,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 318.3550317771733,
    "1": 328.20564694702625
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 329.11751279607415,
      "cases": 71
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 328.20564694702625,
      "cases": 73
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 2.8180058816413873,
    "ci95": [
     2.7424014608000893,
     2.9754803811986856
    ],
    "cases": 144
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 2.778950335773854,
     "ci95": [
      2.778950335773854,
      2.778950335773854
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 3.1341091549695808,
     "ci95": [
      2.748947956357568,
      3.304775274269939
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 2.7641277900270778,
     "ci95": [
      2.6654741915354996,
      2.9994304471032516
     ],
     "cases": 49
    }
   },
   "accepted": true,
   "rejected_for": [],
   "valid_share": {
    "0": 0.993103448275862,
    "1": 0.993103448275862
   },
   "agreement_with_scored": {
    "share": 1.0,
    "agree": 330,
    "n": 330,
    "undetermined": 4
   },
   "host": "its lab's API",
   "provider": "gemini-2.5-flash-lite",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 144,
     "failed_cases": 1,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 1,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 144,
     "failed_cases": 1,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 1,
     "off_host": 0,
     "empty": 0,
     "retried": 0
    }
   },
   "early_terminated": {},
   "cost_usd": 0.11896878000000005
  },
  {
   "system": "gemini-2.5-flash-lite",
   "class": "hosted end-to-end, 16 workers (check only)",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a cheap closed API (hosted only)",
   "score": 0.7353249504019783,
   "score_ci95": [
    0.7206292975102984,
    0.7470186420956758
   ],
   "ms": 306.2196699902415,
   "ci95_ms": [
    298.09756204485893,
    315.761836245656
   ],
   "p95_ms": 607.6560146175325,
   "cases": 145,
   "units": 145,
   "decisions": 168,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 1,
   "pass_ms": {
    "0": 306.2196699902415
   },
   "note": "every graded decision of the case, 16 requests in flight across cases; slowest of the case"
  },
  {
   "system": "MiMo-V2.6-Distill-Qwen-9B",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.7255335875994714,
   "score_ci95": [
    0.7095730468235484,
    0.7388775675767806
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "MiMo-V2.6-Distill-Qwen-9B",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.7255335875994714,
   "score_ci95": [
    0.7095730468235484,
    0.7388775675767806
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "gemma-4-31B",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.7224120495973064,
   "score_ci95": [
    0.7074478963332282,
    0.7345523942103023
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "gemma-4-31B",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.7224120495973064,
   "score_ci95": [
    0.7074478963332282,
    0.7345523942103023
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "Qwen3.5-9B-Base",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.718734722223428,
   "score_ci95": [
    0.7029932598465805,
    0.7315063800111872
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "Qwen3.5-9B-Base",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.718734722223428,
   "score_ci95": [
    0.7029932598465805,
    0.7315063800111872
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "gpt-oss-120b",
   "class": "local interactive",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "on the ranked systems' cost frontier (its listing forces reasoning: local only)",
   "score": 0.7024470961701496,
   "score_ci95": [
    0.6881012243306083,
    0.714259183044212
   ],
   "ms": 90.67696100100875,
   "ci95_ms": [
    88.53641804307699,
    93.07051729410887
   ],
   "p95_ms": 802.3229511454701,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 90.51682380959392,
    "1": 88.86481216177344,
    "2": 94.16966093704104
   },
   "ratio_to_jev": {
    "ratio": 0.7664549238418906,
    "ci95": [
     0.7468976710546394,
     0.7870366987346185
    ],
    "cases": 583
   },
   "accepted": false,
   "rejected_for": [
    "agreement with the scored run below 0.95: 633 of 680 decisions (646 needed); cause: prompts and engine settings match the scored run byte for byte, but the model's vLLM forward pass on L40S (MXFP4 experts on the Marlin MoE kernel) returns different log-probabilities for identical inputs; 58 of 680 decisions change across the timing run's own three passes"
   ],
   "speed_valid": true,
   "speed_note": "2 of 2 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9308823529411765,
    "agree": 633,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 2,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T02:38:09Z",
   "ended": "2026-10-03T02:52:56Z",
   "load_s": 537.449057654012
  },
  {
   "system": "gpt-oss-120b",
   "class": "hosted end-to-end",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "on the ranked systems' cost frontier (its listing forces reasoning: local only)",
   "score": 0.7024470961701496,
   "score_ci95": [
    0.6881012243306083,
    0.714259183044212
   ],
   "gap": "its listing forces reasoning, so it cannot be read as the configuration we scored"
  },
  {
   "system": "granite-4.2-8b",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.7007768838911129,
   "score_ci95": [
    0.685757237553442,
    0.7124861213458213
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 17.305466517708705,
     "cards": 1
    }
   }
  },
  {
   "system": "granite-4.2-8b",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.7007768838911129,
   "score_ci95": [
    0.685757237553442,
    0.7124861213458213
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "granite-4.2-8b",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.7007768838911129,
   "score_ci95": [
    0.685757237553442,
    0.7124861213458213
   ],
   "gap": "outside the hosted subset (not called in the full passes)"
  },
  {
   "system": "LLaDA-1.5",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.6981088753351562,
   "score_ci95": [
    0.6834635056358637,
    0.7097268354887726
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "LLaDA-1.5",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.6981088753351562,
   "score_ci95": [
    0.6834635056358637,
    0.7097268354887726
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "LLaDA-8B-Instruct",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.6937160500253516,
   "score_ci95": [
    0.6787392743194056,
    0.705672719427702
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "LLaDA-8B-Instruct",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.6937160500253516,
   "score_ci95": [
    0.6787392743194056,
    0.705672719427702
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "Mistral-Nemo-Instruct-2407-FP8",
   "class": "local interactive",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the cheapest point of the cost figure's frontier",
   "score": 0.6329811335453825,
   "score_ci95": [
    0.6142428224122574,
    0.6486122272841062
   ],
   "ms": 61.17001874372363,
   "ci95_ms": [
    58.98630898445845,
    63.680148648563765
   ],
   "p95_ms": 716.0985791124403,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 60.17092289403081,
    "1": 61.99559662491083,
    "2": 61.17738923057914
   },
   "ratio_to_jev": {
    "ratio": 0.5215164480195789,
    "ci95": [
     0.5070194033754524,
     0.5373669537183019
    ],
    "cases": 583
   },
   "accepted": false,
   "rejected_for": [
    "agreement with the scored run below 0.95: 641 of 680 decisions (646 needed); cause: same prompts, checkpoint, engine settings and card type, but the timing run used a separately compiled model (a different autotuned RMSNorm reduction) and one case per call instead of whole-benchmark batches; these rounding-level changes reversed 39 near-tie decisions, 32 of them decided by 0.5 nats or less in the scored run"
   ],
   "speed_valid": true,
   "speed_note": "1 of 1 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9426470588235294,
    "agree": 641,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 1,
   "cards": [
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T03:21:02Z",
   "ended": "2026-10-03T03:27:07Z",
   "load_s": 73.92830347595736
  },
  {
   "system": "Mistral-Nemo-Instruct-2407-FP8",
   "class": "hosted end-to-end",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "the cheapest point of the cost figure's frontier",
   "score": 0.6329811335453825,
   "score_ci95": [
    0.6142428224122574,
    0.6486122272841062
   ],
   "ms": 813.7736502103508,
   "ci95_ms": [
    768.3029226027429,
    895.1979428529739
   ],
   "p95_ms": 3134.508367627859,
   "cases": 144,
   "units": 144,
   "decisions": 167,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 2,
   "pass_ms": {
    "0": 755.7489792816341,
    "1": 915.4474521055818
   },
   "fragment_ms": {
    "1": {
     "first session (04:45 to 06:26Z)": {
      "ms": 832.651344127953,
      "cases": 71
     },
     "second session (16:26 to 16:45Z)": {
      "ms": 1106.843275949359,
      "cases": 73
     }
    }
   },
   "ratio_to_jev": {
    "ratio": 7.841112386875288,
    "ci95": [
     6.852518264276555,
     8.704125523838599
    ],
    "cases": 144
   },
   "ratio_to_jev_by_part": {
    "classification": {
     "ratio": 5.218480602018849,
     "ci95": [
      5.218480602018849,
      5.218480602018849
     ],
     "cases": 45
    },
    "workflows": {
     "ratio": 9.827706944277162,
     "ci95": [
      7.638673417070825,
      11.134217009756043
     ],
     "cases": 50
    },
    "typesafe": {
     "ratio": 8.644001110956738,
     "ci95": [
      7.047568425028995,
      10.538395012086088
     ],
     "cases": 49
    }
   },
   "accepted": false,
   "rejected_for": [
    "agreement with the scored configuration below 0.9: 292 of 328 (296 needed)"
   ],
   "valid_share": {
    "0": 0.993103448275862,
    "1": 0.993103448275862
   },
   "agreement_with_scored": {
    "share": 0.8902439024390244,
    "agree": 292,
    "n": 328,
    "undetermined": 6
   },
   "host": "dekallm/fp8",
   "provider": "DekaLLM",
   "per_pass": {
    "0": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 144,
     "failed_cases": 1,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 1,
     "off_host": 0,
     "empty": 0,
     "retried": 7
    },
    "1": {
     "cases": 145,
     "requests": 370,
     "valid_cases": 144,
     "failed_cases": 1,
     "uncertain_cases": 0,
     "unattempted_cases": 0,
     "incomplete_cases": 0,
     "not_collected_cases": 0,
     "recorded_requests": 370,
     "uncertain_requests": 0,
     "unattempted_requests": 0,
     "not_collected_requests": 0,
     "http_failed": 0,
     "no_label": 1,
     "off_host": 0,
     "empty": 0,
     "retried": 16
    }
   },
   "early_terminated": {},
   "cost_usd": 0.021173820000000006
  },
  {
   "system": "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.5909457724978816,
   "score_ci95": [
    0.5774473156003284,
    0.6029932162486913
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.5909457724978816,
   "score_ci95": [
    0.5774473156003284,
    0.6029932162486913
   ],
   "gap": "outside the hosted subset (not called in the full passes)"
  },
  {
   "system": "Qwen3.5-0.8B-Base",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.4451333224873288,
   "score_ci95": [
    0.42994255934176706,
    0.4573823803207741
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 79.6153597357339,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 57.11316197339959,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.5-0.8B-Base",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.4451333224873288,
   "score_ci95": [
    0.42994255934176706,
    0.4573823803207741
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "Qwen3.5-0.8B-Base",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.4451333224873288,
   "score_ci95": [
    0.42994255934176706,
    0.4573823803207741
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "ModernBERT-Large-Instruct",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.355420883737227,
   "score_ci95": [
    0.3419539789426237,
    0.3680246427553993
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 39.954445004080796,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 21.990778363308106,
     "cards": 1
    }
   }
  },
  {
   "system": "ModernBERT-Large-Instruct",
   "class": "local interactive",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.355420883737227,
   "score_ci95": [
    0.3419539789426237,
    0.3680246427553993
   ],
   "gap": "outside the chosen subset: not timed"
  },
  {
   "system": "ModernBERT-Large-Instruct",
   "class": "hosted end-to-end",
   "subset": "outside the subset",
   "role": null,
   "subset_reason": "not in the chosen subset (the matching models, their quantized checkpoints and a few references); not timed",
   "score": 0.355420883737227,
   "score_ci95": [
    0.3419539789426237,
    0.3680246427553993
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "Qwen3.5-0.8B",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a small model for scale, and the contention control",
   "score": 0.34107652743094036,
   "score_ci95": [
    0.33070251067974654,
    0.35029468360210136
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 90.8597649361233,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 65.27210727159262,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.5-0.8B",
   "class": "local interactive",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a small model for scale, and the contention control",
   "score": 0.34107652743094036,
   "score_ci95": [
    0.33070251067974654,
    0.35029468360210136
   ],
   "ms": 73.7910270690918,
   "ci95_ms": [
    73.08234954252839,
    74.33510012924671
   ],
   "p95_ms": 145.01922111958265,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 75.57572098448873,
    "1": 72.40885589271784,
    "2": 73.45910184085369
   },
   "ratio_to_jev": {
    "ratio": 0.5768973572344323,
    "ci95": [
     0.5578052810446821,
     0.5950697511334585
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "1 of 1 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9808823529411764,
    "agree": 667,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 1,
   "cards": [
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T03:09:00Z",
   "ended": "2026-10-03T03:12:30Z",
   "load_s": 61.14905071305111
  },
  {
   "system": "Qwen3.5-0.8B",
   "class": "local interactive (repeat 1: not the point)",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a small model for scale, and the contention control",
   "score": 0.34107652743094036,
   "score_ci95": [
    0.33070251067974654,
    0.35029468360210136
   ],
   "ms": 73.44989478588104,
   "ci95_ms": [
    72.79633404687047,
    73.93692787736653
   ],
   "p95_ms": 147.1458156593144,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 73.10872804373503,
    "1": 72.03731406480074,
    "2": 74.64960776269436
   },
   "ratio_to_jev": {
    "ratio": 0.5699685772647102,
    "ci95": [
     0.5546342847955401,
     0.5841898285225046
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "1 of 1 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9808823529411764,
    "agree": 667,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 1,
   "cards": [
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T03:12:40Z",
   "ended": "2026-10-03T03:15:57Z",
   "load_s": 48.89314363710582
  },
  {
   "system": "Qwen3.5-0.8B",
   "class": "hosted end-to-end",
   "subset": "chosen: reference",
   "role": "reference",
   "subset_reason": "a small model for scale, and the contention control",
   "score": 0.34107652743094036,
   "score_ci95": [
    0.33070251067974654,
    0.35029468360210136
   ],
   "gap": "no OpenRouter listing"
  },
  {
   "system": "Qwen3.6-35B-A3B-NVFP4",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint that still matches; the cheapest matching setup in all four cost views",
   "score": 0.7812583100285201,
   "score_ci95": [
    0.7686887658087446,
    0.791675497037336
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 28.654816036696847,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 14.60968181833648,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.6-35B-A3B-NVFP4",
   "class": "local interactive",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint that still matches; the cheapest matching setup in all four cost views",
   "score": 0.7812583100285201,
   "score_ci95": [
    0.7686887658087446,
    0.791675497037336
   ],
   "ms": 140.72824362665415,
   "ci95_ms": [
    139.84115701168776,
    141.81105885654688
   ],
   "p95_ms": 396.36129373684525,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 139.68089036643505,
    "1": 140.72824362665415,
    "2": 143.25649989768863
   },
   "ratio_to_jev": {
    "ratio": 1.1164572476061823,
    "ci95": [
     1.079479330682558,
     1.1574502289283075
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "1 of 1 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9735294117647059,
    "agree": 662,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 1,
   "cards": [
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T03:27:14Z",
   "ended": "2026-10-03T03:34:38Z",
   "load_s": 137.04288628092036
  },
  {
   "system": "Qwen3.6-35B-A3B-NVFP4",
   "class": "hosted end-to-end",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint that still matches; the cheapest matching setup in all four cost views",
   "score": 0.7812583100285201,
   "score_ci95": [
    0.7686887658087446,
    0.791675497037336
   ],
   "gap": "its only fp4 listing (Darkbloom) failed our agreement test for hosts"
  },
  {
   "system": "gemma-4-26B-A4B-it-NVFP4",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint that still matches; the cheapest gemma-4-26B setup on rented cards",
   "score": 0.78246722170387,
   "score_ci95": [
    0.7707839856619857,
    0.7921707623794505
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 18.39190979223921,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 9.44185023756491,
     "cards": 1
    }
   }
  },
  {
   "system": "gemma-4-26B-A4B-it-NVFP4",
   "class": "local interactive",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint that still matches; the cheapest gemma-4-26B setup on rented cards",
   "score": 0.78246722170387,
   "score_ci95": [
    0.7707839856619857,
    0.7921707623794505
   ],
   "ms": 62.83682398498058,
   "ci95_ms": [
    61.945497058331966,
    63.15324082970619
   ],
   "p95_ms": 834.3431558459997,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 63.0367211997509,
    "1": 63.95499920472503,
    "2": 60.65096613019705
   },
   "ratio_to_jev": {
    "ratio": 0.5515031440846064,
    "ci95": [
     0.531824997283286,
     0.5718735642005421
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "1 of 1 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9897058823529412,
    "agree": 673,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 1,
   "cards": [
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T03:16:06Z",
   "ended": "2026-10-03T03:23:17Z",
   "load_s": 106.48828919837251
  },
  {
   "system": "gemma-4-26B-A4B-it-NVFP4",
   "class": "hosted end-to-end",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint that still matches; the cheapest gemma-4-26B setup on rented cards",
   "score": 0.78246722170387,
   "score_ci95": [
    0.7707839856619857,
    0.7921707623794505
   ],
   "gap": "no listing at its precision (DeepInfra lists fp8)"
  },
  {
   "system": "Qwen3.8-27B-NVFP4",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint of Qwen3.8-27B that still matches, on one card",
   "score": 0.778924154415155,
   "score_ci95": [
    0.7649402181066304,
    0.7902250463819499
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 4.808413380433476,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 2.512189170343147,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.8-27B-NVFP4",
   "class": "local interactive",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint of Qwen3.8-27B that still matches, on one card",
   "score": 0.778924154415155,
   "score_ci95": [
    0.7649402181066304,
    0.7902250463819499
   ],
   "ms": 235.50380999222398,
   "ci95_ms": [
    234.46404980495572,
    236.81257059797645
   ],
   "p95_ms": 2167.3928131349385,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 234.58342999219894,
    "1": 236.30482703447342,
    "2": 236.2151239067316
   },
   "ratio_to_jev": {
    "ratio": 1.8463893942002763,
    "ci95": [
     1.7674673536402299,
     1.9152034322560594
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "1 of 1 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9764705882352941,
    "agree": 664,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 1,
   "cards": [
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T03:23:24Z",
   "ended": "2026-10-03T03:41:24Z",
   "load_s": 129.44151468528435
  },
  {
   "system": "Qwen3.8-27B-NVFP4",
   "class": "hosted end-to-end",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "4-bit checkpoint of Qwen3.8-27B that still matches, on one card",
   "score": 0.778924154415155,
   "score_ci95": [
    0.7649402181066304,
    0.7902250463819499
   ],
   "gap": "the cost figure prices it with Qwen3.8-27B's Wafer listing, timed under Qwen3.8-27B"
  },
  {
   "system": "Qwen3.8-27B-FP8",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "FP8 checkpoint of Qwen3.8-27B that still matches (native FP8 on the L40S)",
   "score": 0.7830433845402016,
   "score_ci95": [
    0.768943457657841,
    0.7939840901421737
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 3.6391905867981413,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 2.1640267350043705,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.8-27B-FP8",
   "class": "local interactive",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "FP8 checkpoint of Qwen3.8-27B that still matches (native FP8 on the L40S)",
   "score": 0.7830433845402016,
   "score_ci95": [
    0.768943457657841,
    0.7939840901421737
   ],
   "ms": 212.35587820410728,
   "ci95_ms": [
    209.81437526643276,
    214.78518890216947
   ],
   "p95_ms": 1699.7037860564888,
   "cases": 583,
   "units": 582,
   "decisions": 680,
   "parts": [
    "classification",
    "typesafe",
    "workflows"
   ],
   "blocks": 25,
   "passes": 3,
   "pass_ms": {
    "0": 211.68074198067188,
    "1": 213.08154985308647,
    "2": 213.49830040708184
   },
   "ratio_to_jev": {
    "ratio": 1.6804361679684705,
    "ci95": [
     1.5928347466770987,
     1.7273192358683824
    ],
    "cases": 583
   },
   "accepted": true,
   "rejected_for": [],
   "speed_valid": true,
   "speed_note": "2 of 2 cards reserved; other processes: none",
   "agreement_with_scored": {
    "share": 0.9926470588235294,
    "agree": 675,
    "n": 680,
    "undetermined": 0
   },
   "engine": "vllm",
   "engine_version": "0.30.0",
   "tp": 2,
   "cards": [
    "NVIDIA L40S",
    "NVIDIA L40S"
   ],
   "started": "2026-10-03T02:53:05Z",
   "ended": "2026-10-03T03:08:35Z",
   "load_s": 123.97940973378718
  },
  {
   "system": "Qwen3.8-27B-FP8",
   "class": "hosted end-to-end",
   "subset": "chosen: checkpoint",
   "role": "copy",
   "subset_reason": "FP8 checkpoint of Qwen3.8-27B that still matches (native FP8 on the L40S)",
   "score": 0.7830433845402016,
   "score_ci95": [
    0.768943457657841,
    0.7939840901421737
   ],
   "gap": "the cost figure prices it with Qwen3.8-27B's Wafer listing, timed under Qwen3.8-27B (Ionstream's fp8 listing is timed there as a check)"
  },
  {
   "system": "Qwen3.8-27B (UD-Q4_K_XL)",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "omitted",
   "role": null,
   "subset_reason": "a second 4-bit checkpoint of Qwen3.8-27B (the NVFP4 checkpoint is timed); llama.cpp, about 55 min of one L40S; its only frontier place, the median-host view, prices it at the median of Wafer's and Darkbloom fp4's listings: Wafer is timed hosted, and Darkbloom failed our agreement test for hosts",
   "score": 0.783892885280164,
   "score_ci95": [
    0.7701054814290179,
    0.7946565631689237
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 1.3179805951632895,
     "cards": 1
    },
    "a6000": {
     "decisions_per_s": 0.7337570732612309,
     "cards": 1
    }
   }
  },
  {
   "system": "Qwen3.8-27B (UD-Q4_K_XL)",
   "class": "local interactive",
   "subset": "omitted",
   "role": null,
   "subset_reason": "a second 4-bit checkpoint of Qwen3.8-27B (the NVFP4 checkpoint is timed); llama.cpp, about 55 min of one L40S; its only frontier place, the median-host view, prices it at the median of Wafer's and Darkbloom fp4's listings: Wafer is timed hosted, and Darkbloom failed our agreement test for hosts",
   "score": 0.783892885280164,
   "score_ci95": [
    0.7701054814290179,
    0.7946565631689237
   ],
   "gap": "a second 4-bit checkpoint of Qwen3.8-27B (the NVFP4 checkpoint is timed); llama.cpp, about 55 min of one L40S; its only frontier place, the median-host view, prices it at the median of Wafer's and Darkbloom fp4's listings: Wafer is timed hosted, and Darkbloom failed our agreement test for hosts"
  },
  {
   "system": "Qwen3.8-27B (UD-Q4_K_XL)",
   "class": "hosted end-to-end",
   "subset": "omitted",
   "role": null,
   "subset_reason": "a second 4-bit checkpoint of Qwen3.8-27B (the NVFP4 checkpoint is timed); llama.cpp, about 55 min of one L40S; its only frontier place, the median-host view, prices it at the median of Wafer's and Darkbloom fp4's listings: Wafer is timed hosted, and Darkbloom failed our agreement test for hosts",
   "score": 0.783892885280164,
   "score_ci95": [
    0.7701054814290179,
    0.7946565631689237
   ],
   "gap": "a second 4-bit checkpoint of Qwen3.8-27B (the NVFP4 checkpoint is timed); llama.cpp, about 55 min of one L40S; its only frontier place, the median-host view, prices it at the median of Wafer's and Darkbloom fp4's listings: Wafer is timed hosted, and Darkbloom failed our agreement test for hosts"
  },
  {
   "system": "Qwen3.8-27B (BF16)",
   "class": "batch throughput (never drawn, never a latency)",
   "subset": "omitted",
   "role": null,
   "subset_reason": "the base model's own weights as a llama.cpp GGUF; the costliest setup on rented cards",
   "score": 0.7866084598361619,
   "score_ci95": [
    0.7728583753196265,
    0.7971628446858173
   ],
   "throughput": {
    "l40s": {
     "decisions_per_s": 1.4325353683374715,
     "cards": 2
    },
    "a6000": {
     "decisions_per_s": 0.8593011981689043,
     "cards": 2
    }
   }
  },
  {
   "system": "Qwen3.8-27B (BF16)",
   "class": "local interactive",
   "subset": "omitted",
   "role": null,
   "subset_reason": "the base model's own weights as a llama.cpp GGUF; the costliest setup on rented cards",
   "score": 0.7866084598361619,
   "score_ci95": [
    0.7728583753196265,
    0.7971628446858173
   ],
   "gap": "the base model's own weights as a llama.cpp GGUF; the costliest setup on rented cards"
  },
  {
   "system": "Qwen3.8-27B (BF16)",
   "class": "hosted end-to-end",
   "subset": "omitted",
   "role": null,
   "subset_reason": "the base model's own weights as a llama.cpp GGUF; the costliest setup on rented cards",
   "score": 0.7866084598361619,
   "score_ci95": [
    0.7728583753196265,
    0.7971628446858173
   ],
   "gap": "the base model's own weights as a llama.cpp GGUF; the costliest setup on rented cards"
  },
  {
   "system": "Qwen3.8-27B (UD-IQ1_S)",
   "class": "local interactive",
   "subset": "omitted",
   "role": null,
   "subset_reason": "~1-bit checkpoint; does not match Jev",
   "score": 0.6139667893350875,
   "score_ci95": [
    0.596885373281207,
    0.6280683343125044
   ],
   "gap": "~1-bit checkpoint; does not match Jev"
  },
  {
   "system": "Qwen3.8-27B (UD-IQ1_S)",
   "class": "hosted end-to-end",
   "subset": "omitted",
   "role": null,
   "subset_reason": "~1-bit checkpoint; does not match Jev",
   "score": 0.6139667893350875,
   "score_ci95": [
    0.596885373281207,
    0.6280683343125044
   ],
   "gap": "~1-bit checkpoint; does not match Jev"
  }
 ]
}