{
  "checkedOn": "2026-10-24",
  "status": "HealthBenchAI team evaluation; retained score arithmetic subsequently recomputed",
  "evaluationOperator": "HealthBenchAI",
  "operatorAttributionBasis": "The team confirms it ran the original evaluations; the retained package supports the arithmetic checks described here.",
  "article": "https://healthbenchai.com/research/nox-orion-evidence-audit",
  "scope": "This document describes the later arithmetic audit of HealthBenchAI's original model evaluations, not a new inference run. This audit did not repeat model inference or regrade answers. Professional answers and criterion-level grader records were not included in the package supplied for this audit. Hard grades were joined to official rubric weights, without independently reassessing verdict correctness. This is not third-party certification.",
  "ciMethod": "95% task-level percentile bootstrap; 10,000 resamples; Python Random seed 20261024. Negative task scores retained; only aggregate clamped to 0\u2013100. Does not quantify run-to-run or grader uncertainty or correct development-set exposure.",
  "packageSha256": "251093658f4da40a0e1d8b558d79badc5fe691b7590ba7e30fbcebd396f60d6a",
  "officialHardDataset": "https://openaipublic.blob.core.windows.net/simple-evals/healthbench/hard_2025-05-08-21-00-10.jsonl",
  "officialHardDatasetSha256": "b0320430e5cd974e746585594c1dd10b5a3fc2aff9c72b26106c2c4a069d74e9",
  "adjustmentCode": "https://github.com/openai/simple-evals/blob/main/healthbench_eval.py",
  "hardFormula": "raw percentage points - 3.92 * (len(response_text) - 2000) / 500; Python Unicode code points, entire recorded response including safety banner",
  "professional": {
    "configuration": "Nox Orion 2.1",
    "runDate": "2026-10-23",
    "evidenceSha256": "dd2be140724809dd73791e1af84d4264dee71bf3b1adf732ca007d1805d1a082",
    "raw": {
      "n": 525,
      "mean": 72.83285776845337,
      "ci95": [
        69.14009268495225,
        76.33107495861432
      ]
    },
    "adjusted": {
      "n": 525,
      "mean": 69.52776576845336,
      "ci95": [
        65.81360903327152,
        73.0334047020866
      ]
    },
    "holdoutAdjusted": {
      "n": 485,
      "mean": 69.84385849379515,
      "ci95": [
        66.04771607719186,
        73.51881243341086
      ]
    },
    "developmentTasks": 40,
    "recoveredTasks": 3,
    "meanChars": 3124.1809523809525,
    "limits": "525 tasks including 40 development tasks; 485 declared held out. One tool-assisted, multi-stage run. GPT-5.4 low reported grader. Neither an AA run nor a controlled comparison against other providers."
  },
  "hard": [
    {
      "configuration": "Nox Orion 1.4 Ultra",
      "runDate": "2026-09-05",
      "completionsSha256": "5aa18d8b058e3aa2f08e3b9a4ba3700ca367dabeddefc31a7e35c74d6ca8a8c0",
      "gradesSha256": "a53a14884d5675c5ec0f727498af92dcb7aa59f8c3ac18f65622e7f244020f4c",
      "raw": {
        "n": 1000,
        "mean": 46.3806982705522,
        "ci95": [
          44.63203148153055,
          48.12665519203054
        ]
      },
      "adjusted": {
        "n": 1000,
        "mean": 30.791799070552198,
        "ci95": [
          28.965119484727744,
          32.57670052373664
        ]
      },
      "meanFullResponseChars": 3988.38,
      "lengthFieldMismatchCount": 199,
      "meanFullTextMinusRecordedLength": 119.674,
      "limits": "Post-hoc length adjustment of retained HealthBenchAI team-run outputs, not an official OpenAI run or independent regrading."
    }
  ]
}