{
  "runId": "professional-2026-10",
  "status": "complete",
  "updatedAt": "2026-10-25T01:48:17.255735+00:00",
  "finishedAt": "2026-10-25T01:48:17.255806+00:00",
  "budgetLimitUsd": 379,
  "estimatedCostUsd": 120.8941735,
  "reservedUsd": 3.0570125,
  "models": [
    {
      "modelId": "claude-fable-5-1",
      "name": "Claude Fable 5.1 (high)",
      "organization": "Anthropic",
      "effort": "high",
      "planned": 525,
      "generated": 519,
      "graded": 525,
      "rubricGraded": 519,
      "refusals": 6,
      "truncated": 2,
      "emptyAnswers": 0,
      "failed": 0,
      "limitUsd": 259.0,
      "estimatedCostUsd": 87.7688775,
      "reservedUsd": 0.0,
      "status": "complete",
      "rawScore": 64.57204627982988,
      "adjustedScore": 57.97139347982988,
      "confidenceInterval": [
        53.38281951177377,
        62.48406632988331
      ]
    },
    {
      "modelId": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol (max)",
      "organization": "OpenAI",
      "effort": "max",
      "planned": 525,
      "generated": 525,
      "graded": 525,
      "rubricGraded": 525,
      "refusals": 0,
      "truncated": 8,
      "emptyAnswers": 2,
      "failed": 0,
      "limitUsd": 120.0,
      "estimatedCostUsd": 33.125296,
      "reservedUsd": 3.0570125,
      "status": "complete",
      "rawScore": 67.5570802522315,
      "adjustedScore": 64.6344346522315,
      "confidenceInterval": [
        60.77679492779105,
        68.34566779058238
      ]
    }
  ],
  "protocol": {
    "tasksPerModel": 525,
    "repeats": 1,
    "grader": "gpt-5.4-2026-03-05",
    "graderEffort": "low",
    "maxOutputTokens": 16384,
    "lengthAdjustment": "raw_percentage - 1.47 * (len(full_final_response) - 2000) / 500",
    "rubricGrading": "Public OpenAI HealthBench reference prompt; one independent GPT-5.4 low request per rubric item; weighted positive and negative points.",
    "limitations": [
      "Official internal Professional implementation is not released; this uses its documented adjustment and the public HealthBench rubric grading prompt.",
      "Fable uses high (official default), not max; GPT uses max; not a replication of every published vendor setting.",
      "Grader is also an OpenAI model, a potential evaluator-family bias.",
      "525 official public tasks per model, one repeat each; no preferential task selection.",
      "Original pilot completions are reused with identical model/effort/token cap/grader settings.",
      "Up to four retries for rejected HTTP 429 requests. Ambiguous requests have at most one normal recovery retry plus one explicitly enabled long-wait recovery; every original full cost reservation is retained.",
      "Visible answers truncated at the unchanged 16,384-output-token limit are graded exactly as generated, not regenerated or extended; marked rubric_graded_truncated. No hidden reasoning is graded.",
      "Generation network timeout is 1200 seconds; model token limits and reasoning settings remain unchanged.",
      "Failures and budget pauses prevent completion and ranking publication; they are not silently discarded.",
      "Local disclosed failure policy: an explicit API refusal returning no answer receives zero raw and adjusted points, without a short-answer bonus or invented rubric verdicts. This override is not presented as an official automatic-zero rule. Visible refusal text is rubric-graded normally. Safety refusals are never retried to bypass safeguards.",
      "A cached token-limited completion with an empty final answer can be rubric-graded as the actual empty assistant message, with normal official length adjustment and no fixed-zero override. Outcome rubric_graded_empty_truncated; hidden reasoning is never passed to the grader. No regeneration or output-cap change.",
      "Task-level bootstrap interval, 10,000 resamples; not grader or run-to-run uncertainty.",
      "Provider bills may differ from conservative uncached list-price usage estimates.",
      "Raw prompts, physician responses, rubrics and model transcripts remain private to avoid contamination."
    ]
  },
  "dataset": {
    "repository": "https://huggingface.co/datasets/openai/healthbench-professional",
    "url": "https://huggingface.co/datasets/openai/healthbench-professional/resolve/349962fd46dd02343a0d8a606491baf59154ea1a/healthbench_professional_eval.jsonl",
    "revision": "349962fd46dd02343a0d8a606491baf59154ea1a",
    "sha256": "d44b08e6e952e04c945e2c406f02533d9e7a989a84e35820ee7efdff20c9e4e2",
    "retrievedAt": "2026-10-24T23:20:21.828707+00:00",
    "count": 525
  },
  "evidenceUrl": "/reports/professional-2026-10-evidence.json",
  "recentEvents": [
    {
      "at": "2026-10-25T01:48:16.664863+00:00",
      "modelId": "gpt-6.1-sol",
      "event": "Task graded.",
      "taskId": "6eec0aa68d532306ea61f809bca7c2c9"
    },
    {
      "at": "2026-10-25T01:48:16.796873+00:00",
      "modelId": "gpt-6.1-sol",
      "event": "Task graded.",
      "taskId": "880a9e8174d618b24158983f4e7d515b"
    }
  ]
}