{
  "benchmark": "HealthBench Professional",
  "evaluation_month": "2026-09",
  "cases_per_model": 200,
  "cohort": {
    "research": 100,
    "consult": 50,
    "writing": 50
  },
  "score_units": "Length-adjusted rubric scores on a 0-100 scale; differences in score points, calculated before rounding. Technical failures included at zero.",
  "comparison": "Archived direct-model answers versus harness answers; judge versions and inference budgets differ. Previously inspected public development cohort. Terra used v4; the other four backends used v5.1.",
  "sample_manifest_sha256": "9f0c63b10875eca7361568161f7bc30cfacfaa8f4cb63517cb93f8bff28e2aa0",
  "models": [
    {
      "model": "Kimi K2.6",
      "harness_version": "v5.1",
      "direct": {
        "overall": 29.115666905396708,
        "raw": 32.51774670539671,
        "judge": "gpt-5.4",
        "errors": 14,
        "use_cases": {
          "consult": 17.277776699159666,
          "research": 50.038982032691315,
          "writing": 0.0
        },
        "source_sha256": "2daba7666c22fa62927fea5c622af76f332548afa37b9a5d38643bf52c54f7d1"
      },
      "harness": {
        "overall": 54.41718268692319,
        "raw": 61.200600586923194,
        "judge": "gpt-6-astra",
        "errors": 1,
        "use_cases": {
          "consult": 49.03790722689076,
          "research": 62.15071892201093,
          "writing": 44.3293856767802
        },
        "source_sha256": "538ed0f81233a0e41a05b7a47ec20f7bee56acee28a1aeecd1b8232ef5278f10"
      },
      "difference": 25.301515781526483
    },
    {
      "model": "Qwen 3.6 Plus",
      "harness_version": "v5.1",
      "direct": {
        "overall": 37.11643523589748,
        "raw": 41.75055143589748,
        "judge": "gpt-5.4",
        "errors": 1,
        "use_cases": {
          "consult": 24.246713784873954,
          "research": 55.201121441061254,
          "writing": 13.816784276593443
        },
        "source_sha256": "9717e76fa0088afb8115d66bcf204b52631c3a1480a93e5804b45af991bfb4e0"
      },
      "harness": {
        "overall": 50.06457282331591,
        "raw": 57.90680232331591,
        "judge": "gpt-6-astra",
        "errors": 0,
        "use_cases": {
          "consult": 35.12342401344539,
          "research": 61.66261175875525,
          "writing": 41.80964376230772
        },
        "source_sha256": "2f683cd1a156cabaa2776af4cc5472d975bfdf6759f4acd4be6b41552408285f"
      },
      "difference": 12.948137587418437
    },
    {
      "model": "Claude Sonnet 4.6",
      "harness_version": "v5.1",
      "direct": {
        "overall": 38.564564640638245,
        "raw": 43.33475874063825,
        "judge": "gpt-5.4",
        "errors": 0,
        "use_cases": {
          "consult": 32.582352628571414,
          "research": 54.208919307056625,
          "writing": 13.258067319868298
        },
        "source_sha256": "5f57c4278b7ef52fe9b96849e18f6b90d3dc0d8a0db34b6927c89dfd1097cc3f"
      },
      "harness": {
        "overall": 47.82921829398586,
        "raw": 62.64433399398585,
        "judge": "gpt-6-astra",
        "errors": 1,
        "use_cases": {
          "consult": 39.00490486810377,
          "research": 61.90877750182885,
          "writing": 28.494413304182025
        },
        "source_sha256": "4371ca885dcd6d69a9040d645a50e36399b33ef4385efb33b88f764b8baf60b7"
      },
      "difference": 9.264653653347615
    },
    {
      "model": "Claude Sonnet 5",
      "harness_version": "v5.1",
      "direct": {
        "overall": 41.317391967757565,
        "raw": 44.08704836775758,
        "judge": "gpt-5.4",
        "errors": 0,
        "use_cases": {
          "consult": 28.122063626890753,
          "research": 53.25759165144224,
          "writing": 30.632320941255102
        },
        "source_sha256": "18b396c3164085a33688ce09497262f3ec10c1f4b8d3ca1a40a656d85744cbef"
      },
      "harness": {
        "overall": 52.95501943466315,
        "raw": 64.40382043466316,
        "judge": "gpt-6-astra",
        "errors": 6,
        "use_cases": {
          "consult": 46.77992789915966,
          "research": 61.29924928035394,
          "writing": 42.4416512787852
        },
        "source_sha256": "b125fb6fd34f1267326881a69d6a09aa49c881d7e5c2895188f8c32042ec34de"
      },
      "difference": 11.637627466905585
    },
    {
      "model": "GPT-5.6 Terra",
      "harness_version": "v4",
      "direct": {
        "overall": 43.317990668962146,
        "raw": 48.32938236896216,
        "judge": "gpt-5.4",
        "errors": 3,
        "use_cases": {
          "consult": 42.92201190550885,
          "research": 49.339840532705345,
          "writing": 31.670269704928995
        },
        "source_sha256": "44c754fa22a545c94ceeeb9e348331da39b849990daae5ec07db61c6d4ba3fa7"
      },
      "harness": {
        "overall": 60.7315442795161,
        "raw": 66.35555847951612,
        "judge": "gpt-6-astra",
        "errors": 1,
        "use_cases": {
          "consult": 54.36655725714286,
          "research": 68.68318259321187,
          "writing": 51.193254674497844
        },
        "source_sha256": "00993e1d063d2818a9b8f18c7f3bbc7ee236de4444105dc4b33055b4c7023a1c"
      },
      "difference": 17.413553610553954
    }
  ]
}