{
  "meta": {
    "updated": "16 Jul 2026",
    "generated": "16 Jul 2026, 13:38",
    "models": 8,
    "tracks": 2,
    "judge_model": "qwen/qwen3-max",
    "rubric_model": "anthropic/claude-opus-4.8"
  },
  "tracks": [
    {
      "key": "closed",
      "label": "Closed-book (recall from case name)",
      "leaderboard": [
        {
          "model": "Claude Opus 4.8",
          "developer": "Anthropic",
          "n": 50,
          "n_valid": 47,
          "cosine": {
            "facts": 0.7977698418829177,
            "issue": 0.5923556014895439,
            "decision": 0.4711046108733053,
            "reasons": 0.7836426179459755,
            "ratio": 0.6269938576627861
          },
          "judge": {
            "facts": 0.7833511111111111,
            "issue": 0.8967391304347828,
            "decision": 0.7463804347826087,
            "reasons": 0.7233978723404256,
            "ratio": 0.7935636363636364
          },
          "rubric": {
            "facts": 0.5261893617021276,
            "issue": 0.8900702127659574,
            "decision": 0.7702893617021276,
            "reasons": 0.6240425531914894,
            "ratio": 0.5810787234042553
          },
          "cosine_overall": 0.6544,
          "judge_overall": 0.7887,
          "rubric_overall": 0.6328,
          "primary_overall": 0.6328,
          "ci_lo": 0.5801,
          "ci_hi": 0.6825,
          "strict_overall": 0.5948,
          "groundedness": 0.8311,
          "format_rate": 0.94,
          "refusal_rate": 0.06,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0261117,
          "avg_latency": 17.15492,
          "tied_above": false,
          "gap_above": null
        },
        {
          "model": "GPT-5.6",
          "developer": "OpenAI",
          "n": 50,
          "n_valid": 42,
          "cosine": {
            "facts": 0.789128287536342,
            "issue": 0.5422392634720337,
            "decision": 0.4968769891959865,
            "reasons": 0.7125856961522784,
            "ratio": 0.5457061873748899
          },
          "judge": {
            "facts": 0.6768390243902439,
            "issue": 0.8150414634146341,
            "decision": 0.676829268292683,
            "reasons": 0.6805571428571427,
            "ratio": 0.7250025
          },
          "rubric": {
            "facts": 0.46706904761904766,
            "issue": 0.7817452380952381,
            "decision": 0.7153047619047619,
            "reasons": 0.5124428571428571,
            "ratio": 0.47959047619047623
          },
          "cosine_overall": 0.6173,
          "judge_overall": 0.7149,
          "rubric_overall": 0.543,
          "primary_overall": 0.543,
          "ci_lo": 0.4824,
          "ci_hi": 0.6023,
          "strict_overall": 0.4561,
          "groundedness": 0.7533,
          "format_rate": 0.84,
          "refusal_rate": 0.08,
          "truncation_rate": 0.0,
          "error_rate": 0.08,
          "avg_cost": 0.0518288,
          "avg_latency": 57.79246000000001,
          "tied_above": false,
          "gap_above": {
            "diff": 0.0851,
            "lo": 0.0387,
            "hi": 0.1346,
            "n": 41,
            "significant": true
          }
        },
        {
          "model": "Grok 4.5",
          "developer": "xAI",
          "n": 50,
          "n_valid": 49,
          "cosine": {
            "facts": 0.7896772914744438,
            "issue": 0.5439348717530569,
            "decision": 0.5528564273069302,
            "reasons": 0.7338568258042238,
            "ratio": 0.5519010567146799
          },
          "judge": {
            "facts": 0.5904340425531914,
            "issue": 0.7708354166666666,
            "decision": 0.60416875,
            "reasons": 0.5289142857142857,
            "ratio": 0.615941304347826
          },
          "rubric": {
            "facts": 0.4736714285714286,
            "issue": 0.758504081632653,
            "decision": 0.6279142857142858,
            "reasons": 0.43748367346938777,
            "ratio": 0.45874489795918366
          },
          "cosine_overall": 0.6344,
          "judge_overall": 0.6221,
          "rubric_overall": 0.502,
          "primary_overall": 0.502,
          "ci_lo": 0.4484,
          "ci_hi": 0.5548,
          "strict_overall": 0.492,
          "groundedness": 0.6528,
          "format_rate": 0.98,
          "refusal_rate": 0.02,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.01185624,
          "avg_latency": 22.080759999999994,
          "tied_above": true,
          "gap_above": {
            "diff": 0.0419,
            "lo": -0.0012,
            "hi": 0.082,
            "n": 42,
            "significant": false
          }
        },
        {
          "model": "Claude Sonnet 4.6",
          "developer": "Anthropic",
          "n": 50,
          "n_valid": 46,
          "cosine": {
            "facts": 0.7850507633252577,
            "issue": 0.5490796814362208,
            "decision": 0.4699347900019752,
            "reasons": 0.7429266831149226,
            "ratio": 0.5780913906041966
          },
          "judge": {
            "facts": 0.5189431818181818,
            "issue": 0.7018488888888889,
            "decision": 0.5333333333333333,
            "reasons": 0.49818478260869553,
            "ratio": 0.5813930232558139
          },
          "rubric": {
            "facts": 0.4039521739130435,
            "issue": 0.7137673913043479,
            "decision": 0.6236021739130435,
            "reasons": 0.40982391304347826,
            "ratio": 0.45737608695652177
          },
          "cosine_overall": 0.625,
          "judge_overall": 0.5667,
          "rubric_overall": 0.4679,
          "primary_overall": 0.4679,
          "ci_lo": 0.4046,
          "ci_hi": 0.5328,
          "strict_overall": 0.4305,
          "groundedness": 0.5704,
          "format_rate": 0.92,
          "refusal_rate": 0.08,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.009753359999999999,
          "avg_latency": 17.2607,
          "tied_above": false,
          "gap_above": {
            "diff": 0.043,
            "lo": 0.003,
            "hi": 0.0867,
            "n": 45,
            "significant": true
          }
        },
        {
          "model": "GPT-5.5",
          "developer": "OpenAI",
          "n": 50,
          "n_valid": 48,
          "cosine": {
            "facts": 0.7837191165789313,
            "issue": 0.5339737804012096,
            "decision": 0.4664150168921085,
            "reasons": 0.7163283669700226,
            "ratio": 0.5490242842170927
          },
          "judge": {
            "facts": 0.6902304347826087,
            "issue": 0.7961085106382978,
            "decision": 0.6666744680851064,
            "reasons": 0.6562583333333333,
            "ratio": 0.7111222222222223
          },
          "rubric": {
            "facts": 0.3710645833333333,
            "issue": 0.7586791666666667,
            "decision": 0.5235104166666666,
            "reasons": 0.44686458333333334,
            "ratio": 0.45825833333333327
          },
          "cosine_overall": 0.6099,
          "judge_overall": 0.7041,
          "rubric_overall": 0.4671,
          "primary_overall": 0.4671,
          "ci_lo": 0.4156,
          "ci_hi": 0.5224,
          "strict_overall": 0.4484,
          "groundedness": 0.7659,
          "format_rate": 0.96,
          "refusal_rate": 0.04,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0762939,
          "avg_latency": 65.8793,
          "tied_above": true,
          "gap_above": {
            "diff": -0.0092,
            "lo": -0.0507,
            "hi": 0.0346,
            "n": 46,
            "significant": false
          }
        },
        {
          "model": "Gemini 2.5 Pro",
          "developer": "Google",
          "n": 50,
          "n_valid": 50,
          "cosine": {
            "facts": 0.7573040698965391,
            "issue": 0.5018251427278226,
            "decision": 0.5372641080496262,
            "reasons": 0.7096784377098083,
            "ratio": 0.5304473828038444
          },
          "judge": {
            "facts": 0.44097708333333335,
            "issue": 0.5697285714285714,
            "decision": 0.47278979591836734,
            "reasons": 0.425,
            "ratio": 0.4710152173913043
          },
          "rubric": {
            "facts": 0.4138780000000001,
            "issue": 0.566668,
            "decision": 0.601284,
            "reasons": 0.447256,
            "ratio": 0.388928
          },
          "cosine_overall": 0.6073,
          "judge_overall": 0.4759,
          "rubric_overall": 0.464,
          "primary_overall": 0.464,
          "ci_lo": 0.3989,
          "ci_hi": 0.5313,
          "strict_overall": 0.464,
          "groundedness": 0.4685,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0211078,
          "avg_latency": 22.071780000000004,
          "tied_above": true,
          "gap_above": {
            "diff": -0.0037,
            "lo": -0.0593,
            "hi": 0.0506,
            "n": 48,
            "significant": false
          }
        },
        {
          "model": "DeepSeek Chat v3.1",
          "developer": "DeepSeek",
          "n": 50,
          "n_valid": 50,
          "cosine": {
            "facts": 0.7026232263694207,
            "issue": 0.47356793649342593,
            "decision": 0.47400282445002573,
            "reasons": 0.6309932976961136,
            "ratio": 0.48176309753857227
          },
          "judge": {
            "facts": 0.28125625,
            "issue": 0.3928551020408163,
            "decision": 0.2840142857142857,
            "reasons": 0.271668,
            "ratio": 0.32608478260869567
          },
          "rubric": {
            "facts": 0.22244600000000003,
            "issue": 0.363332,
            "decision": 0.41656999999999994,
            "reasons": 0.29084,
            "ratio": 0.288786
          },
          "cosine_overall": 0.5526,
          "judge_overall": 0.3112,
          "rubric_overall": 0.2931,
          "primary_overall": 0.2931,
          "ci_lo": 0.2471,
          "ci_hi": 0.3418,
          "strict_overall": 0.2931,
          "groundedness": 0.319,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.00036208000000000003,
          "avg_latency": 22.422859999999996,
          "tied_above": false,
          "gap_above": {
            "diff": 0.1709,
            "lo": 0.1278,
            "hi": 0.2187,
            "n": 50,
            "significant": true
          }
        },
        {
          "model": "Llama 4 Maverick",
          "developer": "Meta",
          "n": 50,
          "n_valid": 49,
          "cosine": {
            "facts": 0.7105613192345234,
            "issue": 0.4636174825330575,
            "decision": 0.4368174923583865,
            "reasons": 0.6125441710559689,
            "ratio": 0.44032709209455384
          },
          "judge": {
            "facts": 0.23581914893617023,
            "issue": 0.2881958333333333,
            "decision": 0.2604208333333333,
            "reasons": 0.17517551020408165,
            "ratio": 0.2129688888888889
          },
          "rubric": {
            "facts": 0.18833469387755103,
            "issue": 0.2190469387755102,
            "decision": 0.30626734693877555,
            "reasons": 0.19641836734693877,
            "ratio": 0.14278367346938775
          },
          "cosine_overall": 0.5328,
          "judge_overall": 0.2345,
          "rubric_overall": 0.2078,
          "primary_overall": 0.2078,
          "ci_lo": 0.172,
          "ci_hi": 0.2479,
          "strict_overall": 0.2037,
          "groundedness": 0.25,
          "format_rate": 0.98,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.02,
          "avg_cost": 0.00020863999999999998,
          "avg_latency": 9.2897,
          "tied_above": false,
          "gap_above": {
            "diff": 0.0898,
            "lo": 0.0532,
            "hi": 0.1263,
            "n": 49,
            "significant": true
          }
        }
      ]
    },
    {
      "key": "open",
      "label": "Open-book (brief from decision text)",
      "leaderboard": [
        {
          "model": "Claude Opus 4.8",
          "developer": "Anthropic",
          "n": 20,
          "n_valid": 20,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.9750099999999999,
            "issue": 0.9916699999999998,
            "decision": 0.9791650000000001,
            "reasons": 0.9666750000000001,
            "ratio": 0.88751
          },
          "rubric": {
            "facts": 0.94923,
            "issue": 1.0,
            "decision": 0.97561,
            "reasons": 0.984845,
            "ratio": 1.0
          },
          "cosine_overall": null,
          "judge_overall": 0.96,
          "rubric_overall": 0.9739,
          "primary_overall": 0.9739,
          "ci_lo": 0.9632,
          "ci_hi": 0.9838,
          "strict_overall": 0.9739,
          "groundedness": 0.98,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.12469599999999999,
          "avg_latency": 25.17555,
          "tied_above": false,
          "gap_above": null
        },
        {
          "model": "Claude Sonnet 4.6",
          "developer": "Anthropic",
          "n": 20,
          "n_valid": 20,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.9750099999999999,
            "issue": 0.9958349999999999,
            "decision": 0.9958349999999999,
            "reasons": 0.9750099999999999,
            "ratio": 0.9125099999999999
          },
          "rubric": {
            "facts": 0.9290700000000001,
            "issue": 0.9600000000000002,
            "decision": 0.9801550000000001,
            "reasons": 0.9039550000000001,
            "ratio": 0.9675
          },
          "cosine_overall": null,
          "judge_overall": 0.9708,
          "rubric_overall": 0.9338,
          "primary_overall": 0.9338,
          "ci_lo": 0.9164,
          "ci_hi": 0.9491,
          "strict_overall": 0.9338,
          "groundedness": 0.99,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.053957099999999994,
          "avg_latency": 30.924349999999993,
          "tied_above": false,
          "gap_above": {
            "diff": 0.0401,
            "lo": 0.024,
            "hi": 0.0571,
            "n": 20,
            "significant": true
          }
        },
        {
          "model": "Grok 4.5",
          "developer": "xAI",
          "n": 20,
          "n_valid": 20,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.9416899999999998,
            "issue": 0.979175,
            "decision": 0.9791650000000001,
            "reasons": 0.9250149999999999,
            "ratio": 0.83751
          },
          "rubric": {
            "facts": 0.8199500000000001,
            "issue": 1.0,
            "decision": 0.972075,
            "reasons": 0.9201050000000001,
            "ratio": 0.9675
          },
          "cosine_overall": null,
          "judge_overall": 0.9325,
          "rubric_overall": 0.9138,
          "primary_overall": 0.9138,
          "ci_lo": 0.8887,
          "ci_hi": 0.9383,
          "strict_overall": 0.9138,
          "groundedness": 0.97,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0258338,
          "avg_latency": 9.730300000000002,
          "tied_above": true,
          "gap_above": {
            "diff": 0.02,
            "lo": -0.0085,
            "hi": 0.0477,
            "n": 20,
            "significant": false
          }
        },
        {
          "model": "GPT-5.5",
          "developer": "OpenAI",
          "n": 20,
          "n_valid": 20,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.9583499999999999,
            "issue": 0.99167,
            "decision": 0.9791650000000001,
            "reasons": 0.9416799999999999,
            "ratio": 0.90418
          },
          "rubric": {
            "facts": 0.8281050000000001,
            "issue": 0.9800000000000001,
            "decision": 0.9669599999999999,
            "reasons": 0.88703,
            "ratio": 0.9675
          },
          "cosine_overall": null,
          "judge_overall": 0.955,
          "rubric_overall": 0.9014,
          "primary_overall": 0.9014,
          "ci_lo": 0.8754,
          "ci_hi": 0.9273,
          "strict_overall": 0.9014,
          "groundedness": 0.98,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0860715,
          "avg_latency": 19.269399999999997,
          "tied_above": true,
          "gap_above": {
            "diff": 0.0123,
            "lo": -0.0297,
            "hi": 0.0509,
            "n": 20,
            "significant": false
          }
        },
        {
          "model": "GPT-5.6",
          "developer": "OpenAI",
          "n": 20,
          "n_valid": 20,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.9500199999999998,
            "issue": 0.9958349999999999,
            "decision": 0.9791650000000001,
            "reasons": 0.9416849999999999,
            "ratio": 0.90001
          },
          "rubric": {
            "facts": 0.7722450000000001,
            "issue": 0.9800000000000001,
            "decision": 0.962415,
            "reasons": 0.89909,
            "ratio": 0.9475
          },
          "cosine_overall": null,
          "judge_overall": 0.9533,
          "rubric_overall": 0.8879,
          "primary_overall": 0.8879,
          "ci_lo": 0.8534,
          "ci_hi": 0.9184,
          "strict_overall": 0.8879,
          "groundedness": 0.9825,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0817185,
          "avg_latency": 18.267300000000002,
          "tied_above": true,
          "gap_above": {
            "diff": 0.0135,
            "lo": -0.0125,
            "hi": 0.0375,
            "n": 20,
            "significant": false
          }
        },
        {
          "model": "Gemini 2.5 Pro",
          "developer": "Google",
          "n": 20,
          "n_valid": 18,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.9213277777777777,
            "issue": 0.9814888888888889,
            "decision": 0.9907444444444445,
            "reasons": 0.9120611111111112,
            "ratio": 0.8379777777777777
          },
          "rubric": {
            "facts": 0.570472222222222,
            "issue": 0.9480166666666666,
            "decision": 0.8655555555555555,
            "reasons": 0.8058944444444446,
            "ratio": 0.9555555555555555
          },
          "cosine_overall": null,
          "judge_overall": 0.9287,
          "rubric_overall": 0.7778,
          "primary_overall": 0.7778,
          "ci_lo": 0.749,
          "ci_hi": 0.8047,
          "strict_overall": 0.7001,
          "groundedness": 0.9833,
          "format_rate": 0.9,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.1,
          "avg_cost": 0.038079,
          "avg_latency": 35.90375000000001,
          "tied_above": false,
          "gap_above": {
            "diff": 0.1085,
            "lo": 0.0693,
            "hi": 0.151,
            "n": 18,
            "significant": true
          }
        },
        {
          "model": "DeepSeek Chat v3.1",
          "developer": "DeepSeek",
          "n": 20,
          "n_valid": 20,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.8916849999999998,
            "issue": 0.979175,
            "decision": 0.975,
            "reasons": 0.8875149999999999,
            "ratio": 0.775005
          },
          "rubric": {
            "facts": 0.5419149999999999,
            "issue": 0.896965,
            "decision": 0.83542,
            "reasons": 0.68658,
            "ratio": 0.8625
          },
          "cosine_overall": null,
          "judge_overall": 0.9017,
          "rubric_overall": 0.7025,
          "primary_overall": 0.7025,
          "ci_lo": 0.6582,
          "ci_hi": 0.7425,
          "strict_overall": 0.7025,
          "groundedness": 0.955,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0025058,
          "avg_latency": 22.699149999999996,
          "tied_above": false,
          "gap_above": {
            "diff": 0.0832,
            "lo": 0.0364,
            "hi": 0.1324,
            "n": 18,
            "significant": true
          }
        },
        {
          "model": "Llama 4 Maverick",
          "developer": "Meta",
          "n": 20,
          "n_valid": 20,
          "cosine": {
            "facts": null,
            "issue": null,
            "decision": null,
            "reasons": null,
            "ratio": null
          },
          "judge": {
            "facts": 0.86252,
            "issue": 0.95001,
            "decision": 0.99167,
            "reasons": 0.795845,
            "ratio": 0.72502
          },
          "rubric": {
            "facts": 0.50505,
            "issue": 0.7894650000000001,
            "decision": 0.8321350000000001,
            "reasons": 0.5042599999999999,
            "ratio": 0.7925
          },
          "cosine_overall": null,
          "judge_overall": 0.865,
          "rubric_overall": 0.6093,
          "primary_overall": 0.6093,
          "ci_lo": 0.5728,
          "ci_hi": 0.6487,
          "strict_overall": 0.6093,
          "groundedness": 0.9275,
          "format_rate": 1.0,
          "refusal_rate": 0.0,
          "truncation_rate": 0.0,
          "error_rate": 0.0,
          "avg_cost": 0.0017912999999999998,
          "avg_latency": 12.144099999999998,
          "tied_above": false,
          "gap_above": {
            "diff": 0.0932,
            "lo": 0.0493,
            "hi": 0.1348,
            "n": 20,
            "significant": true
          }
        }
      ]
    }
  ]
}