{"bundle_id":"mini-benchmark-v1","bundle_hash":"825e262c4771a1512da32bad7f2a071bd36ab1e18f8ab71e96e88f9610b5f05e","category":null,"rows":[{"rank":1,"model_id":"openai/gpt-5.6-luna","score":9.7875,"provisional":true,"complete_runs":1,"disagreement_mean":0.5,"success_rate":1,"avg_cost_usd_per_run":0.004381740000000001,"avg_latency_ms":7668.5,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[0.5],"category_medians":{"roleplay":10,"coding":9.625,"math":10,"research":9.75,"marketing":9.875,"poster":9.5,"story":9.8,"judging":9.75},"category_detail":{"roleplay":{"median":10,"spread":0.25,"validator_pass_rate":1},"coding":{"median":9.625,"spread":0.5,"validator_pass_rate":1},"math":{"median":10,"spread":0,"validator_pass_rate":1},"research":{"median":9.75,"spread":0.5,"validator_pass_rate":1},"marketing":{"median":9.875,"spread":1,"validator_pass_rate":1},"poster":{"median":9.5,"spread":0.5,"validator_pass_rate":1},"story":{"median":9.8,"spread":0.5,"validator_pass_rate":1},"judging":{"median":9.75,"spread":0.75,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":2,"model_id":"openai/gpt-5.6-luna-pro","score":9.184895833333334,"provisional":true,"complete_runs":2,"disagreement_mean":0.3984375,"success_rate":1,"avg_cost_usd_per_run":0.019365588000000003,"avg_latency_ms":16299.375,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[0.421875,0.375],"category_medians":{"roleplay":9.75,"coding":8,"math":10,"research":9.625,"marketing":8.229166666666666,"poster":8.916666666666668,"story":8.958333333333332,"judging":10},"category_detail":{"roleplay":{"median":9.75,"spread":0.4375,"validator_pass_rate":1},"coding":{"median":8,"spread":0.375,"validator_pass_rate":1},"math":{"median":10,"spread":0.0625,"validator_pass_rate":1},"research":{"median":9.625,"spread":0.625,"validator_pass_rate":1},"marketing":{"median":8.229166666666666,"spread":0.125,"validator_pass_rate":1},"poster":{"median":8.916666666666668,"spread":0.625,"validator_pass_rate":1},"story":{"median":8.958333333333332,"spread":0.4375,"validator_pass_rate":1},"judging":{"median":10,"spread":0.5,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":3,"model_id":"x-ai/grok-4.5","score":8.727083333333333,"provisional":true,"complete_runs":1,"disagreement_mean":0.3687499999999999,"success_rate":1,"avg_cost_usd_per_run":0.069170508,"avg_latency_ms":22391.625,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[0.3687499999999999],"category_medians":{"roleplay":7.5,"coding":7,"math":10,"research":9.525,"marketing":6.166666666666667,"poster":9.75,"story":10,"judging":9.875},"category_detail":{"roleplay":{"median":7.5,"spread":1,"validator_pass_rate":1},"coding":{"median":7,"spread":0,"validator_pass_rate":1},"math":{"median":10,"spread":0,"validator_pass_rate":1},"research":{"median":9.525,"spread":0.5,"validator_pass_rate":1},"marketing":{"median":6.166666666666667,"spread":0.5,"validator_pass_rate":1},"poster":{"median":9.75,"spread":0.1999999999999993,"validator_pass_rate":1},"story":{"median":10,"spread":0.34999999999999964,"validator_pass_rate":1},"judging":{"median":9.875,"spread":0.40000000000000036,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":4,"model_id":"anthropic/claude-sonnet-4.6","score":8.328125,"provisional":true,"complete_runs":1,"disagreement_mean":1.50625,"success_rate":1,"avg_cost_usd_per_run":0.11532771,"avg_latency_ms":29857.5,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[1.50625],"category_medians":{"roleplay":9,"coding":7.125,"math":3.75,"research":9.5,"marketing":9,"poster":9,"story":9.25,"judging":10},"category_detail":{"roleplay":{"median":9,"spread":1.25,"validator_pass_rate":1},"coding":{"median":7.125,"spread":2.5,"validator_pass_rate":1},"math":{"median":3.75,"spread":0.25,"validator_pass_rate":0.625},"research":{"median":9.5,"spread":1.5,"validator_pass_rate":1},"marketing":{"median":9,"spread":4,"validator_pass_rate":1},"poster":{"median":9,"spread":1,"validator_pass_rate":1},"story":{"median":9.25,"spread":1.0500000000000007,"validator_pass_rate":1},"judging":{"median":10,"spread":0.5,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":5,"model_id":"deepseek/deepseek-v4-flash","score":7.802083333333333,"provisional":true,"complete_runs":1,"disagreement_mean":1.3125,"success_rate":1,"avg_cost_usd_per_run":0.001739169432,"avg_latency_ms":29779.125,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[1.3125],"category_medians":{"roleplay":9.5,"coding":7.5,"math":4.5,"research":9.25,"marketing":7,"poster":8.75,"story":6.041666666666667,"judging":9.875},"category_detail":{"roleplay":{"median":9.5,"spread":1.25,"validator_pass_rate":1},"coding":{"median":7.5,"spread":2.375,"validator_pass_rate":1},"math":{"median":4.5,"spread":1.5,"validator_pass_rate":0.75},"research":{"median":9.25,"spread":0.75,"validator_pass_rate":1},"marketing":{"median":7,"spread":2.25,"validator_pass_rate":1},"poster":{"median":8.75,"spread":1,"validator_pass_rate":1},"story":{"median":6.041666666666667,"spread":1.125,"validator_pass_rate":0},"judging":{"median":9.875,"spread":0.25,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":6,"model_id":"anthropic/claude-sonnet-5","score":7.40625,"provisional":true,"complete_runs":1,"disagreement_mean":0.99375,"success_rate":1,"avg_cost_usd_per_run":0.1109889,"avg_latency_ms":16306.375,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[0.99375],"category_medians":{"roleplay":9,"coding":3,"math":10,"research":9.5,"marketing":8.75,"poster":2.5,"story":7,"judging":9.5},"category_detail":{"roleplay":{"median":9,"spread":1.75,"validator_pass_rate":1},"coding":{"median":3,"spread":0.5,"validator_pass_rate":0},"math":{"median":10,"spread":0,"validator_pass_rate":1},"research":{"median":9.5,"spread":1.25,"validator_pass_rate":1},"marketing":{"median":8.75,"spread":1.5,"validator_pass_rate":1},"poster":{"median":2.5,"spread":1.25,"validator_pass_rate":0},"story":{"median":7,"spread":0.9500000000000002,"validator_pass_rate":0},"judging":{"median":9.5,"spread":0.75,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":7,"model_id":"anthropic/claude-opus-5","score":6.885416666666666,"provisional":true,"complete_runs":2,"disagreement_mean":1.078125,"success_rate":1,"avg_cost_usd_per_run":0.315649125,"avg_latency_ms":22680.25,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[0.6875,1.46875],"category_medians":{"roleplay":9.875,"coding":1.25,"math":10,"research":3.75,"marketing":9.875,"poster":9.5,"story":2.5,"judging":8.333333333333334},"category_detail":{"roleplay":{"median":9.875,"spread":1,"validator_pass_rate":1},"coding":{"median":1.25,"spread":0.625,"validator_pass_rate":0},"math":{"median":10,"spread":0,"validator_pass_rate":1},"research":{"median":3.75,"spread":1.875,"validator_pass_rate":0},"marketing":{"median":9.875,"spread":2.125,"validator_pass_rate":1},"poster":{"median":9.5,"spread":0.5,"validator_pass_rate":1},"story":{"median":2.5,"spread":1.75,"validator_pass_rate":0},"judging":{"median":8.333333333333334,"spread":0.75,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":8,"model_id":"nvidia/nemotron-3-ultra-550b-a55b:free","score":6.838541666666668,"provisional":true,"complete_runs":2,"disagreement_mean":0.84375,"success_rate":1,"avg_cost_usd_per_run":0,"avg_latency_ms":16759.375,"last_evaluated_at":"2026-07-23T08:06:09.458Z","spread_history":[1.03125,0.65625],"category_medians":{"roleplay":8.291666666666668,"coding":6.458333333333334,"math":6.416666666666667,"research":7.958333333333334,"marketing":7.125,"poster":3.9583333333333335,"story":6.166666666666667,"judging":8.333333333333334},"category_detail":{"roleplay":{"median":8.291666666666668,"spread":0.125,"validator_pass_rate":1},"coding":{"median":6.458333333333334,"spread":1,"validator_pass_rate":1},"math":{"median":6.416666666666667,"spread":1,"validator_pass_rate":0.5},"research":{"median":7.958333333333334,"spread":1.125,"validator_pass_rate":1},"marketing":{"median":7.125,"spread":0.375,"validator_pass_rate":1},"poster":{"median":3.9583333333333335,"spread":1.375,"validator_pass_rate":0},"story":{"median":6.166666666666667,"spread":1.75,"validator_pass_rate":0.5},"judging":{"median":8.333333333333334,"spread":0,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":9,"model_id":"deepseek/deepseek-v4-flash-0731","score":6.59375,"provisional":true,"complete_runs":2,"disagreement_mean":1.140625,"success_rate":1,"avg_cost_usd_per_run":0.0030483941399999997,"avg_latency_ms":23254.125,"last_evaluated_at":"2026-08-01T15:29:34.647Z","spread_history":[0.65625,1.625],"category_medians":{"roleplay":9.375,"coding":1.25,"math":10,"research":9.5,"marketing":3.125,"poster":9.75,"story":0.625,"judging":9.125},"category_detail":{"roleplay":{"median":9.375,"spread":1.125,"validator_pass_rate":1},"coding":{"median":1.25,"spread":1.25,"validator_pass_rate":0},"math":{"median":10,"spread":0,"validator_pass_rate":1},"research":{"median":9.5,"spread":0.625,"validator_pass_rate":1},"marketing":{"median":3.125,"spread":1.125,"validator_pass_rate":0.5},"poster":{"median":9.75,"spread":0.5,"validator_pass_rate":1},"story":{"median":0.625,"spread":2.25,"validator_pass_rate":0},"judging":{"median":9.125,"spread":2.25,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":10,"model_id":"poolside/laguna-s-2.1:free","score":4.471354166666667,"provisional":true,"complete_runs":1,"disagreement_mean":1.0375,"success_rate":0.625,"avg_cost_usd_per_run":0,"avg_latency_ms":23825.2,"last_evaluated_at":"2026-07-23T08:51:53.319Z","spread_history":[1.0375],"category_medians":{"roleplay":0,"coding":7.083333333333334,"math":6.125,"research":3.9583333333333335,"marketing":0,"poster":7.5,"story":3.25,"judging":7.854166666666667},"category_detail":{"roleplay":{"median":0,"spread":0,"validator_pass_rate":0},"coding":{"median":7.083333333333334,"spread":0.625,"validator_pass_rate":1},"math":{"median":6.125,"spread":1.5,"validator_pass_rate":0.8125},"research":{"median":3.9583333333333335,"spread":1.25,"validator_pass_rate":1},"marketing":{"median":0,"spread":0,"validator_pass_rate":0},"poster":{"median":7.5,"spread":0.25,"validator_pass_rate":1},"story":{"median":3.25,"spread":1.5,"validator_pass_rate":0},"judging":{"median":7.854166666666667,"spread":1.4375,"validator_pass_rate":1}},"coverage":0.625,"penalized_tasks":6,"excluded_tasks":0},{"rank":11,"model_id":"qwen/qwen3.7-plus","score":3.640625,"provisional":true,"complete_runs":1,"disagreement_mean":1.671875,"success_rate":1,"avg_cost_usd_per_run":0.0179616096,"avg_latency_ms":30045.875,"last_evaluated_at":"2026-08-04T13:16:50.763Z","spread_history":[1.671875],"category_medians":{"roleplay":1.25,"coding":3.5,"math":10,"research":1.25,"marketing":1.25,"poster":1.25,"story":1.25,"judging":9.375},"category_detail":{"roleplay":{"median":1.25,"spread":2.5,"validator_pass_rate":0},"coding":{"median":3.5,"spread":1.75,"validator_pass_rate":0},"math":{"median":10,"spread":0.125,"validator_pass_rate":1},"research":{"median":1.25,"spread":0.75,"validator_pass_rate":0},"marketing":{"median":1.25,"spread":2,"validator_pass_rate":0},"poster":{"median":1.25,"spread":2.5,"validator_pass_rate":0},"story":{"median":1.25,"spread":2.5,"validator_pass_rate":0},"judging":{"median":9.375,"spread":1.25,"validator_pass_rate":1}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":12,"model_id":"qwen/qwen3.8-max","score":2.375,"provisional":true,"complete_runs":1,"disagreement_mean":1.75,"success_rate":1,"avg_cost_usd_per_run":0.0891396,"avg_latency_ms":43808.625,"last_evaluated_at":"2026-08-04T13:16:50.763Z","spread_history":[1.75],"category_medians":{"roleplay":0,"coding":0,"math":5,"research":9.5,"marketing":0,"poster":1.25,"story":0.25,"judging":3},"category_detail":{"roleplay":{"median":0,"spread":2.5,"validator_pass_rate":0},"coding":{"median":0,"spread":2.5,"validator_pass_rate":0},"math":{"median":5,"spread":1,"validator_pass_rate":0},"research":{"median":9.5,"spread":0.75,"validator_pass_rate":1},"marketing":{"median":0,"spread":2.5,"validator_pass_rate":0},"poster":{"median":1.25,"spread":2,"validator_pass_rate":0},"story":{"median":0.25,"spread":2.5,"validator_pass_rate":0},"judging":{"median":3,"spread":0.25,"validator_pass_rate":0}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":13,"model_id":"qwen/qwen3.7-max","score":2.1875,"provisional":true,"complete_runs":1,"disagreement_mean":1.921875,"success_rate":1,"avg_cost_usd_per_run":0.0669349395,"avg_latency_ms":34137.25,"last_evaluated_at":"2026-08-04T13:16:50.763Z","spread_history":[1.921875],"category_medians":{"roleplay":1.25,"coding":2.75,"math":10,"research":1.25,"marketing":0,"poster":0.5,"story":1.25,"judging":0.5},"category_detail":{"roleplay":{"median":1.25,"spread":2,"validator_pass_rate":0},"coding":{"median":2.75,"spread":0.75,"validator_pass_rate":0},"math":{"median":10,"spread":0.125,"validator_pass_rate":1},"research":{"median":1.25,"spread":2.5,"validator_pass_rate":0},"marketing":{"median":0,"spread":2.5,"validator_pass_rate":0},"poster":{"median":0.5,"spread":2.5,"validator_pass_rate":0},"story":{"median":1.25,"spread":2.5,"validator_pass_rate":0},"judging":{"median":0.5,"spread":2.5,"validator_pass_rate":0}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0},{"rank":14,"model_id":"qwen/qwen3.6-plus","score":0.6875,"provisional":true,"complete_runs":1,"disagreement_mean":2.375,"success_rate":1,"avg_cost_usd_per_run":0.02927442375,"avg_latency_ms":32405.375,"last_evaluated_at":"2026-08-04T13:16:50.763Z","spread_history":[2.375],"category_medians":{"roleplay":1.25,"coding":1.25,"math":0,"research":0,"marketing":1.25,"poster":0,"story":1.25,"judging":0.5},"category_detail":{"roleplay":{"median":1.25,"spread":2,"validator_pass_rate":0},"coding":{"median":1.25,"spread":2.5,"validator_pass_rate":0},"math":{"median":0,"spread":2.5,"validator_pass_rate":0},"research":{"median":0,"spread":2.5,"validator_pass_rate":0},"marketing":{"median":1.25,"spread":2,"validator_pass_rate":0},"poster":{"median":0,"spread":2.5,"validator_pass_rate":0},"story":{"median":1.25,"spread":2.5,"validator_pass_rate":0},"judging":{"median":0.5,"spread":2.5,"validator_pass_rate":0}},"coverage":1,"penalized_tasks":0,"excluded_tasks":0}]}