{"benchmarks":[{"benchmark":"HumanEval","evaluation_elapsed_seconds":128.77665689203423,"gpu_lease_seconds":null,"gpu_slot_seconds":0,"limitation":null,"metric":"pass@1","models":2,"name":"Code generation","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":0,"b_only":4,"both_fail":8,"both_pass":20,"delta_b_minus_a":0.125,"disagreement_rate":0.125,"interval_95":[0.03125,0.25],"mcnemar_exact_p":0.125,"n":32,"sample_variance":0.11290322580645161,"se":0.05939887041393643,"separating_examples":4,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":128.77665689203423,"gpu_lease_seconds":null,"gpu_slot_seconds":null,"item_cost_model":{"basis":"Observed per-item wall times plus fixed loading/setup residual; assumes constant per-item cost when scaling. Not GPU-active time.","fixed_pair_seconds":2.250539105851203,"per_item_pair_seconds":3.9539411808182194},"limitation":null,"run_a":"264775b8e13a45705e029825","run_b":"501f999a9189ca77b745a388"}],"runs_with_gpu_timing":0,"runs_with_lease_timing":0,"sample_limit":32,"scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","task":"code-generation"},{"benchmark":"GSM8K","evaluation_elapsed_seconds":27.00559265015181,"gpu_lease_seconds":40,"gpu_slot_seconds":38.43697674712166,"limitation":null,"metric":"exact_match","models":2,"name":"Math word problems","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":1,"b_only":3,"both_fail":28,"both_pass":0,"delta_b_minus_a":0.0625,"disagreement_rate":0.125,"interval_95":[-0.0625,0.1875],"mcnemar_exact_p":0.625,"n":32,"sample_variance":0.125,"se":0.0625,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":27.00559265015181,"gpu_lease_seconds":40,"gpu_slot_seconds":38.43697674712166,"item_cost_model":null,"limitation":null,"run_a":"4749a153422d5cefccd16747","run_b":"aa5bc8b59a1fd979e61724eb"}],"runs_with_gpu_timing":2,"runs_with_lease_timing":2,"sample_limit":32,"scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","task":"math"},{"benchmark":"BoolQ","evaluation_elapsed_seconds":3.3253459139959887,"gpu_lease_seconds":21,"gpu_slot_seconds":15.154583084047772,"limitation":null,"metric":"accuracy","models":2,"name":"Reading comprehension","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":5,"b_only":5,"both_fail":5,"both_pass":17,"delta_b_minus_a":0.0,"disagreement_rate":0.3125,"interval_95":[-0.1875,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.3225806451612903,"se":0.10040241611281235,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3253459139959887,"gpu_lease_seconds":21,"gpu_slot_seconds":15.154583084047772,"item_cost_model":null,"limitation":null,"run_a":"ff345349801249c15997d2a6","run_b":"c6d083942f8d93dc672e019b"}],"runs_with_gpu_timing":2,"runs_with_lease_timing":2,"sample_limit":32,"scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","task":"reading"},{"benchmark":"HellaSwag","evaluation_elapsed_seconds":2.9554261500015855,"gpu_lease_seconds":9,"gpu_slot_seconds":8.992982605937868,"limitation":null,"metric":"accuracy","models":2,"name":"Commonsense completion","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":2,"b_only":6,"both_fail":14,"both_pass":10,"delta_b_minus_a":0.125,"disagreement_rate":0.25,"interval_95":[-0.03125,0.28125],"mcnemar_exact_p":0.2890625,"n":32,"sample_variance":0.24193548387096775,"se":0.08695104295503155,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.9554261500015855,"gpu_lease_seconds":9,"gpu_slot_seconds":8.992982605937868,"item_cost_model":null,"limitation":null,"run_a":"2f544548b89cd6fed2d4e61d","run_b":"db55a33c25d74960ff389d24"}],"runs_with_gpu_timing":2,"runs_with_lease_timing":2,"sample_limit":32,"scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","task":"commonsense"},{"benchmark":"SST-2","evaluation_elapsed_seconds":2.6582569651072845,"gpu_lease_seconds":21,"gpu_slot_seconds":15.203442082973197,"limitation":null,"metric":"accuracy","models":2,"name":"Sentiment classification","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":11,"b_only":1,"both_fail":1,"both_pass":19,"delta_b_minus_a":-0.3125,"disagreement_rate":0.375,"interval_95":[-0.5,-0.125],"mcnemar_exact_p":0.00634765625,"n":32,"sample_variance":0.2862903225806452,"se":0.0945863234333863,"separating_examples":12,"separation":"observed"},"evaluation_elapsed_seconds":2.6582569651072845,"gpu_lease_seconds":21,"gpu_slot_seconds":15.203442082973197,"item_cost_model":null,"limitation":null,"run_a":"efa7055f4d6480dd542a3abf","run_b":"b700ec1eed7b5ab1a5d28d49"}],"runs_with_gpu_timing":2,"runs_with_lease_timing":2,"sample_limit":32,"scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","task":"sentiment"},{"benchmark":"SNLI","evaluation_elapsed_seconds":2.81733395694755,"gpu_lease_seconds":15,"gpu_slot_seconds":14.404187698033638,"limitation":null,"metric":"accuracy","models":2,"name":"Natural language inference","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":0,"b_only":0,"both_fail":20,"both_pass":12,"delta_b_minus_a":0.0,"disagreement_rate":0.0,"interval_95":[0.0,0.0],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.0,"se":0.0,"separating_examples":0,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.81733395694755,"gpu_lease_seconds":15,"gpu_slot_seconds":14.404187698033638,"item_cost_model":null,"limitation":null,"run_a":"2933b280610cb03f5f2345c2","run_b":"bddd3154bd9cf23f3dece3fb"}],"runs_with_gpu_timing":2,"runs_with_lease_timing":2,"sample_limit":32,"scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","task":"entailment"},{"benchmark":"STS Benchmark","evaluation_elapsed_seconds":0.8793051829561591,"gpu_lease_seconds":null,"gpu_slot_seconds":0,"limitation":"At least two compatible models are required.","metric":"spearman","models":1,"name":"Sentence similarity","pairs":[],"runs_with_gpu_timing":0,"runs_with_lease_timing":0,"sample_limit":32,"scope":"5048133e477a3c13bf97c614b99b2d36a458391bb2a9480cd9b38189f2608b67","task":"similarity"},{"benchmark":"IAM lines","evaluation_elapsed_seconds":8.352338270982727,"gpu_lease_seconds":null,"gpu_slot_seconds":0,"limitation":"At least two compatible models are required.","metric":"cer","models":1,"name":"Handwriting recognition","pairs":[],"runs_with_gpu_timing":0,"runs_with_lease_timing":0,"sample_limit":32,"scope":"87f7f645d4ae1c902e4f3c4d821290e2fa08df318242d5d78a563467763e36ec","task":"handwriting"},{"benchmark":"LibriSpeech test-clean","evaluation_elapsed_seconds":5.0269540110602975,"gpu_lease_seconds":null,"gpu_slot_seconds":0,"limitation":"At least two compatible models are required.","metric":"wer","models":1,"name":"Speech transcription","pairs":[],"runs_with_gpu_timing":0,"runs_with_lease_timing":0,"sample_limit":32,"scope":"9c365ee78df50fa725b4c03d695f65d2b638be00c4687dc66be70b284c865a34","task":"transcription"},{"benchmark":"CIFAR-10","evaluation_elapsed_seconds":1.9501927399542183,"gpu_lease_seconds":null,"gpu_slot_seconds":0,"limitation":"At least two compatible models are required.","metric":"accuracy","models":1,"name":"Image classification","pairs":[],"runs_with_gpu_timing":0,"runs_with_lease_timing":0,"sample_limit":32,"scope":"d4a779c6ba33be115fc5e3302e45bb70d7c90040f954b89dab97beb5a2914649","task":"image-classification"},{"benchmark":"AI2 ARC-Challenge (test)","evaluation_elapsed_seconds":3.1335540000582114,"gpu_lease_seconds":9,"gpu_slot_seconds":9.042805926059373,"limitation":null,"metric":"accuracy","models":2,"name":"AI2 ARC-Challenge (test)","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":1,"b_only":6,"both_fail":16,"both_pass":9,"delta_b_minus_a":0.15625,"disagreement_rate":0.21875,"interval_95":[0.0,0.3125],"mcnemar_exact_p":0.125,"n":32,"sample_variance":0.2006048387096774,"se":0.07917639300749574,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.1335540000582114,"gpu_lease_seconds":9,"gpu_slot_seconds":9.042805926059373,"item_cost_model":null,"limitation":null,"run_a":"0d8c049a046e18e772d16952","run_b":"216e346c83988ade77ddfff0"}],"runs_with_gpu_timing":2,"runs_with_lease_timing":2,"sample_limit":32,"scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","task":"arc-challenge-choice"},{"benchmark":"OpenBookQA main (test)","evaluation_elapsed_seconds":4.055169674102217,"gpu_lease_seconds":12,"gpu_slot_seconds":10.344536917866208,"limitation":null,"metric":"accuracy","models":2,"name":"OpenBookQA main (test)","pairs":[{"a":"Qwen/Qwen2.5-0.5B","b":"Qwen/Qwen2.5-1.5B","diagnostic":{"a_only":2,"b_only":2,"both_fail":17,"both_pass":11,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.055169674102217,"gpu_lease_seconds":12,"gpu_slot_seconds":10.344536917866208,"item_cost_model":null,"limitation":null,"run_a":"f08b963541b415694a99b37b","run_b":"9fd52efbe4976b40a9fd6646"}],"runs_with_gpu_timing":2,"runs_with_lease_timing":2,"sample_limit":32,"scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","task":"openbookqa-main-choice"},{"benchmark":"ARC-Easy","evaluation_elapsed_seconds":20.86332618596498,"gpu_lease_seconds":55,"gpu_slot_seconds":53.29512007709127,"limitation":null,"metric":"accuracy","models":11,"name":"ARC-Easy / sub-1B cohort","pairs":[{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"HuggingFaceTB/SmolLM2-360M-Instruct","diagnostic":{"a_only":8,"b_only":4,"both_fail":10,"both_pass":10,"delta_b_minus_a":-0.125,"disagreement_rate":0.375,"interval_95":[-0.34375,0.09375],"mcnemar_exact_p":0.3876953125,"n":32,"sample_variance":0.3709677419354839,"se":0.10766959615176362,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.2591248189564794,"gpu_lease_seconds":8,"gpu_slot_seconds":8.241824457072653,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"933abc78ddc9563ed3bfe5c3"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"Qwen/Qwen2.5-0.5B-Instruct","diagnostic":{"a_only":4,"b_only":5,"both_fail":9,"both_pass":14,"delta_b_minus_a":0.03125,"disagreement_rate":0.28125,"interval_95":[-0.15625,0.21875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.28931451612903225,"se":0.09508458670590234,"separating_examples":9,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.4895555530674756,"gpu_lease_seconds":8,"gpu_slot_seconds":8.59251348208636,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"da843c3af7d2f039d8dcc997"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":6,"b_only":6,"both_fail":8,"both_pass":12,"delta_b_minus_a":0.0,"disagreement_rate":0.375,"interval_95":[-0.21875,0.21875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.3870967741935484,"se":0.10998533626601496,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.2453677129233256,"gpu_lease_seconds":9,"gpu_slot_seconds":9.195484533091076,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"a5bd023e62dfa94900c68aff"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":4,"b_only":6,"both_fail":8,"both_pass":14,"delta_b_minus_a":0.0625,"disagreement_rate":0.3125,"interval_95":[-0.125,0.25],"mcnemar_exact_p":0.75390625,"n":32,"sample_variance":0.3185483870967742,"se":0.0997729276746663,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.0494012279668823,"gpu_lease_seconds":9,"gpu_slot_seconds":9.042783517972566,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"4f8d3c01030b9ec948e6d8be"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":7,"b_only":1,"both_fail":13,"both_pass":11,"delta_b_minus_a":-0.1875,"disagreement_rate":0.25,"interval_95":[-0.34375,-0.03125],"mcnemar_exact_p":0.0703125,"n":32,"sample_variance":0.2217741935483871,"se":0.08324928557283298,"separating_examples":8,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.069074310013093,"gpu_lease_seconds":9,"gpu_slot_seconds":8.944257690105587,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"81740f9404ca2161b03fab45"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":4,"b_only":6,"both_fail":8,"both_pass":14,"delta_b_minus_a":0.0625,"disagreement_rate":0.3125,"interval_95":[-0.125,0.25],"mcnemar_exact_p":0.75390625,"n":32,"sample_variance":0.3185483870967742,"se":0.0997729276746663,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.083627366926521,"gpu_lease_seconds":9,"gpu_slot_seconds":9.142746297060512,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"c2e37f8ec8e7bbe1a93eefe7"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":9,"b_only":5,"both_fail":9,"both_pass":9,"delta_b_minus_a":-0.125,"disagreement_rate":0.4375,"interval_95":[-0.34375,0.09375],"mcnemar_exact_p":0.4239501953125,"n":32,"sample_variance":0.43548387096774194,"se":0.11665706565717285,"separating_examples":14,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.492576366988942,"gpu_lease_seconds":10,"gpu_slot_seconds":9.343206124030985,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"e66c128ba1817c735cfe0094"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":1,"b_only":8,"both_fail":6,"both_pass":17,"delta_b_minus_a":0.21875,"disagreement_rate":0.28125,"interval_95":[0.0625,0.375],"mcnemar_exact_p":0.0390625,"n":32,"sample_variance":0.24092741935483872,"se":0.08676970585889242,"separating_examples":9,"separation":"observed"},"evaluation_elapsed_seconds":3.1560652839252725,"gpu_lease_seconds":10,"gpu_slot_seconds":9.193171379039995,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"95f7f19d55290d88873e6474"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":5,"b_only":7,"both_fail":7,"both_pass":13,"delta_b_minus_a":0.0625,"disagreement_rate":0.375,"interval_95":[-0.15625,0.28125],"mcnemar_exact_p":0.7744140625,"n":32,"sample_variance":0.38306451612903225,"se":0.10941099638076722,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.2125441710231826,"gpu_lease_seconds":10,"gpu_slot_seconds":9.143269876018167,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":4,"b_only":6,"both_fail":8,"both_pass":14,"delta_b_minus_a":0.0625,"disagreement_rate":0.3125,"interval_95":[-0.125,0.25],"mcnemar_exact_p":0.75390625,"n":32,"sample_variance":0.3185483870967742,"se":0.0997729276746663,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.1980282040312886,"gpu_lease_seconds":9,"gpu_slot_seconds":9.094066599034704,"item_cost_model":null,"limitation":null,"run_a":"83ca3844b75d12aea722afae","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"Qwen/Qwen2.5-0.5B-Instruct","diagnostic":{"a_only":2,"b_only":7,"both_fail":11,"both_pass":12,"delta_b_minus_a":0.15625,"disagreement_rate":0.28125,"interval_95":[-0.03125,0.34375],"mcnemar_exact_p":0.1796875,"n":32,"sample_variance":0.2651209677419355,"se":0.091022141492801,"separating_examples":9,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.6615606320556253,"gpu_lease_seconds":8,"gpu_slot_seconds":8.692514855065383,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"da843c3af7d2f039d8dcc997"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":5,"b_only":9,"both_fail":9,"both_pass":9,"delta_b_minus_a":0.125,"disagreement_rate":0.4375,"interval_95":[-0.09375,0.34375],"mcnemar_exact_p":0.4239501953125,"n":32,"sample_variance":0.43548387096774194,"se":0.11665706565717285,"separating_examples":14,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.4173727919114754,"gpu_lease_seconds":9,"gpu_slot_seconds":9.295485906070098,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"a5bd023e62dfa94900c68aff"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":3,"b_only":9,"both_fail":9,"both_pass":11,"delta_b_minus_a":0.1875,"disagreement_rate":0.375,"interval_95":[-0.03125,0.40625],"mcnemar_exact_p":0.14599609375,"n":32,"sample_variance":0.35080645161290325,"se":0.10470292074676439,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.221406306955032,"gpu_lease_seconds":9,"gpu_slot_seconds":9.142784890951589,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"4f8d3c01030b9ec948e6d8be"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":7,"b_only":5,"both_fail":13,"both_pass":7,"delta_b_minus_a":-0.0625,"disagreement_rate":0.375,"interval_95":[-0.28125,0.15625],"mcnemar_exact_p":0.7744140625,"n":32,"sample_variance":0.38306451612903225,"se":0.10941099638076722,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.241079389001243,"gpu_lease_seconds":9,"gpu_slot_seconds":9.04425906308461,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"81740f9404ca2161b03fab45"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":3,"b_only":9,"both_fail":9,"both_pass":11,"delta_b_minus_a":0.1875,"disagreement_rate":0.375,"interval_95":[-0.03125,0.40625],"mcnemar_exact_p":0.14599609375,"n":32,"sample_variance":0.35080645161290325,"se":0.10470292074676439,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.255632445914671,"gpu_lease_seconds":9,"gpu_slot_seconds":9.242747670039535,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"c2e37f8ec8e7bbe1a93eefe7"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":6,"b_only":6,"both_fail":12,"both_pass":8,"delta_b_minus_a":0.0,"disagreement_rate":0.375,"interval_95":[-0.21875,0.21875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.3870967741935484,"se":0.10998533626601496,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.664581445977092,"gpu_lease_seconds":10,"gpu_slot_seconds":9.443207497010008,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"e66c128ba1817c735cfe0094"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":0,"b_only":11,"both_fail":7,"both_pass":14,"delta_b_minus_a":0.34375,"disagreement_rate":0.34375,"interval_95":[0.1875,0.5],"mcnemar_exact_p":0.0009765625,"n":32,"sample_variance":0.23286290322580644,"se":0.08530513305661303,"separating_examples":11,"separation":"observed"},"evaluation_elapsed_seconds":3.3280703629134223,"gpu_lease_seconds":10,"gpu_slot_seconds":9.293172752019018,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"95f7f19d55290d88873e6474"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":4,"b_only":10,"both_fail":8,"both_pass":10,"delta_b_minus_a":0.1875,"disagreement_rate":0.4375,"interval_95":[-0.03125,0.40625],"mcnemar_exact_p":0.1795654296875,"n":32,"sample_variance":0.4153225806451613,"se":0.11392467092408602,"separating_examples":14,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3845492500113323,"gpu_lease_seconds":10,"gpu_slot_seconds":9.243271248997189,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":3,"b_only":9,"both_fail":9,"both_pass":11,"delta_b_minus_a":0.1875,"disagreement_rate":0.375,"interval_95":[-0.03125,0.40625],"mcnemar_exact_p":0.14599609375,"n":32,"sample_variance":0.35080645161290325,"se":0.10470292074676439,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3700332830194384,"gpu_lease_seconds":9,"gpu_slot_seconds":9.194067972013727,"item_cost_model":null,"limitation":null,"run_a":"933abc78ddc9563ed3bfe5c3","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":4,"b_only":3,"both_fail":10,"both_pass":15,"delta_b_minus_a":-0.03125,"disagreement_rate":0.21875,"interval_95":[-0.1875,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.2247983870967742,"se":0.08381497239022509,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.6478035260224715,"gpu_lease_seconds":9,"gpu_slot_seconds":9.646174931083806,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"a5bd023e62dfa94900c68aff"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":3,"b_only":4,"both_fail":9,"both_pass":16,"delta_b_minus_a":0.03125,"disagreement_rate":0.21875,"interval_95":[-0.125,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.2247983870967742,"se":0.08381497239022509,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.451837041066028,"gpu_lease_seconds":9,"gpu_slot_seconds":9.493473915965296,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"4f8d3c01030b9ec948e6d8be"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":10,"b_only":3,"both_fail":10,"both_pass":9,"delta_b_minus_a":-0.21875,"disagreement_rate":0.40625,"interval_95":[-0.4375,0.0],"mcnemar_exact_p":0.09228515625,"n":32,"sample_variance":0.3699596774193548,"se":0.1075232064224037,"separating_examples":13,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.471510123112239,"gpu_lease_seconds":9,"gpu_slot_seconds":9.394948088098317,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"81740f9404ca2161b03fab45"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":3,"b_only":4,"both_fail":9,"both_pass":16,"delta_b_minus_a":0.03125,"disagreement_rate":0.21875,"interval_95":[-0.125,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.2247983870967742,"se":0.08381497239022509,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.486063180025667,"gpu_lease_seconds":9,"gpu_slot_seconds":9.593436695053242,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"c2e37f8ec8e7bbe1a93eefe7"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":7,"b_only":2,"both_fail":11,"both_pass":12,"delta_b_minus_a":-0.15625,"disagreement_rate":0.28125,"interval_95":[-0.34375,0.03125],"mcnemar_exact_p":0.1796875,"n":32,"sample_variance":0.2651209677419355,"se":0.091022141492801,"separating_examples":9,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.895012180088088,"gpu_lease_seconds":10,"gpu_slot_seconds":9.793896522023715,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"e66c128ba1817c735cfe0094"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":1,"b_only":7,"both_fail":6,"both_pass":18,"delta_b_minus_a":0.1875,"disagreement_rate":0.25,"interval_95":[0.03125,0.34375],"mcnemar_exact_p":0.0703125,"n":32,"sample_variance":0.2217741935483871,"se":0.08324928557283298,"separating_examples":8,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.5585010970244184,"gpu_lease_seconds":10,"gpu_slot_seconds":9.643861777032726,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"95f7f19d55290d88873e6474"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":3,"b_only":4,"both_fail":9,"both_pass":16,"delta_b_minus_a":0.03125,"disagreement_rate":0.21875,"interval_95":[-0.125,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.2247983870967742,"se":0.08381497239022509,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.6149799841223285,"gpu_lease_seconds":10,"gpu_slot_seconds":9.593960274010897,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":2,"b_only":3,"both_fail":10,"both_pass":17,"delta_b_minus_a":0.03125,"disagreement_rate":0.15625,"interval_95":[-0.09375,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.6004640171304345,"gpu_lease_seconds":9,"gpu_slot_seconds":9.544756997027434,"item_cost_model":null,"limitation":null,"run_a":"da843c3af7d2f039d8dcc997","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":1,"b_only":3,"both_fail":11,"both_pass":17,"delta_b_minus_a":0.0625,"disagreement_rate":0.125,"interval_95":[-0.0625,0.1875],"mcnemar_exact_p":0.625,"n":32,"sample_variance":0.125,"se":0.0625,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.207649200921878,"gpu_lease_seconds":10,"gpu_slot_seconds":10.096444966970012,"item_cost_model":null,"limitation":null,"run_a":"a5bd023e62dfa94900c68aff","run_b":"4f8d3c01030b9ec948e6d8be"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":8,"b_only":2,"both_fail":12,"both_pass":10,"delta_b_minus_a":-0.1875,"disagreement_rate":0.3125,"interval_95":[-0.375,0.0],"mcnemar_exact_p":0.109375,"n":32,"sample_variance":0.2862903225806452,"se":0.0945863234333863,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.227322282968089,"gpu_lease_seconds":10,"gpu_slot_seconds":9.997919139103033,"item_cost_model":null,"limitation":null,"run_a":"a5bd023e62dfa94900c68aff","run_b":"81740f9404ca2161b03fab45"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":1,"b_only":3,"both_fail":11,"both_pass":17,"delta_b_minus_a":0.0625,"disagreement_rate":0.125,"interval_95":[-0.0625,0.1875],"mcnemar_exact_p":0.625,"n":32,"sample_variance":0.125,"se":0.0625,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.241875339881517,"gpu_lease_seconds":10,"gpu_slot_seconds":10.196407746057957,"item_cost_model":null,"limitation":null,"run_a":"a5bd023e62dfa94900c68aff","run_b":"c2e37f8ec8e7bbe1a93eefe7"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149","diagnostic":{"a_only":6,"b_only":2,"both_fail":12,"both_pass":12,"delta_b_minus_a":-0.125,"disagreement_rate":0.25,"interval_95":[-0.3125,0.03125],"mcnemar_exact_p":0.2890625,"n":32,"sample_variance":0.24193548387096775,"se":0.08695104295503155,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.650824339943938,"gpu_lease_seconds":11,"gpu_slot_seconds":10.39686757302843,"item_cost_model":null,"limitation":null,"run_a":"a5bd023e62dfa94900c68aff","run_b":"e66c128ba1817c735cfe0094"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":0,"b_only":7,"both_fail":7,"both_pass":18,"delta_b_minus_a":0.21875,"disagreement_rate":0.21875,"interval_95":[0.09375,0.375],"mcnemar_exact_p":0.015625,"n":32,"sample_variance":0.17641129032258066,"se":0.07424858801742054,"separating_examples":7,"separation":"observed"},"evaluation_elapsed_seconds":4.3143132568802685,"gpu_lease_seconds":11,"gpu_slot_seconds":10.24683282803744,"item_cost_model":null,"limitation":null,"run_a":"a5bd023e62dfa94900c68aff","run_b":"95f7f19d55290d88873e6474"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":0,"b_only":2,"both_fail":12,"both_pass":18,"delta_b_minus_a":0.0625,"disagreement_rate":0.0625,"interval_95":[0.0,0.15625],"mcnemar_exact_p":0.5,"n":32,"sample_variance":0.06048387096774194,"se":0.04347552147751577,"separating_examples":2,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.3707921439781785,"gpu_lease_seconds":11,"gpu_slot_seconds":10.196931325015612,"item_cost_model":null,"limitation":null,"run_a":"a5bd023e62dfa94900c68aff","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":0,"b_only":2,"both_fail":12,"both_pass":18,"delta_b_minus_a":0.0625,"disagreement_rate":0.0625,"interval_95":[0.0,0.15625],"mcnemar_exact_p":0.5,"n":32,"sample_variance":0.06048387096774194,"se":0.04347552147751577,"separating_examples":2,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.3562761769862846,"gpu_lease_seconds":10,"gpu_slot_seconds":10.14772804803215,"item_cost_model":null,"limitation":null,"run_a":"a5bd023e62dfa94900c68aff","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":8,"b_only":0,"both_fail":12,"both_pass":12,"delta_b_minus_a":-0.25,"disagreement_rate":0.25,"interval_95":[-0.40625,-0.125],"mcnemar_exact_p":0.0078125,"n":32,"sample_variance":0.1935483870967742,"se":0.0777713771047819,"separating_examples":8,"separation":"observed"},"evaluation_elapsed_seconds":4.031355798011646,"gpu_lease_seconds":10,"gpu_slot_seconds":9.845218123984523,"item_cost_model":null,"limitation":null,"run_a":"4f8d3c01030b9ec948e6d8be","run_b":"81740f9404ca2161b03fab45"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":0,"b_only":0,"both_fail":12,"both_pass":20,"delta_b_minus_a":0.0,"disagreement_rate":0.0,"interval_95":[0.0,0.0],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.0,"se":0.0,"separating_examples":0,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.045908854925074,"gpu_lease_seconds":10,"gpu_slot_seconds":10.043706730939448,"item_cost_model":null,"limitation":null,"run_a":"4f8d3c01030b9ec948e6d8be","run_b":"c2e37f8ec8e7bbe1a93eefe7"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149","diagnostic":{"a_only":8,"b_only":2,"both_fail":10,"both_pass":12,"delta_b_minus_a":-0.1875,"disagreement_rate":0.3125,"interval_95":[-0.375,0.0],"mcnemar_exact_p":0.109375,"n":32,"sample_variance":0.2862903225806452,"se":0.0945863234333863,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.454857854987495,"gpu_lease_seconds":11,"gpu_slot_seconds":10.24416655790992,"item_cost_model":null,"limitation":null,"run_a":"4f8d3c01030b9ec948e6d8be","run_b":"e66c128ba1817c735cfe0094"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":1,"b_only":6,"both_fail":6,"both_pass":19,"delta_b_minus_a":0.15625,"disagreement_rate":0.21875,"interval_95":[0.0,0.3125],"mcnemar_exact_p":0.125,"n":32,"sample_variance":0.2006048387096774,"se":0.07917639300749574,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.118346771923825,"gpu_lease_seconds":11,"gpu_slot_seconds":10.094131812918931,"item_cost_model":null,"limitation":null,"run_a":"4f8d3c01030b9ec948e6d8be","run_b":"95f7f19d55290d88873e6474"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":1,"b_only":1,"both_fail":11,"both_pass":19,"delta_b_minus_a":0.0,"disagreement_rate":0.0625,"interval_95":[-0.09375,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.06451612903225806,"se":0.04490132550669373,"separating_examples":2,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.174825659021735,"gpu_lease_seconds":11,"gpu_slot_seconds":10.044230309897102,"item_cost_model":null,"limitation":null,"run_a":"4f8d3c01030b9ec948e6d8be","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":2,"b_only":2,"both_fail":10,"both_pass":18,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.160309692029841,"gpu_lease_seconds":10,"gpu_slot_seconds":9.99502703291364,"item_cost_model":null,"limitation":null,"run_a":"4f8d3c01030b9ec948e6d8be","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":0,"b_only":8,"both_fail":12,"both_pass":12,"delta_b_minus_a":0.25,"disagreement_rate":0.25,"interval_95":[0.125,0.40625],"mcnemar_exact_p":0.0078125,"n":32,"sample_variance":0.1935483870967742,"se":0.0777713771047819,"separating_examples":8,"separation":"observed"},"evaluation_elapsed_seconds":4.0655819369712844,"gpu_lease_seconds":10,"gpu_slot_seconds":9.945180903072469,"item_cost_model":null,"limitation":null,"run_a":"81740f9404ca2161b03fab45","run_b":"c2e37f8ec8e7bbe1a93eefe7"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149","diagnostic":{"a_only":3,"b_only":5,"both_fail":15,"both_pass":9,"delta_b_minus_a":0.0625,"disagreement_rate":0.25,"interval_95":[-0.125,0.25],"mcnemar_exact_p":0.7265625,"n":32,"sample_variance":0.2540322580645161,"se":0.08909830562090465,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.474530937033705,"gpu_lease_seconds":11,"gpu_slot_seconds":10.145640730042942,"item_cost_model":null,"limitation":null,"run_a":"81740f9404ca2161b03fab45","run_b":"e66c128ba1817c735cfe0094"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":0,"b_only":13,"both_fail":7,"both_pass":12,"delta_b_minus_a":0.40625,"disagreement_rate":0.40625,"interval_95":[0.25,0.59375],"mcnemar_exact_p":0.000244140625,"n":32,"sample_variance":0.24899193548387097,"se":0.0882099653319905,"separating_examples":13,"separation":"observed"},"evaluation_elapsed_seconds":4.138019853970036,"gpu_lease_seconds":11,"gpu_slot_seconds":9.995605985051952,"item_cost_model":null,"limitation":null,"run_a":"81740f9404ca2161b03fab45","run_b":"95f7f19d55290d88873e6474"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":1,"b_only":9,"both_fail":11,"both_pass":11,"delta_b_minus_a":0.25,"disagreement_rate":0.3125,"interval_95":[0.09375,0.4375],"mcnemar_exact_p":0.021484375,"n":32,"sample_variance":0.25806451612903225,"se":0.08980265101338746,"separating_examples":10,"separation":"observed"},"evaluation_elapsed_seconds":4.194498741067946,"gpu_lease_seconds":11,"gpu_slot_seconds":9.945704482030123,"item_cost_model":null,"limitation":null,"run_a":"81740f9404ca2161b03fab45","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":1,"b_only":9,"both_fail":11,"both_pass":11,"delta_b_minus_a":0.25,"disagreement_rate":0.3125,"interval_95":[0.09375,0.4375],"mcnemar_exact_p":0.021484375,"n":32,"sample_variance":0.25806451612903225,"se":0.08980265101338746,"separating_examples":10,"separation":"observed"},"evaluation_elapsed_seconds":4.179982774076052,"gpu_lease_seconds":10,"gpu_slot_seconds":9.896501205046661,"item_cost_model":null,"limitation":null,"run_a":"81740f9404ca2161b03fab45","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149","diagnostic":{"a_only":8,"b_only":2,"both_fail":10,"both_pass":12,"delta_b_minus_a":-0.1875,"disagreement_rate":0.3125,"interval_95":[-0.375,0.0],"mcnemar_exact_p":0.109375,"n":32,"sample_variance":0.2862903225806452,"se":0.0945863234333863,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.489083993947133,"gpu_lease_seconds":11,"gpu_slot_seconds":10.344129336997867,"item_cost_model":null,"limitation":null,"run_a":"c2e37f8ec8e7bbe1a93eefe7","run_b":"e66c128ba1817c735cfe0094"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":1,"b_only":6,"both_fail":6,"both_pass":19,"delta_b_minus_a":0.15625,"disagreement_rate":0.21875,"interval_95":[0.0,0.3125],"mcnemar_exact_p":0.125,"n":32,"sample_variance":0.2006048387096774,"se":0.07917639300749574,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.152572910883464,"gpu_lease_seconds":11,"gpu_slot_seconds":10.194094592006877,"item_cost_model":null,"limitation":null,"run_a":"c2e37f8ec8e7bbe1a93eefe7","run_b":"95f7f19d55290d88873e6474"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":1,"b_only":1,"both_fail":11,"both_pass":19,"delta_b_minus_a":0.0,"disagreement_rate":0.0625,"interval_95":[-0.09375,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.06451612903225806,"se":0.04490132550669373,"separating_examples":2,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.209051797981374,"gpu_lease_seconds":11,"gpu_slot_seconds":10.144193088985048,"item_cost_model":null,"limitation":null,"run_a":"c2e37f8ec8e7bbe1a93eefe7","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":2,"b_only":2,"both_fail":10,"both_pass":18,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.19453583098948,"gpu_lease_seconds":10,"gpu_slot_seconds":10.094989812001586,"item_cost_model":null,"limitation":null,"run_a":"c2e37f8ec8e7bbe1a93eefe7","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":0,"b_only":11,"both_fail":7,"both_pass":14,"delta_b_minus_a":0.34375,"disagreement_rate":0.34375,"interval_95":[0.1875,0.5],"mcnemar_exact_p":0.0009765625,"n":32,"sample_variance":0.23286290322580644,"se":0.08530513305661303,"separating_examples":11,"separation":"observed"},"evaluation_elapsed_seconds":4.561521910945885,"gpu_lease_seconds":12,"gpu_slot_seconds":10.39455441897735,"item_cost_model":null,"limitation":null,"run_a":"e66c128ba1817c735cfe0094","run_b":"95f7f19d55290d88873e6474"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":2,"b_only":8,"both_fail":10,"both_pass":12,"delta_b_minus_a":0.1875,"disagreement_rate":0.3125,"interval_95":[0.0,0.375],"mcnemar_exact_p":0.109375,"n":32,"sample_variance":0.2862903225806452,"se":0.0945863234333863,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.618000798043795,"gpu_lease_seconds":12,"gpu_slot_seconds":10.344652915955521,"item_cost_model":null,"limitation":null,"run_a":"e66c128ba1817c735cfe0094","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":2,"b_only":8,"both_fail":10,"both_pass":12,"delta_b_minus_a":0.1875,"disagreement_rate":0.3125,"interval_95":[0.0,0.375],"mcnemar_exact_p":0.109375,"n":32,"sample_variance":0.2862903225806452,"se":0.0945863234333863,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.603484831051901,"gpu_lease_seconds":11,"gpu_slot_seconds":10.295449638972059,"item_cost_model":null,"limitation":null,"run_a":"e66c128ba1817c735cfe0094","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"SlayerLab/Slayer149-ARC","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":6,"b_only":1,"both_fail":6,"both_pass":19,"delta_b_minus_a":-0.15625,"disagreement_rate":0.21875,"interval_95":[-0.3125,0.0],"mcnemar_exact_p":0.125,"n":32,"sample_variance":0.2006048387096774,"se":0.07917639300749574,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.281489714980125,"gpu_lease_seconds":12,"gpu_slot_seconds":10.194618170964532,"item_cost_model":null,"limitation":null,"run_a":"95f7f19d55290d88873e6474","run_b":"5ebdb1c54bc9de4a483665de"},{"a":"SlayerLab/Slayer149-ARC","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":5,"b_only":0,"both_fail":7,"both_pass":20,"delta_b_minus_a":-0.15625,"disagreement_rate":0.15625,"interval_95":[-0.28125,-0.03125],"mcnemar_exact_p":0.0625,"n":32,"sample_variance":0.13608870967741934,"se":0.06521328221627366,"separating_examples":5,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":4.2669737479882315,"gpu_lease_seconds":11,"gpu_slot_seconds":10.14541489398107,"item_cost_model":null,"limitation":null,"run_a":"95f7f19d55290d88873e6474","run_b":"adeb8d2f845d86b97ddb885e"},{"a":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":1,"b_only":1,"both_fail":11,"both_pass":19,"delta_b_minus_a":0.0,"disagreement_rate":0.0625,"interval_95":[-0.09375,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.06451612903225806,"se":0.04490132550669373,"separating_examples":2,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.3234526350861415,"gpu_lease_seconds":11,"gpu_slot_seconds":10.09551339095924,"item_cost_model":null,"limitation":null,"run_a":"5ebdb1c54bc9de4a483665de","run_b":"adeb8d2f845d86b97ddb885e"}],"runs_with_gpu_timing":11,"runs_with_lease_timing":11,"sample_limit":32,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","task":"sub1b-arc-easy"},{"benchmark":"HellaSwag","evaluation_elapsed_seconds":20.898243353352882,"gpu_lease_seconds":55,"gpu_slot_seconds":53.442520158947445,"limitation":null,"metric":"accuracy","models":11,"name":"HellaSwag / sub-1B cohort","pairs":[{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"HuggingFaceTB/SmolLM2-360M-Instruct","diagnostic":{"a_only":1,"b_only":7,"both_fail":16,"both_pass":8,"delta_b_minus_a":0.1875,"disagreement_rate":0.25,"interval_95":[0.03125,0.34375],"mcnemar_exact_p":0.0703125,"n":32,"sample_variance":0.2217741935483871,"se":0.08324928557283298,"separating_examples":8,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":2.207809524028562,"gpu_lease_seconds":8,"gpu_slot_seconds":8.14108703203965,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"9733ef9ec08a5e033ded7a5e"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"Qwen/Qwen2.5-0.5B-Instruct","diagnostic":{"a_only":0,"b_only":4,"both_fail":19,"both_pass":9,"delta_b_minus_a":0.125,"disagreement_rate":0.125,"interval_95":[0.03125,0.25],"mcnemar_exact_p":0.125,"n":32,"sample_variance":0.11290322580645161,"se":0.05939887041393643,"separating_examples":4,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":2.498483041068539,"gpu_lease_seconds":9,"gpu_slot_seconds":8.592398616950959,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"5182981eed2309237804d9c3"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":2,"b_only":3,"both_fail":20,"both_pass":7,"delta_b_minus_a":0.03125,"disagreement_rate":0.15625,"interval_95":[-0.09375,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.141676811967045,"gpu_lease_seconds":9,"gpu_slot_seconds":9.093538927962072,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"60b94d2f3d3bccb39d7277ca"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":1,"b_only":3,"both_fail":20,"both_pass":8,"delta_b_minus_a":0.0625,"disagreement_rate":0.125,"interval_95":[-0.0625,0.1875],"mcnemar_exact_p":0.625,"n":32,"sample_variance":0.125,"se":0.0625,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.909357605036348,"gpu_lease_seconds":9,"gpu_slot_seconds":8.9435228779912,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"384fccc53f471e23eb9acac8"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":0,"b_only":3,"both_fail":20,"both_pass":9,"delta_b_minus_a":0.09375,"disagreement_rate":0.09375,"interval_95":[0.0,0.21875],"mcnemar_exact_p":0.25,"n":32,"sample_variance":0.08770161290322581,"se":0.052351460373382196,"separating_examples":3,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.968581094057299,"gpu_lease_seconds":9,"gpu_slot_seconds":8.89302276307717,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"75ea5d8ae94de8a86a7a0777"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":1,"b_only":3,"both_fail":20,"both_pass":8,"delta_b_minus_a":0.0625,"disagreement_rate":0.125,"interval_95":[-0.0625,0.1875],"mcnemar_exact_p":0.625,"n":32,"sample_variance":0.125,"se":0.0625,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.928811965044588,"gpu_lease_seconds":9,"gpu_slot_seconds":8.893092222046107,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"8677abc8861650a471b4ea7c"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":1,"b_only":3,"both_fail":20,"both_pass":8,"delta_b_minus_a":0.0625,"disagreement_rate":0.125,"interval_95":[-0.0625,0.1875],"mcnemar_exact_p":0.625,"n":32,"sample_variance":0.125,"se":0.0625,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.357762887957506,"gpu_lease_seconds":9,"gpu_slot_seconds":9.294418806093745,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"ec41e48212ad955d3ea4c8f6"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":3,"b_only":5,"both_fail":18,"both_pass":6,"delta_b_minus_a":0.0625,"disagreement_rate":0.25,"interval_95":[-0.125,0.25],"mcnemar_exact_p":0.7265625,"n":32,"sample_variance":0.2540322580645161,"se":0.08909830562090465,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.067592384060845,"gpu_lease_seconds":9,"gpu_slot_seconds":9.043636059039272,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"4bab151d17bfaf2727299f60"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":1,"b_only":4,"both_fail":19,"both_pass":8,"delta_b_minus_a":0.09375,"disagreement_rate":0.15625,"interval_95":[-0.03125,0.21875],"mcnemar_exact_p":0.375,"n":32,"sample_variance":0.15221774193548387,"se":0.0689695906576505,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.1849458240903914,"gpu_lease_seconds":10,"gpu_slot_seconds":9.144076748983935,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"93979b5fa00514cce3d96032"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":2,"b_only":4,"both_fail":19,"both_pass":7,"delta_b_minus_a":0.0625,"disagreement_rate":0.1875,"interval_95":[-0.09375,0.21875],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.1103180940262973,"gpu_lease_seconds":10,"gpu_slot_seconds":9.14327565394342,"item_cost_model":null,"limitation":null,"run_a":"a0a1b916aa43bef9dc349f52","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"Qwen/Qwen2.5-0.5B-Instruct","diagnostic":{"a_only":4,"b_only":2,"both_fail":15,"both_pass":11,"delta_b_minus_a":-0.0625,"disagreement_rate":0.1875,"interval_95":[-0.21875,0.09375],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.822493481100537,"gpu_lease_seconds":9,"gpu_slot_seconds":8.79136352695059,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"5182981eed2309237804d9c3"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":6,"b_only":1,"both_fail":16,"both_pass":9,"delta_b_minus_a":-0.15625,"disagreement_rate":0.21875,"interval_95":[-0.3125,0.0],"mcnemar_exact_p":0.125,"n":32,"sample_variance":0.2006048387096774,"se":0.07917639300749574,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.465687251999043,"gpu_lease_seconds":9,"gpu_slot_seconds":9.292503837961704,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"60b94d2f3d3bccb39d7277ca"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":7,"b_only":3,"both_fail":14,"both_pass":8,"delta_b_minus_a":-0.125,"disagreement_rate":0.3125,"interval_95":[-0.3125,0.0625],"mcnemar_exact_p":0.34375,"n":32,"sample_variance":0.3064516129032258,"se":0.09786017015735159,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.233368045068346,"gpu_lease_seconds":9,"gpu_slot_seconds":9.14248778799083,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"384fccc53f471e23eb9acac8"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":6,"b_only":3,"both_fail":14,"both_pass":9,"delta_b_minus_a":-0.09375,"disagreement_rate":0.28125,"interval_95":[-0.28125,0.09375],"mcnemar_exact_p":0.5078125,"n":32,"sample_variance":0.28125,"se":0.09375,"separating_examples":9,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.292591534089297,"gpu_lease_seconds":9,"gpu_slot_seconds":9.091987673076801,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"75ea5d8ae94de8a86a7a0777"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":7,"b_only":3,"both_fail":14,"both_pass":8,"delta_b_minus_a":-0.125,"disagreement_rate":0.3125,"interval_95":[-0.3125,0.0625],"mcnemar_exact_p":0.34375,"n":32,"sample_variance":0.3064516129032258,"se":0.09786017015735159,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.2528224050765857,"gpu_lease_seconds":9,"gpu_slot_seconds":9.092057132045738,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"8677abc8861650a471b4ea7c"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":7,"b_only":3,"both_fail":14,"both_pass":8,"delta_b_minus_a":-0.125,"disagreement_rate":0.3125,"interval_95":[-0.3125,0.0625],"mcnemar_exact_p":0.34375,"n":32,"sample_variance":0.3064516129032258,"se":0.09786017015735159,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.6817733279895037,"gpu_lease_seconds":9,"gpu_slot_seconds":9.493383716093376,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"ec41e48212ad955d3ea4c8f6"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":7,"b_only":3,"both_fail":14,"both_pass":8,"delta_b_minus_a":-0.125,"disagreement_rate":0.3125,"interval_95":[-0.31328124999999973,0.0625],"mcnemar_exact_p":0.34375,"n":32,"sample_variance":0.3064516129032258,"se":0.09786017015735159,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3916028240928426,"gpu_lease_seconds":9,"gpu_slot_seconds":9.242600969038904,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"4bab151d17bfaf2727299f60"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":5,"b_only":2,"both_fail":15,"both_pass":10,"delta_b_minus_a":-0.09375,"disagreement_rate":0.21875,"interval_95":[-0.25,0.0625],"mcnemar_exact_p":0.453125,"n":32,"sample_variance":0.21673387096774194,"se":0.08229783391889446,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.5089562641223893,"gpu_lease_seconds":10,"gpu_slot_seconds":9.343041658983566,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"93979b5fa00514cce3d96032"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":6,"b_only":2,"both_fail":15,"both_pass":9,"delta_b_minus_a":-0.125,"disagreement_rate":0.25,"interval_95":[-0.3125,0.03125],"mcnemar_exact_p":0.2890625,"n":32,"sample_variance":0.24193548387096775,"se":0.08695104295503155,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.434328534058295,"gpu_lease_seconds":10,"gpu_slot_seconds":9.34224056394305,"item_cost_model":null,"limitation":null,"run_a":"9733ef9ec08a5e033ded7a5e","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":4,"b_only":1,"both_fail":18,"both_pass":9,"delta_b_minus_a":-0.09375,"disagreement_rate":0.15625,"interval_95":[-0.25,0.03125],"mcnemar_exact_p":0.375,"n":32,"sample_variance":0.15221774193548387,"se":0.0689695906576505,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.75636076903902,"gpu_lease_seconds":10,"gpu_slot_seconds":9.743815422873013,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"60b94d2f3d3bccb39d7277ca"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":4,"b_only":2,"both_fail":17,"both_pass":9,"delta_b_minus_a":-0.0625,"disagreement_rate":0.1875,"interval_95":[-0.21875,0.09375],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.524041562108323,"gpu_lease_seconds":10,"gpu_slot_seconds":9.59379937290214,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"384fccc53f471e23eb9acac8"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":3,"b_only":2,"both_fail":17,"both_pass":10,"delta_b_minus_a":-0.03125,"disagreement_rate":0.15625,"interval_95":[-0.15625,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.583265051129274,"gpu_lease_seconds":10,"gpu_slot_seconds":9.54329925798811,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"75ea5d8ae94de8a86a7a0777"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":4,"b_only":2,"both_fail":17,"both_pass":9,"delta_b_minus_a":-0.0625,"disagreement_rate":0.1875,"interval_95":[-0.21875,0.09375],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.5434959221165627,"gpu_lease_seconds":10,"gpu_slot_seconds":9.543368716957048,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"8677abc8861650a471b4ea7c"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":4,"b_only":2,"both_fail":17,"both_pass":9,"delta_b_minus_a":-0.0625,"disagreement_rate":0.1875,"interval_95":[-0.21875,0.09375],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.9724468450294808,"gpu_lease_seconds":10,"gpu_slot_seconds":9.944695301004685,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"ec41e48212ad955d3ea4c8f6"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":5,"b_only":3,"both_fail":16,"both_pass":8,"delta_b_minus_a":-0.0625,"disagreement_rate":0.25,"interval_95":[-0.25,0.09375],"mcnemar_exact_p":0.7265625,"n":32,"sample_variance":0.2540322580645161,"se":0.08909830562090465,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.6822763411328197,"gpu_lease_seconds":10,"gpu_slot_seconds":9.693912553950213,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"4bab151d17bfaf2727299f60"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":3,"b_only":2,"both_fail":17,"both_pass":10,"delta_b_minus_a":-0.03125,"disagreement_rate":0.15625,"interval_95":[-0.1875,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.7996297811623663,"gpu_lease_seconds":11,"gpu_slot_seconds":9.794353243894875,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"93979b5fa00514cce3d96032"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":4,"b_only":2,"both_fail":17,"both_pass":9,"delta_b_minus_a":-0.0625,"disagreement_rate":0.1875,"interval_95":[-0.21875,0.09375],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.725002051098272,"gpu_lease_seconds":11,"gpu_slot_seconds":9.79355214885436,"item_cost_model":null,"limitation":null,"run_a":"5182981eed2309237804d9c3","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":2,"b_only":3,"both_fail":19,"both_pass":8,"delta_b_minus_a":0.03125,"disagreement_rate":0.15625,"interval_95":[-0.09375,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.167235333006829,"gpu_lease_seconds":10,"gpu_slot_seconds":10.094939683913253,"item_cost_model":null,"limitation":null,"run_a":"60b94d2f3d3bccb39d7277ca","run_b":"384fccc53f471e23eb9acac8"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":2,"b_only":4,"both_fail":18,"both_pass":8,"delta_b_minus_a":0.0625,"disagreement_rate":0.1875,"interval_95":[-0.0625,0.21875],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.22645882202778,"gpu_lease_seconds":10,"gpu_slot_seconds":10.044439568999223,"item_cost_model":null,"limitation":null,"run_a":"60b94d2f3d3bccb39d7277ca","run_b":"75ea5d8ae94de8a86a7a0777"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":2,"b_only":3,"both_fail":19,"both_pass":8,"delta_b_minus_a":0.03125,"disagreement_rate":0.15625,"interval_95":[-0.09375,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.186689693015069,"gpu_lease_seconds":10,"gpu_slot_seconds":10.04450902796816,"item_cost_model":null,"limitation":null,"run_a":"60b94d2f3d3bccb39d7277ca","run_b":"8677abc8861650a471b4ea7c"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149","diagnostic":{"a_only":3,"b_only":4,"both_fail":18,"both_pass":7,"delta_b_minus_a":0.03125,"disagreement_rate":0.21875,"interval_95":[-0.125,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.2247983870967742,"se":0.08381497239022509,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.615640615927987,"gpu_lease_seconds":10,"gpu_slot_seconds":10.445835612015799,"item_cost_model":null,"limitation":null,"run_a":"60b94d2f3d3bccb39d7277ca","run_b":"ec41e48212ad955d3ea4c8f6"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":1,"b_only":2,"both_fail":20,"both_pass":9,"delta_b_minus_a":0.03125,"disagreement_rate":0.09375,"interval_95":[-0.0625,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.09576612903225806,"se":0.05470549819038361,"separating_examples":3,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.325470112031326,"gpu_lease_seconds":10,"gpu_slot_seconds":10.195052864961326,"item_cost_model":null,"limitation":null,"run_a":"60b94d2f3d3bccb39d7277ca","run_b":"4bab151d17bfaf2727299f60"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":0,"b_only":2,"both_fail":20,"both_pass":10,"delta_b_minus_a":0.0625,"disagreement_rate":0.0625,"interval_95":[0.0,0.15625],"mcnemar_exact_p":0.5,"n":32,"sample_variance":0.06048387096774194,"se":0.04347552147751577,"separating_examples":2,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.442823552060872,"gpu_lease_seconds":11,"gpu_slot_seconds":10.295493554905988,"item_cost_model":null,"limitation":null,"run_a":"60b94d2f3d3bccb39d7277ca","run_b":"93979b5fa00514cce3d96032"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":0,"b_only":1,"both_fail":21,"both_pass":10,"delta_b_minus_a":0.03125,"disagreement_rate":0.03125,"interval_95":[0.0,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.03125,"se":0.03125,"separating_examples":1,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.368195821996778,"gpu_lease_seconds":11,"gpu_slot_seconds":10.294692459865473,"item_cost_model":null,"limitation":null,"run_a":"60b94d2f3d3bccb39d7277ca","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":0,"b_only":1,"both_fail":20,"both_pass":11,"delta_b_minus_a":0.03125,"disagreement_rate":0.03125,"interval_95":[0.0,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.03125,"se":0.03125,"separating_examples":1,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.994139615097083,"gpu_lease_seconds":10,"gpu_slot_seconds":9.89442351902835,"item_cost_model":null,"limitation":null,"run_a":"384fccc53f471e23eb9acac8","run_b":"75ea5d8ae94de8a86a7a0777"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":0,"b_only":0,"both_fail":21,"both_pass":11,"delta_b_minus_a":0.0,"disagreement_rate":0.0,"interval_95":[0.0,0.0],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.0,"se":0.0,"separating_examples":0,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.954370486084372,"gpu_lease_seconds":10,"gpu_slot_seconds":9.894492977997288,"item_cost_model":null,"limitation":null,"run_a":"384fccc53f471e23eb9acac8","run_b":"8677abc8861650a471b4ea7c"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149","diagnostic":{"a_only":3,"b_only":3,"both_fail":18,"both_pass":8,"delta_b_minus_a":0.0,"disagreement_rate":0.1875,"interval_95":[-0.15625,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.1935483870967742,"se":0.0777713771047819,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.38332140899729,"gpu_lease_seconds":10,"gpu_slot_seconds":10.295819562044926,"item_cost_model":null,"limitation":null,"run_a":"384fccc53f471e23eb9acac8","run_b":"ec41e48212ad955d3ea4c8f6"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":2,"b_only":2,"both_fail":19,"both_pass":9,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.093150905100629,"gpu_lease_seconds":10,"gpu_slot_seconds":10.045036814990453,"item_cost_model":null,"limitation":null,"run_a":"384fccc53f471e23eb9acac8","run_b":"4bab151d17bfaf2727299f60"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":2,"b_only":3,"both_fail":18,"both_pass":9,"delta_b_minus_a":0.03125,"disagreement_rate":0.15625,"interval_95":[-0.09375,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.210504345130175,"gpu_lease_seconds":11,"gpu_slot_seconds":10.145477504935116,"item_cost_model":null,"limitation":null,"run_a":"384fccc53f471e23eb9acac8","run_b":"93979b5fa00514cce3d96032"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":2,"b_only":2,"both_fail":19,"both_pass":9,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.135876615066081,"gpu_lease_seconds":11,"gpu_slot_seconds":10.1446764098946,"item_cost_model":null,"limitation":null,"run_a":"384fccc53f471e23eb9acac8","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":1,"b_only":0,"both_fail":20,"both_pass":11,"delta_b_minus_a":-0.03125,"disagreement_rate":0.03125,"interval_95":[-0.09375,0.0],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.03125,"se":0.03125,"separating_examples":1,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.013593975105323,"gpu_lease_seconds":10,"gpu_slot_seconds":9.843992863083258,"item_cost_model":null,"limitation":null,"run_a":"75ea5d8ae94de8a86a7a0777","run_b":"8677abc8861650a471b4ea7c"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149","diagnostic":{"a_only":3,"b_only":2,"both_fail":18,"both_pass":9,"delta_b_minus_a":-0.03125,"disagreement_rate":0.15625,"interval_95":[-0.15625,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.442544898018241,"gpu_lease_seconds":10,"gpu_slot_seconds":10.245319447130896,"item_cost_model":null,"limitation":null,"run_a":"75ea5d8ae94de8a86a7a0777","run_b":"ec41e48212ad955d3ea4c8f6"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":3,"b_only":2,"both_fail":18,"both_pass":9,"delta_b_minus_a":-0.03125,"disagreement_rate":0.15625,"interval_95":[-0.1875,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.15237439412158,"gpu_lease_seconds":10,"gpu_slot_seconds":9.994536700076424,"item_cost_model":null,"limitation":null,"run_a":"75ea5d8ae94de8a86a7a0777","run_b":"4bab151d17bfaf2727299f60"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":2,"b_only":2,"both_fail":18,"both_pass":10,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.2697278341511264,"gpu_lease_seconds":11,"gpu_slot_seconds":10.094977390021086,"item_cost_model":null,"limitation":null,"run_a":"75ea5d8ae94de8a86a7a0777","run_b":"93979b5fa00514cce3d96032"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":3,"b_only":2,"both_fail":18,"both_pass":9,"delta_b_minus_a":-0.03125,"disagreement_rate":0.15625,"interval_95":[-0.15625,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.195100104087032,"gpu_lease_seconds":11,"gpu_slot_seconds":10.09417629498057,"item_cost_model":null,"limitation":null,"run_a":"75ea5d8ae94de8a86a7a0777","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149","diagnostic":{"a_only":3,"b_only":3,"both_fail":18,"both_pass":8,"delta_b_minus_a":0.0,"disagreement_rate":0.1875,"interval_95":[-0.15625,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.1935483870967742,"se":0.0777713771047819,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.40277576900553,"gpu_lease_seconds":10,"gpu_slot_seconds":10.245388906099834,"item_cost_model":null,"limitation":null,"run_a":"8677abc8861650a471b4ea7c","run_b":"ec41e48212ad955d3ea4c8f6"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":2,"b_only":2,"both_fail":19,"both_pass":9,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.1126052651088685,"gpu_lease_seconds":10,"gpu_slot_seconds":9.994606159045361,"item_cost_model":null,"limitation":null,"run_a":"8677abc8861650a471b4ea7c","run_b":"4bab151d17bfaf2727299f60"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":2,"b_only":3,"both_fail":18,"both_pass":9,"delta_b_minus_a":0.03125,"disagreement_rate":0.15625,"interval_95":[-0.09375,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.16028225806451613,"se":0.07077302144543589,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.229958705138415,"gpu_lease_seconds":11,"gpu_slot_seconds":10.095046848990023,"item_cost_model":null,"limitation":null,"run_a":"8677abc8861650a471b4ea7c","run_b":"93979b5fa00514cce3d96032"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":2,"b_only":2,"both_fail":19,"both_pass":9,"delta_b_minus_a":0.0,"disagreement_rate":0.125,"interval_95":[-0.125,0.125],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.12903225806451613,"se":0.06350006350009525,"separating_examples":4,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.155330975074321,"gpu_lease_seconds":11,"gpu_slot_seconds":10.094245753949508,"item_cost_model":null,"limitation":null,"run_a":"8677abc8861650a471b4ea7c","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":3,"b_only":3,"both_fail":18,"both_pass":8,"delta_b_minus_a":0.0,"disagreement_rate":0.1875,"interval_95":[-0.15625,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.1935483870967742,"se":0.0777713771047819,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.5415561880217865,"gpu_lease_seconds":10,"gpu_slot_seconds":10.395932743092999,"item_cost_model":null,"limitation":null,"run_a":"ec41e48212ad955d3ea4c8f6","run_b":"4bab151d17bfaf2727299f60"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":3,"b_only":4,"both_fail":17,"both_pass":8,"delta_b_minus_a":0.03125,"disagreement_rate":0.21875,"interval_95":[-0.125,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.2247983870967742,"se":0.08381497239022509,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.658909628051333,"gpu_lease_seconds":11,"gpu_slot_seconds":10.496373433037661,"item_cost_model":null,"limitation":null,"run_a":"ec41e48212ad955d3ea4c8f6","run_b":"93979b5fa00514cce3d96032"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":4,"b_only":4,"both_fail":17,"both_pass":7,"delta_b_minus_a":0.0,"disagreement_rate":0.25,"interval_95":[-0.1875,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.25806451612903225,"se":0.08980265101338746,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.584281897987239,"gpu_lease_seconds":11,"gpu_slot_seconds":10.495572337997146,"item_cost_model":null,"limitation":null,"run_a":"ec41e48212ad955d3ea4c8f6","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"SlayerLab/Slayer149-ARC","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":1,"b_only":2,"both_fail":19,"both_pass":10,"delta_b_minus_a":0.03125,"disagreement_rate":0.09375,"interval_95":[-0.0625,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.09576612903225806,"se":0.05470549819038361,"separating_examples":3,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.368739124154672,"gpu_lease_seconds":11,"gpu_slot_seconds":10.245590685983188,"item_cost_model":null,"limitation":null,"run_a":"4bab151d17bfaf2727299f60","run_b":"93979b5fa00514cce3d96032"},{"a":"SlayerLab/Slayer149-ARC","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":1,"b_only":1,"both_fail":20,"both_pass":10,"delta_b_minus_a":0.0,"disagreement_rate":0.0625,"interval_95":[-0.09375,0.09375],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.06451612903225806,"se":0.04490132550669373,"separating_examples":2,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.294111394090578,"gpu_lease_seconds":11,"gpu_slot_seconds":10.244789590942673,"item_cost_model":null,"limitation":null,"run_a":"4bab151d17bfaf2727299f60","run_b":"d6a751bd1fb70f34af6e88c0"},{"a":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":1,"b_only":0,"both_fail":20,"both_pass":11,"delta_b_minus_a":-0.03125,"disagreement_rate":0.03125,"interval_95":[-0.09375,0.0],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.03125,"se":0.03125,"separating_examples":1,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.411464834120125,"gpu_lease_seconds":12,"gpu_slot_seconds":10.345230280887336,"item_cost_model":null,"limitation":null,"run_a":"93979b5fa00514cce3d96032","run_b":"d6a751bd1fb70f34af6e88c0"}],"runs_with_gpu_timing":11,"runs_with_lease_timing":11,"sample_limit":32,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","task":"sub1b-hellaswag"},{"benchmark":"BoolQ","evaluation_elapsed_seconds":18.078722105012275,"gpu_lease_seconds":82,"gpu_slot_seconds":80.18851390609052,"limitation":null,"metric":"accuracy","models":11,"name":"BoolQ / sub-1B cohort","pairs":[{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"HuggingFaceTB/SmolLM2-360M-Instruct","diagnostic":{"a_only":1,"b_only":5,"both_fail":20,"both_pass":6,"delta_b_minus_a":0.125,"disagreement_rate":0.1875,"interval_95":[0.0,0.28125],"mcnemar_exact_p":0.21875,"n":32,"sample_variance":0.1774193548387097,"se":0.07446042464765883,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":2.7122159810969606,"gpu_lease_seconds":14,"gpu_slot_seconds":14.049712387030013,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"eb5517fac2e0ec0cb50196c5"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"Qwen/Qwen2.5-0.5B-Instruct","diagnostic":{"a_only":3,"b_only":19,"both_fail":6,"both_pass":4,"delta_b_minus_a":0.5,"disagreement_rate":0.6875,"interval_95":[0.28125,0.71875],"mcnemar_exact_p":0.0008554458618164062,"n":32,"sample_variance":0.45161290322580644,"se":0.11879774082787287,"separating_examples":22,"separation":"observed"},"evaluation_elapsed_seconds":2.9953374569304287,"gpu_lease_seconds":15,"gpu_slot_seconds":14.501307431026362,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"141ceb6c09dc8f0156f70c0c"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":6,"b_only":21,"both_fail":4,"both_pass":1,"delta_b_minus_a":0.46875,"disagreement_rate":0.84375,"interval_95":[0.1875,0.71875],"mcnemar_exact_p":0.005924612283706665,"n":32,"sample_variance":0.6441532258064516,"se":0.14187948515007945,"separating_examples":27,"separation":"observed"},"evaluation_elapsed_seconds":2.7491487390361726,"gpu_lease_seconds":15,"gpu_slot_seconds":14.100977687048726,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"d12d9fe7b986a834ea0d1098"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":4,"b_only":15,"both_fail":10,"both_pass":3,"delta_b_minus_a":0.34375,"disagreement_rate":0.59375,"interval_95":[0.12421875000000027,0.5625],"mcnemar_exact_p":0.0192108154296875,"n":32,"sample_variance":0.4909274193548387,"se":0.12386073572702008,"separating_examples":19,"separation":"observed"},"evaluation_elapsed_seconds":2.579376925015822,"gpu_lease_seconds":14,"gpu_slot_seconds":13.75089794502128,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"a64f0553519f2724df9f01c1"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":5,"b_only":22,"both_fail":3,"both_pass":2,"delta_b_minus_a":0.53125,"disagreement_rate":0.84375,"interval_95":[0.25,0.78125],"mcnemar_exact_p":0.0015137195587158203,"n":32,"sample_variance":0.5796370967741935,"se":0.13458699518970452,"separating_examples":27,"separation":"observed"},"evaluation_elapsed_seconds":2.6041842399863526,"gpu_lease_seconds":14,"gpu_slot_seconds":13.600595474010333,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"ff71c6e3b5a06ff21384b00e"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":4,"b_only":15,"both_fail":10,"both_pass":3,"delta_b_minus_a":0.34375,"disagreement_rate":0.59375,"interval_95":[0.12421875000000027,0.5625],"mcnemar_exact_p":0.0192108154296875,"n":32,"sample_variance":0.4909274193548387,"se":0.12386073572702008,"separating_examples":19,"separation":"observed"},"evaluation_elapsed_seconds":2.764403481967747,"gpu_lease_seconds":15,"gpu_slot_seconds":14.04990980005823,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"18802f73c41e24cdd29f5a2f"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":5,"b_only":20,"both_fail":5,"both_pass":2,"delta_b_minus_a":0.46875,"disagreement_rate":0.78125,"interval_95":[0.1875,0.71875],"mcnemar_exact_p":0.004077315330505371,"n":32,"sample_variance":0.5796370967741935,"se":0.13458699518970452,"separating_examples":25,"separation":"observed"},"evaluation_elapsed_seconds":3.127241977956146,"gpu_lease_seconds":15,"gpu_slot_seconds":14.351973386015743,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"509bc4c9edce51c91c7e5e39"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":4,"b_only":13,"both_fail":12,"both_pass":3,"delta_b_minus_a":0.28125,"disagreement_rate":0.53125,"interval_95":[0.03125,0.5],"mcnemar_exact_p":0.049041748046875,"n":32,"sample_variance":0.46673387096774194,"se":0.12077016795443292,"separating_examples":17,"separation":"observed"},"evaluation_elapsed_seconds":2.6918435159604996,"gpu_lease_seconds":15,"gpu_slot_seconds":13.801519487984478,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"dce70413303e4781f185a0ac"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":3,"b_only":21,"both_fail":4,"both_pass":4,"delta_b_minus_a":0.5625,"disagreement_rate":0.75,"interval_95":[0.3125,0.78125],"mcnemar_exact_p":0.0002771615982055664,"n":32,"sample_variance":0.4475806451612903,"se":0.11826620464566504,"separating_examples":24,"separation":"observed"},"evaluation_elapsed_seconds":2.7076060319086537,"gpu_lease_seconds":14,"gpu_slot_seconds":13.849983530002646,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"HuggingFaceTB/SmolLM2-135M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":4,"b_only":17,"both_fail":8,"both_pass":3,"delta_b_minus_a":0.40625,"disagreement_rate":0.65625,"interval_95":[0.15625,0.625],"mcnemar_exact_p":0.007197380065917969,"n":32,"sample_variance":0.5070564516129032,"se":0.12587896612581162,"separating_examples":21,"separation":"observed"},"evaluation_elapsed_seconds":2.6817595900502056,"gpu_lease_seconds":14,"gpu_slot_seconds":13.750957283074968,"item_cost_model":null,"limitation":null,"run_a":"3b9bc33f16b6187cceb7819f","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"Qwen/Qwen2.5-0.5B-Instruct","diagnostic":{"a_only":3,"b_only":15,"both_fail":6,"both_pass":8,"delta_b_minus_a":0.375,"disagreement_rate":0.5625,"interval_95":[0.125,0.59375],"mcnemar_exact_p":0.007537841796875,"n":32,"sample_variance":0.43548387096774194,"se":0.11665706565717285,"separating_examples":18,"separation":"observed"},"evaluation_elapsed_seconds":3.588798808050342,"gpu_lease_seconds":15,"gpu_slot_seconds":15.302281928015873,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"141ceb6c09dc8f0156f70c0c"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":5,"b_only":16,"both_fail":5,"both_pass":6,"delta_b_minus_a":0.34375,"disagreement_rate":0.65625,"interval_95":[0.0625,0.59375],"mcnemar_exact_p":0.02660369873046875,"n":32,"sample_variance":0.5554435483870968,"se":0.131748286087891,"separating_examples":21,"separation":"observed"},"evaluation_elapsed_seconds":3.342610090156086,"gpu_lease_seconds":15,"gpu_slot_seconds":14.901952184038237,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"d12d9fe7b986a834ea0d1098"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":4,"b_only":11,"both_fail":10,"both_pass":7,"delta_b_minus_a":0.21875,"disagreement_rate":0.46875,"interval_95":[0.0,0.4375],"mcnemar_exact_p":0.11846923828125,"n":32,"sample_variance":0.4344758064516129,"se":0.11652196767825758,"separating_examples":15,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.172838276135735,"gpu_lease_seconds":14,"gpu_slot_seconds":14.55187244201079,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"a64f0553519f2724df9f01c1"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":5,"b_only":18,"both_fail":3,"both_pass":6,"delta_b_minus_a":0.40625,"disagreement_rate":0.71875,"interval_95":[0.125,0.65625],"mcnemar_exact_p":0.010622024536132812,"n":32,"sample_variance":0.5715725806451613,"se":0.13364745843135697,"separating_examples":23,"separation":"observed"},"evaluation_elapsed_seconds":3.197645591106266,"gpu_lease_seconds":14,"gpu_slot_seconds":14.401569970999844,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"ff71c6e3b5a06ff21384b00e"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":4,"b_only":11,"both_fail":10,"both_pass":7,"delta_b_minus_a":0.21875,"disagreement_rate":0.46875,"interval_95":[0.0,0.4375],"mcnemar_exact_p":0.11846923828125,"n":32,"sample_variance":0.4344758064516129,"se":0.11652196767825758,"separating_examples":15,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3578648330876604,"gpu_lease_seconds":15,"gpu_slot_seconds":14.850884297047742,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"18802f73c41e24cdd29f5a2f"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":7,"b_only":18,"both_fail":3,"both_pass":4,"delta_b_minus_a":0.34375,"disagreement_rate":0.78125,"interval_95":[0.0625,0.625],"mcnemar_exact_p":0.043285250663757324,"n":32,"sample_variance":0.6844758064516129,"se":0.1462527570735434,"separating_examples":25,"separation":"observed"},"evaluation_elapsed_seconds":3.720703329076059,"gpu_lease_seconds":15,"gpu_slot_seconds":15.152947883005254,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"509bc4c9edce51c91c7e5e39"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":3,"b_only":8,"both_fail":13,"both_pass":8,"delta_b_minus_a":0.15625,"disagreement_rate":0.34375,"interval_95":[-0.03125,0.34375],"mcnemar_exact_p":0.2265625,"n":32,"sample_variance":0.32963709677419356,"se":0.10149462682424892,"separating_examples":11,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.285304867080413,"gpu_lease_seconds":15,"gpu_slot_seconds":14.60249398497399,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"dce70413303e4781f185a0ac"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":3,"b_only":17,"both_fail":4,"both_pass":8,"delta_b_minus_a":0.4375,"disagreement_rate":0.625,"interval_95":[0.1875,0.65625],"mcnemar_exact_p":0.0025768280029296875,"n":32,"sample_variance":0.4475806451612903,"se":0.11826620464566504,"separating_examples":20,"separation":"observed"},"evaluation_elapsed_seconds":3.301067383028567,"gpu_lease_seconds":14,"gpu_slot_seconds":14.650958026992157,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"HuggingFaceTB/SmolLM2-360M-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":3,"b_only":12,"both_fail":9,"both_pass":8,"delta_b_minus_a":0.28125,"disagreement_rate":0.46875,"interval_95":[0.0625,0.5],"mcnemar_exact_p":0.03515625,"n":32,"sample_variance":0.4022177419354839,"se":0.11211290931683056,"separating_examples":15,"separation":"observed"},"evaluation_elapsed_seconds":3.2752209411701187,"gpu_lease_seconds":14,"gpu_slot_seconds":14.551931780064479,"item_cost_model":null,"limitation":null,"run_a":"eb5517fac2e0ec0cb50196c5","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer-149M-Decision-v1","diagnostic":{"a_only":8,"b_only":7,"both_fail":2,"both_pass":15,"delta_b_minus_a":-0.03125,"disagreement_rate":0.46875,"interval_95":[-0.25,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.48286290322580644,"se":0.12283918644230127,"separating_examples":15,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.625731565989554,"gpu_lease_seconds":16,"gpu_slot_seconds":15.353547228034586,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"d12d9fe7b986a834ea0d1098"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":10,"b_only":5,"both_fail":4,"both_pass":13,"delta_b_minus_a":-0.15625,"disagreement_rate":0.46875,"interval_95":[-0.375,0.0625],"mcnemar_exact_p":0.3017578125,"n":32,"sample_variance":0.4586693548387097,"se":0.11972225080873512,"separating_examples":15,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.4559597519692034,"gpu_lease_seconds":15,"gpu_slot_seconds":15.003467486007139,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"a64f0553519f2724df9f01c1"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":6,"b_only":7,"both_fail":2,"both_pass":17,"delta_b_minus_a":0.03125,"disagreement_rate":0.40625,"interval_95":[-0.1875,0.25],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.4183467741935484,"se":0.11433869289767304,"separating_examples":13,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.480767066939734,"gpu_lease_seconds":15,"gpu_slot_seconds":14.853165014996193,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"ff71c6e3b5a06ff21384b00e"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":10,"b_only":5,"both_fail":4,"both_pass":13,"delta_b_minus_a":-0.15625,"disagreement_rate":0.46875,"interval_95":[-0.375,0.0625],"mcnemar_exact_p":0.3017578125,"n":32,"sample_variance":0.4586693548387097,"se":0.11972225080873512,"separating_examples":15,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.6409863089211285,"gpu_lease_seconds":16,"gpu_slot_seconds":15.30247934104409,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"18802f73c41e24cdd29f5a2f"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149","diagnostic":{"a_only":8,"b_only":7,"both_fail":2,"both_pass":15,"delta_b_minus_a":-0.03125,"disagreement_rate":0.46875,"interval_95":[-0.25,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.48286290322580644,"se":0.12283918644230127,"separating_examples":15,"separation":"inconclusive"},"evaluation_elapsed_seconds":4.003824804909527,"gpu_lease_seconds":16,"gpu_slot_seconds":15.604542927001603,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"509bc4c9edce51c91c7e5e39"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":10,"b_only":3,"both_fail":6,"both_pass":13,"delta_b_minus_a":-0.21875,"disagreement_rate":0.40625,"interval_95":[-0.4375,0.0],"mcnemar_exact_p":0.09228515625,"n":32,"sample_variance":0.3699596774193548,"se":0.1075232064224037,"separating_examples":13,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.568426342913881,"gpu_lease_seconds":16,"gpu_slot_seconds":15.054089028970338,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"dce70413303e4781f185a0ac"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":7,"b_only":9,"both_fail":0,"both_pass":16,"delta_b_minus_a":0.0625,"disagreement_rate":0.5,"interval_95":[-0.1875,0.3125],"mcnemar_exact_p":0.803619384765625,"n":32,"sample_variance":0.5120967741935484,"se":0.12650306001654027,"separating_examples":16,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.584188858862035,"gpu_lease_seconds":15,"gpu_slot_seconds":15.102553070988506,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"Qwen/Qwen2.5-0.5B-Instruct","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":10,"b_only":7,"both_fail":2,"both_pass":13,"delta_b_minus_a":-0.09375,"disagreement_rate":0.53125,"interval_95":[-0.34375,0.15625],"mcnemar_exact_p":0.629058837890625,"n":32,"sample_variance":0.5393145161290323,"se":0.12982133348965516,"separating_examples":17,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.558342417003587,"gpu_lease_seconds":15,"gpu_slot_seconds":15.003526824060827,"item_cost_model":null,"limitation":null,"run_a":"141ceb6c09dc8f0156f70c0c","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.01","diagnostic":{"a_only":7,"b_only":3,"both_fail":7,"both_pass":15,"delta_b_minus_a":-0.125,"disagreement_rate":0.3125,"interval_95":[-0.3125,0.0625],"mcnemar_exact_p":0.34375,"n":32,"sample_variance":0.3064516129032258,"se":0.09786017015735159,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.2097710340749472,"gpu_lease_seconds":15,"gpu_slot_seconds":14.603137742029503,"item_cost_model":null,"limitation":null,"run_a":"d12d9fe7b986a834ea0d1098","run_b":"a64f0553519f2724df9f01c1"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":2,"b_only":4,"both_fail":6,"both_pass":20,"delta_b_minus_a":0.0625,"disagreement_rate":0.1875,"interval_95":[-0.09375,0.21875],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.234578349045478,"gpu_lease_seconds":15,"gpu_slot_seconds":14.452835271018557,"item_cost_model":null,"limitation":null,"run_a":"d12d9fe7b986a834ea0d1098","run_b":"ff71c6e3b5a06ff21384b00e"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":7,"b_only":3,"both_fail":7,"both_pass":15,"delta_b_minus_a":-0.125,"disagreement_rate":0.3125,"interval_95":[-0.3125,0.0625],"mcnemar_exact_p":0.34375,"n":32,"sample_variance":0.3064516129032258,"se":0.09786017015735159,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3947975910268724,"gpu_lease_seconds":16,"gpu_slot_seconds":14.902149597066455,"item_cost_model":null,"limitation":null,"run_a":"d12d9fe7b986a834ea0d1098","run_b":"18802f73c41e24cdd29f5a2f"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149","diagnostic":{"a_only":3,"b_only":3,"both_fail":7,"both_pass":19,"delta_b_minus_a":0.0,"disagreement_rate":0.1875,"interval_95":[-0.15625,0.15625],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.1935483870967742,"se":0.0777713771047819,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.757636087015271,"gpu_lease_seconds":16,"gpu_slot_seconds":15.204213183023967,"item_cost_model":null,"limitation":null,"run_a":"d12d9fe7b986a834ea0d1098","run_b":"509bc4c9edce51c91c7e5e39"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":9,"b_only":3,"both_fail":7,"both_pass":13,"delta_b_minus_a":-0.1875,"disagreement_rate":0.375,"interval_95":[-0.40625,0.0],"mcnemar_exact_p":0.14599609375,"n":32,"sample_variance":0.35080645161290325,"se":0.10470292074676439,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.322237625019625,"gpu_lease_seconds":16,"gpu_slot_seconds":14.653759284992702,"item_cost_model":null,"limitation":null,"run_a":"d12d9fe7b986a834ea0d1098","run_b":"dce70413303e4781f185a0ac"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":1,"b_only":4,"both_fail":6,"both_pass":21,"delta_b_minus_a":0.09375,"disagreement_rate":0.15625,"interval_95":[-0.03125,0.21875],"mcnemar_exact_p":0.375,"n":32,"sample_variance":0.15221774193548387,"se":0.0689695906576505,"separating_examples":5,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.338000140967779,"gpu_lease_seconds":15,"gpu_slot_seconds":14.70222332701087,"item_cost_model":null,"limitation":null,"run_a":"d12d9fe7b986a834ea0d1098","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"SlayerLab/Slayer-149M-Decision-v1","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":4,"b_only":2,"both_fail":8,"both_pass":18,"delta_b_minus_a":-0.0625,"disagreement_rate":0.1875,"interval_95":[-0.21875,0.09375],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3121536991093308,"gpu_lease_seconds":15,"gpu_slot_seconds":14.603197080083191,"item_cost_model":null,"limitation":null,"run_a":"d12d9fe7b986a834ea0d1098","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer139-1.01b","diagnostic":{"a_only":1,"b_only":7,"both_fail":7,"both_pass":17,"delta_b_minus_a":0.1875,"disagreement_rate":0.25,"interval_95":[0.03125,0.34375],"mcnemar_exact_p":0.0703125,"n":32,"sample_variance":0.2217741935483871,"se":0.08324928557283298,"separating_examples":8,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.0648065350251272,"gpu_lease_seconds":14,"gpu_slot_seconds":14.10275552899111,"item_cost_model":null,"limitation":null,"run_a":"a64f0553519f2724df9f01c1","run_b":"ff71c6e3b5a06ff21384b00e"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":0,"b_only":0,"both_fail":14,"both_pass":18,"delta_b_minus_a":0.0,"disagreement_rate":0.0,"interval_95":[0.0,0.0],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.0,"se":0.0,"separating_examples":0,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.225025777006522,"gpu_lease_seconds":15,"gpu_slot_seconds":14.552069855039008,"item_cost_model":null,"limitation":null,"run_a":"a64f0553519f2724df9f01c1","run_b":"18802f73c41e24cdd29f5a2f"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149","diagnostic":{"a_only":4,"b_only":8,"both_fail":6,"both_pass":14,"delta_b_minus_a":0.125,"disagreement_rate":0.375,"interval_95":[-0.0625,0.34375],"mcnemar_exact_p":0.3876953125,"n":32,"sample_variance":0.3709677419354839,"se":0.10766959615176362,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.5878642729949206,"gpu_lease_seconds":15,"gpu_slot_seconds":14.85413344099652,"item_cost_model":null,"limitation":null,"run_a":"a64f0553519f2724df9f01c1","run_b":"509bc4c9edce51c91c7e5e39"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":5,"b_only":3,"both_fail":11,"both_pass":13,"delta_b_minus_a":-0.0625,"disagreement_rate":0.25,"interval_95":[-0.21875,0.125],"mcnemar_exact_p":0.7265625,"n":32,"sample_variance":0.2540322580645161,"se":0.08909830562090465,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.1524658109992743,"gpu_lease_seconds":15,"gpu_slot_seconds":14.303679542965256,"item_cost_model":null,"limitation":null,"run_a":"a64f0553519f2724df9f01c1","run_b":"dce70413303e4781f185a0ac"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":2,"b_only":9,"both_fail":5,"both_pass":16,"delta_b_minus_a":0.21875,"disagreement_rate":0.34375,"interval_95":[0.03125,0.40625],"mcnemar_exact_p":0.0654296875,"n":32,"sample_variance":0.30544354838709675,"se":0.09769908334829336,"separating_examples":11,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.1682283269474283,"gpu_lease_seconds":14,"gpu_slot_seconds":14.352143584983423,"item_cost_model":null,"limitation":null,"run_a":"a64f0553519f2724df9f01c1","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"SlayerLab/Slayer139-1.01","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":4,"b_only":6,"both_fail":8,"both_pass":14,"delta_b_minus_a":0.0625,"disagreement_rate":0.3125,"interval_95":[-0.125,0.25],"mcnemar_exact_p":0.75390625,"n":32,"sample_variance":0.3185483870967742,"se":0.0997729276746663,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.14238188508898,"gpu_lease_seconds":14,"gpu_slot_seconds":14.253117338055745,"item_cost_model":null,"limitation":null,"run_a":"a64f0553519f2724df9f01c1","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer139-1.02","diagnostic":{"a_only":7,"b_only":1,"both_fail":7,"both_pass":17,"delta_b_minus_a":-0.1875,"disagreement_rate":0.25,"interval_95":[-0.34375,-0.03125],"mcnemar_exact_p":0.0703125,"n":32,"sample_variance":0.2217741935483871,"se":0.08324928557283298,"separating_examples":8,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.2498330919770524,"gpu_lease_seconds":15,"gpu_slot_seconds":14.401767384028062,"item_cost_model":null,"limitation":null,"run_a":"ff71c6e3b5a06ff21384b00e","run_b":"18802f73c41e24cdd29f5a2f"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149","diagnostic":{"a_only":4,"b_only":2,"both_fail":6,"both_pass":20,"delta_b_minus_a":-0.0625,"disagreement_rate":0.1875,"interval_95":[-0.21875,0.09375],"mcnemar_exact_p":0.6875,"n":32,"sample_variance":0.18951612903225806,"se":0.0769569946935174,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.612671587965451,"gpu_lease_seconds":15,"gpu_slot_seconds":14.703830969985574,"item_cost_model":null,"limitation":null,"run_a":"ff71c6e3b5a06ff21384b00e","run_b":"509bc4c9edce51c91c7e5e39"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":11,"b_only":3,"both_fail":5,"both_pass":13,"delta_b_minus_a":-0.25,"disagreement_rate":0.4375,"interval_95":[-0.46875,-0.03125],"mcnemar_exact_p":0.057373046875,"n":32,"sample_variance":0.3870967741935484,"se":0.10998533626601496,"separating_examples":14,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.177273125969805,"gpu_lease_seconds":15,"gpu_slot_seconds":14.15337707195431,"item_cost_model":null,"limitation":null,"run_a":"ff71c6e3b5a06ff21384b00e","run_b":"dce70413303e4781f185a0ac"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":3,"b_only":4,"both_fail":4,"both_pass":21,"delta_b_minus_a":0.03125,"disagreement_rate":0.21875,"interval_95":[-0.125,0.1875],"mcnemar_exact_p":1.0,"n":32,"sample_variance":0.2247983870967742,"se":0.08381497239022509,"separating_examples":7,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.193035641917959,"gpu_lease_seconds":14,"gpu_slot_seconds":14.201841113972478,"item_cost_model":null,"limitation":null,"run_a":"ff71c6e3b5a06ff21384b00e","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"SlayerLab/Slayer139-1.01b","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":8,"b_only":4,"both_fail":4,"both_pass":16,"delta_b_minus_a":-0.125,"disagreement_rate":0.375,"interval_95":[-0.34375,0.09375],"mcnemar_exact_p":0.3876953125,"n":32,"sample_variance":0.3709677419354839,"se":0.10766959615176362,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.1671892000595108,"gpu_lease_seconds":14,"gpu_slot_seconds":14.102814867044799,"item_cost_model":null,"limitation":null,"run_a":"ff71c6e3b5a06ff21384b00e","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149","diagnostic":{"a_only":4,"b_only":8,"both_fail":6,"both_pass":14,"delta_b_minus_a":0.125,"disagreement_rate":0.375,"interval_95":[-0.0625,0.34375],"mcnemar_exact_p":0.3876953125,"n":32,"sample_variance":0.3709677419354839,"se":0.10766959615176362,"separating_examples":12,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.7728908299468458,"gpu_lease_seconds":16,"gpu_slot_seconds":15.153145296033472,"item_cost_model":null,"limitation":null,"run_a":"18802f73c41e24cdd29f5a2f","run_b":"509bc4c9edce51c91c7e5e39"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":5,"b_only":3,"both_fail":11,"both_pass":13,"delta_b_minus_a":-0.0625,"disagreement_rate":0.25,"interval_95":[-0.21875,0.125],"mcnemar_exact_p":0.7265625,"n":32,"sample_variance":0.2540322580645161,"se":0.08909830562090465,"separating_examples":8,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3374923679511994,"gpu_lease_seconds":16,"gpu_slot_seconds":14.602691398002207,"item_cost_model":null,"limitation":null,"run_a":"18802f73c41e24cdd29f5a2f","run_b":"dce70413303e4781f185a0ac"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":2,"b_only":9,"both_fail":5,"both_pass":16,"delta_b_minus_a":0.21875,"disagreement_rate":0.34375,"interval_95":[0.03125,0.40625],"mcnemar_exact_p":0.0654296875,"n":32,"sample_variance":0.30544354838709675,"se":0.09769908334829336,"separating_examples":11,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.3532548838993534,"gpu_lease_seconds":15,"gpu_slot_seconds":14.651155440020375,"item_cost_model":null,"limitation":null,"run_a":"18802f73c41e24cdd29f5a2f","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"SlayerLab/Slayer139-1.02","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":4,"b_only":6,"both_fail":8,"both_pass":14,"delta_b_minus_a":0.0625,"disagreement_rate":0.3125,"interval_95":[-0.125,0.25],"mcnemar_exact_p":0.75390625,"n":32,"sample_variance":0.3185483870967742,"se":0.0997729276746663,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.3274084420409054,"gpu_lease_seconds":15,"gpu_slot_seconds":14.552129193092696,"item_cost_model":null,"limitation":null,"run_a":"18802f73c41e24cdd29f5a2f","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-ARC","diagnostic":{"a_only":11,"b_only":5,"both_fail":5,"both_pass":11,"delta_b_minus_a":-0.1875,"disagreement_rate":0.5,"interval_95":[-0.4375,0.0625],"mcnemar_exact_p":0.210113525390625,"n":32,"sample_variance":0.4798387096774194,"se":0.12245390837951786,"separating_examples":16,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.700330863939598,"gpu_lease_seconds":16,"gpu_slot_seconds":14.90475498395972,"item_cost_model":null,"limitation":null,"run_a":"509bc4c9edce51c91c7e5e39","run_b":"dce70413303e4781f185a0ac"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":3,"b_only":6,"both_fail":4,"both_pass":19,"delta_b_minus_a":0.09375,"disagreement_rate":0.28125,"interval_95":[-0.09375,0.28125],"mcnemar_exact_p":0.5078125,"n":32,"sample_variance":0.28125,"se":0.09375,"separating_examples":9,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.7160933798877522,"gpu_lease_seconds":15,"gpu_slot_seconds":14.953219025977887,"item_cost_model":null,"limitation":null,"run_a":"509bc4c9edce51c91c7e5e39","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"SlayerLab/Slayer149","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":6,"b_only":4,"both_fail":6,"both_pass":16,"delta_b_minus_a":-0.0625,"disagreement_rate":0.3125,"interval_95":[-0.25,0.125],"mcnemar_exact_p":0.75390625,"n":32,"sample_variance":0.3185483870967742,"se":0.0997729276746663,"separating_examples":10,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.690246938029304,"gpu_lease_seconds":15,"gpu_slot_seconds":14.854192779050209,"item_cost_model":null,"limitation":null,"run_a":"509bc4c9edce51c91c7e5e39","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"SlayerLab/Slayer149-ARC","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","diagnostic":{"a_only":1,"b_only":10,"both_fail":6,"both_pass":15,"delta_b_minus_a":0.28125,"disagreement_rate":0.34375,"interval_95":[0.09375,0.46875],"mcnemar_exact_p":0.01171875,"n":32,"sample_variance":0.27318548387096775,"se":0.09239613829034059,"separating_examples":11,"separation":"observed"},"evaluation_elapsed_seconds":3.280694917892106,"gpu_lease_seconds":15,"gpu_slot_seconds":14.402765127946623,"item_cost_model":null,"limitation":null,"run_a":"dce70413303e4781f185a0ac","run_b":"3fb07acb70ad720b7ea6387f"},{"a":"SlayerLab/Slayer149-ARC","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":1,"b_only":5,"both_fail":11,"both_pass":15,"delta_b_minus_a":0.125,"disagreement_rate":0.1875,"interval_95":[0.0,0.28125],"mcnemar_exact_p":0.21875,"n":32,"sample_variance":0.1774193548387097,"se":0.07446042464765883,"separating_examples":6,"separation":"inconclusive"},"evaluation_elapsed_seconds":3.254848476033658,"gpu_lease_seconds":15,"gpu_slot_seconds":14.303738881018944,"item_cost_model":null,"limitation":null,"run_a":"dce70413303e4781f185a0ac","run_b":"3ce0ba3d66280b6cbd47845b"},{"a":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","b":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","diagnostic":{"a_only":5,"b_only":0,"both_fail":7,"both_pass":20,"delta_b_minus_a":-0.15625,"disagreement_rate":0.15625,"interval_95":[-0.28125,-0.03125],"mcnemar_exact_p":0.0625,"n":32,"sample_variance":0.13608870967741934,"se":0.06521328221627366,"separating_examples":5,"separation":"bootstrap_only"},"evaluation_elapsed_seconds":3.270610991981812,"gpu_lease_seconds":14,"gpu_slot_seconds":14.352202923037112,"item_cost_model":null,"limitation":null,"run_a":"3fb07acb70ad720b7ea6387f","run_b":"3ce0ba3d66280b6cbd47845b"}],"runs_with_gpu_timing":11,"runs_with_lease_timing":11,"sample_limit":32,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","task":"sub1b-boolq"}],"method":{"cost":"GPU lease seconds come from server claim-to-finish timestamps for one reserved GPU slot per job (one-second clock resolution); they include loading, polling and upload overhead. Historical upload-only or inconsistent leases are excluded, not interpreted as zero-cost evaluations. Instrumented GPU-slot seconds are evaluator subprocess wall seconds \u00d7 one reserved GPU, including setup, data/model loading and CPU scoring. They are not kernel-active seconds. Historical evaluation elapsed is a distinct proxy and not retroactively relabelled.","interval":"Paired percentile resampling over aligned binary correctness; B minus A; exploratory, no multiple-comparison correction","normal_factor_80_alpha05":2.801585218112968,"resamples":4000,"scope":"First valid examples in pinned order, not a random representative sample. The interval describes sensitivity within this pilot, not population confidence on the full benchmark.","seed":42,"value":"Correctness disagreement and score separation describe this measured model pair. High disagreement alone does not prove a useful or reliable benchmark. Redundancy, ranking stability and broad information gain need a larger shared model panel and repeated/wider evaluations."}}