{"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193158,"id":"4ecdb6321390abddd6ac658d","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3143732736,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"commonsense","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Commonsense completion","protocol_version":"codesota-pilot-v1","source":"https://rowanzellers.com/hellaswag/","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","score":0.5,"seconds":2.7435219429899007,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"commonsense"}