{"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193396,"id":"c5b99d0e49512667f8c0c91f","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1012201984,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/grade-school-math","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","score":0.03125,"seconds":32.15862700995058,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math"}