{"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evidence":"CodeSOTA execution on White","finished":1792192784,"id":"0ab8b157bc6f3029a19271ab","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3109161472,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/grade-school-math","split":"test"},"runner_sha256":"fd358150cb0cf5b9dc73de627f01f0f194d800a3e6355193039d335ce043b9a3","scope":"931808a3f82fdda3ddddbf7a73eec8abfdf78e7c460ffa9c507a3ac8693138ff","score":0.09375,"seconds":124.60450974502601,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math"}