{"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193772,"gpu":0,"id":"ca6a72065b0a3873beda997b","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"32-example CodeSOTA declarative multiple-choice pilot. Not an official full-benchmark score.","peak_gpu_bytes":1049815552,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"arc-challenge-choice","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_choice.py","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test"},"runner_sha256":"4f6810ca652d8b80ae0c5e19f2a7e313e68357bb645a88a4d866e4c1141a0ed9","scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","score":0.3125,"seconds":2.2367252570111305,"seed":42,"selection":"first 32 examples in pinned parquet order","status":"completed","task":"arc-challenge-choice","worker":"white-gpu0"}