{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.12616999598685652},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.01283452904317528},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.01111921004485339},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.012109637958928943},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.011846847017295659},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.011934920912608504},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.012371324934065342},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.012757446034811437},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.014081505010835826},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.014609239995479584}],"dataset_revision":"94630fe30dad47192a8546eb75f094926d47e155","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792251726,"gpu":1,"id":"34e846dc3d7bd71b87d6f19d","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1041294848,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Reasoning","benchmark":"CommonsenseQA (validation)","contract":"Question + candidate answers \u2192 selected answer","dataset":"tau/commonsense_qa","dataset_config":"default","dataset_revision":"94630fe30dad47192a8546eb75f094926d47e155","dataset_url":"https://huggingface.co/datasets/tau/commonsense_qa","direction":"max","id":"commonsenseqa-validation-choice","license":["mit"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"CommonsenseQA (validation)","parent_task":"commonsense","protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"data/validation-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_fast.py","source":"https://huggingface.co/datasets/tau/commonsense_qa","split":"validation"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.169544448028319,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"f0998ace0736b4da3a3e3949ec0597035cf98f4c1796febce8344e4f727cc90a","score":0.4375,"seconds":1.2005699990550056,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"commonsenseqa-validation-choice","worker":"white-gpu1"}