{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11999208689667284},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.04712149198167026},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.053831106051802635},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.060813799034804106},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.062681361916475},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.07165342499502003},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.07059212494641542},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.09778208506759256}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198142,"gpu":0,"id":"216e346c83988ade77ddfff0","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3253701120,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"arc-challenge-choice","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_fast.py","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.971455986960791,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","score":0.46875,"seconds":1.8925830170046538,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"arc-challenge-choice","worker":"white-gpu0"}