{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"examples":[1,18,3,24,30,23,6,21],"generated_tokens":2048,"kind":"generation","padded_input_tokens":504,"seconds":3.3634832510724664},{"examples":[28,14,16,22,5,2,13,17],"generated_tokens":2048,"kind":"generation","padded_input_tokens":600,"seconds":3.2442861669696867},{"examples":[27,9,10,20,31,0,11,19],"generated_tokens":2048,"kind":"generation","padded_input_tokens":664,"seconds":3.398668773006648},{"examples":[12,25,29,26,7,15,8,4],"generated_tokens":2048,"kind":"generation","padded_input_tokens":1032,"seconds":3.292171653942205}],"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198057,"gpu":1,"id":"aa5bc8b59a1fd979e61724eb","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3223840256,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://github.com/openai/grade-school-math","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":20.445043627056293,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","score":0.09375,"seconds":14.718171672080643,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math","worker":"white-gpu1"}