{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.30749484000261873},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13433970394544303},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13469515705946833},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13649503397755325},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13486212899442762},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13734989799559116},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13695090496912599},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13868639094289392}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199350,"gpu":0,"id":"5ebdb1c54bc9de4a483665de","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.072358333971351,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"seconds":2.1689843010390177,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"}