{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.2908450149698183},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11340139200910926},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11413722299039364},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11324444797355682},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11335861205589026},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11393208999652416},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11398995807394385},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1180888400413096}],"dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792205202,"gpu":1,"id":"1a93c5498bda553a7673f19c","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":665454592,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"OpenBookQA","catalogue_id":"openbookqa","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","direction":"max","id":"sub1b-openbookqa","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"OpenBookQA / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.822887591901235,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.25,"seconds":1.9651672700420022,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-openbookqa","worker":"white-gpu1"}