{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.3079821140272543},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13468963908962905},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.12939720193389803},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13092658296227455},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13222664699424058},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13175314397085458},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13212954497430474},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13372248504310846}],"dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792205404,"gpu":1,"id":"f60e6c2b8fa9ae016b0a7999","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":701987840,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"OpenBookQA","catalogue_id":"openbookqa","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","direction":"max","id":"sub1b-openbookqa","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"OpenBookQA / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.971450684941374,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.28125,"seconds":2.1226394310360774,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-openbookqa","worker":"white-gpu1"}