{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.09920341498218477},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.033430437091737986},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04072352894581854},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.046183633035980165},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0429107389645651},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05098394490778446},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.055247246054932475},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.07813984109088778}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199450,"gpu":0,"id":"9733ef9ec08a5e033ded7a5e","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":1660646912,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.170025971019641,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.46875,"seconds":1.26590998203028,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"}