{"api_version":"codesota.v1","data":[{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/e66c128ba1817c735cfe0094/samples.jsonl","manifest":"/artifacts/e66c128ba1817c735cfe0094/manifest.json","result":"/artifacts/e66c128ba1817c735cfe0094/result.json","source":"/artifacts/e66c128ba1817c735cfe0094/evaluate.py"},"finished":1792199641,"id":"e66c128ba1817c735cfe0094","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.3073595860041678},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12762841605581343},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12597069598268718},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1279401279753074},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12618803093209863},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12855588598176837},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13134093105327338},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12986225401982665}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199641,"gpu":0,"id":"e66c128ba1817c735cfe0094","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":680474112,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.27229458198417,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.4375,"seconds":2.449016497004777,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.4375},"model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","model_version_id":"de9f8c2c54051dd496f8","origin":"codesota","page":"/runs/e66c128ba1817c735cfe0094","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/da843c3af7d2f039d8dcc997/samples.jsonl","manifest":"/artifacts/da843c3af7d2f039d8dcc997/manifest.json","result":"/artifacts/da843c3af7d2f039d8dcc997/result.json","source":"/artifacts/da843c3af7d2f039d8dcc997/evaluate.py"},"finished":1792199620,"id":"da843c3af7d2f039d8dcc997","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.07994207192678005},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02546265791170299},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03141217399388552},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03354428999591619},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03849511395674199},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04128733999095857},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.045172406011261046},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.059367791982367635}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199620,"gpu":1,"id":"da843c3af7d2f039d8dcc997","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B-Instruct","model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":2270253568,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.521601940039545,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.59375,"seconds":1.4459956830833107,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.59375},"model":"Qwen/Qwen2.5-0.5B-Instruct","model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","model_version_id":"8039b74746db8f085616","origin":"codesota","page":"/runs/da843c3af7d2f039d8dcc997","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/c2e37f8ec8e7bbe1a93eefe7/samples.jsonl","manifest":"/artifacts/c2e37f8ec8e7bbe1a93eefe7/manifest.json","result":"/artifacts/c2e37f8ec8e7bbe1a93eefe7/result.json","source":"/artifacts/c2e37f8ec8e7bbe1a93eefe7/evaluate.py"},"finished":1792199620,"id":"c2e37f8ec8e7bbe1a93eefe7","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.31639307306613773},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1166775340680033},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11946902994532138},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13180541794281453},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12876322399824858},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11960899108089507},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11863554595038295},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12067093094810843}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199620,"gpu":0,"id":"c2e37f8ec8e7bbe1a93eefe7","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.02","model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":688101888,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.071834755013697,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"seconds":2.0400674969423562,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.625},"model":"SlayerLab/Slayer139-1.02","model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","model_version_id":"f0e7e46ec1771438dc59","origin":"codesota","page":"/runs/c2e37f8ec8e7bbe1a93eefe7","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/adeb8d2f845d86b97ddb885e/samples.jsonl","manifest":"/artifacts/adeb8d2f845d86b97ddb885e/manifest.json","result":"/artifacts/adeb8d2f845d86b97ddb885e/result.json","source":"/artifacts/adeb8d2f845d86b97ddb885e/evaluate.py"},"finished":1792199600,"id":"adeb8d2f845d86b97ddb885e","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.301861822954379},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13498842308763415},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13530818989966065},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13598228699993342},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13709837896749377},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13612622499931604},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13629421801306307},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1378802970284596}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199600,"gpu":1,"id":"adeb8d2f845d86b97ddb885e","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.023155056987889,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"seconds":2.154468334047124,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.625},"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","model_version_id":"838b8282b8ce3b57f356","origin":"codesota","page":"/runs/adeb8d2f845d86b97ddb885e","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/a5bd023e62dfa94900c68aff/samples.jsonl","manifest":"/artifacts/a5bd023e62dfa94900c68aff/manifest.json","result":"/artifacts/a5bd023e62dfa94900c68aff/result.json","source":"/artifacts/a5bd023e62dfa94900c68aff/evaluate.py"},"finished":1792199600,"id":"a5bd023e62dfa94900c68aff","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.3167141720186919},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.16382455301936716},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.15081263601314276},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14019769907463342},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1395214069634676},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13469981495290995},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1361441989429295},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13518077204935253}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199600,"gpu":0,"id":"a5bd023e62dfa94900c68aff","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.124572991044261,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.5625,"seconds":2.2018078429391608,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.5625},"model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","model_version_id":"3b2aab414cfbc7e30b1d","origin":"codesota","page":"/runs/a5bd023e62dfa94900c68aff","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"boolq","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/ff71c6e3b5a06ff21384b00e/samples.jsonl","manifest":"/artifacts/ff71c6e3b5a06ff21384b00e/manifest.json","result":"/artifacts/ff71c6e3b5a06ff21384b00e/result.json","source":"/artifacts/ff71c6e3b5a06ff21384b00e/evaluate.py"},"finished":1792199580,"id":"ff71c6e3b5a06ff21384b00e","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.26462260202970356},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12735508708283305},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.12937858095392585},{"candidates":16,"kind":"choice","padded_tokens":4672,"seconds":0.16476030799094588}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199580,"gpu":1,"id":"ff71c6e3b5a06ff21384b00e","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":938301952,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":6.976226528990082,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.75,"seconds":1.544806924997829,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.75},"model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","model_version_id":"c73bf44055a8a374a1ae","origin":"codesota","page":"/runs/ff71c6e3b5a06ff21384b00e","protocol_id":"codesota-sub1b-choice-v1","scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5"},{"benchmark_id":"boolq","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/eb5517fac2e0ec0cb50196c5/samples.jsonl","manifest":"/artifacts/eb5517fac2e0ec0cb50196c5/manifest.json","result":"/artifacts/eb5517fac2e0ec0cb50196c5/result.json","source":"/artifacts/eb5517fac2e0ec0cb50196c5/evaluate.py"},"finished":1792199579,"id":"eb5517fac2e0ec0cb50196c5","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.18185965798329562},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.14411108696367592},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.20974927197676152},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.3129351839888841}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199579,"gpu":0,"id":"eb5517fac2e0ec0cb50196c5","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":1681739264,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.425343442009762,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.34375,"seconds":1.6528386661084369,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.34375},"model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","model_version_id":"6bcaa5691b2486184a02","origin":"codesota","page":"/runs/eb5517fac2e0ec0cb50196c5","protocol_id":"codesota-sub1b-choice-v1","scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5"},{"benchmark_id":"boolq","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/dce70413303e4781f185a0ac/samples.jsonl","manifest":"/artifacts/dce70413303e4781f185a0ac/manifest.json","result":"/artifacts/dce70413303e4781f185a0ac/result.json","source":"/artifacts/dce70413303e4781f185a0ac/evaluate.py"},"finished":1792199558,"id":"dce70413303e4781f185a0ac","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.270229447982274},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1432230609934777},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15365709108300507},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.19088799704331905}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199558,"gpu":1,"id":"dce70413303e4781f185a0ac","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":984605696,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.1771505429642275,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5,"seconds":1.632466200971976,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.5},"model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","model_version_id":"56b3a6d6f5ac040e1719","origin":"codesota","page":"/runs/dce70413303e4781f185a0ac","protocol_id":"codesota-sub1b-choice-v1","scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5"},{"benchmark_id":"boolq","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/d12d9fe7b986a834ea0d1098/samples.jsonl","manifest":"/artifacts/d12d9fe7b986a834ea0d1098/manifest.json","result":"/artifacts/d12d9fe7b986a834ea0d1098/result.json","source":"/artifacts/d12d9fe7b986a834ea0d1098/evaluate.py"},"finished":1792199557,"id":"d12d9fe7b986a834ea0d1098","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.2734037881018594},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15291272301692516},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.17487754696048796},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.2079960119444877}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199557,"gpu":0,"id":"d12d9fe7b986a834ea0d1098","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":984605696,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.476608742028475,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.6875,"seconds":1.689771424047649,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.6875},"model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","model_version_id":"3b2aab414cfbc7e30b1d","origin":"codesota","page":"/runs/d12d9fe7b986a834ea0d1098","protocol_id":"codesota-sub1b-choice-v1","scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5"},{"benchmark_id":"boolq","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/a64f0553519f2724df9f01c1/samples.jsonl","manifest":"/artifacts/a64f0553519f2724df9f01c1/manifest.json","result":"/artifacts/a64f0553519f2724df9f01c1/result.json","source":"/artifacts/a64f0553519f2724df9f01c1/evaluate.py"},"finished":1792199535,"id":"a64f0553519f2724df9f01c1","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.2582136089913547},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12060998799279332},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.12783878995105624},{"candidates":16,"kind":"choice","padded_tokens":4672,"seconds":0.15742255293298513}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199535,"gpu":1,"id":"a64f0553519f2724df9f01c1","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01","model_revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":938301952,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.126529000001028,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5625,"seconds":1.5199996100272983,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.5625},"model":"SlayerLab/Slayer139-1.01","model_revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","model_version_id":"5850670525f3ffec6afe","origin":"codesota","page":"/runs/a64f0553519f2724df9f01c1","protocol_id":"codesota-sub1b-choice-v1","scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/95f7f19d55290d88873e6474/samples.jsonl","manifest":"/artifacts/95f7f19d55290d88873e6474/manifest.json","result":"/artifacts/95f7f19d55290d88873e6474/result.json","source":"/artifacts/95f7f19d55290d88873e6474/evaluate.py"},"finished":1792199534,"id":"95f7f19d55290d88873e6474","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.29677336290478706},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13383584399707615},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13450233708135784},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13556442793924361},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13510968000628054},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13517078396398574},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13405970204621553},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1333185089752078}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199534,"gpu":0,"id":"95f7f19d55290d88873e6474","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.12225983699318,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.78125,"seconds":2.1125054139411077,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.78125},"model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","model_version_id":"56b3a6d6f5ac040e1719","origin":"codesota","page":"/runs/95f7f19d55290d88873e6474","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/933abc78ddc9563ed3bfe5c3/samples.jsonl","manifest":"/artifacts/933abc78ddc9563ed3bfe5c3/manifest.json","result":"/artifacts/933abc78ddc9563ed3bfe5c3/result.json","source":"/artifacts/933abc78ddc9563ed3bfe5c3/evaluate.py"},"finished":1792199513,"id":"933abc78ddc9563ed3bfe5c3","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.09018106700386852},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.02608981798402965},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03442929999437183},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03219556505791843},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.035866821999661624},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.046312381979078054},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.04108243901282549},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.06735055800527334}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199513,"gpu":0,"id":"933abc78ddc9563ed3bfe5c3","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":1564464128,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.170912915025838,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.4375,"seconds":1.2155649489723146,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.4375},"model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","model_version_id":"6bcaa5691b2486184a02","origin":"codesota","page":"/runs/933abc78ddc9563ed3bfe5c3","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/83ca3844b75d12aea722afae/samples.jsonl","manifest":"/artifacts/83ca3844b75d12aea722afae/manifest.json","result":"/artifacts/83ca3844b75d12aea722afae/result.json","source":"/artifacts/83ca3844b75d12aea722afae/evaluate.py"},"finished":1792199513,"id":"83ca3844b75d12aea722afae","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.12425816198810935},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014353499049320817},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.01433193904813379},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.02308932098094374},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.016291594016365707},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.01679475198034197},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.01820836798287928},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.024417127948254347}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199513,"gpu":1,"id":"83ca3844b75d12aea722afae","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":657188352,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.070911542046815,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.5625,"seconds":1.0435598699841648,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.5625},"model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","model_version_id":"2ca2f9e9fd30d75e5b9a","origin":"codesota","page":"/runs/83ca3844b75d12aea722afae","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"boolq","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/509bc4c9edce51c91c7e5e39/samples.jsonl","manifest":"/artifacts/509bc4c9edce51c91c7e5e39/manifest.json","result":"/artifacts/509bc4c9edce51c91c7e5e39/result.json","source":"/artifacts/509bc4c9edce51c91c7e5e39/evaluate.py"},"finished":1792199494,"id":"509bc4c9edce51c91c7e5e39","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.28066387795843184},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1456537441117689},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.1672645480139181},{"candidates":16,"kind":"choice","padded_tokens":5136,"seconds":0.20438037905842066}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199494,"gpu":0,"id":"509bc4c9edce51c91c7e5e39","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":818889728,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.727604440995492,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.6875,"seconds":2.0678646629676223,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.6875},"model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","model_version_id":"de9f8c2c54051dd496f8","origin":"codesota","page":"/runs/509bc4c9edce51c91c7e5e39","protocol_id":"codesota-sub1b-choice-v1","scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5"},{"benchmark_id":"arc-easy","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/81740f9404ca2161b03fab45/samples.jsonl","manifest":"/artifacts/81740f9404ca2161b03fab45/manifest.json","result":"/artifacts/81740f9404ca2161b03fab45/result.json","source":"/artifacts/81740f9404ca2161b03fab45/evaluate.py"},"finished":1792199493,"id":"81740f9404ca2161b03fab45","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.30376290902495384},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1181516939541325},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12269497907254845},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1451712850248441},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11987522710114717},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11452228506095707},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11452719301450998},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11412549391388893}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199493,"gpu":1,"id":"81740f9404ca2161b03fab45","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":688101888,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.873346148058772,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.375,"seconds":2.025514440028928,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.375},"model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","model_version_id":"c73bf44055a8a374a1ae","origin":"codesota","page":"/runs/81740f9404ca2161b03fab45","protocol_id":"codesota-sub1b-choice-v1","scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6"},{"benchmark_id":"hellaswag","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/ec41e48212ad955d3ea4c8f6/samples.jsonl","manifest":"/artifacts/ec41e48212ad955d3ea4c8f6/manifest.json","result":"/artifacts/ec41e48212ad955d3ea4c8f6/result.json","source":"/artifacts/ec41e48212ad955d3ea4c8f6/evaluate.py"},"finished":1792199473,"id":"ec41e48212ad955d3ea4c8f6","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.2987431969959289},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12729038705583662},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12539833795744926},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12649443303234875},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12608513899613172},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12715133698657155},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1267648560460657},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.12892765109427273}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199473,"gpu":1,"id":"ec41e48212ad955d3ea4c8f6","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":721014272,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.323357745073736,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"seconds":2.415863345959224,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.34375},"model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","model_version_id":"de9f8c2c54051dd496f8","origin":"codesota","page":"/runs/ec41e48212ad955d3ea4c8f6","protocol_id":"codesota-sub1b-choice-v1","scope_id":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456"},{"benchmark_id":"hellaswag","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/d6a751bd1fb70f34af6e88c0/samples.jsonl","manifest":"/artifacts/d6a751bd1fb70f34af6e88c0/manifest.json","result":"/artifacts/d6a751bd1fb70f34af6e88c0/result.json","source":"/artifacts/d6a751bd1fb70f34af6e88c0/evaluate.py"},"finished":1792199471,"id":"d6a751bd1fb70f34af6e88c0","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.3043450729455799},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13877835101448},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14041231910232455},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13864246802404523},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13872320798691362},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14067755406722426},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14329399401322007},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13903140393085778}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199471,"gpu":0,"id":"d6a751bd1fb70f34af6e88c0","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":794730496,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.17221459292341,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"seconds":2.1684185520280153,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.34375},"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","model_version_id":"838b8282b8ce3b57f356","origin":"codesota","page":"/runs/d6a751bd1fb70f34af6e88c0","protocol_id":"codesota-sub1b-choice-v1","scope_id":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456"},{"benchmark_id":"hellaswag","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/a0a1b916aa43bef9dc349f52/samples.jsonl","manifest":"/artifacts/a0a1b916aa43bef9dc349f52/manifest.json","result":"/artifacts/a0a1b916aa43bef9dc349f52/result.json","source":"/artifacts/a0a1b916aa43bef9dc349f52/evaluate.py"},"finished":1792199452,"id":"a0a1b916aa43bef9dc349f52","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13017428503371775},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.01650666492059827},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.016730963019654155},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.01793303492013365},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.018158614984713495},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.020130372955463827},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.020577267045155168},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.029766182997263968}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199452,"gpu":1,"id":"a0a1b916aa43bef9dc349f52","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":752199680,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":3.971061061020009,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.28125,"seconds":0.9418995419982821,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.28125},"model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","model_version_id":"2ca2f9e9fd30d75e5b9a","origin":"codesota","page":"/runs/a0a1b916aa43bef9dc349f52","protocol_id":"codesota-sub1b-choice-v1","scope_id":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456"},{"benchmark_id":"hellaswag","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/9733ef9ec08a5e033ded7a5e/samples.jsonl","manifest":"/artifacts/9733ef9ec08a5e033ded7a5e/manifest.json","result":"/artifacts/9733ef9ec08a5e033ded7a5e/result.json","source":"/artifacts/9733ef9ec08a5e033ded7a5e/evaluate.py"},"finished":1792199450,"id":"9733ef9ec08a5e033ded7a5e","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.09920341498218477},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.033430437091737986},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04072352894581854},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.046183633035980165},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0429107389645651},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05098394490778446},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.055247246054932475},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.07813984109088778}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199450,"gpu":0,"id":"9733ef9ec08a5e033ded7a5e","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":1660646912,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.170025971019641,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.46875,"seconds":1.26590998203028,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},"metric":{"direction":"max","name":"accuracy","value":0.46875},"model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","model_version_id":"6bcaa5691b2486184a02","origin":"codesota","page":"/runs/9733ef9ec08a5e033ded7a5e","protocol_id":"codesota-sub1b-choice-v1","scope_id":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456"},{"benchmark_id":"boolq","coverage":{"count":32,"kind":"subset","unit":"examples"},"evidence":{"items":"/artifacts/3fb07acb70ad720b7ea6387f/samples.jsonl","manifest":"/artifacts/3fb07acb70ad720b7ea6387f/manifest.json","result":"/artifacts/3fb07acb70ad720b7ea6387f/result.json","source":"/artifacts/3fb07acb70ad720b7ea6387f/evaluate.py"},"finished":1792199433,"id":"3fb07acb70ad720b7ea6387f","is_current_scope":true,"measurement":{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.2658541889395565},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.14040926401503384},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15321300295181572},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.1910893419990316}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199433,"gpu":1,"id":"3fb07acb70ad720b7ea6387f","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":984605696,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.225614584982395,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.78125,"seconds":1.64822871692013,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},"metric":{"direction":"max","name":"accuracy","value":0.78125},"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","model_version_id":"a83031f2f2578d188916","origin":"codesota","page":"/runs/3fb07acb70ad720b7ea6387f","protocol_id":"codesota-sub1b-choice-v1","scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5"}],"generated_at":"2026-10-17T02:25:23.943033+00:00","pagination":{"limit":20,"next":"/api/v1/results?limit=20&offset=20","offset":0,"previous":null,"total":64}}