{"model_metadata":[{"basis":"Unique model parameters, instantiated from pinned Transformers config on meta device","model":"Qwen/Qwen2.5-0.5B","parameters":494032768,"revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","source":"https://huggingface.co/Qwen/Qwen2.5-0.5B/tree/060db6499f32faf8b98477b0a26969ef7d8b9987","updated":1792194501},{"basis":"Unique model parameters, instantiated from pinned Transformers config on meta device","model":"Qwen/Qwen2.5-1.5B","parameters":1543714304,"revision":"8faed761d45a263340a0528343f099c05c9a4323","source":"https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/8faed761d45a263340a0528343f099c05c9a4323","updated":1792194501},{"basis":"Unique model parameters, instantiated from pinned Transformers config on meta device","model":"sentence-transformers/all-MiniLM-L6-v2","parameters":22713216,"revision":"1110a243fdf4706b3f48f1d95db1a4f5529b4d41","source":"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/tree/1110a243fdf4706b3f48f1d95db1a4f5529b4d41","updated":1792194501},{"basis":"Unique model parameters, instantiated from pinned Transformers config on meta device","model":"microsoft/trocr-small-handwritten","parameters":61596672,"revision":"b4648cfa171985a6745f37ddd637e98c0da958ac","source":"https://huggingface.co/microsoft/trocr-small-handwritten/tree/b4648cfa171985a6745f37ddd637e98c0da958ac","updated":1792194501},{"basis":"Unique model parameters, instantiated from pinned Transformers config on meta device","model":"openai/whisper-tiny.en","parameters":37760256,"revision":"87c7102498dcde7456f24cfd30239ca606ed9063","source":"https://huggingface.co/openai/whisper-tiny.en/tree/87c7102498dcde7456f24cfd30239ca606ed9063","updated":1792194501},{"basis":"Unique model parameters, instantiated from pinned Transformers config on meta device","model":"openai/clip-vit-base-patch32","parameters":151277313,"revision":"3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268","source":"https://huggingface.co/openai/clip-vit-base-patch32/tree/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268","updated":1792194501},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"HuggingFaceTB/SmolLM2-135M-Instruct","parameters":134515008,"revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","source":"https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct/tree/12fd25f77366fa6b3b4b768ec3050bf629380bac","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer139-1.01b","parameters":139279760,"revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","source":"https://huggingface.co/SlayerLab/Slayer139-1.01b/tree/283ffb6d620be2d6ea1f2429d721962dc8e73633","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer149","parameters":148910738,"revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","source":"https://huggingface.co/SlayerLab/Slayer149/tree/7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer139-1.02","parameters":139279760,"revision":"c4298d757ba39cba19dd768777da9fadf319ff61","source":"https://huggingface.co/SlayerLab/Slayer139-1.02/tree/c4298d757ba39cba19dd768777da9fadf319ff61","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","parameters":149333081,"revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","source":"https://huggingface.co/SlayerLab/Slayer149-Balanced-ARC-E-ft-45500/tree/e7d4b8dea44125264077c2ce6d2f141379f999cd","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer-149M-Decision-v1","parameters":149333081,"revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","source":"https://huggingface.co/SlayerLab/Slayer-149M-Decision-v1/tree/ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer149-ARC","parameters":149333081,"revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","source":"https://huggingface.co/SlayerLab/Slayer149-ARC/tree/0e6793ecfb8501dccc1c97dcd944731748eb8d17","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer139-1.01","parameters":139279760,"revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","source":"https://huggingface.co/SlayerLab/Slayer139-1.01/tree/2e72c696d7c5cad73dc58606b0751b911dc94495","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"HuggingFaceTB/SmolLM2-360M-Instruct","parameters":361821120,"revision":"a10cc1512eabd3dde888204e902eca88bddb4951","source":"https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/tree/a10cc1512eabd3dde888204e902eca88bddb4951","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"Qwen/Qwen2.5-0.5B-Instruct","parameters":494032768,"revision":"7ae557604adf67be50417f59c2c2f167def9a775","source":"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/tree/7ae557604adf67be50417f59c2c2f167def9a775","updated":1792199477},{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","parameters":149333081,"revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","source":"https://huggingface.co/SlayerLab/Slayer149-Balanced-ARC-E-ft-v2/tree/7d11e8fd63764baf00c276f586b2aa776648ed33","updated":1792199477}],"original_interest":{"basis":"Recorded page_views counters in the returned top-100 pages per period; not unique people or complete site analytics","source":"https://www.codesota.com/api/views","top_pages":[{"path":"/","period":"month","updated":1792202059,"views":1230},{"path":"/ocr/best-for-python","period":"month","updated":1792202059,"views":616},{"path":"/ocr","period":"month","updated":1792202059,"views":479},{"path":"/guides/tts-models","period":"month","updated":1792202059,"views":471},{"path":"/benchmarks","period":"month","updated":1792202059,"views":470},{"path":"/tasks","period":"month","updated":1792202059,"views":447},{"path":"/benchmarks/mteb","period":"month","updated":1792202059,"views":446},{"path":"/ocr/best-for-handwriting","period":"month","updated":1792202059,"views":396},{"path":"/llm","period":"month","updated":1792202059,"views":370},{"path":"/models","period":"month","updated":1792202059,"views":345},{"path":"/submit","period":"month","updated":1792202059,"views":341},{"path":"/code-generation","period":"month","updated":1792202059,"views":309},{"path":"/browse/computer-vision/object-detection/coco","period":"month","updated":1792202059,"views":289},{"path":"/ocr/benchmark/omnidocbench","period":"month","updated":1792202059,"views":276},{"path":"/speech/stt-leaderboard","period":"month","updated":1792202059,"views":273},{"path":"/speech","period":"month","updated":1792202059,"views":252},{"path":"/benchmark/imagenet-1k","period":"month","updated":1792202059,"views":245},{"path":"/tasks/swe-bench","period":"month","updated":1792202059,"views":236},{"path":"/text-to-speech","period":"month","updated":1792202059,"views":221},{"path":"/embedded-ai/hailo-10h-llms","period":"month","updated":1792202059,"views":215}],"topics":[{"id":"documents","name":"OCR / document understanding","observed_views":{"all":34680,"month":2666,"week":621},"priority_bonus":35,"related_task":"handwriting"},{"id":"code","name":"Code / software engineering","observed_views":{"all":15112,"month":935,"week":194},"priority_bonus":35,"related_task":"code-generation"},{"id":"tts","name":"Speech synthesis / TTS","observed_views":{"all":11742,"month":925,"week":150},"priority_bonus":35,"related_task":"transcription"},{"id":"vision","name":"Computer vision","observed_views":{"all":6210,"month":867,"week":207},"priority_bonus":35,"related_task":"image-classification"},{"id":"asr","name":"Speech recognition","observed_views":{"all":5306,"month":600,"week":145},"priority_bonus":34.17,"related_task":"transcription"},{"id":"embeddings","name":"Embeddings / retrieval","observed_views":{"all":3326,"month":446,"week":92},"priority_bonus":32.52,"related_task":"similarity"},{"id":"math","name":"Mathematical reasoning","observed_views":{"all":1464,"month":166,"week":41},"priority_bonus":27.76,"related_task":"math"}],"updated":1792202059},"popular":[{"kind":"Registry","path":"/","title":"All tasks","visits_30d":11,"visits_7d":11},{"kind":"Task","path":"/tasks/code-generation","title":"Code generation","visits_30d":6,"visits_7d":6},{"kind":"Planning","path":"/queue","title":"Evaluation queue","visits_30d":4,"visits_7d":4},{"kind":"Documentation","path":"/ontology","title":"Ontology and evaluation loop","visits_30d":4,"visits_7d":4},{"kind":"Agents","path":"/factory","title":"AI evaluation factory","visits_30d":3,"visits_7d":3},{"kind":"Diagnostics","path":"/benchmark-value","title":"Benchmark cost and discrimination","visits_30d":3,"visits_7d":3},{"kind":"Popularity","path":"/popular","title":"Popular pages","visits_30d":3,"visits_7d":3},{"kind":"Editorial","path":"/sub-1b","title":"Under 1B: useful small models","visits_30d":2,"visits_7d":2},{"kind":"Task","path":"/tasks/arc-challenge-choice","title":"AI2 ARC-Challenge (test)","visits_30d":1,"visits_7d":1},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/alpacaeval-2-lc","title":"AlpacaEval 2.0 \u2014 length-controlled","visits_30d":1,"visits_7d":1},{"kind":"Task","path":"/tasks/sub1b-boolq","title":"BoolQ / sub-1B cohort","visits_30d":1,"visits_7d":1},{"kind":"Run evidence","path":"/runs/501f999a9189ca77b745a388","title":"Code generation / Qwen/Qwen2.5-1.5B","visits_30d":1,"visits_7d":1},{"kind":"Task","path":"/tasks/commonsense","title":"Commonsense completion","visits_30d":1,"visits_7d":1},{"kind":"Run evidence","path":"/runs/4ecdb6321390abddd6ac658d","title":"Commonsense completion / Qwen/Qwen2.5-1.5B","visits_30d":1,"visits_7d":1},{"kind":"Task","path":"/tasks/sub1b-hellaswag","title":"HellaSwag / sub-1B cohort","visits_30d":1,"visits_7d":1},{"kind":"Task","path":"/tasks/math","title":"Math word problems","visits_30d":1,"visits_7d":1},{"kind":"Run evidence","path":"/runs/14e52d72173736c52853bbf1","title":"Math word problems / Qwen/Qwen2.5-1.5B","visits_30d":1,"visits_7d":1},{"kind":"Documentation","path":"/methodology","title":"Methodology","visits_30d":1,"visits_7d":1},{"kind":"Task","path":"/tasks/transcription","title":"Speech transcription","visits_30d":1,"visits_7d":1},{"kind":"Run evidence","path":"/runs/ca6a72065b0a3873beda997b","title":"AI2 ARC-Challenge (test) / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/0d8c049a046e18e772d16952","title":"AI2 ARC-Challenge (test) / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/293bd712648d308f4ffe2539","title":"AI2 ARC-Challenge (test) / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/216e346c83988ade77ddfff0","title":"AI2 ARC-Challenge (test) / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/aime-2025","title":"AIME 2025","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/arc-challenge","title":"ARC-Challenge","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/arc-easy","title":"ARC-Easy","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/sub1b-arc-easy","title":"ARC-Easy / sub-1B cohort","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/83ca3844b75d12aea722afae","title":"ARC-Easy / sub-1B cohort / HuggingFaceTB/SmolLM2-135M-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/933abc78ddc9563ed3bfe5c3","title":"ARC-Easy / sub-1B cohort / HuggingFaceTB/SmolLM2-360M-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/da843c3af7d2f039d8dcc997","title":"ARC-Easy / sub-1B cohort / Qwen/Qwen2.5-0.5B-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/a5bd023e62dfa94900c68aff","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer-149M-Decision-v1","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/4f8d3c01030b9ec948e6d8be","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer139-1.01","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/81740f9404ca2161b03fab45","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer139-1.01b","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/c2e37f8ec8e7bbe1a93eefe7","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer139-1.02","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/e66c128ba1817c735cfe0094","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer149","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/95f7f19d55290d88873e6474","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer149-ARC","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/5ebdb1c54bc9de4a483665de","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/adeb8d2f845d86b97ddb885e","title":"ARC-Easy / sub-1B cohort / SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/bfcl-v3","title":"BFCL v3","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/bfcl-v4","title":"BFCL v4","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/bbh","title":"BIG-Bench Hard \u2014 BBH","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/blimp","title":"BLiMP","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/blimp-supplement","title":"BLiMP Supplement","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/belebele","title":"Belebele","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/boolq","title":"BoolQ","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/3b9bc33f16b6187cceb7819f","title":"BoolQ / sub-1B cohort / HuggingFaceTB/SmolLM2-135M-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/eb5517fac2e0ec0cb50196c5","title":"BoolQ / sub-1B cohort / HuggingFaceTB/SmolLM2-360M-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/141ceb6c09dc8f0156f70c0c","title":"BoolQ / sub-1B cohort / Qwen/Qwen2.5-0.5B-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/d12d9fe7b986a834ea0d1098","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer-149M-Decision-v1","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/a64f0553519f2724df9f01c1","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer139-1.01","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/ff71c6e3b5a06ff21384b00e","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer139-1.01b","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/18802f73c41e24cdd29f5a2f","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer139-1.02","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/509bc4c9edce51c91c7e5e39","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer149","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/dce70413303e4781f185a0ac","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer149-ARC","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/3fb07acb70ad720b7ea6387f","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/3ce0ba3d66280b6cbd47845b","title":"BoolQ / sub-1B cohort / SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/casereportbench","title":"CaseReportBench","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/264775b8e13a45705e029825","title":"Code generation / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/b82a09a698a552ab21a0075b","title":"Commonsense completion / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/2f544548b89cd6fed2d4e61d","title":"Commonsense completion / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/db55a33c25d74960ff389d24","title":"Commonsense completion / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/drop","title":"DROP","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/ewok","title":"EWoK","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/gpqa","title":"GPQA","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/gpqa-diamond","title":"GPQA Diamond","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/gsm-symbolic","title":"GSM-Symbolic","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/gsm8k","title":"GSM8K","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/handwriting","title":"Handwriting recognition","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/1ed98ed973f5e29bf51634c4","title":"Handwriting recognition / microsoft/trocr-small-handwritten","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/hellaswag","title":"HellaSwag","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/a0a1b916aa43bef9dc349f52","title":"HellaSwag / sub-1B cohort / HuggingFaceTB/SmolLM2-135M-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/9733ef9ec08a5e033ded7a5e","title":"HellaSwag / sub-1B cohort / HuggingFaceTB/SmolLM2-360M-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/5182981eed2309237804d9c3","title":"HellaSwag / sub-1B cohort / Qwen/Qwen2.5-0.5B-Instruct","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/60b94d2f3d3bccb39d7277ca","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer-149M-Decision-v1","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/384fccc53f471e23eb9acac8","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer139-1.01","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/75ea5d8ae94de8a86a7a0777","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer139-1.01b","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/8677abc8861650a471b4ea7c","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer139-1.02","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/ec41e48212ad955d3ea4c8f6","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer149","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/4bab151d17bfaf2727299f60","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer149-ARC","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/93979b5fa00514cce3d96032","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/d6a751bd1fb70f34af6e88c0","title":"HellaSwag / sub-1B cohort / SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/humaneval","title":"HumanEval","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/humaneval-plus","title":"HumanEval+","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/ifbench","title":"IFBench","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/ifeval","title":"IFEval","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/image-classification","title":"Image classification","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/ffc11a3556867f2915bcdf77","title":"Image classification / openai/clip-vit-base-patch32","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/lambada","title":"LAMBADA","visits_30d":0,"visits_7d":0},{"kind":"Operations","path":"/gpu","title":"Live GPU processing","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/livecodebench","title":"LiveCodeBench","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/longbench-v2","title":"LongBench v2","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/math-500","title":"MATH-500","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/mbpp","title":"MBPP","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/mbpp-plus","title":"MBPP+","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/mgsm","title":"MGSM","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/mmlu","title":"MMLU","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/mmlu-pro","title":"MMLU-Pro","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/mmmlu","title":"MMMLU \u2014 OpenAI","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/mt-bench","title":"MT-Bench","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/c5b99d0e49512667f8c0c91f","title":"Math word problems / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/4749a153422d5cefccd16747","title":"Math word problems / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/0ab8b157bc6f3029a19271ab","title":"Math word problems / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/aa5bc8b59a1fd979e61724eb","title":"Math word problems / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/multi-if","title":"Multi-IF","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/entailment","title":"Natural language inference","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/272e772751322114921d9ee6","title":"Natural language inference / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/2933b280610cb03f5f2345c2","title":"Natural language inference / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/5a2a497f01a1e75965c07849","title":"Natural language inference / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/bddd3154bd9cf23f3dece3fb","title":"Natural language inference / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/openbookqa","title":"OpenBookQA","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/openbookqa-main-choice","title":"OpenBookQA main (test)","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/40d1459ca17c38921b30a6ea","title":"OpenBookQA main (test) / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/f08b963541b415694a99b37b","title":"OpenBookQA main (test) / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/795ccf7bff6558c30d67ce8c","title":"OpenBookQA main (test) / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/9fd52efbe4976b40a9fd6646","title":"OpenBookQA main (test) / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/heldout-en-pl-code","title":"Own held-out EN / PL / code","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/piqa","title":"PIQA","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/ruler","title":"RULER","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/reading","title":"Reading comprehension","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/7af8b47f7ce3815bea0c442e","title":"Reading comprehension / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/ff345349801249c15997d2a6","title":"Reading comprehension / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/206aa355b6974c770b0b29c4","title":"Reading comprehension / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/c6d083942f8d93dc672e019b","title":"Reading comprehension / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/squad-v2","title":"SQuAD 2.0","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/svamp","title":"SVAMP","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/sciq","title":"SciQ","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/similarity","title":"Sentence similarity","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/f8dd3623ae6e8786284a6ca8","title":"Sentence similarity / sentence-transformers/all-MiniLM-L6-v2","visits_30d":0,"visits_7d":0},{"kind":"Task","path":"/tasks/sentiment","title":"Sentiment classification","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/2baa9449b32fde6f476f7e1f","title":"Sentiment classification / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/efa7055f4d6480dd542a3abf","title":"Sentiment classification / Qwen/Qwen2.5-0.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/9c6acd69de4197860e28ebb2","title":"Sentiment classification / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/b700ec1eed7b5ab1a5d28d49","title":"Sentiment classification / Qwen/Qwen2.5-1.5B","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/simpleqa","title":"SimpleQA","visits_30d":0,"visits_7d":0},{"kind":"Run evidence","path":"/runs/9971178eeb5427d24da23fab","title":"Speech transcription / openai/whisper-tiny.en","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/truthfulqa","title":"TruthfulQA","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/uncheatable-eval","title":"Uncheatable Eval","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/wikitext-2","title":"WikiText-2","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/wildbench","title":"WildBench","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/winogrande","title":"WinoGrande","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/tau2-retail","title":"\u03c4\u00b2-Bench Retail","visits_30d":0,"visits_7d":0},{"kind":"Benchmark plan","path":"/sub-1b/benchmarks/tau2-telecom","title":"\u03c4\u00b2-Bench Telecom","visits_30d":0,"visits_7d":0}],"queue":[{"attempts":1,"created":1792199285,"error":null,"finished":1792199307,"gpu":0,"id":"141ceb6c09dc8f0156f70c0c","last_heartbeat":1792199299,"model":"Qwen/Qwen2.5-0.5B-Instruct","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199299,"status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199307,"gpu":1,"id":"18802f73c41e24cdd29f5a2f","last_heartbeat":1792199299,"model":"SlayerLab/Slayer139-1.02","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199299,"status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199327,"gpu":1,"id":"384fccc53f471e23eb9acac8","last_heartbeat":1792199322,"model":"SlayerLab/Slayer139-1.01","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199322,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199329,"gpu":0,"id":"3b9bc33f16b6187cceb7819f","last_heartbeat":1792199322,"model":"HuggingFaceTB/SmolLM2-135M-Instruct","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199322,"status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199390,"gpu":1,"id":"3ce0ba3d66280b6cbd47845b","last_heartbeat":1792199383,"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199383,"status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199433,"gpu":1,"id":"3fb07acb70ad720b7ea6387f","last_heartbeat":1792199426,"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199426,"status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199368,"gpu":1,"id":"4bab151d17bfaf2727299f60","last_heartbeat":1792199363,"model":"SlayerLab/Slayer149-ARC","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199363,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199494,"gpu":0,"id":"509bc4c9edce51c91c7e5e39","last_heartbeat":1792199486,"model":"SlayerLab/Slayer149","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199486,"status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199370,"gpu":0,"id":"5182981eed2309237804d9c3","last_heartbeat":1792199365,"model":"Qwen/Qwen2.5-0.5B-Instruct","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199365,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199390,"gpu":0,"id":"60b94d2f3d3bccb39d7277ca","last_heartbeat":1792199385,"model":"SlayerLab/Slayer-149M-Decision-v1","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199385,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199410,"gpu":0,"id":"75ea5d8ae94de8a86a7a0777","last_heartbeat":1792199405,"model":"SlayerLab/Slayer139-1.01b","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199405,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199411,"gpu":1,"id":"8677abc8861650a471b4ea7c","last_heartbeat":1792199406,"model":"SlayerLab/Slayer139-1.02","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199406,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199431,"gpu":0,"id":"93979b5fa00514cce3d96032","last_heartbeat":1792199425,"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199425,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199450,"gpu":0,"id":"9733ef9ec08a5e033ded7a5e","last_heartbeat":1792199446,"model":"HuggingFaceTB/SmolLM2-360M-Instruct","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199446,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199452,"gpu":1,"id":"a0a1b916aa43bef9dc349f52","last_heartbeat":1792199448,"model":"HuggingFaceTB/SmolLM2-135M-Instruct","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199448,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199535,"gpu":1,"id":"a64f0553519f2724df9f01c1","last_heartbeat":1792199528,"model":"SlayerLab/Slayer139-1.01","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199528,"status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199557,"gpu":0,"id":"d12d9fe7b986a834ea0d1098","last_heartbeat":1792199549,"model":"SlayerLab/Slayer-149M-Decision-v1","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199549,"status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199471,"gpu":0,"id":"d6a751bd1fb70f34af6e88c0","last_heartbeat":1792199465,"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199465,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199558,"gpu":1,"id":"dce70413303e4781f185a0ac","last_heartbeat":1792199550,"model":"SlayerLab/Slayer149-ARC","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199550,"status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199579,"gpu":0,"id":"eb5517fac2e0ec0cb50196c5","last_heartbeat":1792199572,"model":"HuggingFaceTB/SmolLM2-360M-Instruct","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199572,"status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199473,"gpu":1,"id":"ec41e48212ad955d3ea4c8f6","last_heartbeat":1792199468,"model":"SlayerLab/Slayer149","origin":null,"priority":1006.95,"scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","started":1792199468,"status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199580,"gpu":1,"id":"ff71c6e3b5a06ff21384b00e","last_heartbeat":1792199573,"model":"SlayerLab/Slayer139-1.01b","origin":null,"priority":1006.95,"scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","started":1792199573,"status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199347,"gpu":1,"id":"4f8d3c01030b9ec948e6d8be","last_heartbeat":1792199342,"model":"SlayerLab/Slayer139-1.01","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199342,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199350,"gpu":0,"id":"5ebdb1c54bc9de4a483665de","last_heartbeat":1792199344,"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199344,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199493,"gpu":1,"id":"81740f9404ca2161b03fab45","last_heartbeat":1792199488,"model":"SlayerLab/Slayer139-1.01b","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199488,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199513,"gpu":1,"id":"83ca3844b75d12aea722afae","last_heartbeat":1792199509,"model":"HuggingFaceTB/SmolLM2-135M-Instruct","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199509,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199513,"gpu":0,"id":"933abc78ddc9563ed3bfe5c3","last_heartbeat":1792199509,"model":"HuggingFaceTB/SmolLM2-360M-Instruct","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199509,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199534,"gpu":0,"id":"95f7f19d55290d88873e6474","last_heartbeat":1792199528,"model":"SlayerLab/Slayer149-ARC","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199528,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199600,"gpu":0,"id":"a5bd023e62dfa94900c68aff","last_heartbeat":1792199595,"model":"SlayerLab/Slayer-149M-Decision-v1","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199595,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199600,"gpu":1,"id":"adeb8d2f845d86b97ddb885e","last_heartbeat":1792199595,"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199595,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199620,"gpu":0,"id":"c2e37f8ec8e7bbe1a93eefe7","last_heartbeat":1792199615,"model":"SlayerLab/Slayer139-1.02","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199615,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199620,"gpu":1,"id":"da843c3af7d2f039d8dcc997","last_heartbeat":1792199616,"model":"Qwen/Qwen2.5-0.5B-Instruct","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199616,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"attempts":1,"created":1792199285,"error":null,"finished":1792199641,"gpu":0,"id":"e66c128ba1817c735cfe0094","last_heartbeat":1792199635,"model":"SlayerLab/Slayer149","origin":null,"priority":1000.03,"scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","started":1792199635,"status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"attempts":1,"created":1792192844,"error":null,"finished":1792193082,"gpu":null,"id":"264775b8e13a45705e029825","last_heartbeat":null,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":54.5,"scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","started":1792193082,"status":"completed","task":"code-generation","worker":null},{"attempts":1,"created":1792192844,"error":null,"finished":1792193254,"gpu":null,"id":"501f999a9189ca77b745a388","last_heartbeat":null,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":54.5,"scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","started":1792193254,"status":"completed","task":"code-generation","worker":null},{"attempts":1,"created":1792192844,"error":null,"finished":1792193290,"gpu":null,"id":"9971178eeb5427d24da23fab","last_heartbeat":null,"model":"openai/whisper-tiny.en","origin":null,"priority":41.99,"scope":"9c365ee78df50fa725b4c03d695f65d2b638be00c4687dc66be70b284c865a34","started":1792193290,"status":"completed","task":"transcription","worker":null},{"attempts":2,"created":1792192844,"error":null,"finished":1792192922,"gpu":null,"id":"1ed98ed973f5e29bf51634c4","last_heartbeat":null,"model":"microsoft/trocr-small-handwritten","origin":null,"priority":35.11,"scope":"87f7f645d4ae1c902e4f3c4d821290e2fa08df318242d5d78a563467763e36ec","started":1792192922,"status":"completed","task":"handwriting","worker":null},{"attempts":1,"created":1792192844,"error":null,"finished":1792193340,"gpu":null,"id":"ffc11a3556867f2915bcdf77","last_heartbeat":null,"model":"openai/clip-vit-base-patch32","origin":null,"priority":35.1,"scope":"d4a779c6ba33be115fc5e3302e45bb70d7c90040f954b89dab97beb5a2914649","started":1792193340,"status":"completed","task":"image-classification","worker":null},{"attempts":1,"created":1792198012,"error":null,"finished":1792198055,"gpu":0,"id":"4749a153422d5cefccd16747","last_heartbeat":1792198047,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":34.68,"scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","started":1792198036,"status":"completed","task":"math","worker":"white-gpu0"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198057,"gpu":1,"id":"aa5bc8b59a1fd979e61724eb","last_heartbeat":1792198057,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":34.68,"scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","started":1792198036,"status":"completed","task":"math","worker":"white-gpu1"},{"attempts":1,"created":1792192844,"error":null,"finished":1792193315,"gpu":null,"id":"f8dd3623ae6e8786284a6ca8","last_heartbeat":null,"model":"sentence-transformers/all-MiniLM-L6-v2","origin":null,"priority":32.62,"scope":"5048133e477a3c13bf97c614b99b2d36a458391bb2a9480cd9b38189f2608b67","started":1792193315,"status":"completed","task":"similarity","worker":null},{"attempts":1,"created":1792198034,"error":null,"finished":1792198139,"gpu":1,"id":"0d8c049a046e18e772d16952","last_heartbeat":1792198135,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":6.96,"scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","started":1792198135,"status":"completed","task":"arc-challenge-choice","worker":"white-gpu1"},{"attempts":1,"created":1792198034,"error":null,"finished":1792198142,"gpu":0,"id":"216e346c83988ade77ddfff0","last_heartbeat":1792198137,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":6.96,"scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","started":1792198137,"status":"completed","task":"arc-challenge-choice","worker":"white-gpu0"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198076,"gpu":1,"id":"2f544548b89cd6fed2d4e61d","last_heartbeat":1792198072,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":6.94,"scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","started":1792198072,"status":"completed","task":"commonsense","worker":"white-gpu1"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198097,"gpu":1,"id":"db55a33c25d74960ff389d24","last_heartbeat":1792198092,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":6.94,"scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","started":1792198092,"status":"completed","task":"commonsense","worker":"white-gpu1"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198077,"gpu":0,"id":"2933b280610cb03f5f2345c2","last_heartbeat":1792198070,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":0.05,"scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","started":1792198070,"status":"completed","task":"entailment","worker":"white-gpu0"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198099,"gpu":0,"id":"b700ec1eed7b5ab1a5d28d49","last_heartbeat":1792198092,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":0.05,"scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","started":1792198092,"status":"completed","task":"sentiment","worker":"white-gpu0"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198122,"gpu":0,"id":"bddd3154bd9cf23f3dece3fb","last_heartbeat":1792198114,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":0.05,"scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","started":1792198114,"status":"completed","task":"entailment","worker":"white-gpu0"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198120,"gpu":1,"id":"c6d083942f8d93dc672e019b","last_heartbeat":1792198112,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":0.05,"scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","started":1792198112,"status":"completed","task":"reading","worker":"white-gpu1"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198193,"gpu":1,"id":"efa7055f4d6480dd542a3abf","last_heartbeat":1792198179,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":0.05,"scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","started":1792198179,"status":"completed","task":"sentiment","worker":"white-gpu1"},{"attempts":1,"created":1792198012,"error":null,"finished":1792198193,"gpu":0,"id":"ff345349801249c15997d2a6","last_heartbeat":1792198180,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":0.05,"scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","started":1792198180,"status":"completed","task":"reading","worker":"white-gpu0"},{"attempts":1,"created":1792198034,"error":null,"finished":1792198159,"gpu":1,"id":"9fd52efbe4976b40a9fd6646","last_heartbeat":1792198154,"model":"Qwen/Qwen2.5-1.5B","origin":null,"priority":0.05,"scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","started":1792198154,"status":"completed","task":"openbookqa-main-choice","worker":"white-gpu1"},{"attempts":1,"created":1792198034,"error":null,"finished":1792198164,"gpu":0,"id":"f08b963541b415694a99b37b","last_heartbeat":1792198157,"model":"Qwen/Qwen2.5-0.5B","origin":null,"priority":0.05,"scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","started":1792198157,"status":"completed","task":"openbookqa-main-choice","worker":"white-gpu0"}],"runs":[{"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evidence":"CodeSOTA execution on White","finished":1792192784,"id":"0ab8b157bc6f3029a19271ab","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3109161472,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/grade-school-math","split":"test"},"runner_sha256":"fd358150cb0cf5b9dc73de627f01f0f194d800a3e6355193039d335ce043b9a3","scope":"931808a3f82fdda3ddddbf7a73eec8abfdf78e7c460ffa9c507a3ac8693138ff","score":0.09375,"seconds":124.60450974502601,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math"},{"dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193082,"id":"264775b8e13a45705e029825","measured_cost_usd":null,"metric":"pass@1","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1013277696,"profile":"pilot-32","protocol":{"adapter":"code","area":"Code","benchmark":"HumanEval","contract":"Function signature + docstring \u2192 Python function","dataset":"openai/openai_humaneval","dataset_config":"openai_humaneval","dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","dataset_url":"https://huggingface.co/datasets/openai/openai_humaneval","direction":"max","id":"code-generation","license":["mit"],"limit":32,"metric":"pass@1","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Code generation","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/human-eval","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","sandbox_image":"python@sha256:e88e9763f943ec1834f992a4b51e0f24500486803e8bc534e5767af9ea65f6ce","scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","score":0.625,"seconds":60.507072682026774,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"code-generation"},{"dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193254,"id":"501f999a9189ca77b745a388","measured_cost_usd":null,"metric":"pass@1","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3112684544,"profile":"pilot-32","protocol":{"adapter":"code","area":"Code","benchmark":"HumanEval","contract":"Function signature + docstring \u2192 Python function","dataset":"openai/openai_humaneval","dataset_config":"openai_humaneval","dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","dataset_url":"https://huggingface.co/datasets/openai/openai_humaneval","direction":"max","id":"code-generation","license":["mit"],"limit":32,"metric":"pass@1","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Code generation","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/human-eval","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","sandbox_image":"python@sha256:e88e9763f943ec1834f992a4b51e0f24500486803e8bc534e5767af9ea65f6ce","scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","score":0.75,"seconds":68.26958421000745,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"code-generation"},{"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193396,"id":"c5b99d0e49512667f8c0c91f","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1012201984,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/grade-school-math","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","score":0.03125,"seconds":32.15862700995058,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math"},{"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792192998,"id":"14e52d72173736c52853bbf1","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3109161472,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/grade-school-math","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","score":0.09375,"seconds":49.895180064952,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math"},{"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193447,"id":"7af8b47f7ce3815bea0c442e","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1097589760,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"reading","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Reading comprehension","protocol_version":"codesota-pilot-v1","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","score":0.6875,"seconds":1.5907042749458924,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"reading"},{"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792192887,"id":"206aa355b6974c770b0b29c4","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3194701824,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"reading","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Reading comprehension","protocol_version":"codesota-pilot-v1","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","score":0.6875,"seconds":2.550903692957945,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"reading"},{"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193498,"id":"b82a09a698a552ab21a0075b","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1051707904,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"commonsense","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Commonsense completion","protocol_version":"codesota-pilot-v1","source":"https://rowanzellers.com/hellaswag/","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","score":0.375,"seconds":2.113465471076779,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"commonsense"},{"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193158,"id":"4ecdb6321390abddd6ac658d","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3143732736,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"commonsense","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Commonsense completion","protocol_version":"codesota-pilot-v1","source":"https://rowanzellers.com/hellaswag/","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","score":0.5,"seconds":2.7435219429899007,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"commonsense"},{"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193133,"id":"2baa9449b32fde6f476f7e1f","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1021347840,"profile":"pilot-32","protocol":{"adapter":"sentiment","area":"Language","benchmark":"SST-2","contract":"Movie review \u2192 positive / negative","dataset":"stanfordnlp/sst2","dataset_config":null,"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","dataset_url":"https://huggingface.co/datasets/stanfordnlp/sst2","direction":"max","id":"sentiment","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Sentiment classification","protocol_version":"codesota-pilot-v1","source":"https://nlp.stanford.edu/sentiment/","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","score":0.9375,"seconds":1.55905277794227,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sentiment"},{"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193472,"id":"9c6acd69de4197860e28ebb2","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3114255872,"profile":"pilot-32","protocol":{"adapter":"sentiment","area":"Language","benchmark":"SST-2","contract":"Movie review \u2192 positive / negative","dataset":"stanfordnlp/sst2","dataset_config":null,"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","dataset_url":"https://huggingface.co/datasets/stanfordnlp/sst2","direction":"max","id":"sentiment","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Sentiment classification","protocol_version":"codesota-pilot-v1","source":"https://nlp.stanford.edu/sentiment/","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","score":0.625,"seconds":2.0898517509922385,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sentiment"},{"dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193107,"id":"272e772751322114921d9ee6","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1021412352,"profile":"pilot-32","protocol":{"adapter":"entailment","area":"Language","benchmark":"SNLI","contract":"Premise + hypothesis \u2192 entailment / neutral / contradiction","dataset":"stanfordnlp/snli","dataset_config":"plain_text","dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","dataset_url":"https://huggingface.co/datasets/stanfordnlp/snli","direction":"max","id":"entailment","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Natural language inference","protocol_version":"codesota-pilot-v1","source":"https://nlp.stanford.edu/projects/snli/","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","score":0.375,"seconds":1.8406528600025922,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"entailment"},{"dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193422,"id":"5a2a497f01a1e75965c07849","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3114874368,"profile":"pilot-32","protocol":{"adapter":"entailment","area":"Language","benchmark":"SNLI","contract":"Premise + hypothesis \u2192 entailment / neutral / contradiction","dataset":"stanfordnlp/snli","dataset_config":"plain_text","dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","dataset_url":"https://huggingface.co/datasets/stanfordnlp/snli","direction":"max","id":"entailment","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Natural language inference","protocol_version":"codesota-pilot-v1","source":"https://nlp.stanford.edu/projects/snli/","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","score":0.375,"seconds":2.3908391969744116,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"entailment"},{"dataset_revision":"ab7a5ac0e35aa22088bdcf23e7fd99b220e53308","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193315,"id":"f8dd3623ae6e8786284a6ca8","measured_cost_usd":null,"metric":"spearman","model":"sentence-transformers/all-MiniLM-L6-v2","model_revision":"1110a243fdf4706b3f48f1d95db1a4f5529b4d41","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":107248128,"profile":"pilot-32","protocol":{"adapter":"similarity","area":"Embeddings","benchmark":"STS Benchmark","contract":"Two sentences \u2192 semantic similarity","dataset":"sentence-transformers/stsb","dataset_config":null,"dataset_revision":"ab7a5ac0e35aa22088bdcf23e7fd99b220e53308","dataset_url":"https://huggingface.co/datasets/sentence-transformers/stsb","direction":"max","id":"similarity","license":"See upstream terms","limit":32,"metric":"spearman","models":[{"id":"sentence-transformers/all-MiniLM-L6-v2","revision":"1110a243fdf4706b3f48f1d95db1a4f5529b4d41"}],"name":"Sentence similarity","protocol_version":"codesota-pilot-v1","source":"https://ixa2.si.ehu.eus/stswiki/index.php/STSbenchmark","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"5048133e477a3c13bf97c614b99b2d36a458391bb2a9480cd9b38189f2608b67","score":0.9241385507168905,"seconds":0.8793051829561591,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"similarity"},{"dataset_revision":"fbdad97500ce54635c0d1ba306bf535cb40656cf","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792192922,"id":"1ed98ed973f5e29bf51634c4","measured_cost_usd":null,"metric":"cer","model":"microsoft/trocr-small-handwritten","model_revision":"b4648cfa171985a6745f37ddd637e98c0da958ac","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":270352896,"profile":"pilot-32","protocol":{"adapter":"ocr","area":"Documents","benchmark":"IAM lines","contract":"Handwritten line image \u2192 text","dataset":"Teklia/IAM-line","dataset_config":null,"dataset_revision":"fbdad97500ce54635c0d1ba306bf535cb40656cf","dataset_url":"https://huggingface.co/datasets/Teklia/IAM-line","direction":"min","id":"handwriting","license":"mit","limit":32,"metric":"cer","models":[{"id":"microsoft/trocr-small-handwritten","revision":"b4648cfa171985a6745f37ddd637e98c0da958ac"}],"name":"Handwriting recognition","protocol_version":"codesota-pilot-v1","source":"https://fki.tic.heia-fr.ch/databases/iam-handwriting-database","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"87f7f645d4ae1c902e4f3c4d821290e2fa08df318242d5d78a563467763e36ec","score":0.05858170606372045,"seconds":8.352338270982727,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"handwriting"},{"dataset_revision":"71cacbfb7e2354c4226d01e70d77d5fca3d04ba1","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193290,"id":"9971178eeb5427d24da23fab","measured_cost_usd":null,"metric":"wer","model":"openai/whisper-tiny.en","model_revision":"87c7102498dcde7456f24cfd30239ca606ed9063","n":32,"normalizer":"transformers Whisper BasicTextNormalizer","notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":190671360,"profile":"pilot-32","protocol":{"adapter":"asr","area":"Speech","benchmark":"LibriSpeech test-clean","contract":"English speech recording \u2192 transcript","dataset":"openslr/librispeech_asr","dataset_config":"clean","dataset_revision":"71cacbfb7e2354c4226d01e70d77d5fca3d04ba1","dataset_url":"https://huggingface.co/datasets/openslr/librispeech_asr","direction":"min","id":"transcription","license":["cc-by-4.0"],"limit":32,"metric":"wer","models":[{"id":"openai/whisper-tiny.en","revision":"87c7102498dcde7456f24cfd30239ca606ed9063"}],"name":"Speech transcription","protocol_version":"codesota-pilot-v1","source":"https://www.openslr.org/12","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"9c365ee78df50fa725b4c03d695f65d2b638be00c4687dc66be70b284c865a34","score":0.05087440381558029,"seconds":5.0269540110602975,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"transcription"},{"class_prompts":["a photo of a airplane.","a photo of a automobile.","a photo of a bird.","a photo of a cat.","a photo of a deer.","a photo of a dog.","a photo of a frog.","a photo of a horse.","a photo of a ship.","a photo of a truck."],"dataset_revision":"0b2714987fa478483af9968de7c934580d0bb9a2","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193340,"id":"ffc11a3556867f2915bcdf77","measured_cost_usd":null,"metric":"accuracy","model":"openai/clip-vit-base-patch32","model_revision":"3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":617016320,"profile":"pilot-32","protocol":{"adapter":"vision","area":"Vision","benchmark":"CIFAR-10","contract":"Image \u2192 one of ten object classes","dataset":"uoft-cs/cifar10","dataset_config":"plain_text","dataset_revision":"0b2714987fa478483af9968de7c934580d0bb9a2","dataset_url":"https://huggingface.co/datasets/uoft-cs/cifar10","direction":"max","id":"image-classification","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"openai/clip-vit-base-patch32","revision":"3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"}],"name":"Image classification","protocol_version":"codesota-pilot-v1","source":"https://www.cs.toronto.edu/~kriz/cifar.html","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"d4a779c6ba33be115fc5e3302e45bb70d7c90040f954b89dab97beb5a2914649","score":0.9375,"seconds":1.9501927399542183,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"image-classification","training_regime":"zero-shot CLIP; compare only within this protocol"},{"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193772,"gpu":0,"id":"ca6a72065b0a3873beda997b","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"32-example CodeSOTA declarative multiple-choice pilot. Not an official full-benchmark score.","peak_gpu_bytes":1049815552,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"arc-challenge-choice","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_choice.py","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test"},"runner_sha256":"4f6810ca652d8b80ae0c5e19f2a7e313e68357bb645a88a4d866e4c1141a0ed9","scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","score":0.3125,"seconds":2.2367252570111305,"seed":42,"selection":"first 32 examples in pinned parquet order","status":"completed","task":"arc-challenge-choice","worker":"white-gpu0"},{"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193772,"gpu":1,"id":"293bd712648d308f4ffe2539","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"32-example CodeSOTA declarative multiple-choice pilot. Not an official full-benchmark score.","peak_gpu_bytes":3141840384,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"arc-challenge-choice","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_choice.py","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test"},"runner_sha256":"4f6810ca652d8b80ae0c5e19f2a7e313e68357bb645a88a4d866e4c1141a0ed9","scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","score":0.46875,"seconds":2.842379883979447,"seed":42,"selection":"first 32 examples in pinned parquet order","status":"completed","task":"arc-challenge-choice","worker":"white-gpu1"},{"dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193843,"gpu":1,"id":"40d1459ca17c38921b30a6ea","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"32-example CodeSOTA declarative multiple-choice pilot. Not an official full-benchmark score.","peak_gpu_bytes":1026005504,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"OpenBookQA main (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","direction":"max","id":"openbookqa-main-choice","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"OpenBookQA main (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_choice.py","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test"},"runner_sha256":"4f6810ca652d8b80ae0c5e19f2a7e313e68357bb645a88a4d866e4c1141a0ed9","scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","score":0.40625,"seconds":2.220320450956933,"seed":42,"selection":"first 32 examples in pinned parquet order","status":"completed","task":"openbookqa-main-choice","worker":"white-gpu1"},{"dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193843,"gpu":0,"id":"795ccf7bff6558c30d67ce8c","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"32-example CodeSOTA declarative multiple-choice pilot. Not an official full-benchmark score.","peak_gpu_bytes":3117530112,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"OpenBookQA main (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","direction":"max","id":"openbookqa-main-choice","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"OpenBookQA main (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_choice.py","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test"},"runner_sha256":"4f6810ca652d8b80ae0c5e19f2a7e313e68357bb645a88a4d866e4c1141a0ed9","scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","score":0.40625,"seconds":2.770547211985104,"seed":42,"selection":"first 32 examples in pinned parquet order","status":"completed","task":"openbookqa-main-choice","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"examples":[1,18,3,24,30,23,6,21],"generated_tokens":2048,"kind":"generation","padded_input_tokens":504,"seconds":2.979331613983959},{"examples":[28,14,16,22,5,2,13,17],"generated_tokens":2048,"kind":"generation","padded_input_tokens":600,"seconds":2.733933093957603},{"examples":[27,9,10,20,31,0,11,19],"generated_tokens":2048,"kind":"generation","padded_input_tokens":664,"seconds":2.7375622619874775},{"examples":[12,25,29,26,7,15,8,4],"generated_tokens":2048,"kind":"generation","padded_input_tokens":1032,"seconds":2.879807790974155}],"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198055,"gpu":0,"id":"4749a153422d5cefccd16747","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1058638336,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://github.com/openai/grade-school-math","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":17.99193312006537,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","score":0.03125,"seconds":12.287420978071168,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"examples":[1,18,3,24,30,23,6,21],"generated_tokens":2048,"kind":"generation","padded_input_tokens":504,"seconds":3.3634832510724664},{"examples":[28,14,16,22,5,2,13,17],"generated_tokens":2048,"kind":"generation","padded_input_tokens":600,"seconds":3.2442861669696867},{"examples":[27,9,10,20,31,0,11,19],"generated_tokens":2048,"kind":"generation","padded_input_tokens":664,"seconds":3.398668773006648},{"examples":[12,25,29,26,7,15,8,4],"generated_tokens":2048,"kind":"generation","padded_input_tokens":1032,"seconds":3.292171653942205}],"dataset_revision":"740312add88f781978c0658806c59bc2815b9866","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198057,"gpu":1,"id":"aa5bc8b59a1fd979e61724eb","measured_cost_usd":null,"metric":"exact_match","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3223840256,"profile":"pilot-32","protocol":{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","direction":"max","id":"math","license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://github.com/openai/grade-school-math","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":20.445043627056293,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","score":0.09375,"seconds":14.718171672080643,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"math","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1476720900973305},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.052455789991654456},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.06026104697957635},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.09851725597400218}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198193,"gpu":0,"id":"ff345349801249c15997d2a6","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1176103936,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"reading","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Reading comprehension","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.375335416058078,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","score":0.6875,"seconds":1.2992077759699896,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"reading","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.18977543292567134},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.13158546795602888},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.1604797359323129},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.25351069995667785}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198120,"gpu":1,"id":"c6d083942f8d93dc672e019b","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3407441920,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"reading","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Reading comprehension","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.779247667989694,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","score":0.6875,"seconds":2.026138138025999,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"reading","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13930650800466537},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.016007635043933988},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.016305911005474627},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.01750927499961108},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.018990446929819882},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.02015766396652907},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.02054631896317005},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.02836571296211332}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198076,"gpu":1,"id":"2f544548b89cd6fed2d4e61d","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1216680960,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"commonsense","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Commonsense completion","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://rowanzellers.com/hellaswag/","split":"validation"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.171156510012224,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","score":0.375,"seconds":1.182585975038819,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"commonsense","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1233458510832861},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03480881894938648},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.03560852992814034},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.041067461017519236},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.04721856291871518},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05448357202112675},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.05457581393420696},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.07369675498921424}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198097,"gpu":1,"id":"db55a33c25d74960ff389d24","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3308705792,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"commonsense","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Commonsense completion","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://rowanzellers.com/hellaswag/","split":"validation"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.821826095925644,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","score":0.5,"seconds":1.7728401749627665,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"commonsense","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1297435329761356},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.015189048019237816},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.016184214036911726},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.023662249906919897}],"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198193,"gpu":1,"id":"efa7055f4d6480dd542a3abf","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1036140544,"profile":"pilot-32","protocol":{"adapter":"sentiment","area":"Language","benchmark":"SST-2","contract":"Movie review \u2192 positive / negative","dataset":"stanfordnlp/sst2","dataset_config":null,"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","dataset_url":"https://huggingface.co/datasets/stanfordnlp/sst2","direction":"max","id":"sentiment","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Sentiment classification","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://nlp.stanford.edu/sentiment/","split":"validation"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":8.02642539597582,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","score":0.9375,"seconds":1.1125338861020282,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sentiment","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12251766701228917},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.034768092911690474},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.040776347974315286},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.06278559600468725}],"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198099,"gpu":0,"id":"b700ec1eed7b5ab1a5d28d49","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3151635456,"profile":"pilot-32","protocol":{"adapter":"sentiment","area":"Language","benchmark":"SST-2","contract":"Movie review \u2192 positive / negative","dataset":"stanfordnlp/sst2","dataset_config":null,"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","dataset_url":"https://huggingface.co/datasets/stanfordnlp/sst2","direction":"max","id":"sentiment","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Sentiment classification","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://nlp.stanford.edu/sentiment/","split":"validation"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.177016686997376,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","score":0.625,"seconds":1.5457230790052563,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sentiment","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12068927695509046},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.01626022602431476},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.015007188892923295},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.015376847935840487},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.02187646005768329},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.019569096039049327}],"dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198077,"gpu":0,"id":"2933b280610cb03f5f2345c2","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1035193344,"profile":"pilot-32","protocol":{"adapter":"entailment","area":"Language","benchmark":"SNLI","contract":"Premise + hypothesis \u2192 entailment / neutral / contradiction","dataset":"stanfordnlp/snli","dataset_config":"plain_text","dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","dataset_url":"https://huggingface.co/datasets/stanfordnlp/snli","direction":"max","id":"entailment","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Natural language inference","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://nlp.stanford.edu/projects/snli/","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.026732878992334,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","score":0.375,"seconds":1.1434840289875865,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"entailment","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12951591901946813},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.03552447399124503},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.040724562015384436},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04952972300816327},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04472782800439745},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.05515940301120281}],"dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198122,"gpu":0,"id":"bddd3154bd9cf23f3dece3fb","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3151138816,"profile":"pilot-32","protocol":{"adapter":"entailment","area":"Language","benchmark":"SNLI","contract":"Premise + hypothesis \u2192 entailment / neutral / contradiction","dataset":"stanfordnlp/snli","dataset_config":"plain_text","dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","dataset_url":"https://huggingface.co/datasets/stanfordnlp/snli","direction":"max","id":"entailment","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Natural language inference","protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","source":"https://nlp.stanford.edu/projects/snli/","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.377454819041304,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","score":0.375,"seconds":1.6738499279599637,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"entailment","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12666251696646214},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.01758025703020394},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.01823836495168507},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.01971449691336602},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.021584945963695645},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.024363117991015315},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.027675545075908303},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.03174972301349044}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198139,"gpu":1,"id":"0d8c049a046e18e772d16952","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1161676288,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"arc-challenge-choice","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_fast.py","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.071349939098582,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","score":0.3125,"seconds":1.2409709830535576,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"arc-challenge-choice","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11999208689667284},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.04712149198167026},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.053831106051802635},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.060813799034804106},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.062681361916475},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.07165342499502003},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.07059212494641542},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.09778208506759256}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198142,"gpu":0,"id":"216e346c83988ade77ddfff0","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3253701120,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"arc-challenge-choice","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_fast.py","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.971455986960791,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","score":0.46875,"seconds":1.8925830170046538,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"arc-challenge-choice","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13906778290402144},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.0142781789181754},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.011920462013222277},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.012994835036806762},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.01210576796438545},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.013348915963433683},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.015171325067058206},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.02136441390030086}],"dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198164,"gpu":0,"id":"f08b963541b415694a99b37b","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":1071536640,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"OpenBookQA main (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","direction":"max","id":"openbookqa-main-choice","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"OpenBookQA main (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_fast.py","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.5228236329276115,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","score":0.40625,"seconds":2.3172576900105923,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"openbookqa-main-choice","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1246656629955396},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.021783375996164978},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.022670169943012297},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02873803593683988},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.02725917799398303},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.030699505005031824},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.034868279937654734},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.055141029064543545}],"dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"batched-v2","finished":1792198159,"gpu":1,"id":"9fd52efbe4976b40a9fd6646","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, CodeSOTA batched-v2 execution. Separate scope from serial v1; unchanged prompt/tokenization/metric contracts.","peak_gpu_bytes":3163561472,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"OpenBookQA main (test)","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","direction":"max","id":"openbookqa-main-choice","license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"OpenBookQA main (test)","parent_task":"reading","protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_fast.py","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.821713284938596,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bc899fb160f6881408863cd8bcdf96102dd6d8b10559a5f781a8cd816103da77","scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","score":0.40625,"seconds":1.7379119840916246,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"openbookqa-main-choice","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.12425816198810935},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014353499049320817},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.01433193904813379},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.02308932098094374},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.016291594016365707},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.01679475198034197},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.01820836798287928},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.024417127948254347}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199513,"gpu":1,"id":"83ca3844b75d12aea722afae","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":657188352,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.070911542046815,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.5625,"seconds":1.0435598699841648,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.09018106700386852},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.02608981798402965},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03442929999437183},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03219556505791843},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.035866821999661624},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.046312381979078054},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.04108243901282549},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.06735055800527334}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199513,"gpu":0,"id":"933abc78ddc9563ed3bfe5c3","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":1564464128,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.170912915025838,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.4375,"seconds":1.2155649489723146,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.07994207192678005},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02546265791170299},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03141217399388552},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03354428999591619},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03849511395674199},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04128733999095857},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.045172406011261046},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.059367791982367635}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199620,"gpu":1,"id":"da843c3af7d2f039d8dcc997","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B-Instruct","model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":2270253568,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.521601940039545,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.59375,"seconds":1.4459956830833107,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.29677336290478706},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13383584399707615},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13450233708135784},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13556442793924361},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13510968000628054},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13517078396398574},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13405970204621553},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1333185089752078}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199534,"gpu":0,"id":"95f7f19d55290d88873e6474","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.12225983699318,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.78125,"seconds":2.1125054139411077,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.3073595860041678},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12762841605581343},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12597069598268718},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1279401279753074},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12618803093209863},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12855588598176837},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13134093105327338},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12986225401982665}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199641,"gpu":0,"id":"e66c128ba1817c735cfe0094","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":680474112,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.27229458198417,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.4375,"seconds":2.449016497004777,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.30749484000261873},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13433970394544303},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13469515705946833},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13649503397755325},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13486212899442762},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13734989799559116},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13695090496912599},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13868639094289392}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199350,"gpu":0,"id":"5ebdb1c54bc9de4a483665de","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.072358333971351,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"seconds":2.1689843010390177,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.29653745400719345},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1174258099636063},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11791576398536563},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11664059502072632},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1171089590061456},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11837183404713869},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12155886797700077},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11784734693355858}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199347,"gpu":1,"id":"4f8d3c01030b9ec948e6d8be","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01","model_revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":688101888,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.971871975925751,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"seconds":2.0058413579827175,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.30376290902495384},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1181516939541325},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12269497907254845},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1451712850248441},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11987522710114717},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11452228506095707},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11452719301450998},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11412549391388893}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199493,"gpu":1,"id":"81740f9404ca2161b03fab45","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":688101888,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.873346148058772,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.375,"seconds":2.025514440028928,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.301861822954379},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13498842308763415},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13530818989966065},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13598228699993342},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13709837896749377},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13612622499931604},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13629421801306307},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1378802970284596}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199600,"gpu":1,"id":"adeb8d2f845d86b97ddb885e","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.023155056987889,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"seconds":2.154468334047124,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.31639307306613773},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1166775340680033},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11946902994532138},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13180541794281453},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12876322399824858},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11960899108089507},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11863554595038295},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12067093094810843}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199620,"gpu":0,"id":"c2e37f8ec8e7bbe1a93eefe7","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.02","model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":688101888,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.071834755013697,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"seconds":2.0400674969423562,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.3167141720186919},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.16382455301936716},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.15081263601314276},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14019769907463342},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1395214069634676},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13469981495290995},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1361441989429295},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13518077204935253}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199600,"gpu":0,"id":"a5bd023e62dfa94900c68aff","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":723978240,"profile":"pilot-32","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.124572991044261,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.5625,"seconds":2.2018078429391608,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-arc-easy","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13017428503371775},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.01650666492059827},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.016730963019654155},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.01793303492013365},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.018158614984713495},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.020130372955463827},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.020577267045155168},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.029766182997263968}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199452,"gpu":1,"id":"a0a1b916aa43bef9dc349f52","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":752199680,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":3.971061061020009,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.28125,"seconds":0.9418995419982821,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.09920341498218477},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.033430437091737986},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04072352894581854},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.046183633035980165},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0429107389645651},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05098394490778446},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.055247246054932475},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.07813984109088778}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199450,"gpu":0,"id":"9733ef9ec08a5e033ded7a5e","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":1660646912,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.170025971019641,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.46875,"seconds":1.26590998203028,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0871340159792453},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.039911095052957535},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.05221338791307062},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.04669109906535596},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.05426594603341073},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06833100807853043},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06116744806058705},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.09336118900682777}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199370,"gpu":0,"id":"5182981eed2309237804d9c3","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B-Instruct","model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":2545815040,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.62133755593095,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.40625,"seconds":1.556583499070257,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.30352677893824875},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1339959530159831},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13381280098110437},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1330113069852814},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13166162895504385},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1330211660824716},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13330050697550178},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13452494994271547}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199368,"gpu":1,"id":"4bab151d17bfaf2727299f60","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":794730496,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.072574998019263,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"seconds":2.1256928420625627,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.2987431969959289},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12729038705583662},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12539833795744926},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12649443303234875},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12608513899613172},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12715133698657155},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1267648560460657},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.12892765109427273}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199473,"gpu":1,"id":"ec41e48212ad955d3ea4c8f6","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":721014272,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.323357745073736,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"seconds":2.415863345959224,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.30035811802372336},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13631828397046775},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13461679103784263},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13437864603474736},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13565114000812173},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13505954598076642},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13464067492168397},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13527415599673986}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199431,"gpu":0,"id":"93979b5fa00514cce3d96032","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":794730496,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.173015687963925,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.375,"seconds":2.2430462820921093,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.2908139299834147},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11394369602203369},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11360044591128826},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11670476594008505},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11444447899702936},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11443551303818822},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11699038802180439},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.11520056996960193}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199327,"gpu":1,"id":"384fccc53f471e23eb9acac8","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01","model_revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":761637888,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.97246181697119,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"seconds":1.967458063038066,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.34222346590831876},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.12201374594587833},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1149420770816505},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11583159596193582},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11585439799819142},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11552498396486044},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11585323698818684},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.12013397994451225}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199410,"gpu":0,"id":"75ea5d8ae94de8a86a7a0777","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":761637888,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.92196170205716,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.375,"seconds":2.026681552059017,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.3043450729455799},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13877835101448},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14041231910232455},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13864246802404523},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13872320798691362},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14067755406722426},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14329399401322007},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13903140393085778}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199471,"gpu":0,"id":"d6a751bd1fb70f34af6e88c0","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":794730496,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.17221459292341,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"seconds":2.1684185520280153,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.2905287758912891},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1158063190523535},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11494059406686574},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11834335094317794},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11687655502464622},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11503151396755129},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11441944306716323},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.11583800800144672}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199411,"gpu":1,"id":"8677abc8861650a471b4ea7c","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.02","model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":761637888,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":4.922031161026098,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"seconds":1.9869124230463058,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.30243999301455915},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1421957560814917},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14088747603818774},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1415439499542117},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1665020870277658},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1441936519695446},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14196027698926628},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13688690506387502}],"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199390,"gpu":0,"id":"60b94d2f3d3bccb39d7277ca","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":794730496,"profile":"pilot-32","protocol":{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","direction":"max","id":"sub1b-hellaswag","license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":5.122477866942063,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.3125,"seconds":2.199777269968763,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-hellaswag","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.15573862299788743},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.052442956948652864},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.08554483205080032},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.1200737280305475}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199329,"gpu":0,"id":"3b9bc33f16b6187cceb7819f","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":691466240,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":6.624368945020251,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.21875,"seconds":1.0593773149885237,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.18185965798329562},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.14411108696367592},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.20974927197676152},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.3129351839888841}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199579,"gpu":0,"id":"eb5517fac2e0ec0cb50196c5","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":1681739264,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.425343442009762,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.34375,"seconds":1.6528386661084369,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.16883695300202817},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.15180813299957663},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.20950204296968877},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.33938537701033056}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199307,"gpu":0,"id":"141ceb6c09dc8f0156f70c0c","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B-Instruct","model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":2346900480,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.876938486006111,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.71875,"seconds":1.935960141941905,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.270229447982274},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1432230609934777},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15365709108300507},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.19088799704331905}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199558,"gpu":1,"id":"dce70413303e4781f185a0ac","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":984605696,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.1771505429642275,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5,"seconds":1.632466200971976,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.28066387795843184},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1456537441117689},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.1672645480139181},{"candidates":16,"kind":"choice","padded_tokens":5136,"seconds":0.20438037905842066}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199494,"gpu":0,"id":"509bc4c9edce51c91c7e5e39","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":818889728,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.727604440995492,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.6875,"seconds":2.0678646629676223,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.2658541889395565},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.14040926401503384},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15321300295181572},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.1910893419990316}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199433,"gpu":1,"id":"3fb07acb70ad720b7ea6387f","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":984605696,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.225614584982395,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.78125,"seconds":1.64822871692013,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.2582136089913547},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12060998799279332},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.12783878995105624},{"candidates":16,"kind":"choice","padded_tokens":4672,"seconds":0.15742255293298513}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199535,"gpu":1,"id":"a64f0553519f2724df9f01c1","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01","model_revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":938301952,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.126529000001028,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5625,"seconds":1.5199996100272983,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.26462260202970356},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12735508708283305},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.12937858095392585},{"candidates":16,"kind":"choice","padded_tokens":4672,"seconds":0.16476030799094588}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199580,"gpu":1,"id":"ff71c6e3b5a06ff21384b00e","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":938301952,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":6.976226528990082,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.75,"seconds":1.544806924997829,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.26295868994202465},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.14198489300906658},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15262418508064002},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.19119064207188785}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199390,"gpu":1,"id":"3ce0ba3d66280b6cbd47845b","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":984605696,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.126588338054717,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.625,"seconds":1.6223822750616819,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.3098644999554381},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12282925797626376},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.12925180501770228},{"candidates":16,"kind":"choice","padded_tokens":4672,"seconds":0.15686873102094978}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199307,"gpu":1,"id":"18802f73c41e24cdd29f5a2f","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.02","model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":938301952,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.42554085503798,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5625,"seconds":1.7050261669792235,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu1"},{"batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.2734037881018594},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15291272301692516},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.17487754696048796},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.2079960119444877}],"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-choice-v1","finished":1792199557,"gpu":0,"id":"d12d9fe7b986a834ea0d1098","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":32,"notes":"Sub-1B 32-example pilot; FP32, 1024-token input ceiling, mean answer-token log likelihood. Native masked-padding adapter uses unpadded row forwards. Compare only this cohort protocol.","peak_gpu_bytes":984605696,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"sub1b-boolq","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":7.476608742028475,"measurement_version":"gpu-slot-v1"},"runner_sha256":"5011da874931804d8f5159762d18d33b71feb02044d808b106cca5da28178509","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.6875,"seconds":1.689771424047649,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"sub1b-boolq","worker":"white-gpu0"}],"tasks":[{"adapter":"code","area":"Code","benchmark":"HumanEval","completed":2,"contract":"Function signature + docstring \u2192 Python function","dataset":"openai/openai_humaneval","dataset_config":"openai_humaneval","dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","dataset_url":"https://huggingface.co/datasets/openai/openai_humaneval","demand":5.96,"direction":"max","id":"code-generation","legacy_bonus":35,"license":["mit"],"limit":32,"metric":"pass@1","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Code generation","priority":54.5,"protocol_version":"codesota-pilot-v1","scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","source":"https://github.com/openai/human-eval","split":"test","visits_30d":6},{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","completed":5,"contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","demand":0.99,"direction":"max","id":"math","legacy_bonus":27.76,"license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","priority":34.68,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","source":"https://github.com/openai/grade-school-math","split":"test","visits_30d":1},{"adapter":"reading","area":"Language","benchmark":"BoolQ","completed":4,"contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","demand":0.0,"direction":"max","id":"reading","legacy_bonus":0,"license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Reading comprehension","priority":0.05,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","visits_30d":0},{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","completed":4,"contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","demand":0.99,"direction":"max","id":"commonsense","legacy_bonus":0,"license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Commonsense completion","priority":6.94,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","source":"https://rowanzellers.com/hellaswag/","split":"validation","visits_30d":1},{"adapter":"sentiment","area":"Language","benchmark":"SST-2","completed":4,"contract":"Movie review \u2192 positive / negative","dataset":"stanfordnlp/sst2","dataset_config":null,"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","dataset_url":"https://huggingface.co/datasets/stanfordnlp/sst2","demand":0.0,"direction":"max","id":"sentiment","legacy_bonus":0,"license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Sentiment classification","priority":0.05,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","source":"https://nlp.stanford.edu/sentiment/","split":"validation","visits_30d":0},{"adapter":"entailment","area":"Language","benchmark":"SNLI","completed":4,"contract":"Premise + hypothesis \u2192 entailment / neutral / contradiction","dataset":"stanfordnlp/snli","dataset_config":"plain_text","dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","dataset_url":"https://huggingface.co/datasets/stanfordnlp/snli","demand":0.0,"direction":"max","id":"entailment","legacy_bonus":0,"license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Natural language inference","priority":0.05,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","source":"https://nlp.stanford.edu/projects/snli/","split":"test","visits_30d":0},{"adapter":"similarity","area":"Embeddings","benchmark":"STS Benchmark","completed":1,"contract":"Two sentences \u2192 semantic similarity","dataset":"sentence-transformers/stsb","dataset_config":null,"dataset_revision":"ab7a5ac0e35aa22088bdcf23e7fd99b220e53308","dataset_url":"https://huggingface.co/datasets/sentence-transformers/stsb","demand":0.0,"direction":"max","id":"similarity","legacy_bonus":32.52,"license":"See upstream terms","limit":32,"metric":"spearman","models":[{"id":"sentence-transformers/all-MiniLM-L6-v2","revision":"1110a243fdf4706b3f48f1d95db1a4f5529b4d41"}],"name":"Sentence similarity","priority":32.62,"protocol_version":"codesota-pilot-v1","scope":"5048133e477a3c13bf97c614b99b2d36a458391bb2a9480cd9b38189f2608b67","source":"https://ixa2.si.ehu.eus/stswiki/index.php/STSbenchmark","split":"test","visits_30d":0},{"adapter":"ocr","area":"Documents","benchmark":"IAM lines","completed":1,"contract":"Handwritten line image \u2192 text","dataset":"Teklia/IAM-line","dataset_config":null,"dataset_revision":"fbdad97500ce54635c0d1ba306bf535cb40656cf","dataset_url":"https://huggingface.co/datasets/Teklia/IAM-line","demand":0.0,"direction":"min","id":"handwriting","legacy_bonus":35,"license":"mit","limit":32,"metric":"cer","models":[{"id":"microsoft/trocr-small-handwritten","revision":"b4648cfa171985a6745f37ddd637e98c0da958ac"}],"name":"Handwriting recognition","priority":35.11,"protocol_version":"codesota-pilot-v1","scope":"87f7f645d4ae1c902e4f3c4d821290e2fa08df318242d5d78a563467763e36ec","source":"https://fki.tic.heia-fr.ch/databases/iam-handwriting-database","split":"test","visits_30d":0},{"adapter":"asr","area":"Speech","benchmark":"LibriSpeech test-clean","completed":1,"contract":"English speech recording \u2192 transcript","dataset":"openslr/librispeech_asr","dataset_config":"clean","dataset_revision":"71cacbfb7e2354c4226d01e70d77d5fca3d04ba1","dataset_url":"https://huggingface.co/datasets/openslr/librispeech_asr","demand":0.99,"direction":"min","id":"transcription","legacy_bonus":35,"license":["cc-by-4.0"],"limit":32,"metric":"wer","models":[{"id":"openai/whisper-tiny.en","revision":"87c7102498dcde7456f24cfd30239ca606ed9063"}],"name":"Speech transcription","priority":41.99,"protocol_version":"codesota-pilot-v1","scope":"9c365ee78df50fa725b4c03d695f65d2b638be00c4687dc66be70b284c865a34","source":"https://www.openslr.org/12","split":"test","visits_30d":1},{"adapter":"vision","area":"Vision","benchmark":"CIFAR-10","completed":1,"contract":"Image \u2192 one of ten object classes","dataset":"uoft-cs/cifar10","dataset_config":"plain_text","dataset_revision":"0b2714987fa478483af9968de7c934580d0bb9a2","dataset_url":"https://huggingface.co/datasets/uoft-cs/cifar10","demand":0.0,"direction":"max","id":"image-classification","legacy_bonus":35,"license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"openai/clip-vit-base-patch32","revision":"3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"}],"name":"Image classification","priority":35.1,"protocol_version":"codesota-pilot-v1","scope":"d4a779c6ba33be115fc5e3302e45bb70d7c90040f954b89dab97beb5a2914649","source":"https://www.cs.toronto.edu/~kriz/cifar.html","split":"test","visits_30d":0},{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","completed":4,"contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","demand":1.0,"direction":"max","id":"arc-challenge-choice","legacy_bonus":0,"license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","priority":6.96,"protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_fast.py","scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","visits_30d":1},{"adapter":"choice","area":"Language","benchmark":"OpenBookQA main (test)","completed":4,"contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","demand":0.0,"direction":"max","id":"openbookqa-main-choice","legacy_bonus":0,"license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"OpenBookQA main (test)","parent_task":"reading","priority":0.05,"protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_fast.py","scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test","visits_30d":0},{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","catalogue_id":"arc-easy","completed":11,"contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","demand":0.0,"direction":"max","id":"sub1b-arc-easy","legacy_bonus":0,"license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","priority":1000.03,"protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","visits_30d":0,"workstream":"sub1b"},{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","catalogue_id":"hellaswag","completed":11,"contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","demand":1.0,"direction":"max","id":"sub1b-hellaswag","legacy_bonus":0,"license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","priority":1006.95,"protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","visits_30d":1,"workstream":"sub1b"},{"adapter":"reading","area":"Language","benchmark":"BoolQ","catalogue_id":"boolq","completed":11,"contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","demand":1.0,"direction":"max","id":"sub1b-boolq","legacy_bonus":0,"license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","priority":1006.95,"protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","visits_30d":1,"workstream":"sub1b"}],"traffic":{"daily":[{"date":"2026-10-17","partial":true,"unique_visitors":10,"views":55}],"hourly":[{"date":"2026-10-17T00:00Z","unique_visitors":7,"views":29},{"date":"2026-10-17T01:00Z","unique_visitors":4,"views":26}],"limitations":"Daily estimates, not identified people. Shared IP/browser combinations can merge visitors; changing networks/browsers can split them. Visitor-days sum daily uniques and do not deduplicate people across days. Known bots, prefetches, assets, APIs, HEAD requests, DNT and GPC are excluded; undetected bots may remain.","method":"Server HTML GET responses; cookieless daily HMAC(IP + User-Agent), with a fresh random key each UTC day","pages":{"30":[{"kind":"Registry","page":"_home","path":"/","title":"All tasks","views":20,"visitor_days":9},{"kind":"Diagnostics","page":"_benchmark_value","path":"/benchmark-value","title":"Benchmark cost and discrimination","views":7,"visitor_days":2},{"kind":"Editorial","page":"_sub1b","path":"/sub-1b","title":"Under 1B: useful small models","views":7,"visitor_days":1},{"kind":"Planning","page":"_queue","path":"/queue","title":"Evaluation queue","views":5,"visitor_days":1},{"kind":"Task","page":"code-generation","path":"/tasks/code-generation","title":"Code generation","views":5,"visitor_days":1},{"kind":"Documentation","page":"_ontology","path":"/ontology","title":"Ontology and evaluation loop","views":2,"visitor_days":1},{"kind":"Popularity","page":"_popular","path":"/popular","title":"Popular pages","views":2,"visitor_days":1},{"kind":"Task","page":"arc-challenge-choice","path":"/tasks/arc-challenge-choice","title":"AI2 ARC-Challenge (test)","views":2,"visitor_days":2},{"kind":"Task","page":"sub1b-boolq","path":"/tasks/sub1b-boolq","title":"BoolQ / sub-1B cohort","views":2,"visitor_days":1},{"kind":"Agents","page":"_factory","path":"/factory","title":"AI evaluation factory","views":1,"visitor_days":1},{"kind":"Benchmark plan","page":"sub1b-benchmark-alpacaeval-2-lc","path":"/sub-1b/benchmarks/alpacaeval-2-lc","title":"AlpacaEval 2.0 \u2014 length-controlled","views":1,"visitor_days":1},{"kind":"Task","page":"sub1b-hellaswag","path":"/tasks/sub1b-hellaswag","title":"HellaSwag / sub-1B cohort","views":1,"visitor_days":1}],"7":[{"kind":"Registry","page":"_home","path":"/","title":"All tasks","views":20,"visitor_days":9},{"kind":"Diagnostics","page":"_benchmark_value","path":"/benchmark-value","title":"Benchmark cost and discrimination","views":7,"visitor_days":2},{"kind":"Editorial","page":"_sub1b","path":"/sub-1b","title":"Under 1B: useful small models","views":7,"visitor_days":1},{"kind":"Planning","page":"_queue","path":"/queue","title":"Evaluation queue","views":5,"visitor_days":1},{"kind":"Task","page":"code-generation","path":"/tasks/code-generation","title":"Code generation","views":5,"visitor_days":1},{"kind":"Documentation","page":"_ontology","path":"/ontology","title":"Ontology and evaluation loop","views":2,"visitor_days":1},{"kind":"Popularity","page":"_popular","path":"/popular","title":"Popular pages","views":2,"visitor_days":1},{"kind":"Task","page":"arc-challenge-choice","path":"/tasks/arc-challenge-choice","title":"AI2 ARC-Challenge (test)","views":2,"visitor_days":2},{"kind":"Task","page":"sub1b-boolq","path":"/tasks/sub1b-boolq","title":"BoolQ / sub-1B cohort","views":2,"visitor_days":1},{"kind":"Agents","page":"_factory","path":"/factory","title":"AI evaluation factory","views":1,"visitor_days":1},{"kind":"Benchmark plan","page":"sub1b-benchmark-alpacaeval-2-lc","path":"/sub-1b/benchmarks/alpacaeval-2-lc","title":"AlpacaEval 2.0 \u2014 length-controlled","views":1,"visitor_days":1},{"kind":"Task","page":"sub1b-hellaswag","path":"/tasks/sub1b-hellaswag","title":"HellaSwag / sub-1B cohort","views":1,"visitor_days":1}]},"started":1792196301,"timezone":"UTC","today":{"date":"2026-10-17","partial":true,"unique_visitors":10,"views":55},"totals":{"30":{"views":55,"visitor_days":10},"7":{"views":55,"visitor_days":10}},"updated":1792202117},"worker":{"running":false}}