{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer-149M-Decision-v1","parameters":149333081,"revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","source":"https://huggingface.co/SlayerLab/Slayer-149M-Decision-v1/tree/ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","updated":1792199477}],"model":"SlayerLab/Slayer-149M-Decision-v1","page":"/models/SlayerLab/Slayer-149M-Decision-v1","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":2.134203830966726,"finished":1792205247,"gpu_slot_seconds":4.971983639057726,"manifest_url":"/artifacts/300738db7c15b71f9d333a83/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","run_id":"300738db7c15b71f9d333a83","run_url":"/runs/300738db7c15b71f9d333a83","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.28125,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":2.2018078429391608,"finished":1792199600,"gpu_slot_seconds":5.124572991044261,"manifest_url":"/artifacts/a5bd023e62dfa94900c68aff/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","run_id":"a5bd023e62dfa94900c68aff","run_url":"/runs/a5bd023e62dfa94900c68aff","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.5625,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":82.21051689004526,"finished":1792205904,"gpu_slot_seconds":85.12563666200731,"manifest_url":"/artifacts/3cb7075070775d0cbe5ae4ae/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","run_id":"3cb7075070775d0cbe5ae4ae","run_url":"/runs/3cb7075070775d0cbe5ae4ae","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.5635521885521886,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.689771424047649,"finished":1792199557,"gpu_slot_seconds":7.476608742028475,"manifest_url":"/artifacts/d12d9fe7b986a834ea0d1098/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","run_id":"d12d9fe7b986a834ea0d1098","run_url":"/runs/d12d9fe7b986a834ea0d1098","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.6875,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":2.199777269968763,"finished":1792199390,"gpu_slot_seconds":5.122477866942063,"manifest_url":"/artifacts/60b94d2f3d3bccb39d7277ca/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","run_id":"60b94d2f3d3bccb39d7277ca","run_url":"/runs/60b94d2f3d3bccb39d7277ca","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.3125,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":2.1226394310360774,"finished":1792205404,"gpu_slot_seconds":4.971450684941374,"manifest_url":"/artifacts/f60e6c2b8fa9ae016b0a7999/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","run_id":"f60e6c2b8fa9ae016b0a7999","run_url":"/runs/f60e6c2b8fa9ae016b0a7999","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.28125,"task":"sub1b-openbookqa"}],"revisions":["ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"],"source":"https://huggingface.co/SlayerLab/Slayer-149M-Decision-v1"}}