{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer149","parameters":148910738,"revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","source":"https://huggingface.co/SlayerLab/Slayer149/tree/7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","updated":1792199477}],"model":"SlayerLab/Slayer149","page":"/models/SlayerLab/Slayer149","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":2.4085759710287675,"finished":1792205287,"gpu_slot_seconds":5.272166696959175,"manifest_url":"/artifacts/6e76c70adf6c0d200a203282/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","run_id":"6e76c70adf6c0d200a203282","run_url":"/runs/6e76c70adf6c0d200a203282","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.28125,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":2.449016497004777,"finished":1792199641,"gpu_slot_seconds":5.27229458198417,"manifest_url":"/artifacts/e66c128ba1817c735cfe0094/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","run_id":"e66c128ba1817c735cfe0094","run_url":"/runs/e66c128ba1817c735cfe0094","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.4375,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":78.64737620996311,"finished":1792205713,"gpu_slot_seconds":81.5715113170445,"manifest_url":"/artifacts/0fccdec668aa67893d88f83b/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","run_id":"0fccdec668aa67893d88f83b","run_url":"/runs/0fccdec668aa67893d88f83b","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.5677609427609428,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":2.0678646629676223,"finished":1792199494,"gpu_slot_seconds":7.727604440995492,"manifest_url":"/artifacts/509bc4c9edce51c91c7e5e39/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","run_id":"509bc4c9edce51c91c7e5e39","run_url":"/runs/509bc4c9edce51c91c7e5e39","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.6875,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":2.415863345959224,"finished":1792199473,"gpu_slot_seconds":5.323357745073736,"manifest_url":"/artifacts/ec41e48212ad955d3ea4c8f6/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","run_id":"ec41e48212ad955d3ea4c8f6","run_url":"/runs/ec41e48212ad955d3ea4c8f6","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":2.4314531040145084,"finished":1792205243,"gpu_slot_seconds":5.374162771040574,"manifest_url":"/artifacts/1e88a8202b9cf33d251d7c91/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","run_id":"1e88a8202b9cf33d251d7c91","run_url":"/runs/1e88a8202b9cf33d251d7c91","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.21875,"task":"sub1b-openbookqa"}],"revisions":["7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"],"source":"https://huggingface.co/SlayerLab/Slayer149"}}