{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer139-1.02","parameters":139279760,"revision":"c4298d757ba39cba19dd768777da9fadf319ff61","source":"https://huggingface.co/SlayerLab/Slayer139-1.02/tree/c4298d757ba39cba19dd768777da9fadf319ff61","updated":1792199477}],"model":"SlayerLab/Slayer139-1.02","page":"/models/SlayerLab/Slayer139-1.02","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":1.969372360035777,"finished":1792205222,"gpu_slot_seconds":4.822982860961929,"manifest_url":"/artifacts/1c3898213c12a124369e6294/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"c4298d757ba39cba19dd768777da9fadf319ff61","run_id":"1c3898213c12a124369e6294","run_url":"/runs/1c3898213c12a124369e6294","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.3125,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":2.0400674969423562,"finished":1792199620,"gpu_slot_seconds":5.071834755013697,"manifest_url":"/artifacts/c2e37f8ec8e7bbe1a93eefe7/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"c4298d757ba39cba19dd768777da9fadf319ff61","run_id":"c2e37f8ec8e7bbe1a93eefe7","run_url":"/runs/c2e37f8ec8e7bbe1a93eefe7","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":71.06391803000588,"finished":1792205993,"gpu_slot_seconds":74.03766193194315,"manifest_url":"/artifacts/a355f6d6302b0acac66e9d1a/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"c4298d757ba39cba19dd768777da9fadf319ff61","run_id":"a355f6d6302b0acac66e9d1a","run_url":"/runs/a355f6d6302b0acac66e9d1a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6590909090909091,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.7050261669792235,"finished":1792199307,"gpu_slot_seconds":7.42554085503798,"manifest_url":"/artifacts/18802f73c41e24cdd29f5a2f/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"c4298d757ba39cba19dd768777da9fadf319ff61","run_id":"18802f73c41e24cdd29f5a2f","run_url":"/runs/18802f73c41e24cdd29f5a2f","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5625,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":1.9869124230463058,"finished":1792199411,"gpu_slot_seconds":4.922031161026098,"manifest_url":"/artifacts/8677abc8861650a471b4ea7c/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"c4298d757ba39cba19dd768777da9fadf319ff61","run_id":"8677abc8861650a471b4ea7c","run_url":"/runs/8677abc8861650a471b4ea7c","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":1.9813053660327569,"finished":1792205383,"gpu_slot_seconds":4.771652588038705,"manifest_url":"/artifacts/b45cdfda82db9ac358a1240b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"c4298d757ba39cba19dd768777da9fadf319ff61","run_id":"b45cdfda82db9ac358a1240b","run_url":"/runs/b45cdfda82db9ac358a1240b","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.28125,"task":"sub1b-openbookqa"}],"revisions":["c4298d757ba39cba19dd768777da9fadf319ff61"],"source":"https://huggingface.co/SlayerLab/Slayer139-1.02"}}