{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","parameters":149333081,"revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","source":"https://huggingface.co/SlayerLab/Slayer149-Balanced-ARC-E-ft-v2/tree/7d11e8fd63764baf00c276f586b2aa776648ed33","updated":1792199477}],"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","page":"/models/SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":2.133822920965031,"finished":1792205327,"gpu_slot_seconds":4.973189837997779,"manifest_url":"/artifacts/94a2a3354da20599aafb07f3/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","run_id":"94a2a3354da20599aafb07f3","run_url":"/runs/94a2a3354da20599aafb07f3","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.25,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":2.154468334047124,"finished":1792199600,"gpu_slot_seconds":5.023155056987889,"manifest_url":"/artifacts/adeb8d2f845d86b97ddb885e/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","run_id":"adeb8d2f845d86b97ddb885e","run_url":"/runs/adeb8d2f845d86b97ddb885e","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":83.2209709190065,"finished":1792205718,"gpu_slot_seconds":86.19639519799966,"manifest_url":"/artifacts/0d25cdfcbb8238cbd864f8cb/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","run_id":"0d25cdfcbb8238cbd864f8cb","run_url":"/runs/0d25cdfcbb8238cbd864f8cb","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6216329966329966,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.6223822750616819,"finished":1792199390,"gpu_slot_seconds":7.126588338054717,"manifest_url":"/artifacts/3ce0ba3d66280b6cbd47845b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","run_id":"3ce0ba3d66280b6cbd47845b","run_url":"/runs/3ce0ba3d66280b6cbd47845b","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.625,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":2.1684185520280153,"finished":1792199471,"gpu_slot_seconds":5.17221459292341,"manifest_url":"/artifacts/d6a751bd1fb70f34af6e88c0/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","run_id":"d6a751bd1fb70f34af6e88c0","run_url":"/runs/d6a751bd1fb70f34af6e88c0","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":2.1103981600608677,"finished":1792205323,"gpu_slot_seconds":4.972976600052789,"manifest_url":"/artifacts/87581709c5578840b0dcb8ea/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","run_id":"87581709c5578840b0dcb8ea","run_url":"/runs/87581709c5578840b0dcb8ea","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.28125,"task":"sub1b-openbookqa"}],"revisions":["7d11e8fd63764baf00c276f586b2aa776648ed33"],"source":"https://huggingface.co/SlayerLab/Slayer149-Balanced-ARC-E-ft-v2"}}