{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","parameters":149333081,"revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","source":"https://huggingface.co/SlayerLab/Slayer149-Balanced-ARC-E-ft-45500/tree/e7d4b8dea44125264077c2ce6d2f141379f999cd","updated":1792199477}],"model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","page":"/models/SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":2.152657873928547,"finished":1792205368,"gpu_slot_seconds":4.921444610925391,"manifest_url":"/artifacts/b360f606b667041a2a0d2aec/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","run_id":"b360f606b667041a2a0d2aec","run_url":"/runs/b360f606b667041a2a0d2aec","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.28125,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":2.1689843010390177,"finished":1792199350,"gpu_slot_seconds":5.072358333971351,"manifest_url":"/artifacts/5ebdb1c54bc9de4a483665de/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","run_id":"5ebdb1c54bc9de4a483665de","run_url":"/runs/5ebdb1c54bc9de4a483665de","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":84.23893190396484,"finished":1792205972,"gpu_slot_seconds":87.13233877101447,"manifest_url":"/artifacts/7e4ed737ed0c83699e490690/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","run_id":"7e4ed737ed0c83699e490690","run_url":"/runs/7e4ed737ed0c83699e490690","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.5934343434343434,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.64822871692013,"finished":1792199433,"gpu_slot_seconds":7.225614584982395,"manifest_url":"/artifacts/3fb07acb70ad720b7ea6387f/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","run_id":"3fb07acb70ad720b7ea6387f","run_url":"/runs/3fb07acb70ad720b7ea6387f","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.78125,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":2.2430462820921093,"finished":1792199431,"gpu_slot_seconds":5.173015687963925,"manifest_url":"/artifacts/93979b5fa00514cce3d96032/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","run_id":"93979b5fa00514cce3d96032","run_url":"/runs/93979b5fa00514cce3d96032","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.375,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":2.12483947200235,"finished":1792205303,"gpu_slot_seconds":5.021943978965282,"manifest_url":"/artifacts/7179dfb3e9e0c22b1ad58922/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","run_id":"7179dfb3e9e0c22b1ad58922","run_url":"/runs/7179dfb3e9e0c22b1ad58922","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.21875,"task":"sub1b-openbookqa"}],"revisions":["e7d4b8dea44125264077c2ce6d2f141379f999cd"],"source":"https://huggingface.co/SlayerLab/Slayer149-Balanced-ARC-E-ft-45500"}}