{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"SlayerLab/Slayer139-1.01","parameters":139279760,"revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","source":"https://huggingface.co/SlayerLab/Slayer139-1.01/tree/2e72c696d7c5cad73dc58606b0751b911dc94495","updated":1792199477}],"model":"SlayerLab/Slayer139-1.01","page":"/models/SlayerLab/Slayer139-1.01","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":1.9915327730122954,"finished":1792205307,"gpu_slot_seconds":4.823453077930026,"manifest_url":"/artifacts/74187fb9fcce65b425d80906/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","run_id":"74187fb9fcce65b425d80906","run_url":"/runs/74187fb9fcce65b425d80906","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.28125,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":2.0058413579827175,"finished":1792199347,"gpu_slot_seconds":4.971871975925751,"manifest_url":"/artifacts/4f8d3c01030b9ec948e6d8be/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","run_id":"4f8d3c01030b9ec948e6d8be","run_url":"/runs/4f8d3c01030b9ec948e6d8be","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.625,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":71.23654929595068,"finished":1792205803,"gpu_slot_seconds":74.19250445999205,"manifest_url":"/artifacts/11238d1b571f7ff20cbc8c5e/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","run_id":"11238d1b571f7ff20cbc8c5e","run_url":"/runs/11238d1b571f7ff20cbc8c5e","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6308922558922558,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.5199996100272983,"finished":1792199535,"gpu_slot_seconds":7.126529000001028,"manifest_url":"/artifacts/a64f0553519f2724df9f01c1/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","run_id":"a64f0553519f2724df9f01c1","run_url":"/runs/a64f0553519f2724df9f01c1","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5625,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":1.967458063038066,"finished":1792199327,"gpu_slot_seconds":4.97246181697119,"manifest_url":"/artifacts/384fccc53f471e23eb9acac8/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","run_id":"384fccc53f471e23eb9acac8","run_url":"/runs/384fccc53f471e23eb9acac8","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":1.9850552669959143,"finished":1792205348,"gpu_slot_seconds":4.771677452954464,"manifest_url":"/artifacts/9f341286de510514b18c1e3b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","run_id":"9f341286de510514b18c1e3b","run_url":"/runs/9f341286de510514b18c1e3b","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.28125,"task":"sub1b-openbookqa"}],"revisions":["2e72c696d7c5cad73dc58606b0751b911dc94495"],"source":"https://huggingface.co/SlayerLab/Slayer139-1.01"}}