{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"HuggingFaceTB/SmolLM2-360M-Instruct","parameters":361821120,"revision":"a10cc1512eabd3dde888204e902eca88bddb4951","source":"https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct/tree/a10cc1512eabd3dde888204e902eca88bddb4951","updated":1792199477}],"model":"HuggingFaceTB/SmolLM2-360M-Instruct","page":"/models/HuggingFaceTB/SmolLM2-360M-Instruct","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":1.3765633009606972,"finished":1792205226,"gpu_slot_seconds":4.271645738044754,"manifest_url":"/artifacts/2c30cf2ceb6a87338eb33d7b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"a10cc1512eabd3dde888204e902eca88bddb4951","run_id":"2c30cf2ceb6a87338eb33d7b","run_url":"/runs/2c30cf2ceb6a87338eb33d7b","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.28125,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":1.2155649489723146,"finished":1792199513,"gpu_slot_seconds":4.170912915025838,"manifest_url":"/artifacts/933abc78ddc9563ed3bfe5c3/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"a10cc1512eabd3dde888204e902eca88bddb4951","run_id":"933abc78ddc9563ed3bfe5c3","run_url":"/runs/933abc78ddc9563ed3bfe5c3","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.4375,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":23.65220459201373,"finished":1792206061,"gpu_slot_seconds":26.592118525062688,"manifest_url":"/artifacts/e494a3b38b3224efe73f9491/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"a10cc1512eabd3dde888204e902eca88bddb4951","run_id":"e494a3b38b3224efe73f9491","run_url":"/runs/e494a3b38b3224efe73f9491","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.5652356902356902,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.6528386661084369,"finished":1792199579,"gpu_slot_seconds":7.425343442009762,"manifest_url":"/artifacts/eb5517fac2e0ec0cb50196c5/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"a10cc1512eabd3dde888204e902eca88bddb4951","run_id":"eb5517fac2e0ec0cb50196c5","run_url":"/runs/eb5517fac2e0ec0cb50196c5","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.34375,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":1.26590998203028,"finished":1792199450,"gpu_slot_seconds":4.170025971019641,"manifest_url":"/artifacts/9733ef9ec08a5e033ded7a5e/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"a10cc1512eabd3dde888204e902eca88bddb4951","run_id":"9733ef9ec08a5e033ded7a5e","run_url":"/runs/9733ef9ec08a5e033ded7a5e","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.46875,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":1.1336964400252327,"finished":1792205343,"gpu_slot_seconds":4.0714760549599305,"manifest_url":"/artifacts/9758fdf769c2fbc0173271ee/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"a10cc1512eabd3dde888204e902eca88bddb4951","run_id":"9758fdf769c2fbc0173271ee","run_url":"/runs/9758fdf769c2fbc0173271ee","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.4375,"task":"sub1b-openbookqa"}],"revisions":["a10cc1512eabd3dde888204e902eca88bddb4951"],"source":"https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct"}}