{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"HuggingFaceTB/SmolLM2-135M-Instruct","parameters":134515008,"revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","source":"https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct/tree/12fd25f77366fa6b3b4b768ec3050bf629380bac","updated":1792199477}],"model":"HuggingFaceTB/SmolLM2-135M-Instruct","page":"/models/HuggingFaceTB/SmolLM2-135M-Instruct","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":1.027235188987106,"finished":1792205283,"gpu_slot_seconds":4.020824210019782,"manifest_url":"/artifacts/59fbacde6accd669b97a31f6/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","run_id":"59fbacde6accd669b97a31f6","run_url":"/runs/59fbacde6accd669b97a31f6","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.3125,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":1.0435598699841648,"finished":1792199513,"gpu_slot_seconds":4.070911542046815,"manifest_url":"/artifacts/83ca3844b75d12aea722afae/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","run_id":"83ca3844b75d12aea722afae","run_url":"/runs/83ca3844b75d12aea722afae","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.5625,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":13.22382045397535,"finished":1792206004,"gpu_slot_seconds":16.103333471924998,"manifest_url":"/artifacts/ab13c5aaf35df5134fcb5566/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","run_id":"ab13c5aaf35df5134fcb5566","run_url":"/runs/ab13c5aaf35df5134fcb5566","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.539983164983165,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.0593773149885237,"finished":1792199329,"gpu_slot_seconds":6.624368945020251,"manifest_url":"/artifacts/3b9bc33f16b6187cceb7819f/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","run_id":"3b9bc33f16b6187cceb7819f","run_url":"/runs/3b9bc33f16b6187cceb7819f","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.21875,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":0.9418995419982821,"finished":1792199452,"gpu_slot_seconds":3.971061061020009,"manifest_url":"/artifacts/a0a1b916aa43bef9dc349f52/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","run_id":"a0a1b916aa43bef9dc349f52","run_url":"/runs/a0a1b916aa43bef9dc349f52","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.28125,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":0.8846409409306943,"finished":1792205266,"gpu_slot_seconds":3.769673411967233,"manifest_url":"/artifacts/4aece19226d58f5232e9053f/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","run_id":"4aece19226d58f5232e9053f","run_url":"/runs/4aece19226d58f5232e9053f","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.28125,"task":"sub1b-openbookqa"}],"revisions":["12fd25f77366fa6b3b4b768ec3050bf629380bac"],"source":"https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct"}}