{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Actual unique parameter count from pinned model loaded in CodeSOTA FP32 live-BPB evaluation","model":"Qwen/Qwen2.5-0.5B-Instruct","parameters":494032768,"revision":"7ae557604adf67be50417f59c2c2f167def9a775","source":"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/tree/7ae557604adf67be50417f59c2c2f167def9a775","updated":1792199477}],"model":"Qwen/Qwen2.5-0.5B-Instruct","page":"/models/Qwen/Qwen2.5-0.5B-Instruct","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":1.596422306029126,"finished":1792205388,"gpu_slot_seconds":4.4209835240617394,"manifest_url":"/artifacts/df8c19749a220ea7a936f73e/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7ae557604adf67be50417f59c2c2f167def9a775","run_id":"df8c19749a220ea7a936f73e","run_url":"/runs/df8c19749a220ea7a936f73e","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.40625,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":1.4459956830833107,"finished":1792199620,"gpu_slot_seconds":4.521601940039545,"manifest_url":"/artifacts/da843c3af7d2f039d8dcc997/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7ae557604adf67be50417f59c2c2f167def9a775","run_id":"da843c3af7d2f039d8dcc997","run_url":"/runs/da843c3af7d2f039d8dcc997","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.59375,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":30.392575137899257,"finished":1792205767,"gpu_slot_seconds":33.362681315047666,"manifest_url":"/artifacts/13bc7aefda65a9714768d863/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"7ae557604adf67be50417f59c2c2f167def9a775","run_id":"13bc7aefda65a9714768d863","run_url":"/runs/13bc7aefda65a9714768d863","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6557239057239057,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.935960141941905,"finished":1792199307,"gpu_slot_seconds":7.876938486006111,"manifest_url":"/artifacts/141ceb6c09dc8f0156f70c0c/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7ae557604adf67be50417f59c2c2f167def9a775","run_id":"141ceb6c09dc8f0156f70c0c","run_url":"/runs/141ceb6c09dc8f0156f70c0c","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.71875,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":1.556583499070257,"finished":1792199370,"gpu_slot_seconds":4.62133755593095,"manifest_url":"/artifacts/5182981eed2309237804d9c3/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7ae557604adf67be50417f59c2c2f167def9a775","run_id":"5182981eed2309237804d9c3","run_url":"/runs/5182981eed2309237804d9c3","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.40625,"task":"sub1b-hellaswag"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":1.3361344808945432,"finished":1792205407,"gpu_slot_seconds":4.2211276579182595,"manifest_url":"/artifacts/f7313eab76420c052774eccf/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"7ae557604adf67be50417f59c2c2f167def9a775","run_id":"f7313eab76420c052774eccf","run_url":"/runs/f7313eab76420c052774eccf","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.34375,"task":"sub1b-openbookqa"}],"revisions":["7ae557604adf67be50417f59c2c2f167def9a775"],"source":"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct"}}