{"api_version":"codesota.v1","data":{"archived_results":[{"benchmark":"GSM8K","coverage":"subset","direction":"max","evaluation_seconds":32.15862700995058,"finished":1792193396,"gpu_slot_seconds":null,"manifest_url":"/artifacts/c5b99d0e49512667f8c0c91f/manifest.json","metric":"exact_match","n":32,"protocol_version":"codesota-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"c5b99d0e49512667f8c0c91f","run_url":"/runs/c5b99d0e49512667f8c0c91f","scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","score":0.03125,"task":"math"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.5907042749458924,"finished":1792193447,"gpu_slot_seconds":null,"manifest_url":"/artifacts/7af8b47f7ce3815bea0c442e/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"7af8b47f7ce3815bea0c442e","run_url":"/runs/7af8b47f7ce3815bea0c442e","scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","score":0.6875,"task":"reading"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":2.113465471076779,"finished":1792193498,"gpu_slot_seconds":null,"manifest_url":"/artifacts/b82a09a698a552ab21a0075b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"b82a09a698a552ab21a0075b","run_url":"/runs/b82a09a698a552ab21a0075b","scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","score":0.375,"task":"commonsense"},{"benchmark":"SST-2","coverage":"subset","direction":"max","evaluation_seconds":1.55905277794227,"finished":1792193133,"gpu_slot_seconds":null,"manifest_url":"/artifacts/2baa9449b32fde6f476f7e1f/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"2baa9449b32fde6f476f7e1f","run_url":"/runs/2baa9449b32fde6f476f7e1f","scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","score":0.9375,"task":"sentiment"},{"benchmark":"SNLI","coverage":"subset","direction":"max","evaluation_seconds":1.8406528600025922,"finished":1792193107,"gpu_slot_seconds":null,"manifest_url":"/artifacts/272e772751322114921d9ee6/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"272e772751322114921d9ee6","run_url":"/runs/272e772751322114921d9ee6","scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","score":0.375,"task":"entailment"},{"benchmark":"AI2 ARC-Challenge (test)","coverage":"subset","direction":"max","evaluation_seconds":2.2367252570111305,"finished":1792193772,"gpu_slot_seconds":null,"manifest_url":"/artifacts/ca6a72065b0a3873beda997b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"ca6a72065b0a3873beda997b","run_url":"/runs/ca6a72065b0a3873beda997b","scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","score":0.3125,"task":"arc-challenge-choice"},{"benchmark":"OpenBookQA main (test)","coverage":"subset","direction":"max","evaluation_seconds":2.220320450956933,"finished":1792193843,"gpu_slot_seconds":null,"manifest_url":"/artifacts/40d1459ca17c38921b30a6ea/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"40d1459ca17c38921b30a6ea","run_url":"/runs/40d1459ca17c38921b30a6ea","scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","score":0.40625,"task":"openbookqa-main-choice"}],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Unique parameters counted in the loaded evaluator model on White","model":"Qwen/Qwen2.5-0.5B","parameters":494032768,"revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","source":"https://huggingface.co/Qwen/Qwen2.5-0.5B/tree/060db6499f32faf8b98477b0a26969ef7d8b9987","updated":1792207103}],"model":"Qwen/Qwen2.5-0.5B","page":"/models/Qwen/Qwen2.5-0.5B","pending":[],"results":[{"benchmark":"AI2 ARC-Challenge (test)","coverage":"subset","direction":"max","evaluation_seconds":1.2409709830535576,"finished":1792198139,"gpu_slot_seconds":4.071349939098582,"manifest_url":"/artifacts/0d8c049a046e18e772d16952/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v2-batched","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"0d8c049a046e18e772d16952","run_url":"/runs/0d8c049a046e18e772d16952","scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","score":0.3125,"task":"arc-challenge-choice"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.2992077759699896,"finished":1792198193,"gpu_slot_seconds":7.375335416058078,"manifest_url":"/artifacts/ff345349801249c15997d2a6/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"ff345349801249c15997d2a6","run_url":"/runs/ff345349801249c15997d2a6","scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","score":0.6875,"task":"reading"},{"benchmark":"GSM8K","coverage":"subset","direction":"max","evaluation_seconds":12.287420978071168,"finished":1792198055,"gpu_slot_seconds":17.99193312006537,"manifest_url":"/artifacts/4749a153422d5cefccd16747/manifest.json","metric":"exact_match","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"4749a153422d5cefccd16747","run_url":"/runs/4749a153422d5cefccd16747","scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","score":0.03125,"task":"math"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":1.182585975038819,"finished":1792198076,"gpu_slot_seconds":4.171156510012224,"manifest_url":"/artifacts/2f544548b89cd6fed2d4e61d/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"2f544548b89cd6fed2d4e61d","run_url":"/runs/2f544548b89cd6fed2d4e61d","scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","score":0.375,"task":"commonsense"},{"benchmark":"HumanEval","coverage":"subset","direction":"max","evaluation_seconds":60.507072682026774,"finished":1792193082,"gpu_slot_seconds":null,"manifest_url":"/artifacts/264775b8e13a45705e029825/manifest.json","metric":"pass@1","n":32,"protocol_version":"codesota-pilot-v1","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"264775b8e13a45705e029825","run_url":"/runs/264775b8e13a45705e029825","scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","score":0.625,"task":"code-generation"},{"benchmark":"HumanEval (full 164, serial greedy pass@1)","coverage":"full","direction":"max","evaluation_seconds":341.7888783799717,"finished":1792207103,"gpu_slot_seconds":343.3049716670066,"manifest_url":"/artifacts/227324969d59c1a426063471/manifest.json","metric":"pass@1","n":164,"protocol_version":"codesota-humaneval-full-v2","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"227324969d59c1a426063471","run_url":"/runs/227324969d59c1a426063471","scope":"7388be4229d45cb22b346201154469558e2015772bcb2abde7f2fb7b0b7ad294","score":0.2865853658536585,"task":"code-generation-full"},{"benchmark":"OpenBookQA main (test)","coverage":"subset","direction":"max","evaluation_seconds":2.3172576900105923,"finished":1792198164,"gpu_slot_seconds":5.5228236329276115,"manifest_url":"/artifacts/f08b963541b415694a99b37b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v2-batched","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"f08b963541b415694a99b37b","run_url":"/runs/f08b963541b415694a99b37b","scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","score":0.40625,"task":"openbookqa-main-choice"},{"benchmark":"SNLI","coverage":"subset","direction":"max","evaluation_seconds":1.1434840289875865,"finished":1792198077,"gpu_slot_seconds":7.026732878992334,"manifest_url":"/artifacts/2933b280610cb03f5f2345c2/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"2933b280610cb03f5f2345c2","run_url":"/runs/2933b280610cb03f5f2345c2","scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","score":0.375,"task":"entailment"},{"benchmark":"SST-2","coverage":"subset","direction":"max","evaluation_seconds":1.1125338861020282,"finished":1792198193,"gpu_slot_seconds":8.02642539597582,"manifest_url":"/artifacts/efa7055f4d6480dd542a3abf/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","run_id":"efa7055f4d6480dd542a3abf","run_url":"/runs/efa7055f4d6480dd542a3abf","scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","score":0.9375,"task":"sentiment"}],"revisions":["060db6499f32faf8b98477b0a26969ef7d8b9987"],"source":"https://huggingface.co/Qwen/Qwen2.5-0.5B"}}