{"api_version":"codesota.v1","data":{"archived_results":[{"benchmark":"GSM8K","coverage":"subset","direction":"max","evaluation_seconds":124.60450974502601,"finished":1792192784,"gpu_slot_seconds":null,"manifest_url":"/artifacts/0ab8b157bc6f3029a19271ab/manifest.json","metric":"exact_match","n":32,"protocol_version":"codesota-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"0ab8b157bc6f3029a19271ab","run_url":"/runs/0ab8b157bc6f3029a19271ab","scope":"931808a3f82fdda3ddddbf7a73eec8abfdf78e7c460ffa9c507a3ac8693138ff","score":0.09375,"task":"math"},{"benchmark":"GSM8K","coverage":"subset","direction":"max","evaluation_seconds":49.895180064952,"finished":1792192998,"gpu_slot_seconds":null,"manifest_url":"/artifacts/14e52d72173736c52853bbf1/manifest.json","metric":"exact_match","n":32,"protocol_version":"codesota-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"14e52d72173736c52853bbf1","run_url":"/runs/14e52d72173736c52853bbf1","scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","score":0.09375,"task":"math"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":2.550903692957945,"finished":1792192887,"gpu_slot_seconds":null,"manifest_url":"/artifacts/206aa355b6974c770b0b29c4/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"206aa355b6974c770b0b29c4","run_url":"/runs/206aa355b6974c770b0b29c4","scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","score":0.6875,"task":"reading"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":2.7435219429899007,"finished":1792193158,"gpu_slot_seconds":null,"manifest_url":"/artifacts/4ecdb6321390abddd6ac658d/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"4ecdb6321390abddd6ac658d","run_url":"/runs/4ecdb6321390abddd6ac658d","scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","score":0.5,"task":"commonsense"},{"benchmark":"SST-2","coverage":"subset","direction":"max","evaluation_seconds":2.0898517509922385,"finished":1792193472,"gpu_slot_seconds":null,"manifest_url":"/artifacts/9c6acd69de4197860e28ebb2/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"9c6acd69de4197860e28ebb2","run_url":"/runs/9c6acd69de4197860e28ebb2","scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","score":0.625,"task":"sentiment"},{"benchmark":"SNLI","coverage":"subset","direction":"max","evaluation_seconds":2.3908391969744116,"finished":1792193422,"gpu_slot_seconds":null,"manifest_url":"/artifacts/5a2a497f01a1e75965c07849/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"5a2a497f01a1e75965c07849","run_url":"/runs/5a2a497f01a1e75965c07849","scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","score":0.375,"task":"entailment"},{"benchmark":"AI2 ARC-Challenge (test)","coverage":"subset","direction":"max","evaluation_seconds":2.842379883979447,"finished":1792193772,"gpu_slot_seconds":null,"manifest_url":"/artifacts/293bd712648d308f4ffe2539/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"293bd712648d308f4ffe2539","run_url":"/runs/293bd712648d308f4ffe2539","scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","score":0.46875,"task":"arc-challenge-choice"},{"benchmark":"OpenBookQA main (test)","coverage":"subset","direction":"max","evaluation_seconds":2.770547211985104,"finished":1792193843,"gpu_slot_seconds":null,"manifest_url":"/artifacts/795ccf7bff6558c30d67ce8c/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"795ccf7bff6558c30d67ce8c","run_url":"/runs/795ccf7bff6558c30d67ce8c","scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","score":0.40625,"task":"openbookqa-main-choice"}],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Unique parameters counted in the loaded evaluator model on White","model":"Qwen/Qwen2.5-1.5B","parameters":1543714304,"revision":"8faed761d45a263340a0528343f099c05c9a4323","source":"https://huggingface.co/Qwen/Qwen2.5-1.5B/tree/8faed761d45a263340a0528343f099c05c9a4323","updated":1792207883}],"model":"Qwen/Qwen2.5-1.5B","page":"/models/Qwen/Qwen2.5-1.5B","pending":[],"results":[{"benchmark":"AI2 ARC-Challenge (test)","coverage":"subset","direction":"max","evaluation_seconds":1.8925830170046538,"finished":1792198142,"gpu_slot_seconds":4.971455986960791,"manifest_url":"/artifacts/216e346c83988ade77ddfff0/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v2-batched","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"216e346c83988ade77ddfff0","run_url":"/runs/216e346c83988ade77ddfff0","scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","score":0.46875,"task":"arc-challenge-choice"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":2.026138138025999,"finished":1792198120,"gpu_slot_seconds":7.779247667989694,"manifest_url":"/artifacts/c6d083942f8d93dc672e019b/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"c6d083942f8d93dc672e019b","run_url":"/runs/c6d083942f8d93dc672e019b","scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","score":0.6875,"task":"reading"},{"benchmark":"GSM8K","coverage":"subset","direction":"max","evaluation_seconds":14.718171672080643,"finished":1792198057,"gpu_slot_seconds":20.445043627056293,"manifest_url":"/artifacts/aa5bc8b59a1fd979e61724eb/manifest.json","metric":"exact_match","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"aa5bc8b59a1fd979e61724eb","run_url":"/runs/aa5bc8b59a1fd979e61724eb","scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","score":0.09375,"task":"math"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":1.7728401749627665,"finished":1792198097,"gpu_slot_seconds":4.821826095925644,"manifest_url":"/artifacts/db55a33c25d74960ff389d24/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"db55a33c25d74960ff389d24","run_url":"/runs/db55a33c25d74960ff389d24","scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","score":0.5,"task":"commonsense"},{"benchmark":"HumanEval","coverage":"subset","direction":"max","evaluation_seconds":68.26958421000745,"finished":1792193254,"gpu_slot_seconds":null,"manifest_url":"/artifacts/501f999a9189ca77b745a388/manifest.json","metric":"pass@1","n":32,"protocol_version":"codesota-pilot-v1","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"501f999a9189ca77b745a388","run_url":"/runs/501f999a9189ca77b745a388","scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","score":0.75,"task":"code-generation"},{"benchmark":"HumanEval (full 164, serial greedy pass@1)","coverage":"full","direction":"max","evaluation_seconds":516.8492759029614,"finished":1792207883,"gpu_slot_seconds":518.2742866390618,"manifest_url":"/artifacts/599a2e85e8b3a5c55f01d349/manifest.json","metric":"pass@1","n":164,"protocol_version":"codesota-humaneval-full-v2","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"599a2e85e8b3a5c55f01d349","run_url":"/runs/599a2e85e8b3a5c55f01d349","scope":"7388be4229d45cb22b346201154469558e2015772bcb2abde7f2fb7b0b7ad294","score":0.3597560975609756,"task":"code-generation-full"},{"benchmark":"OpenBookQA main (test)","coverage":"subset","direction":"max","evaluation_seconds":1.7379119840916246,"finished":1792198159,"gpu_slot_seconds":4.821713284938596,"manifest_url":"/artifacts/9fd52efbe4976b40a9fd6646/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-choice-pilot-v2-batched","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"9fd52efbe4976b40a9fd6646","run_url":"/runs/9fd52efbe4976b40a9fd6646","scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","score":0.40625,"task":"openbookqa-main-choice"},{"benchmark":"SNLI","coverage":"subset","direction":"max","evaluation_seconds":1.6738499279599637,"finished":1792198122,"gpu_slot_seconds":7.377454819041304,"manifest_url":"/artifacts/bddd3154bd9cf23f3dece3fb/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"bddd3154bd9cf23f3dece3fb","run_url":"/runs/bddd3154bd9cf23f3dece3fb","scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","score":0.375,"task":"entailment"},{"benchmark":"SST-2","coverage":"subset","direction":"max","evaluation_seconds":1.5457230790052563,"finished":1792198099,"gpu_slot_seconds":7.177016686997376,"manifest_url":"/artifacts/b700ec1eed7b5ab1a5d28d49/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-pilot-v2-batched","revision":"8faed761d45a263340a0528343f099c05c9a4323","run_id":"b700ec1eed7b5ab1a5d28d49","run_url":"/runs/b700ec1eed7b5ab1a5d28d49","scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","score":0.625,"task":"sentiment"}],"revisions":["8faed761d45a263340a0528343f099c05c9a4323"],"source":"https://huggingface.co/Qwen/Qwen2.5-1.5B"}}