{"api_version":"codesota.v1","data":{"archived_results":[],"evidence_status":"CodeSOTA measurements only; model-card claims are not benchmark results","measured_parameters":[{"basis":"Unique parameters counted in the loaded evaluator model on White","model":"SlayerLab/Slayer149-ARC","parameters":149333081,"revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","source":"https://huggingface.co/SlayerLab/Slayer149-ARC/tree/0e6793ecfb8501dccc1c97dcd944731748eb8d17","updated":1792207349}],"model":"SlayerLab/Slayer149-ARC","page":"/models/SlayerLab/Slayer149-ARC","pending":[],"results":[{"benchmark":"ARC-Challenge","coverage":"subset","direction":"max","evaluation_seconds":2.1412392569473013,"finished":1792205363,"gpu_slot_seconds":4.971379082999192,"manifest_url":"/artifacts/a16311ef88b7fee9cfea8917/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","run_id":"a16311ef88b7fee9cfea8917","run_url":"/runs/a16311ef88b7fee9cfea8917","scope":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","score":0.40625,"task":"sub1b-arc-challenge"},{"benchmark":"ARC-Easy","coverage":"subset","direction":"max","evaluation_seconds":2.1125054139411077,"finished":1792199534,"gpu_slot_seconds":5.12225983699318,"manifest_url":"/artifacts/95f7f19d55290d88873e6474/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","run_id":"95f7f19d55290d88873e6474","run_url":"/runs/95f7f19d55290d88873e6474","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","score":0.78125,"task":"sub1b-arc-easy"},{"benchmark":"ARC-Easy (full test split, raw accuracy)","coverage":"full","direction":"max","evaluation_seconds":83.8724024950061,"finished":1792205869,"gpu_slot_seconds":86.7796474751085,"manifest_url":"/artifacts/da7edd3fa6604be87c71349a/manifest.json","metric":"accuracy","n":2376,"protocol_version":"codesota-sub1b-arc-easy-standard-v2","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","run_id":"da7edd3fa6604be87c71349a","run_url":"/runs/da7edd3fa6604be87c71349a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6658249158249159,"task":"sub1b-arc-easy-full"},{"benchmark":"BoolQ","coverage":"subset","direction":"max","evaluation_seconds":1.632466200971976,"finished":1792199558,"gpu_slot_seconds":7.1771505429642275,"manifest_url":"/artifacts/dce70413303e4781f185a0ac/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","run_id":"dce70413303e4781f185a0ac","run_url":"/runs/dce70413303e4781f185a0ac","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","score":0.5,"task":"sub1b-boolq"},{"benchmark":"HellaSwag","coverage":"subset","direction":"max","evaluation_seconds":2.1256928420625627,"finished":1792199368,"gpu_slot_seconds":5.072574998019263,"manifest_url":"/artifacts/4bab151d17bfaf2727299f60/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","run_id":"4bab151d17bfaf2727299f60","run_url":"/runs/4bab151d17bfaf2727299f60","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","score":0.34375,"task":"sub1b-hellaswag"},{"benchmark":"HumanEval (full 164, serial greedy pass@1)","coverage":"full","direction":"max","evaluation_seconds":588.2384040479083,"finished":1792207349,"gpu_slot_seconds":589.406429530005,"manifest_url":"/artifacts/b5ac56547d5088dab7e49bc4/manifest.json","metric":"pass@1","n":164,"protocol_version":"codesota-humaneval-full-v2","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","run_id":"b5ac56547d5088dab7e49bc4","run_url":"/runs/b5ac56547d5088dab7e49bc4","scope":"7388be4229d45cb22b346201154469558e2015772bcb2abde7f2fb7b0b7ad294","score":0.006097560975609756,"task":"code-generation-full"},{"benchmark":"OpenBookQA","coverage":"subset","direction":"max","evaluation_seconds":2.1152150300331414,"finished":1792205206,"gpu_slot_seconds":4.976126773050055,"manifest_url":"/artifacts/1b9463f9997376b742df0888/manifest.json","metric":"accuracy","n":32,"protocol_version":"codesota-sub1b-choice-v1","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","run_id":"1b9463f9997376b742df0888","run_url":"/runs/1b9463f9997376b742df0888","scope":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","score":0.375,"task":"sub1b-openbookqa"}],"revisions":["0e6793ecfb8501dccc1c97dcd944731748eb8d17"],"source":"https://huggingface.co/SlayerLab/Slayer149-ARC"}}