{"dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193107,"id":"272e772751322114921d9ee6","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1021412352,"profile":"pilot-32","protocol":{"adapter":"entailment","area":"Language","benchmark":"SNLI","contract":"Premise + hypothesis \u2192 entailment / neutral / contradiction","dataset":"stanfordnlp/snli","dataset_config":"plain_text","dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","dataset_url":"https://huggingface.co/datasets/stanfordnlp/snli","direction":"max","id":"entailment","license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Natural language inference","protocol_version":"codesota-pilot-v1","source":"https://nlp.stanford.edu/projects/snli/","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","score":0.375,"seconds":1.8406528600025922,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"entailment"}