{"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792192887,"id":"206aa355b6974c770b0b29c4","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-1.5B","model_revision":"8faed761d45a263340a0528343f099c05c9a4323","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":3194701824,"profile":"pilot-32","protocol":{"adapter":"reading","area":"Language","benchmark":"BoolQ","contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","direction":"max","id":"reading","license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Reading comprehension","protocol_version":"codesota-pilot-v1","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","score":0.6875,"seconds":2.550903692957945,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"reading"}