{"dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792193082,"id":"264775b8e13a45705e029825","measured_cost_usd":null,"metric":"pass@1","model":"Qwen/Qwen2.5-0.5B","model_revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","n":32,"notes":"Pilot subset, custom CodeSOTA protocol. Not an official full-benchmark score.","peak_gpu_bytes":1013277696,"profile":"pilot-32","protocol":{"adapter":"code","area":"Code","benchmark":"HumanEval","contract":"Function signature + docstring \u2192 Python function","dataset":"openai/openai_humaneval","dataset_config":"openai_humaneval","dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","dataset_url":"https://huggingface.co/datasets/openai/openai_humaneval","direction":"max","id":"code-generation","license":["mit"],"limit":32,"metric":"pass@1","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Code generation","protocol_version":"codesota-pilot-v1","source":"https://github.com/openai/human-eval","split":"test"},"runner_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","sandbox_image":"python@sha256:e88e9763f943ec1834f992a4b51e0f24500486803e8bc534e5767af9ea65f6ce","scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","score":0.625,"seconds":60.507072682026774,"seed":42,"selection":"first 32 valid examples in pinned split order","status":"completed","task":"code-generation"}