{"dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","dependency_sources":{"humaneval-systems.json":"1ec4c8db6d55ce5d03aa99e1910844139f24dc4c6146e312cdd2eaeed81da271","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"humaneval-full-v2","finished":1792207349,"generation":{"add_bos":false,"chat_template":false,"hf_kv_cache":true,"max_new_tokens":256,"native_kv_cache":false,"precision":"float32","prompt_token_limit":768,"samples_per_problem":1,"serial":true,"strategy":"greedy","tf32":false,"total_context_limit":1024},"gpu":0,"id":"b5ac56547d5088dab7e49bc4","measured_cost_usd":null,"metric":"pass@1","model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":164,"notes":"Full 164-problem HumanEval; one greedy 256-token completion per problem; pass@1 is fraction passing original isolated unit tests. FP32; no chat template/BOS. Fixed top-level completion extraction; no retries or repair. Native generation recomputes context per token; HF uses a serial KV cache. Not HumanEval+ or an unrestricted-generation score.","parameters":149333081,"peak_gpu_bytes":677814784,"profile":"full","protocol":{"adapter":"code","area":"Code","benchmark":"HumanEval (full 164, serial greedy pass@1)","catalogue_id":"humaneval","contract":"Function signature + docstring \u2192 Python function","coverage":"full","dataset":"openai/openai_humaneval","dataset_config":"openai_humaneval","dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","dataset_url":"https://huggingface.co/datasets/openai/openai_humaneval","direction":"max","id":"code-generation-full","license":["mit"],"limit":164,"metric":"pass@1","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-Coder-0.5B","revision":"8123ea2e9354afb7ffcc6c8641d1b2f5ecf18301"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"},{"id":"Qwen/Qwen2.5-Coder-1.5B","revision":"df3ce67c0e24480f20468b6ef2894622d69eb73b"}],"name":"HumanEval / full native and HF code comparison","parent_task":"code-generation","priority_models":["SlayerLab/Slayer149-ARC"],"protocol_version":"codesota-humaneval-full-v2","runner_file":"evaluate_humaneval_native.py","selection":"All 164 problems; one serial greedy 256-token completion each; original isolated HumanEval unit tests","source":"https://github.com/openai/human-eval","split":"test","workstream":"coding"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":589.406429530005,"measurement_version":"gpu-slot-v1"},"runner_sha256":"496a63534a2dcc2f9bf860166acaaeb0f15f41abd137c89881b8c7674bb321d6","sandbox_image":"python@sha256:e88e9763f943ec1834f992a4b51e0f24500486803e8bc534e5767af9ea65f6ce","scope":"7388be4229d45cb22b346201154469558e2015772bcb2abde7f2fb7b0b7ad294","score":0.006097560975609756,"seconds":588.2384040479083,"seed":42,"selection":"full pinned HumanEval test split; all 164 distinct problems in source order","status":"completed","task":"code-generation-full","worker":"white-gpu0"}