{"batch_size":4,"data_loading_seconds":0.08758749696426094,"dataset_revision":"92a54bd1384387c178d5a07140a2d85e0a3d12e1","dtype":"bfloat16","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792347434,"gpu":0,"id":"b25ac4100084fad49c9f3938","metadata_sha256":"c1ccebb13035e4a37357fe3df4132152d674704d3df9bc07747f76a6cfd4c677","metric":"accuracy","model":"Qwen/Qwen2.5-VL-3B-Instruct","model_load_seconds":3.6028056730283424,"model_revision":"66285546d2b821cf421d4f5eb2576359d3770cd3","n":1000,"notes":"All 1,000 OCRBench v1 questions. Upstream answer-containment scoring; response verbosity can game this metric. Separate visual-text QA protocol, not document parsing/layout or table reconstruction. Image/decoding budgets and model chat template are CodeSOTA-pinned, not a claim of vendor inference parity.","parameters":3754622976,"peak_gpu_bytes":8416196608,"profile":"full","protocol":{"adapter":"ocrbench","area":"Documents","benchmark":"OCRBench v1 / full 1,000 questions","contract":"Image + question \u2192 answer","coverage":"full","dataset":"echo840/OCRBench","dataset_config":null,"dataset_revision":"92a54bd1384387c178d5a07140a2d85e0a3d12e1","dataset_url":"https://huggingface.co/datasets/echo840/OCRBench","direction":"max","id":"ocrbench-v1-full","license":"Upstream benchmark code MIT; component data retain their source terms","limit":1000,"metric":"accuracy","models":[{"id":"Qwen/Qwen2-VL-2B-Instruct","revision":"895c3a49bc3fa70a340399125c650a463535e71c"},{"id":"Qwen/Qwen2.5-VL-3B-Instruct","revision":"66285546d2b821cf421d4f5eb2576359d3770cd3"}],"name":"Visual text understanding / OCRBench v1","parent_task":null,"protocol_version":"codesota-ocrbench-v1-full-bf16-qwen-batch4-v1","recipe":{"attn_implementation":"sdpa","batch_size":4,"do_sample":false,"dtype":"bfloat16","max_new_tokens":128,"max_pixels":802816,"metadata_sha256":"c1ccebb13035e4a37357fe3df4132152d674704d3df9bc07747f76a6cfd4c677","min_pixels":200704,"num_beams":1,"prompt":"Original question only, using the pinned model chat template","upstream_revision":"4e363b3f499811b43d792ef0bb33932ef05eca27"},"runner_file":"evaluate_ocrbench.py","selection":"All 1,000 questions; question, category, dataset and answer metadata matched against the pinned upstream list","source":"https://github.com/Yuliang-Liu/MultimodalOCR","split":"test","workstream":"ocr"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":372.21522576897405,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd9caa01cf3bb32f2b5f470f953d191efed4632fd9263d386b21939ba5dc3eb1","scope":"d5b62220da83f9ebb4c271f820fa35b964ad4a0400651e30e94f5727f4a14007","score":0.607,"scoring_seconds":365.3466421830235,"seconds":369.03743726201355,"seed":42,"status":"completed","subgroups":{"Artistic Text Recognition":{"correct":42,"n":50},"Digit String Recognition":{"correct":27,"n":50},"Doc-oriented VQA":{"correct":109,"n":200},"Handwriting Recognition":{"correct":28,"n":50},"Handwritten Mathematical Expression Recognition":{"correct":25,"n":100},"Irregular Text Recognition":{"correct":39,"n":50},"Key Information Extraction":{"correct":119,"n":200},"Non-Semantic Text Recognition":{"correct":40,"n":50},"Regular Text Recognition":{"correct":43,"n":50},"Scene Text-centric VQA":{"correct":135,"n":200}},"task":"ocrbench-v1-full","tf32":false,"upstream_max_score":1000,"upstream_total_score":607,"worker":"white-gpu0"}