{"batch_size":4,"data_loading_seconds":0.08789016108494252,"dataset_revision":"92a54bd1384387c178d5a07140a2d85e0a3d12e1","dtype":"bfloat16","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792347330,"gpu":1,"id":"c6ad7d2fef906cfb7173ab6f","metadata_sha256":"c1ccebb13035e4a37357fe3df4132152d674704d3df9bc07747f76a6cfd4c677","metric":"accuracy","model":"Qwen/Qwen2-VL-2B-Instruct","model_load_seconds":2.8704930680105463,"model_revision":"895c3a49bc3fa70a340399125c650a463535e71c","n":1000,"notes":"All 1,000 OCRBench v1 questions. Upstream answer-containment scoring; response verbosity can game this metric. Separate visual-text QA protocol, not document parsing/layout or table reconstruction. Image/decoding budgets and model chat template are CodeSOTA-pinned, not a claim of vendor inference parity.","parameters":2208985600,"peak_gpu_bytes":5899851264,"profile":"full","protocol":{"adapter":"ocrbench","area":"Documents","benchmark":"OCRBench v1 / full 1,000 questions","contract":"Image + question \u2192 answer","coverage":"full","dataset":"echo840/OCRBench","dataset_config":null,"dataset_revision":"92a54bd1384387c178d5a07140a2d85e0a3d12e1","dataset_url":"https://huggingface.co/datasets/echo840/OCRBench","direction":"max","id":"ocrbench-v1-full","license":"Upstream benchmark code MIT; component data retain their source terms","limit":1000,"metric":"accuracy","models":[{"id":"Qwen/Qwen2-VL-2B-Instruct","revision":"895c3a49bc3fa70a340399125c650a463535e71c"},{"id":"Qwen/Qwen2.5-VL-3B-Instruct","revision":"66285546d2b821cf421d4f5eb2576359d3770cd3"}],"name":"Visual text understanding / OCRBench v1","parent_task":null,"protocol_version":"codesota-ocrbench-v1-full-bf16-qwen-batch4-v1","recipe":{"attn_implementation":"sdpa","batch_size":4,"do_sample":false,"dtype":"bfloat16","max_new_tokens":128,"max_pixels":802816,"metadata_sha256":"c1ccebb13035e4a37357fe3df4132152d674704d3df9bc07747f76a6cfd4c677","min_pixels":200704,"num_beams":1,"prompt":"Original question only, using the pinned model chat template","upstream_revision":"4e363b3f499811b43d792ef0bb33932ef05eca27"},"runner_file":"evaluate_ocrbench.py","selection":"All 1,000 questions; question, category, dataset and answer metadata matched against the pinned upstream list","source":"https://github.com/Yuliang-Liu/MultimodalOCR","split":"test","workstream":"ocr"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":268.25494880997576,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd9caa01cf3bb32f2b5f470f953d191efed4632fd9263d386b21939ba5dc3eb1","scope":"d5b62220da83f9ebb4c271f820fa35b964ad4a0400651e30e94f5727f4a14007","score":0.49,"scoring_seconds":262.23469278099947,"seconds":265.19395936699584,"seed":42,"status":"completed","subgroups":{"Artistic Text Recognition":{"correct":31,"n":50},"Digit String Recognition":{"correct":17,"n":50},"Doc-oriented VQA":{"correct":89,"n":200},"Handwriting Recognition":{"correct":27,"n":50},"Handwritten Mathematical Expression Recognition":{"correct":13,"n":100},"Irregular Text Recognition":{"correct":35,"n":50},"Key Information Extraction":{"correct":90,"n":200},"Non-Semantic Text Recognition":{"correct":26,"n":50},"Regular Text Recognition":{"correct":41,"n":50},"Scene Text-centric VQA":{"correct":121,"n":200}},"task":"ocrbench-v1-full","tf32":false,"upstream_max_score":1000,"upstream_total_score":490,"worker":"white-gpu1"}