{"batch_size":1,"data_loading_seconds":0.07965102000162005,"dataset_revision":"92a54bd1384387c178d5a07140a2d85e0a3d12e1","dtype":"bfloat16","evaluator_source":true,"evidence":"CodeSOTA execution on White","finished":1792348351,"gpu":0,"id":"32dde2b675d08afa871097ec","metadata_sha256":"c1ccebb13035e4a37357fe3df4132152d674704d3df9bc07747f76a6cfd4c677","metric":"accuracy","model":"Qwen/Qwen2.5-VL-3B-Instruct","model_load_seconds":2.9611786449095234,"model_revision":"66285546d2b821cf421d4f5eb2576359d3770cd3","n":1000,"notes":"All 1,000 OCRBench v1 questions. Upstream answer-containment scoring; response verbosity can game this metric. Separate visual-text QA protocol, not document parsing/layout or table reconstruction. Image/decoding budgets and model chat template are CodeSOTA-pinned, not a claim of vendor inference parity.","parameters":3754622976,"peak_gpu_bytes":10769312768,"profile":"full","protocol":{"adapter":"ocrbench","area":"Documents","benchmark":"OCRBench v1 / full serial reference","contract":"Image + question \u2192 answer","coverage":"full","dataset":"echo840/OCRBench","dataset_config":null,"dataset_revision":"92a54bd1384387c178d5a07140a2d85e0a3d12e1","dataset_url":"https://huggingface.co/datasets/echo840/OCRBench","direction":"max","id":"ocrbench-v1-full-serial","license":"Upstream benchmark code MIT; component data retain their source terms","limit":1000,"metric":"accuracy","models":[{"id":"Qwen/Qwen2-VL-2B-Instruct","revision":"895c3a49bc3fa70a340399125c650a463535e71c"},{"id":"Qwen/Qwen2.5-VL-3B-Instruct","revision":"66285546d2b821cf421d4f5eb2576359d3770cd3"}],"name":"Visual text understanding / serial OCRBench reference","parent_task":null,"protocol_version":"codesota-ocrbench-v1-full-native-resolution-serial-v1","recipe":{"attn_implementation":"sdpa","batch_size":1,"do_sample":false,"dtype":"bfloat16","max_new_tokens":128,"max_pixels":12845056,"metadata_sha256":"c1ccebb13035e4a37357fe3df4132152d674704d3df9bc07747f76a6cfd4c677","min_pixels":3136,"num_beams":1,"prompt":"Original question only, using the pinned model chat template","upstream_revision":"4e363b3f499811b43d792ef0bb33932ef05eca27"},"runner_file":"evaluate_ocrbench_serial.py","selection":"All 1,000 questions; question, category, dataset and answer metadata matched against the pinned upstream list","source":"https://github.com/Yuliang-Liu/MultimodalOCR","split":"test","workstream":"ocr"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":683.3304858480114,"measurement_version":"gpu-slot-v1"},"runner_sha256":"50e622b9a313b1ee83d79c785095f871b4e136fedc03ef0a73e48ac22c46cce5","scope":"dcccfe2d1c65816985fdcc2bb76bfad20fcc76719fc61e7a4fd8a82df9ec1d29","score":0.813,"scoring_seconds":677.2785062999465,"seconds":680.3199762400473,"seed":42,"status":"completed","subgroups":{"Artistic Text Recognition":{"correct":45,"n":50},"Digit String Recognition":{"correct":39,"n":50},"Doc-oriented VQA":{"correct":162,"n":200},"Handwriting Recognition":{"correct":37,"n":50},"Handwritten Mathematical Expression Recognition":{"correct":46,"n":100},"Irregular Text Recognition":{"correct":46,"n":50},"Key Information Extraction":{"correct":176,"n":200},"Non-Semantic Text Recognition":{"correct":44,"n":50},"Regular Text Recognition":{"correct":48,"n":50},"Scene Text-centric VQA":{"correct":170,"n":200}},"task":"ocrbench-v1-full-serial","tf32":false,"upstream_max_score":1000,"upstream_total_score":813,"worker":"white-gpu0"}