{"api_version":"codesota.v1","data":{"areas":[{"id":"language-knowledge","name":"Language & Knowledge","tasks":[{"id":"multi-step-reasoning","legacy_url":"https://www.codesota.com/tasks/multi-step-reasoning","name":"Multi-step Reasoning"},{"id":"mathematical-reasoning","legacy_url":"https://www.codesota.com/tasks/mathematical-reasoning","name":"Mathematical Reasoning"},{"id":"commonsense-reasoning","legacy_url":"https://www.codesota.com/tasks/commonsense-reasoning","name":"Commonsense Reasoning"},{"id":"question-answering","legacy_url":"https://www.codesota.com/tasks/question-answering","name":"Question Answering"},{"id":"feature-extraction","legacy_url":"https://www.codesota.com/tasks/feature-extraction","name":"Text Embeddings"},{"id":"text-summarization","legacy_url":"https://www.codesota.com/tasks/text-summarization","name":"Text Summarization"},{"id":"logical-reasoning","legacy_url":"https://www.codesota.com/tasks/logical-reasoning","name":"Logical Reasoning"},{"id":"text-ranking","legacy_url":"https://www.codesota.com/tasks/text-ranking","name":"Text Ranking"},{"id":"natural-language-inference","legacy_url":"https://www.codesota.com/tasks/natural-language-inference","name":"Natural Language Inference"},{"id":"named-entity-recognition","legacy_url":"https://www.codesota.com/tasks/named-entity-recognition","name":"Named Entity Recognition"},{"id":"arithmetic-reasoning","legacy_url":"https://www.codesota.com/tasks/arithmetic-reasoning","name":"Arithmetic Reasoning"},{"id":"entity-linking","legacy_url":"https://www.codesota.com/tasks/entity-linking","name":"Entity Linking"},{"id":"knowledge-graph-completion","legacy_url":"https://www.codesota.com/tasks/knowledge-graph-completion","name":"Knowledge Graph Completion"},{"id":"relation-extraction","legacy_url":"https://www.codesota.com/tasks/relation-extraction","name":"Relation Extraction"},{"id":"semantic-similarity","legacy_url":"https://www.codesota.com/tasks/semantic-similarity","name":"Semantic Textual Similarity"},{"id":"table-question-answering","legacy_url":"https://www.codesota.com/tasks/table-question-answering","name":"Table Question Answering"},{"id":"reading-comprehension","legacy_url":"https://www.codesota.com/tasks/reading-comprehension","name":"Reading Comprehension"}]},{"id":"vision-documents","name":"Vision & Documents","tasks":[{"id":"document-ocr","legacy_url":"https://www.codesota.com/tasks/document-ocr","name":"Document OCR"},{"id":"scene-text-detection","legacy_url":"https://www.codesota.com/tasks/scene-text-detection","name":"Scene Text Detection"},{"id":"document-parsing","legacy_url":"https://www.codesota.com/tasks/document-parsing","name":"Document Parsing"},{"id":"document-layout-analysis","legacy_url":"https://www.codesota.com/tasks/document-layout-analysis","name":"Document Layout Analysis"},{"id":"scene-text-recognition","legacy_url":"https://www.codesota.com/tasks/scene-text-recognition","name":"Scene Text Recognition"},{"id":"object-detection","legacy_url":"https://www.codesota.com/tasks/object-detection","name":"Object Detection"},{"id":"image-classification","legacy_url":"https://www.codesota.com/tasks/image-classification","name":"Image Classification"},{"id":"table-recognition","legacy_url":"https://www.codesota.com/tasks/table-recognition","name":"Table Recognition"},{"id":"ocr-capabilities","legacy_url":"https://www.codesota.com/tasks/ocr-capabilities","name":"General OCR Capabilities"},{"id":"document-classification","legacy_url":"https://www.codesota.com/tasks/document-classification","name":"Document Image Classification"},{"id":"handwriting-recognition","legacy_url":"https://www.codesota.com/tasks/handwriting-recognition","name":"Handwriting Recognition"},{"id":"document-understanding","legacy_url":"https://www.codesota.com/tasks/document-understanding","name":"Document Understanding"},{"id":"semantic-segmentation","legacy_url":"https://www.codesota.com/tasks/semantic-segmentation","name":"Semantic Segmentation"},{"id":"video-classification","legacy_url":"https://www.codesota.com/tasks/video-classification","name":"Video classification"},{"id":"image-segmentation","legacy_url":"https://www.codesota.com/tasks/image-segmentation","name":"Image segmentation"},{"id":"keypoint-detection","legacy_url":"https://www.codesota.com/tasks/keypoint-detection","name":"Keypoint Detection"},{"id":"ocr","legacy_url":"https://www.codesota.com/tasks/ocr","name":"OCR"}]},{"id":"audio-speech","name":"Audio & Speech","tasks":[{"id":"speech-recognition","legacy_url":"https://www.codesota.com/tasks/speech-recognition","name":"Speech Recognition"},{"id":"audio-captioning","legacy_url":"https://www.codesota.com/tasks/audio-captioning","name":"Audio Captioning"},{"id":"music-generation","legacy_url":"https://www.codesota.com/tasks/music-generation","name":"Music Generation"},{"id":"sound-event-detection","legacy_url":"https://www.codesota.com/tasks/sound-event-detection","name":"Sound Event Detection"},{"id":"speaker-verification","legacy_url":"https://www.codesota.com/tasks/speaker-verification","name":"Speaker Verification"},{"id":"speech-translation","legacy_url":"https://www.codesota.com/tasks/speech-translation","name":"Speech Translation"},{"id":"speech-enhancement","legacy_url":"https://www.codesota.com/tasks/speech-enhancement","name":"Speech Enhancement"}]},{"id":"multimodal-media","name":"Multimodal Media","tasks":[{"id":"visual-question-answering","legacy_url":"https://www.codesota.com/tasks/visual-question-answering","name":"Visual Question Answering"},{"id":"video-understanding","legacy_url":"https://www.codesota.com/tasks/video-understanding","name":"Video Understanding"},{"id":"text-to-image","legacy_url":"https://www.codesota.com/tasks/text-to-image","name":"Text-to-Image Generation"},{"id":"image-captioning","legacy_url":"https://www.codesota.com/tasks/image-captioning","name":"Image Captioning"}]},{"id":"code-software-engineering","name":"Code & Software Engineering","tasks":[{"id":"code-generation","legacy_url":"https://www.codesota.com/tasks/code-generation","name":"Code Generation"},{"id":"react-native-code-generation","legacy_url":"https://www.codesota.com/tasks/react-native-code-generation","name":"React Native Code Generation"},{"id":"code-translation","legacy_url":"https://www.codesota.com/tasks/code-translation","name":"Code Translation"},{"id":"bug-detection","legacy_url":"https://www.codesota.com/tasks/bug-detection","name":"Bug Detection"},{"id":"code-completion","legacy_url":"https://www.codesota.com/tasks/code-completion","name":"Code Completion"},{"id":"program-repair","legacy_url":"https://www.codesota.com/tasks/program-repair","name":"Program Repair"},{"id":"code-summarization","legacy_url":"https://www.codesota.com/tasks/code-summarization","name":"Code Summarization"}]},{"id":"agents-tool-use","name":"Agents & Tool Use","tasks":[{"id":"swe-bench","legacy_url":"https://www.codesota.com/tasks/swe-bench","name":"SWE-bench"},{"id":"agents","legacy_url":"https://www.codesota.com/tasks/agents","name":"Task agents"},{"id":"web-agents","legacy_url":"https://www.codesota.com/tasks/web-agents","name":"Web & Desktop Agents"},{"id":"autonomous-coding","legacy_url":"https://www.codesota.com/tasks/autonomous-coding","name":"Autonomous Coding"},{"id":"tool-use","legacy_url":"https://www.codesota.com/tasks/tool-use","name":"Tool Use"},{"id":"hcast","legacy_url":"https://www.codesota.com/tasks/hcast","name":"HCAST"},{"id":"re-bench","legacy_url":"https://www.codesota.com/tasks/re-bench","name":"RE-Bench"},{"id":"time-horizon","legacy_url":"https://www.codesota.com/tasks/time-horizon","name":"Time Horizon"},{"id":"bioinformatics-agents","legacy_url":"https://www.codesota.com/tasks/bioinformatics-agents","name":"Bioinformatics Agents"}]},{"id":"structured-data-forecasting","name":"Structured Data & Forecasting","tasks":[{"id":"node-classification","legacy_url":"https://www.codesota.com/tasks/node-classification","name":"Node Classification"},{"id":"tabular-classification","legacy_url":"https://www.codesota.com/tasks/tabular-classification","name":"Tabular Classification"},{"id":"link-prediction","legacy_url":"https://www.codesota.com/tasks/link-prediction","name":"Link Prediction"},{"id":"molecular-property-prediction","legacy_url":"https://www.codesota.com/tasks/molecular-property-prediction","name":"Molecular Property Prediction"},{"id":"tabular-regression","legacy_url":"https://www.codesota.com/tasks/tabular-regression","name":"Tabular Regression"},{"id":"tabular-ml","legacy_url":"https://www.codesota.com/tasks/tabular-ml","name":"Tabular Machine Learning"}]},{"id":"robotics-control-rl","name":"Robotics, Control & RL","tasks":[{"id":"atari-games","legacy_url":"https://www.codesota.com/tasks/atari-games","name":"Atari Games"},{"id":"continuous-control","legacy_url":"https://www.codesota.com/tasks/continuous-control","name":"Continuous Control"}]},{"id":"science-medicine-industry","name":"Science, Medicine & Industry","tasks":[{"id":"disease-classification","legacy_url":"https://www.codesota.com/tasks/disease-classification","name":"Disease Classification"},{"id":"anomaly-detection","legacy_url":"https://www.codesota.com/tasks/anomaly-detection","name":"Anomaly Detection"},{"id":"medical-image-segmentation","legacy_url":"https://www.codesota.com/tasks/medical-image-segmentation","name":"Medical Image Segmentation"}]}],"beta_extensions":[{"area_id":"language-knowledge","id":"sentiment-classification","name":"Sentiment classification"}],"evaluation_profiles":[{"area_id":"code-software-engineering","canonical_task_ids":["code-generation"],"coverage":"subset","domains":[],"id":"code-generation","measured_runs":2,"methods":["code"],"modalities":["text"],"models":2,"name":"HumanEval","safety_properties":[],"scope_id":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","url":"/tasks/code-generation"},{"area_id":"language-knowledge","canonical_task_ids":["mathematical-reasoning"],"coverage":"subset","domains":[],"id":"math","measured_runs":2,"methods":["math"],"modalities":["text"],"models":2,"name":"GSM8K","safety_properties":[],"scope_id":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","url":"/tasks/math"},{"area_id":"language-knowledge","canonical_task_ids":["reading-comprehension"],"coverage":"subset","domains":[],"id":"reading","measured_runs":2,"methods":["reading"],"modalities":["text"],"models":2,"name":"BoolQ","safety_properties":[],"scope_id":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","url":"/tasks/reading"},{"area_id":"language-knowledge","canonical_task_ids":["commonsense-reasoning"],"coverage":"subset","domains":[],"id":"commonsense","measured_runs":2,"methods":["commonsense"],"modalities":["text"],"models":2,"name":"HellaSwag","safety_properties":[],"scope_id":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","url":"/tasks/commonsense"},{"area_id":"language-knowledge","canonical_task_ids":["sentiment-classification"],"coverage":"subset","domains":[],"id":"sentiment","measured_runs":2,"methods":["sentiment"],"modalities":["text"],"models":2,"name":"SST-2","safety_properties":[],"scope_id":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","url":"/tasks/sentiment"},{"area_id":"language-knowledge","canonical_task_ids":["natural-language-inference"],"coverage":"subset","domains":[],"id":"entailment","measured_runs":2,"methods":["entailment"],"modalities":["text"],"models":2,"name":"SNLI","safety_properties":[],"scope_id":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","url":"/tasks/entailment"},{"area_id":"language-knowledge","canonical_task_ids":["semantic-similarity"],"coverage":"subset","domains":[],"id":"similarity","measured_runs":1,"methods":["similarity"],"modalities":["text"],"models":1,"name":"STS Benchmark","safety_properties":[],"scope_id":"5048133e477a3c13bf97c614b99b2d36a458391bb2a9480cd9b38189f2608b67","url":"/tasks/similarity"},{"area_id":"vision-documents","canonical_task_ids":["handwriting-recognition"],"coverage":"subset","domains":[],"id":"handwriting","measured_runs":1,"methods":["ocr"],"modalities":["image"],"models":1,"name":"IAM lines","safety_properties":[],"scope_id":"87f7f645d4ae1c902e4f3c4d821290e2fa08df318242d5d78a563467763e36ec","url":"/tasks/handwriting"},{"area_id":"audio-speech","canonical_task_ids":["speech-recognition"],"coverage":"subset","domains":[],"id":"transcription","measured_runs":1,"methods":["asr"],"modalities":["audio"],"models":1,"name":"LibriSpeech test-clean","safety_properties":[],"scope_id":"9c365ee78df50fa725b4c03d695f65d2b638be00c4687dc66be70b284c865a34","url":"/tasks/transcription"},{"area_id":"vision-documents","canonical_task_ids":["image-classification"],"coverage":"subset","domains":[],"id":"image-classification","measured_runs":1,"methods":["vision"],"modalities":["image"],"models":1,"name":"CIFAR-10","safety_properties":[],"scope_id":"d4a779c6ba33be115fc5e3302e45bb70d7c90040f954b89dab97beb5a2914649","url":"/tasks/image-classification"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"subset","domains":["science"],"id":"arc-challenge-choice","measured_runs":2,"methods":["choice"],"modalities":["text"],"models":2,"name":"AI2 ARC-Challenge (test)","safety_properties":[],"scope_id":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","url":"/tasks/arc-challenge-choice"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"subset","domains":["science"],"id":"openbookqa-main-choice","measured_runs":2,"methods":["choice"],"modalities":["text"],"models":2,"name":"OpenBookQA main (test)","safety_properties":[],"scope_id":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","url":"/tasks/openbookqa-main-choice"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"subset","domains":["science"],"id":"sub1b-arc-easy","measured_runs":11,"methods":["choice"],"modalities":["text"],"models":11,"name":"ARC-Easy","safety_properties":[],"scope_id":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","url":"/tasks/sub1b-arc-easy"},{"area_id":"language-knowledge","canonical_task_ids":["commonsense-reasoning"],"coverage":"subset","domains":[],"id":"sub1b-hellaswag","measured_runs":11,"methods":["commonsense"],"modalities":["text"],"models":11,"name":"HellaSwag","safety_properties":[],"scope_id":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","url":"/tasks/sub1b-hellaswag"},{"area_id":"language-knowledge","canonical_task_ids":["reading-comprehension"],"coverage":"subset","domains":[],"id":"sub1b-boolq","measured_runs":11,"methods":["reading"],"modalities":["text"],"models":11,"name":"BoolQ","safety_properties":[],"scope_id":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","url":"/tasks/sub1b-boolq"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"subset","domains":["science"],"id":"sub1b-arc-challenge","measured_runs":11,"methods":["choice"],"modalities":["text"],"models":11,"name":"ARC-Challenge","safety_properties":[],"scope_id":"e14efec58a56b426d1075a48d781531cc86194c0ae545ccae25ab8c742c4cdfa","url":"/tasks/sub1b-arc-challenge"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"subset","domains":["science"],"id":"sub1b-openbookqa","measured_runs":11,"methods":["choice"],"modalities":["text"],"models":11,"name":"OpenBookQA","safety_properties":[],"scope_id":"14346d6c23a8cd6b6720f20dbe91c6ebdbfbb7721023028eb9705c7ca8a51fa7","url":"/tasks/sub1b-openbookqa"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"full","domains":["science"],"id":"sub1b-arc-easy-full","measured_runs":11,"methods":["choice"],"modalities":["text"],"models":11,"name":"ARC-Easy (full test split, raw accuracy)","safety_properties":[],"scope_id":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","url":"/tasks/sub1b-arc-easy-full"},{"area_id":"code-software-engineering","canonical_task_ids":["code-generation"],"coverage":"full","domains":[],"id":"code-generation-full","measured_runs":5,"methods":["code"],"modalities":["text"],"models":5,"name":"HumanEval (full 164, serial greedy pass@1)","safety_properties":[],"scope_id":"7388be4229d45cb22b346201154469558e2015772bcb2abde7f2fb7b0b7ad294","url":"/tasks/code-generation-full"},{"area_id":"vision-documents","canonical_task_ids":["handwriting-recognition"],"coverage":"subset-256","domains":[],"id":"ocr-handwriting-256","measured_runs":3,"methods":["ocr"],"modalities":["image"],"models":3,"name":"IAM lines / 256-line subset","safety_properties":[],"scope_id":"43b6e2903c8e8a7a053dbd24dd6eda620db738365eeb9319b6c582768fa103f4","url":"/tasks/ocr-handwriting-256"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"full","domains":["science"],"id":"qwen35-arc-easy-full","measured_runs":15,"methods":["choice"],"modalities":["text"],"models":15,"name":"ARC-Easy / Qwen3.5 same-base cohort","safety_properties":[],"scope_id":"84fe51c6790b19376ad4a234c742b80e0f73e3eb0d6a7257d48e5315d6a7c078","url":"/tasks/qwen35-arc-easy-full"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"full","domains":["science"],"id":"qwen35-arc-challenge-full","measured_runs":15,"methods":["choice"],"modalities":["text"],"models":15,"name":"ARC-Challenge / Qwen3.5 same-base cohort","safety_properties":[],"scope_id":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","url":"/tasks/qwen35-arc-challenge-full"},{"area_id":"language-knowledge","canonical_task_ids":["question-answering"],"coverage":"full","domains":["science"],"id":"native-arc-challenge-common-core-full","measured_runs":11,"methods":["choice"],"modalities":["text"],"models":11,"name":"ARC-Challenge (full test, raw likelihood accuracy)","safety_properties":[],"scope_id":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","url":"/tasks/native-arc-challenge-common-core-full"},{"area_id":"language-knowledge","canonical_task_ids":["commonsense-reasoning"],"coverage":"full","domains":[],"id":"native-hellaswag-common-core-full","measured_runs":11,"methods":["choice"],"modalities":["text"],"models":11,"name":"HellaSwag (full validation, character-normalized accuracy)","safety_properties":[],"scope_id":"9ba01dfd8954831947fc9d9e821f5c27a78509f522b87d05d07b1c4f172ae0e6","url":"/tasks/native-hellaswag-common-core-full"},{"area_id":"language-knowledge","canonical_task_ids":["reading-comprehension"],"coverage":"full","domains":[],"id":"native-boolq-common-core-full","measured_runs":11,"methods":["choice"],"modalities":["text"],"models":11,"name":"BoolQ (full validation, raw likelihood accuracy)","safety_properties":[],"scope_id":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","url":"/tasks/native-boolq-common-core-full"},{"area_id":"language-knowledge","canonical_task_ids":["commonsense-reasoning"],"coverage":"full","domains":[],"id":"qwen35-hellaswag-common-core-full","measured_runs":12,"methods":["choice"],"modalities":["text"],"models":15,"name":"HellaSwag (full validation, character-normalized accuracy)","safety_properties":[],"scope_id":"e83048a65cc6e110ed09bc40c410945ce72105f15f101b049aa100bf096278dd","url":"/tasks/qwen35-hellaswag-common-core-full"},{"area_id":"language-knowledge","canonical_task_ids":["reading-comprehension"],"coverage":"full","domains":[],"id":"qwen35-boolq-common-core-full","measured_runs":2,"methods":["choice"],"modalities":["text"],"models":15,"name":"BoolQ (full validation, raw likelihood accuracy)","safety_properties":[],"scope_id":"087725fb28ea8991a3f6bacbedcecfe584a0305a5e4d876496d753c3718af752","url":"/tasks/qwen35-boolq-common-core-full"},{"area_id":"language-knowledge","canonical_task_ids":["commonsense-reasoning"],"coverage":"subset","domains":[],"id":"commonsenseqa-validation-choice","measured_runs":2,"methods":["choice"],"modalities":["text"],"models":2,"name":"CommonsenseQA (validation)","safety_properties":[],"scope_id":"f0998ace0736b4da3a3e3949ec0597035cf98f4c1796febce8344e4f727cc90a","url":"/tasks/commonsenseqa-validation-choice"},{"area_id":"vision-documents","canonical_task_ids":["handwriting-recognition"],"coverage":"full","domains":[],"id":"ocr-handwriting-full","measured_runs":3,"methods":["ocr"],"modalities":["image"],"models":3,"name":"IAM lines / full 2,915-line test","safety_properties":[],"scope_id":"befde9a399561651c0f51c037036c5ce08be730bb11271b7ec7d78989978c1d2","url":"/tasks/ocr-handwriting-full"},{"area_id":"vision-documents","canonical_task_ids":["ocr-capabilities","visual-question-answering"],"coverage":"full","domains":[],"id":"ocrbench-v1-full","measured_runs":2,"methods":["ocrbench"],"modalities":["image"],"models":2,"name":"OCRBench v1 / full 1,000 questions \u2014 experimental batch-4","safety_properties":[],"scope_id":"d5b62220da83f9ebb4c271f820fa35b964ad4a0400651e30e94f5727f4a14007","url":"/tasks/ocrbench-v1-full"},{"area_id":"vision-documents","canonical_task_ids":["ocr-capabilities","visual-question-answering"],"coverage":"full","domains":[],"id":"ocrbench-v1-full-serial","measured_runs":2,"methods":["ocrbench"],"modalities":["image"],"models":2,"name":"OCRBench v1 / full serial reference","safety_properties":[],"scope_id":"dcccfe2d1c65816985fdcc2bb76bfad20fcc76719fc61e7a4fd8a82df9ec1d29","url":"/tasks/ocrbench-v1-full-serial"}],"filters":["modalities","domains","methods","safety_properties"],"legacy_task_count":72,"note":"Imported discovery tasks are not runnable adapters. Beta extensions and measured evaluation profiles are counted separately.","parsebench":{"announcement":"https://www.llamaindex.ai/blog/parsebench","area_id":"vision-documents","candidate_local_pipelines":["Docling","Dots OCR","Qwen VLM"],"canonical_task_ids":["document-parsing","table-recognition","document-layout-analysis"],"code_license":"Apache-2.0","code_revision":"9af4102e26b51f234ab5f037e19fec05c0696ec6","dataset":"llamaindex/ParseBench","dataset_license_metadata":"apache-2.0","dataset_revision":"2805a1d940f95a203e0ae4b88be9934f7765b3fc","dimensions":["tables","charts","content faithfulness","semantic formatting","visual grounding"],"domains":["enterprise"],"id":"parsebench","integration_requirements":["Pin evaluator, dataset files and split/group identifiers","Adapt each local parser to the required structured output and preserve native coordinates","Keep all five dimensions separate; do not infer document parsing from line CER","Record local model revision, parser configuration, failures, latency and reserved GPU cost","Keep upstream vendor-reported scores separate from CodeSOTA runs"],"methods":["document-parsing"],"modalities":["document-image","pdf"],"name":"ParseBench","note":"Candidate pipeline names from upstream documentation; exact models, weights licences, local hosting and adapter compatibility still require validation. No CodeSOTA ParseBench scores yet.","retrieved_at":"2026-10-18T17:34:11.255724+00:00","runnable":false,"safety_properties":["faithfulness","grounding"],"scoring":"Default evaluator is deterministic; optional judge-based chart normalization must remain disabled or a separate protocol.","source":"https://github.com/run-llama/ParseBench","status":"adapter_required"},"retrieved_at":"2026-10-18T17:33:45.684788+00:00","source":"https://www.codesota.com/tasks","source_sha256":"6c1949e5c3388025821151a7b94ff5c393c0d6b44c967dab6ca282e84a7e7044","version":"codesota-capability-map-v1"}}