{"api_version":"codesota.v1","data":[{"base_or_instruct":"instruct","benchmark_family":"mmlu","benchmark_id":"mmlu","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":101,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2"],"measured_cost":null,"metric":"accuracy","name":"MMLU","note":"Pin data, item selection and scoring protocol before scheduling.","number":1,"planning_cost":"\u015a\u2013W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/hendrycks/test","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"gpqa","benchmark_id":"gpqa","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":102,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2"],"measured_cost":null,"metric":"accuracy","name":"GPQA","note":"Pin data, item selection and scoring protocol before scheduling.","number":2,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/idavidrein/gpqa","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"gpqa","benchmark_id":"gpqa-diamond","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":103,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"accuracy","name":"GPQA Diamond","note":"Subset of GPQA; not an independent capability.","number":3,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/idavidrein/gpqa","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"mmlu-pro","benchmark_id":"mmlu-pro","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":104,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"accuracy","name":"MMLU-Pro","note":"Pin data, item selection and scoring protocol before scheduling.","number":4,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/TIGER-AI-Lab/MMLU-Pro","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"ifeval","benchmark_id":"ifeval","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":105,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2","LFM2.5-350M"],"measured_cost":null,"metric":"instruction_accuracy","name":"IFEval","note":"Pin data, item selection and scoring protocol before scheduling.","number":5,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/google-research/google-research/tree/master/instruction_following_eval","split":null,"status":"planned","task_ids":["instruction-following"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"ifbench","benchmark_id":"ifbench","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":106,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2","LFM2.5-350M"],"measured_cost":null,"metric":"constraint_accuracy","name":"IFBench","note":"Pin data, item selection and scoring protocol before scheduling.","number":6,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/allenai/IFBench","split":null,"status":"planned","task_ids":["instruction-following"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"multi-if","benchmark_id":"multi-if","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":107,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["selected languages; aggregation pending"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"multi_turn_instruction_accuracy","name":"Multi-IF","note":"Pin data, item selection and scoring protocol before scheduling.","number":7,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/facebookresearch/Multi-IF","split":null,"status":"planned","task_ids":["instruction-following"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"gsm8k","benchmark_id":"gsm8k","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":108,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":"math","generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2"],"measured_cost":null,"metric":"exact_match","name":"GSM8K","note":"Pin data, item selection and scoring protocol before scheduling. Existing CodeSOTA pilot adapter is available; loading/scoring native Slayer models still needs validation for this task.","number":8,"planning_cost":"\u015a\u2013W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/openai/grade-school-math","split":null,"status":"adapter_ready","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"mgsm","benchmark_id":"mgsm","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":109,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["selected languages; aggregation pending"],"liquid":["LFM2"],"measured_cost":null,"metric":"exact_match_by_language","name":"MGSM","note":"Pin data, item selection and scoring protocol before scheduling.","number":9,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/google-research/url-nlp/tree/main/mgsm","split":null,"status":"planned","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"mmmlu","benchmark_id":"mmmlu","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":110,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["selected languages; aggregation pending"],"liquid":["LFM2"],"measured_cost":null,"metric":"accuracy_by_language","name":"MMMLU \u2014 OpenAI","note":"Select languages and aggregation explicitly. Liquid LFM2 reports seven languages; do not silently pool all MMMLU languages.","number":10,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/openai/MMMLU","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"casereportbench","benchmark_id":"casereportbench","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":111,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"extraction_score","name":"CaseReportBench","note":"Pin data, item selection and scoring protocol before scheduling.","number":11,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://arxiv.org/abs/2505.17265","split":null,"status":"planned","task_ids":["extraction"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"bfcl","benchmark_id":"bfcl-v3","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":112,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"function_call_accuracy","name":"BFCL v3","note":"Related BFCL versions; preserve separate protocols and shared family.","number":12,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"bfcl","benchmark_id":"bfcl-v4","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":113,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"function_call_accuracy","name":"BFCL v4","note":"Related BFCL versions; agent/environment costs must be included.","number":13,"planning_cost":"W\u2013BW","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"tau2-bench","benchmark_id":"tau2-telecom","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":114,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"task_success","name":"\u03c4\u00b2-Bench Telecom","note":"Telecom domain of tau2-bench; simulator/user model and tool budgets must be pinned.","number":14,"planning_cost":"BW","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/sierra-research/tau2-bench","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"tau2-bench","benchmark_id":"tau2-retail","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":115,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"task_success","name":"\u03c4\u00b2-Bench Retail","note":"Retail domain of tau2-bench; related to Telecom, not an independent benchmark family.","number":15,"planning_cost":"BW","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/sierra-research/tau2-bench","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"wikitext-2","benchmark_id":"wikitext-2","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"min","execution_order":5,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"bits_per_utf8_byte","name":"WikiText-2","note":"Pin data, item selection and scoring protocol before scheduling.","number":16,"planning_cost":"N","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/Salesforce/wikitext","split":null,"status":"planned","task_ids":["language-modeling"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"uncheatable-eval","benchmark_id":"uncheatable-eval","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"min","execution_order":15,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"likelihood_compression","name":"Uncheatable Eval","note":"Official upstream base-model benchmark. The CodeSOTA arXiv BPB pilot is a separate protocol and has no official UncheatableEval score.","number":17,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/Jellyfish042/uncheatable_eval","split":null,"status":"planned","task_ids":["language-modeling"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"heldout-en-pl-code","benchmark_id":"heldout-en-pl-code","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"min","execution_order":14,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en","pl","code"],"liquid":[],"measured_cost":null,"metric":"bits_per_utf8_byte_by_domain","name":"Own held-out EN / PL / code","note":"Separate source-disjoint EN, PL and code streams; not yet collected. No cross-domain aggregate.","number":18,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://beta.codesota.com/sub-1b","split":null,"status":"planned","task_ids":["language-modeling"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"blimp","benchmark_id":"blimp","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":4,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"minimal_pair_accuracy","name":"BLiMP","note":"Pin data, item selection and scoring protocol before scheduling.","number":19,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/alexwarstadt/blimp","split":null,"status":"planned","task_ids":["grammar"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"blimp","benchmark_id":"blimp-supplement","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":12,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy_by_subtask","name":"BLiMP Supplement","note":"Preserve BabyLM version, subtask definitions and dataset terms.","number":20,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/babylm-org/babylm-eval","split":null,"status":"planned","task_ids":["grammar"],"visits_30d":0}],"generated_at":"2026-10-17T02:25:13.164516+00:00","pagination":{"limit":20,"next":"/api/v1/benchmarks?limit=20&offset=20","offset":0,"previous":null,"total":57}}