{"benchmarks":[{"base_or_instruct":"instruct","benchmark_family":"mmlu","benchmark_id":"mmlu","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":101,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2"],"measured_cost":null,"metric":"accuracy","name":"MMLU","note":"Pin data, item selection and scoring protocol before scheduling.","number":1,"planning_cost":"\u015a\u2013W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/hendrycks/test","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"gpqa","benchmark_id":"gpqa","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":102,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2"],"measured_cost":null,"metric":"accuracy","name":"GPQA","note":"Pin data, item selection and scoring protocol before scheduling.","number":2,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/idavidrein/gpqa","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"gpqa","benchmark_id":"gpqa-diamond","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":103,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"accuracy","name":"GPQA Diamond","note":"Subset of GPQA; not an independent capability.","number":3,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/idavidrein/gpqa","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"mmlu-pro","benchmark_id":"mmlu-pro","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":104,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"accuracy","name":"MMLU-Pro","note":"Pin data, item selection and scoring protocol before scheduling.","number":4,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/TIGER-AI-Lab/MMLU-Pro","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"ifeval","benchmark_id":"ifeval","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":105,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2","LFM2.5-350M"],"measured_cost":null,"metric":"instruction_accuracy","name":"IFEval","note":"Pin data, item selection and scoring protocol before scheduling.","number":5,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/google-research/google-research/tree/master/instruction_following_eval","split":null,"status":"planned","task_ids":["instruction-following"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"ifbench","benchmark_id":"ifbench","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":106,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2","LFM2.5-350M"],"measured_cost":null,"metric":"constraint_accuracy","name":"IFBench","note":"Pin data, item selection and scoring protocol before scheduling.","number":6,"planning_cost":"\u015a","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/allenai/IFBench","split":null,"status":"planned","task_ids":["instruction-following"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"multi-if","benchmark_id":"multi-if","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":107,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["selected languages; aggregation pending"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"multi_turn_instruction_accuracy","name":"Multi-IF","note":"Pin data, item selection and scoring protocol before scheduling.","number":7,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/facebookresearch/Multi-IF","split":null,"status":"planned","task_ids":["instruction-following"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"gsm8k","benchmark_id":"gsm8k","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":108,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":"math","generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2"],"measured_cost":null,"metric":"exact_match","name":"GSM8K","note":"Pin data, item selection and scoring protocol before scheduling. Existing CodeSOTA pilot adapter is available; loading/scoring native Slayer models still needs validation for this task.","number":8,"planning_cost":"\u015a\u2013W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/openai/grade-school-math","split":null,"status":"adapter_ready","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"mgsm","benchmark_id":"mgsm","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":109,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["selected languages; aggregation pending"],"liquid":["LFM2"],"measured_cost":null,"metric":"exact_match_by_language","name":"MGSM","note":"Pin data, item selection and scoring protocol before scheduling.","number":9,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/google-research/url-nlp/tree/main/mgsm","split":null,"status":"planned","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"mmmlu","benchmark_id":"mmmlu","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":110,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["selected languages; aggregation pending"],"liquid":["LFM2"],"measured_cost":null,"metric":"accuracy_by_language","name":"MMMLU \u2014 OpenAI","note":"Select languages and aggregation explicitly. Liquid LFM2 reports seven languages; do not silently pool all MMMLU languages.","number":10,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/openai/MMMLU","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"casereportbench","benchmark_id":"casereportbench","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":111,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"extraction_score","name":"CaseReportBench","note":"Pin data, item selection and scoring protocol before scheduling.","number":11,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://arxiv.org/abs/2505.17265","split":null,"status":"planned","task_ids":["extraction"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"bfcl","benchmark_id":"bfcl-v3","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":112,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"function_call_accuracy","name":"BFCL v3","note":"Related BFCL versions; preserve separate protocols and shared family.","number":12,"planning_cost":"W","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"bfcl","benchmark_id":"bfcl-v4","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":113,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"function_call_accuracy","name":"BFCL v4","note":"Related BFCL versions; agent/environment costs must be included.","number":13,"planning_cost":"W\u2013BW","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"tau2-bench","benchmark_id":"tau2-telecom","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":114,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"task_success","name":"\u03c4\u00b2-Bench Telecom","note":"Telecom domain of tau2-bench; simulator/user model and tool budgets must be pinned.","number":14,"planning_cost":"BW","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/sierra-research/tau2-bench","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"tau2-bench","benchmark_id":"tau2-retail","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":115,"execution_priority_reason":"Liquid reference coverage","execution_tier":2,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned tools and simulator required","group":"liquid-reference","item_ids":null,"languages":["en"],"liquid":["LFM2.5-350M"],"measured_cost":null,"metric":"task_success","name":"\u03c4\u00b2-Bench Retail","note":"Retail domain of tau2-bench; related to Telecom, not an independent benchmark family.","number":15,"planning_cost":"BW","presets":["liquid-reference"],"priority":0.0,"protocol_id":null,"source":"https://github.com/sierra-research/tau2-bench","split":null,"status":"planned","task_ids":["tool-use"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"wikitext-2","benchmark_id":"wikitext-2","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"min","execution_order":5,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"bits_per_utf8_byte","name":"WikiText-2","note":"Pin data, item selection and scoring protocol before scheduling.","number":16,"planning_cost":"N","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/Salesforce/wikitext","split":null,"status":"planned","task_ids":["language-modeling"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"uncheatable-eval","benchmark_id":"uncheatable-eval","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"min","execution_order":15,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"likelihood_compression","name":"Uncheatable Eval","note":"Official upstream base-model benchmark. The CodeSOTA arXiv BPB pilot is a separate protocol and has no official UncheatableEval score.","number":17,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/Jellyfish042/uncheatable_eval","split":null,"status":"planned","task_ids":["language-modeling"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"heldout-en-pl-code","benchmark_id":"heldout-en-pl-code","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"min","execution_order":14,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en","pl","code"],"liquid":[],"measured_cost":null,"metric":"bits_per_utf8_byte_by_domain","name":"Own held-out EN / PL / code","note":"Separate source-disjoint EN, PL and code streams; not yet collected. No cross-domain aggregate.","number":18,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://beta.codesota.com/sub-1b","split":null,"status":"planned","task_ids":["language-modeling"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"blimp","benchmark_id":"blimp","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":4,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"minimal_pair_accuracy","name":"BLiMP","note":"Pin data, item selection and scoring protocol before scheduling.","number":19,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/alexwarstadt/blimp","split":null,"status":"planned","task_ids":["grammar"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"blimp","benchmark_id":"blimp-supplement","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":12,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy_by_subtask","name":"BLiMP Supplement","note":"Preserve BabyLM version, subtask definitions and dataset terms.","number":20,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/babylm-org/babylm-eval","split":null,"status":"planned","task_ids":["grammar"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"ewok","benchmark_id":"ewok","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":13,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy","name":"EWoK","note":"Pin data, item selection and scoring protocol before scheduling.","number":21,"planning_cost":"\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/ewok-core/ewok-paper","split":null,"status":"planned","task_ids":["world-modeling"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"arc-easy","benchmark_id":"arc-easy","cohort_profile_id":"sub1b-arc-easy","cohort_progress":{"completed":11,"failed":0,"queued":0,"running":0,"total":11},"comparison_scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","dataset_revision":null,"demand":0,"direction":"max","execution_order":1,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":53.3,"measured_cost_unit":"gpu_slot_seconds","metric":"accuracy","name":"ARC-Easy","note":"Pin data, item selection and scoring protocol before scheduling.","number":22,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/allenai/ai2_arc","split":null,"status":"pilot","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"arc-challenge","benchmark_id":"arc-challenge","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":7,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":"arc-challenge-choice","generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy","name":"ARC-Challenge","note":"Pin data, item selection and scoring protocol before scheduling. Existing CodeSOTA pilot adapter is available; loading/scoring native Slayer models still needs validation for this task.","number":23,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/allenai/ai2_arc","split":null,"status":"adapter_ready","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"hellaswag","benchmark_id":"hellaswag","cohort_profile_id":"sub1b-hellaswag","cohort_progress":{"completed":11,"failed":0,"queued":0,"running":0,"total":11},"comparison_scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","dataset_revision":null,"demand":0,"direction":"max","execution_order":2,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":"commonsense","generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":53.44,"measured_cost_unit":"gpu_slot_seconds","metric":"accuracy","name":"HellaSwag","note":"Pin data, item selection and scoring protocol before scheduling. Existing CodeSOTA pilot adapter is available; loading/scoring native Slayer models still needs validation for this task.","number":24,"planning_cost":"\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/Rowan/hellaswag","split":null,"status":"pilot","task_ids":["commonsense"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"piqa","benchmark_id":"piqa","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":6,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy","name":"PIQA","note":"Pin data, item selection and scoring protocol before scheduling.","number":25,"planning_cost":"N","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/ybisk/ybisk.github.io/tree/master/piqa","split":null,"status":"planned","task_ids":["commonsense"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"winogrande","benchmark_id":"winogrande","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":9,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy","name":"WinoGrande","note":"Pin data, item selection and scoring protocol before scheduling.","number":26,"planning_cost":"N","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/allenai/winogrande","split":null,"status":"planned","task_ids":["commonsense"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"openbookqa","benchmark_id":"openbookqa","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":8,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":"openbookqa-main-choice","generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy","name":"OpenBookQA","note":"Pin data, item selection and scoring protocol before scheduling. Existing CodeSOTA pilot adapter is available; loading/scoring native Slayer models still needs validation for this task.","number":27,"planning_cost":"N","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://github.com/allenai/OpenBookQA","split":null,"status":"adapter_ready","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"sciq","benchmark_id":"sciq","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":10,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy","name":"SciQ","note":"Choose context-supported or context-free protocol explicitly.","number":28,"planning_cost":"N","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/allenai/sciq","split":null,"status":"planned","task_ids":["knowledge"],"visits_30d":0},{"base_or_instruct":"base","benchmark_family":"lambada","benchmark_id":"lambada","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":11,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"small-base","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy_and_likelihood","name":"LAMBADA","note":"Pin data, item selection and scoring protocol before scheduling.","number":29,"planning_cost":"N\u2013\u015a","presets":["small-base"],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/EleutherAI/lambada_openai","split":null,"status":"planned","task_ids":["language-modeling"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"math-500","benchmark_id":"math-500","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":130,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"math-reasoning","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"exact_match","name":"MATH-500","note":"Pin data, item selection and scoring protocol before scheduling.","number":30,"planning_cost":"W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/HuggingFaceH4/MATH-500","split":null,"status":"planned","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"gsm-symbolic","benchmark_id":"gsm-symbolic","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":131,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"math-reasoning","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"exact_match_by_structure","name":"GSM-Symbolic","note":"Pin data, item selection and scoring protocol before scheduling.","number":31,"planning_cost":"\u015a\u2013W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://machinelearning.apple.com/research/gsm-symbolic","split":null,"status":"planned","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"svamp","benchmark_id":"svamp","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":132,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"math-reasoning","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"exact_match","name":"SVAMP","note":"Pin data, item selection and scoring protocol before scheduling.","number":32,"planning_cost":"\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/arkilpatel/SVAMP","split":null,"status":"planned","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"bbh","benchmark_id":"bbh","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":133,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"math-reasoning","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy_by_subtask","name":"BIG-Bench Hard \u2014 BBH","note":"Pin data, item selection and scoring protocol before scheduling.","number":33,"planning_cost":"W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/suzgunmirac/BIG-Bench-Hard","split":null,"status":"planned","task_ids":["reasoning"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"aime-2025","benchmark_id":"aime-2025","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":134,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"math-reasoning","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"exact_match","name":"AIME 2025","note":"Ambitious floor probe; may not separate small models. No presumed discriminative value.","number":34,"planning_cost":"W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/aime","split":null,"status":"planned","task_ids":["math"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"humaneval","benchmark_id":"humaneval","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":135,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":"code-generation","generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Isolated executable tests","group":"coding","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"pass_at_1","name":"HumanEval","note":"Pin data, item selection and scoring protocol before scheduling. Existing CodeSOTA pilot adapter is available; loading/scoring native Slayer models still needs validation for this task.","number":35,"planning_cost":"\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/openai/human-eval","split":null,"status":"adapter_ready","task_ids":["code-generation"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"humaneval","benchmark_id":"humaneval-plus","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":136,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Isolated executable tests","group":"coding","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"pass_at_1","name":"HumanEval+","note":"Reuse HumanEval generations only when prompts, decoding, seeds and item IDs match; score stronger tests separately.","number":36,"planning_cost":"\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/evalplus/evalplus","split":null,"status":"planned","task_ids":["code-generation"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"mbpp","benchmark_id":"mbpp","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":137,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Isolated executable tests","group":"coding","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"pass_at_1","name":"MBPP","note":"Pin data, item selection and scoring protocol before scheduling.","number":37,"planning_cost":"\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/google-research/google-research/tree/master/mbpp","split":null,"status":"planned","task_ids":["code-generation"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"mbpp","benchmark_id":"mbpp-plus","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":138,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Isolated executable tests","group":"coding","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"pass_at_1","name":"MBPP+","note":"Reuse MBPP generations only under a matched generation protocol.","number":38,"planning_cost":"\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/evalplus/evalplus","split":null,"status":"planned","task_ids":["code-generation"],"visits_30d":0},{"base_or_instruct":"base-or-instruct","benchmark_family":"livecodebench","benchmark_id":"livecodebench","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":139,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Isolated executable tests","group":"coding","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"versioned_pass_at_1","name":"LiveCodeBench","note":"Pin release, date window and complete item manifest.","number":39,"planning_cost":"W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/LiveCodeBench/LiveCodeBench","split":null,"status":"planned","task_ids":["code-generation"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"boolq","benchmark_id":"boolq","cohort_profile_id":"sub1b-boolq","cohort_progress":{"completed":11,"failed":0,"queued":0,"running":0,"total":11},"comparison_scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","dataset_revision":null,"demand":0,"direction":"max","execution_order":3,"execution_priority_reason":"Core small-model signal","execution_tier":1,"existing_adapter_task":"reading","generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"reading-context","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":80.19,"measured_cost_unit":"gpu_slot_seconds","metric":"accuracy","name":"BoolQ","note":"Pin data, item selection and scoring protocol before scheduling. Existing CodeSOTA pilot adapter is available; loading/scoring native Slayer models still needs validation for this task.","number":40,"planning_cost":"N\u2013\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/google-research-datasets/boolean-questions","split":null,"status":"pilot","task_ids":["reading"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"squad-v2","benchmark_id":"squad-v2","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":141,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"reading-context","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"exact_match_and_f1","name":"SQuAD 2.0","note":"Pin data, item selection and scoring protocol before scheduling.","number":41,"planning_cost":"\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://rajpurkar.github.io/SQuAD-explorer/","split":null,"status":"planned","task_ids":["reading"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"drop","benchmark_id":"drop","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":142,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"reading-context","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"exact_match_and_f1","name":"DROP","note":"Pin data, item selection and scoring protocol before scheduling.","number":42,"planning_cost":"\u015a\u2013W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://huggingface.co/datasets/ucinlp/drop","split":null,"status":"planned","task_ids":["reading"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"ruler","benchmark_id":"ruler","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":143,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"reading-context","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy_by_context_length","name":"RULER","note":"Evaluate a context-length ladder within each model\u2019s supported context; no silent truncation.","number":43,"planning_cost":"\u015a\u2013W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/NVIDIA/RULER","split":null,"status":"planned","task_ids":["long-context"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"longbench-v2","benchmark_id":"longbench-v2","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":144,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"reading-context","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"accuracy","name":"LongBench v2","note":"Full long-context protocol may exceed the context or memory supported by small models.","number":44,"planning_cost":"BW","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/THUDM/LongBench","split":null,"status":"planned","task_ids":["long-context"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"truthfulqa","benchmark_id":"truthfulqa","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":145,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"trust-preference-multilingual","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"protocol_specific_accuracy","name":"TruthfulQA","note":"Choose MC or judged-generation variant; their scores are not interchangeable.","number":45,"planning_cost":"\u015a","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/sylinrl/TruthfulQA","split":null,"status":"planned","task_ids":["truthfulness"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"simpleqa","benchmark_id":"simpleqa","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":146,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned external judge required","group":"trust-preference-multilingual","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"correctness_and_abstention","name":"SimpleQA","note":"Pin the grader and evaluate abstention separately.","number":46,"planning_cost":"\u015a\u2013W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/openai/simple-evals","split":null,"status":"planned","task_ids":["truthfulness"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"wildbench","benchmark_id":"wildbench","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":147,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned external judge required","group":"trust-preference-multilingual","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"judge_score","name":"WildBench","note":"Judge cost and bias must be measured; not human preference ground truth.","number":47,"planning_cost":"W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/allenai/WildBench","split":null,"status":"planned","task_ids":["preference"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"alpacaeval-2-lc","benchmark_id":"alpacaeval-2-lc","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0.99,"direction":"max","execution_order":148,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned external judge required","group":"trust-preference-multilingual","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"length_controlled_win_rate","name":"AlpacaEval 2.0 \u2014 length-controlled","note":"Pin judge and reference responses; use length-controlled win rate.","number":48,"planning_cost":"W","presets":[],"priority":6.9,"protocol_id":null,"source":"https://github.com/tatsu-lab/alpaca_eval","split":null,"status":"planned","task_ids":["preference"],"visits_30d":1},{"base_or_instruct":"instruct","benchmark_family":"mt-bench","benchmark_id":"mt-bench","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":149,"execution_priority_reason":"Expensive or ambitious extension","execution_tier":4,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Pinned external judge required","group":"trust-preference-multilingual","item_ids":null,"languages":["en"],"liquid":[],"measured_cost":null,"metric":"multi_turn_judge_score","name":"MT-Bench","note":"Pin judge, conversations and multi-turn context; not interchangeable with arena votes.","number":49,"planning_cost":"W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge","split":null,"status":"planned","task_ids":["preference"],"visits_30d":0},{"base_or_instruct":"instruct","benchmark_family":"belebele","benchmark_id":"belebele","comparison_scope":"No sub-1B cohort result for this catalogue entry yet","dataset_revision":null,"demand":0,"direction":"max","execution_order":150,"execution_priority_reason":"Additional task coverage","execution_tier":3,"existing_adapter_task":null,"generation_and_context_budget":{"max_input_tokens":null,"max_output_tokens":null,"sample_count":null},"grader_or_environment":"Deterministic scoring adapter to validate","group":"trust-preference-multilingual","item_ids":null,"languages":["selected languages; aggregation pending"],"liquid":[],"measured_cost":null,"metric":"accuracy_by_language","name":"Belebele","note":"Select language variants, including PL, and disclose aggregation.","number":50,"planning_cost":"\u015a\u2013W","presets":[],"priority":0.0,"protocol_id":null,"source":"https://github.com/facebookresearch/belebele","split":null,"status":"planned","task_ids":["multilingual-reading"],"visits_30d":0}],"description":"50 benchmarks, variants and validation streams; a selection catalogue, not 50 completed evaluations.","liquid_sources":["https://www.liquid.ai/blog/liquid-foundation-models-v2-our-second-series-of-generative-ai-models","https://www.liquid.ai/blog/lfm2-5-350m-no-size-left-behind"],"planning_cost_legend":{"BW":"very high","N":"low","W":"high","\u015a":"medium"},"presets":{"liquid-reference":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],"small-base":[16,17,18,19,20,21,22,23,24,25,26,27,28,29]},"status_values":["planned","adapter_ready","pilot","full","failed"],"tasks":[{"id":"knowledge","input":"Question + choices","name":"Knowledge QA","output":"Selected answer"},{"id":"instruction-following","input":"Instructions + context","name":"Instruction following","output":"Constrained response"},{"id":"math","input":"Math problem","name":"Math reasoning","output":"Solution and answer"},{"id":"extraction","input":"Source text + schema","name":"Structured extraction","output":"Structured fields"},{"id":"tool-use","input":"User goal + tools","name":"Tool use","output":"Actions and final response"},{"id":"language-modeling","input":"Raw text","name":"Language modelling","output":"Next-token probabilities"},{"id":"grammar","input":"Minimal sentence pair","name":"Grammatical preference","output":"Preferred sentence likelihood"},{"id":"world-modeling","input":"Controlled scenarios","name":"World modelling","output":"Selected consistent answer"},{"id":"commonsense","input":"Situation + choices","name":"Commonsense completion","output":"Selected completion"},{"id":"reasoning","input":"Problem","name":"Reasoning","output":"Answer"},{"id":"code-generation","input":"Programming prompt","name":"Code generation","output":"Executable program"},{"id":"reading","input":"Context + question","name":"Reading comprehension","output":"Answer or abstention"},{"id":"long-context","input":"Long document + question","name":"Long-context understanding","output":"Answer"},{"id":"truthfulness","input":"Question","name":"Factual correctness","output":"Answer or abstention"},{"id":"preference","input":"User prompt or conversation","name":"Response quality","output":"Generated response"},{"id":"multilingual-reading","input":"Context + question in selected language","name":"Multilingual reading","output":"Selected answer"}],"updated":"2026-10-17","version":"sub1b-catalog-v1"}