{"api_version":"codesota.v1","data":[{"adapter":"code","area":"Code","benchmark":"HumanEval","benchmark_id":"humaneval","completed":2,"contract":"Function signature + docstring \u2192 Python function","dataset":"openai/openai_humaneval","dataset_config":"openai_humaneval","dataset_revision":"7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544","dataset_url":"https://huggingface.co/datasets/openai/openai_humaneval","demand":5.94,"direction":"max","id":"code-generation","legacy_bonus":35,"license":["mit"],"limit":32,"metric":"pass@1","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Code generation","priority":54.5,"protocol_version":"codesota-pilot-v1","scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","source":"https://github.com/openai/human-eval","split":"test","task_ids":["code-generation"],"visits_30d":6},{"adapter":"math","area":"Reasoning","benchmark":"GSM8K","benchmark_id":"gsm8k","completed":5,"contract":"Word problem \u2192 numerical answer","dataset":"openai/gsm8k","dataset_config":"main","dataset_revision":"740312add88f781978c0658806c59bc2815b9866","dataset_url":"https://huggingface.co/datasets/openai/gsm8k","demand":0.99,"direction":"max","id":"math","legacy_bonus":27.76,"license":["mit"],"limit":32,"metric":"exact_match","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Math word problems","priority":34.69,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"763d57f2f0b0ae9652d317e3336dd413d2a42328ddd6f72de570542909aa13c9","source":"https://github.com/openai/grade-school-math","split":"test","task_ids":["math"],"visits_30d":1},{"adapter":"reading","area":"Language","benchmark":"BoolQ","benchmark_id":"boolq","completed":4,"contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","demand":0.0,"direction":"max","id":"reading","legacy_bonus":0,"license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Reading comprehension","priority":0.07,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"50f1e323d677230fdcedb032817fe2f0387aab1b8fc6c2b0f2282f32d4318845","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","task_ids":["reading"],"visits_30d":0},{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","benchmark_id":"hellaswag","completed":4,"contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","demand":0.99,"direction":"max","id":"commonsense","legacy_bonus":0,"license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Commonsense completion","priority":6.95,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"035178b6b8d089715228ff8d6925d31eefde18bf58d0d0859b2d5dc1f184293b","source":"https://rowanzellers.com/hellaswag/","split":"validation","task_ids":["commonsense"],"visits_30d":1},{"adapter":"sentiment","area":"Language","benchmark":"SST-2","benchmark_id":"sentiment","completed":4,"contract":"Movie review \u2192 positive / negative","dataset":"stanfordnlp/sst2","dataset_config":null,"dataset_revision":"8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb","dataset_url":"https://huggingface.co/datasets/stanfordnlp/sst2","demand":0.0,"direction":"max","id":"sentiment","legacy_bonus":0,"license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Sentiment classification","priority":0.07,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"1985c666f87bc1b3203e78c666ffc282afe39e7e4d54d3a5359a3ccc34bfb465","source":"https://nlp.stanford.edu/sentiment/","split":"validation","task_ids":["sentiment"],"visits_30d":0},{"adapter":"entailment","area":"Language","benchmark":"SNLI","benchmark_id":"entailment","completed":4,"contract":"Premise + hypothesis \u2192 entailment / neutral / contradiction","dataset":"stanfordnlp/snli","dataset_config":"plain_text","dataset_revision":"cdb5c3d5eed6ead6e5a341c8e56e669bb666725b","dataset_url":"https://huggingface.co/datasets/stanfordnlp/snli","demand":0.0,"direction":"max","id":"entailment","legacy_bonus":0,"license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"Natural language inference","priority":0.07,"protocol_version":"codesota-pilot-v2-batched","runner_file":"evaluate_fast.py","scope":"3ef5086324e4a200ffa81c1058aab801250aedfad5e4f11c7c4b8a63c1bf5b3e","source":"https://nlp.stanford.edu/projects/snli/","split":"test","task_ids":["entailment"],"visits_30d":0},{"adapter":"similarity","area":"Embeddings","benchmark":"STS Benchmark","benchmark_id":"similarity","completed":1,"contract":"Two sentences \u2192 semantic similarity","dataset":"sentence-transformers/stsb","dataset_config":null,"dataset_revision":"ab7a5ac0e35aa22088bdcf23e7fd99b220e53308","dataset_url":"https://huggingface.co/datasets/sentence-transformers/stsb","demand":0.0,"direction":"max","id":"similarity","legacy_bonus":32.52,"license":"See upstream terms","limit":32,"metric":"spearman","models":[{"id":"sentence-transformers/all-MiniLM-L6-v2","revision":"1110a243fdf4706b3f48f1d95db1a4f5529b4d41"}],"name":"Sentence similarity","priority":32.64,"protocol_version":"codesota-pilot-v1","scope":"5048133e477a3c13bf97c614b99b2d36a458391bb2a9480cd9b38189f2608b67","source":"https://ixa2.si.ehu.eus/stswiki/index.php/STSbenchmark","split":"test","task_ids":["similarity"],"visits_30d":0},{"adapter":"ocr","area":"Documents","benchmark":"IAM lines","benchmark_id":"handwriting","completed":1,"contract":"Handwritten line image \u2192 text","dataset":"Teklia/IAM-line","dataset_config":null,"dataset_revision":"fbdad97500ce54635c0d1ba306bf535cb40656cf","dataset_url":"https://huggingface.co/datasets/Teklia/IAM-line","demand":0.0,"direction":"min","id":"handwriting","legacy_bonus":35,"license":"mit","limit":32,"metric":"cer","models":[{"id":"microsoft/trocr-small-handwritten","revision":"b4648cfa171985a6745f37ddd637e98c0da958ac"}],"name":"Handwriting recognition","priority":35.13,"protocol_version":"codesota-pilot-v1","scope":"87f7f645d4ae1c902e4f3c4d821290e2fa08df318242d5d78a563467763e36ec","source":"https://fki.tic.heia-fr.ch/databases/iam-handwriting-database","split":"test","task_ids":["handwriting"],"visits_30d":0},{"adapter":"asr","area":"Speech","benchmark":"LibriSpeech test-clean","benchmark_id":"transcription","completed":1,"contract":"English speech recording \u2192 transcript","dataset":"openslr/librispeech_asr","dataset_config":"clean","dataset_revision":"71cacbfb7e2354c4226d01e70d77d5fca3d04ba1","dataset_url":"https://huggingface.co/datasets/openslr/librispeech_asr","demand":0.99,"direction":"min","id":"transcription","legacy_bonus":35,"license":["cc-by-4.0"],"limit":32,"metric":"wer","models":[{"id":"openai/whisper-tiny.en","revision":"87c7102498dcde7456f24cfd30239ca606ed9063"}],"name":"Speech transcription","priority":42.0,"protocol_version":"codesota-pilot-v1","scope":"9c365ee78df50fa725b4c03d695f65d2b638be00c4687dc66be70b284c865a34","source":"https://www.openslr.org/12","split":"test","task_ids":["transcription"],"visits_30d":1},{"adapter":"vision","area":"Vision","benchmark":"CIFAR-10","benchmark_id":"image-classification","completed":1,"contract":"Image \u2192 one of ten object classes","dataset":"uoft-cs/cifar10","dataset_config":"plain_text","dataset_revision":"0b2714987fa478483af9968de7c934580d0bb9a2","dataset_url":"https://huggingface.co/datasets/uoft-cs/cifar10","demand":0.0,"direction":"max","id":"image-classification","legacy_bonus":35,"license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"openai/clip-vit-base-patch32","revision":"3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"}],"name":"Image classification","priority":35.12,"protocol_version":"codesota-pilot-v1","scope":"d4a779c6ba33be115fc5e3302e45bb70d7c90040f954b89dab97beb5a2914649","source":"https://www.cs.toronto.edu/~kriz/cifar.html","split":"test","task_ids":["image-classification"],"visits_30d":0},{"adapter":"choice","area":"Language","benchmark":"AI2 ARC-Challenge (test)","benchmark_id":"arc-challenge","completed":4,"contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","demand":0.99,"direction":"max","id":"arc-challenge-choice","legacy_bonus":0,"license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"AI2 ARC-Challenge (test)","parent_task":"reading","priority":6.97,"protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_fast.py","scope":"b05a21507d67515bfd401ff30911972742a70ff07f5672df4183e50d3877d299","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","task_ids":["knowledge"],"visits_30d":1},{"adapter":"choice","area":"Language","benchmark":"OpenBookQA main (test)","benchmark_id":"openbookqa","completed":4,"contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/openbookqa","dataset_config":"main","dataset_revision":"388097ea7776314e93a529163e0fea805b8a6454","dataset_url":"https://huggingface.co/datasets/allenai/openbookqa","demand":0.0,"direction":"max","id":"openbookqa-main-choice","legacy_bonus":0,"license":["unknown"],"limit":32,"metric":"accuracy","models":[{"id":"Qwen/Qwen2.5-0.5B","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987"},{"id":"Qwen/Qwen2.5-1.5B","revision":"8faed761d45a263340a0528343f099c05c9a4323"}],"name":"OpenBookQA main (test)","parent_task":"reading","priority":0.07,"protocol_version":"codesota-choice-pilot-v2-batched","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"main/test-00000-of-00001.parquet","question_field":"question_stem"},"runner_file":"evaluate_fast.py","scope":"08d620ad5299cfd3b7b752304580d3c9ea87e7f135c266b7ae4e2d94c2fe51f6","source":"https://huggingface.co/datasets/allenai/openbookqa","split":"test","task_ids":["knowledge"],"visits_30d":0},{"adapter":"choice","area":"Language","benchmark":"ARC-Easy","benchmark_id":"arc-easy","catalogue_id":"arc-easy","completed":11,"contract":"Question + candidate answers \u2192 selected answer","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","demand":0.0,"direction":"max","id":"sub1b-arc-easy","legacy_bonus":0,"license":["cc-by-sa-4.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / sub-1B cohort","parent_task":"reading","priority":1000.05,"protocol_version":"codesota-sub1b-choice-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_tasks.py","scope":"44c19edbe693799c7f6c36c5dcae3a6094da38e830a1757f4ce21a6d173e9af6","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","task_ids":["knowledge"],"visits_30d":0,"workstream":"sub1b"},{"adapter":"commonsense","area":"Reasoning","benchmark":"HellaSwag","benchmark_id":"hellaswag","catalogue_id":"hellaswag","completed":11,"contract":"Scenario + four endings \u2192 most plausible ending","dataset":"Rowan/hellaswag","dataset_config":null,"dataset_revision":"218ec52e09a7e7462a5400043bb9a69a41d06b76","dataset_url":"https://huggingface.co/datasets/Rowan/hellaswag","demand":0.99,"direction":"max","id":"sub1b-hellaswag","legacy_bonus":0,"license":"See upstream terms","limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"HellaSwag / sub-1B cohort","priority":1006.96,"protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","scope":"b4821eeee94ae676351fa0a7023207ccfb0c15fd7747debe3ffa69d04edf6456","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://rowanzellers.com/hellaswag/","split":"validation","task_ids":["commonsense"],"visits_30d":1,"workstream":"sub1b"},{"adapter":"reading","area":"Language","benchmark":"BoolQ","benchmark_id":"boolq","catalogue_id":"boolq","completed":11,"contract":"Passage + question \u2192 yes / no","dataset":"google/boolq","dataset_config":null,"dataset_revision":"35b264d03638db9f4ce671b711558bf7ff0f80d5","dataset_url":"https://huggingface.co/datasets/google/boolq","demand":0.99,"direction":"max","id":"sub1b-boolq","legacy_bonus":0,"license":["cc-by-sa-3.0"],"limit":32,"metric":"accuracy","models":[{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / sub-1B cohort","priority":1006.96,"protocol_version":"codesota-sub1b-choice-v1","runner_file":"evaluate_sub1b_tasks.py","scope":"bea6197d37b7692e8660c09f3be4bdc84c65bd6d240a6e23f418cc1d58d63ca5","selection":"First 32 valid examples in pinned split order; FP32 native/HF cohort; not full-benchmark results","source":"https://github.com/google-research-datasets/boolean-questions","split":"validation","task_ids":["reading"],"visits_30d":1,"workstream":"sub1b"}],"generated_at":"2026-10-17T02:25:24.436454+00:00","pagination":{"limit":20,"next":null,"offset":0,"previous":null,"total":15}}