{"acc_norm":0.6143730886850153,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.5443425076452599,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.2742418540874496},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12052999599836767},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11965001502539963},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11937339894939214},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11774588306434453},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11822231207042933},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11764662794303149},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11775654298253357},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12126284290570766},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12143009505234659},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1165882139466703},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11929697601590306},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11827136296778917},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.11759533197619021},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.11810306995175779},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11774160491768271},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.11737095389980823},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11814608995337039},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11751245788764209},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11780792893841863},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.11954421806149185},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1215019189985469},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12246957304887474},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.12275440501980484},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.12110927700996399},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.12265376700088382},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1255627740174532},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1244438859866932},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.12322363106068224},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.12195172801148146},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1235728709725663},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12468940496910363},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12245983607135713},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.12274336395785213},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.12214319501072168},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12173794896807522},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12234445998910815},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12174765800591558},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.12242266605608165},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.1219253990566358},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.12360349902883172},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.12562140298541635},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.1223721019923687},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.12388146796729416},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.1234826430445537},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12257931905332953},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12357278203126043},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12207735201809555},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.12220277602318674},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.12296723201870918},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.12428711401298642},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.12545012403279543},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.12402898294385523},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.12373606592882425},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.12423364399001002},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.12513864203356206},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.1266822909237817},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.12384213402401656},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12333052000030875},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12300457200035453},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12378064007498324},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.12581356998998672},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12366056500468403},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12378302391152829},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12390595092438161},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12381222506519407},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12447239994071424},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.1238887500949204},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.12367541191633791},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.12119190010707825},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.11918679100926965},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.12244068703148514},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.12322248797863722},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1230231870431453},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.12383199506439269},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.12322276900522411},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.12318343610968441},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1234378699446097},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12543735990766436},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12417229008860886},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12281105993315578},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.1235463930061087},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12440216797403991},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12592141306959093},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.1250804039882496},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12385760305915028},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.12325689301360399},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.12429419695399702},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.12407548900227994},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12501246703322977},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12380379193928093},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12371314305346459},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.12369248399045318},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.12331056303810328},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.1235349919879809},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1256393960211426},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.12542260193731636},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.12490577797871083},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.12484712805598974},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.12434072408359498},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.12450161401648074},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.12520241097081453},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.12304968608077615},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.12218719709198922},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.12237604998517781},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.12449144502170384},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.12231922405771911},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.12234592309687287},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1242158809909597},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.12266735290177166},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1232205550186336},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1235771500505507},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.12391790503170341},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.123947549960576},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.12331423000432551},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.12222012795973569},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.12272689398378134},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.12374025397002697},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.12314295000396669},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.12416429491713643},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.12306282995268703},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.12242257595062256},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.12337654607836157},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.12345699605066329},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.12314225896261632},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.12311878497712314},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.12288028001785278},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.12415482697542757},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.12263892008922994},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.12262151704635471},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.12478076503612101},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.12421319598797709},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.1226491789566353},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12285554397385567},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12308183603454381},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.1238846640335396},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12379324296489358},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.12307626497931778},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.12489472702145576},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12379049800802022},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12485876004211605},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12391509010922164},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12311037897597998},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12345975998323411},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12345090496819466},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12325740291271359},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1234184839995578},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12497462704777718},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12489900505170226},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12338797701522708},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12349177990108728},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1252083220751956},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12321757897734642},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12465777504257858},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12430136103648692},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.12470927205868065},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.1243034650105983},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.12357025698293},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.12601203203666955},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.12361581099685282},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.12420069403015077},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.12383920908905566},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.12406761501915753},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.12436366709880531},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.12498016702011228},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1255010690074414},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.12516054301522672},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.12498980399686843},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.12547339592128992},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.12412264698650688},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12450809602160007},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12681013997644186},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12471830809954554},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12380053696688265},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.13176207593642175},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.13789218105375767},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.13806439202744514},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.13767811097204685},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.13726710493210703},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.14255956700071692},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.13908842206001282},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.13729595800396055},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.13746859005186707},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.13785654492676258},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.13900581700727344},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.13849129795562476},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.13829060399439186},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.13814705610275269},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.14277553907595575},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1380446950206533},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.13861747295595706},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.13970495201647282},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.13793029100634158},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1408325950615108},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.13859693496488035},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.13817334501072764},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.14461788604967296},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.1388799330452457},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.14005897205788642},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.13843973190523684},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.13878692011348903},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1409512070240453},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.13889311696402729},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.14032573997974396},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.14369251998141408},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.13939637690782547},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.13865696697030216},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.13905887608416378},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.140163857024163},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.1404227609746158},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.13916778902057558},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.14064682903699577},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.1441729550715536},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.13941480999346823},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.139423556975089},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.13948717492166907},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.1397188580594957},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.13952059799339622},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.13973283302038908},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.13958539895247668},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.14468578295782208},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.14008629298768938},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.1398500429932028},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.14061771507840604},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.14084108103998005},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.14130899403244257},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.14005241997074336},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.14687271101865917},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.14068616193253547},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.1399430149467662},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.13993979000952095},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.14052695396821946},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.1418933339882642},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.14105234504677355},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.14114225306548178},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.14596324495505542},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.14064978493843228},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.14086359296925366},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1411664680344984},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.1420045510167256},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.14151874498929828},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.14191566605586559},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.14726992300711572},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.1416611800668761},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.14114507799968123},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.14164866704959422},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.14140563399996608},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1414552559144795},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.14195804402697831},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.1478564360877499},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.1418057200498879},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.14157657301984727},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.14177159708924592},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.14227579592261463},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.14280885097105056},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.1416390889789909},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.143201373051852},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.14591695799026638},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.15034371602814645},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.1505771310767159},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.1504549429519102},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.15549440297763795},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.15050937398336828},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.15095337305683643},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.15198109997436404},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.15707254700828344},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1512838089838624},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.15116295299958438},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.15121139399707317},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1584065839415416},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.15280092903412879},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.15295798098668456},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.1528345310362056},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.15196843596640974},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.1571001879638061},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.15229616803117096},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.15232281608041376},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.15298516198527068},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.15799656009767205},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.15270679304376245},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.15227148099802434},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.152003351948224},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.15853343100752681},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.152917715953663},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.1581786199240014},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.16197693394497037},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.16535261005628854},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.1616089768940583},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.16118246200494468},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.16773876105435193},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.16280144196934998},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.1614315160550177},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.16575306793674827},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.16077885893173516},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.16158052498940378},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.1621311919298023},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.16837755299638957},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.16186945396475494},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.1617108180653304},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.16682019794825464},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.1649133110186085},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.16249128396157175},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.17049312905874103},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.16377232305239886},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.1633018059656024},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.16849910898599774},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.16441232699435204},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.16411042399704456},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.17014748405199498},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.1631023440277204},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.16293878795113415},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.16963944700546563},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.16338487097527832},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.16401378309819847},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.17007688200101256},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.16493260697461665},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.16387146699707955},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.16979261301457882},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.17184103291947395},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.17211767903063446},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.1773868870222941},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.1719449469819665},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.18120855593588203},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.17215505801141262},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.17836275801528245},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.17222112196031958},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.17288042197469622},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.17984464205801487},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.17348249501083046},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.18027210899163038},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.1741749260108918},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.17403333203401417},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.1800231549423188},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.1735443810466677},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.1791859130607918},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.17521775199566036},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.1792166200466454},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.17456200800370425},{"candidates":16,"kind":"choice","padded_tokens":3360,"seconds":0.1763132750056684},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.18094813008792698},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.17613819905091077},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.17973407497629523},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.17514033697079867},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.17616480798460543},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.18065724801272154},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.17685415293090045},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.1832113809650764},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.1751051110913977},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.1844999250024557},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.17583875893615186},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.18057869095355272},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.1766887950943783},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.1756489760009572},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.1828502380521968},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.17827886110171676},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.18353782000485808},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.17957697191741318},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.18563035293482244},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.17830842593684793},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.186305292067118},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.17988874402362853},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.18488605495076627},{"candidates":16,"kind":"choice","padded_tokens":3760,"seconds":0.17968841001857072},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.1805327848996967},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.1852680470328778},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.1814097510650754},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.18573998799547553},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.18104410904925317},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.18878223199862987},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.18267216498497874},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.18870760407298803},{"candidates":16,"kind":"choice","padded_tokens":4016,"seconds":0.1814893000992015},{"candidates":16,"kind":"choice","padded_tokens":4048,"seconds":0.19040212396066636},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.18229147500824183},{"candidates":16,"kind":"choice","padded_tokens":4112,"seconds":0.20067157794255763},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.19775410706643015},{"candidates":16,"kind":"choice","padded_tokens":4192,"seconds":0.20397607295308262},{"candidates":16,"kind":"choice","padded_tokens":4224,"seconds":0.20526542700827122},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.19777323305606842},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.20349952299147844},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.20546499907504767},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.19921963999513537},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.20399434596765786},{"candidates":16,"kind":"choice","padded_tokens":4448,"seconds":0.20061358995735645},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.20545198500622064},{"candidates":16,"kind":"choice","padded_tokens":4512,"seconds":0.206144907977432},{"candidates":16,"kind":"choice","padded_tokens":4544,"seconds":0.20086458895821124},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.2060200539417565},{"candidates":16,"kind":"choice","padded_tokens":4592,"seconds":0.20784311508759856},{"candidates":16,"kind":"choice","padded_tokens":4640,"seconds":0.20399149099830538},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.21735495701432228},{"candidates":16,"kind":"choice","padded_tokens":4704,"seconds":0.21812014491297305},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.21854489704128355},{"candidates":16,"kind":"choice","padded_tokens":4832,"seconds":0.21428631292656064},{"candidates":16,"kind":"choice","padded_tokens":4880,"seconds":0.21895149594638497},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.22052521002478898},{"candidates":16,"kind":"choice","padded_tokens":5056,"seconds":0.22054410493001342},{"candidates":16,"kind":"choice","padded_tokens":5136,"seconds":0.2196508590131998},{"candidates":16,"kind":"choice","padded_tokens":5168,"seconds":0.2365019009448588},{"candidates":16,"kind":"choice","padded_tokens":5264,"seconds":0.23502446501515806},{"candidates":16,"kind":"choice","padded_tokens":5520,"seconds":0.2376051879255101},{"candidates":16,"kind":"choice","padded_tokens":5760,"seconds":0.23799005593173206},{"candidates":16,"kind":"choice","padded_tokens":6016,"seconds":0.24232329800724983},{"candidates":16,"kind":"choice","padded_tokens":6464,"seconds":0.25527059799060225},{"candidates":16,"kind":"choice","padded_tokens":6784,"seconds":0.267314063035883},{"candidates":16,"kind":"choice","padded_tokens":7120,"seconds":0.2763441860442981},{"candidates":16,"kind":"choice","padded_tokens":7856,"seconds":0.31094474298879504},{"candidates":13,"kind":"choice","padded_tokens":7982,"seconds":0.2769059829879552},{"candidates":11,"kind":"choice","padded_tokens":8041,"seconds":0.2861924149328843},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.14225995703600347}],"chat_template":false,"data_loading_seconds":0.051306247944012284,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251906,"gpu":0,"id":"9d98af19e580b6df631ad79c","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.02","model_load_seconds":1.8982931930804625,"model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":139279760,"peak_gpu_bytes":1330423808,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":65.8047755099833,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.5443425076452599,"scoring_seconds":61.73831770394463,"seconds":64.13624230399728,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu0"}