{"acc_norm":0.6070336391437309,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.42201834862385323,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1042476559523493},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.04046958510298282},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.04332164500374347},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.042759677045978606},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.04418855195399374},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.047233400982804596},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.04865213704761118},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.04923469398636371},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.050500095007009804},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.050710878102108836},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.05065042490605265},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.052834288100712},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.05428130703512579},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.054953780956566334},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.0560847909655422},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.05847760394681245},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.05708195106126368},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.0608875589678064},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.07071500003803521},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.06269474001601338},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.06590956693980843},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.06546352396253496},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.06491488090250641},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.06548531504813582},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.06536176498048007},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.06566878699231893},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.06591817201115191},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.0666822970379144},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.06698563403915614},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.06701850495301187},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.06760683294851333},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.06948819104582071},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.07240282802376896},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.07039617490954697},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.06977228191681206},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.070775281987153},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.07252909406088293},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.07094287499785423},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.07150026503950357},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.07091690693050623},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.07167994999326766},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.07076457294169813},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.07206747203599662},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.07378106901887804},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.07239385100547224},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.07554103201255202},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.07552304805722088},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.07408307096920907},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.07483130693435669},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.07483340194448829},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.07541518693324178},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.07457670208532363},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.08140446804463863},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.08421268593519926},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.08319047896657139},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.08533735398668796},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.08582234801724553},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.08752518403343856},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.08740750490687788},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.08821733598597348},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.0874647629680112},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.0853746630018577},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.08665683504659683},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.08632047800347209},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.08495227608364075},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.08713302400428802},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.0852304040454328},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.08799195499159396},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.09037127299234271},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.08943988604005426},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.09250465198419988},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.08989600697532296},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.09154364001005888},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.09190362202934921},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.09148571209516376},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.08997238893061876},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.09170453995466232},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.09267754503525794},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.09678338305093348},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.09718024299945682},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.09630561305675656},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.09626207093242556},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.09292869304772466},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.09504996007308364},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.09568366291932762},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.09223362698685378},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.09539498295634985},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.095385906053707},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.09825340495444834},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.10201833699829876},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.09999647608492523},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.10288524406496435},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.10194674390368164},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.10205519595183432},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.10267451195977628},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.10210412798915058},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.09954249893780798},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.10295251989737153},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.10420891304966062},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.1060100020840764},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.10558486997615546},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.1048006770433858},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.10508121992461383},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.10505244694650173},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.1027882139896974},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.10550328809767962},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.1053237629821524},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.10301095899194479},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.10665112896822393},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.10750103404279798},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.10349292797036469},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.10646464093588293},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.10696054599247873},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.10343967890366912},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.10692291602026671},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.10787891899235547},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.10801743704359978},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.10879430593922734},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.10632197500672191},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.10933917295187712},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.1087761529488489},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.10600377095397562},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.10934933205135167},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.10400346992537379},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.10832223598845303},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.10902017704211175},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.10609682404901832},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.10988231503870338},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.11061189707834274},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.11094118095934391},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.10982823406811804},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.11085228598676622},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.11491561506409198},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.11541220010258257},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.11165255692321807},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.11647242598701268},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.11177319404669106},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.11783296300563961},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.11747398599982262},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.11736965097952634},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.11279924702830613},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.11704512604046613},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.11982110398821533},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12026735604740679},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.11509260395541787},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12095307500567287},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12099624599795789},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.1219520780723542},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1154271379346028},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12059335398953408},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12161842605564743},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.11603674490470439},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12200026796199381},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12204013299196959},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12225755804684013},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12326795305125415},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.12224011600483209},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.12714675802271813},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.12859428802039474},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12331899802666157},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12858866702299565},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.12385898490902036},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.12984781595878303},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.130380820017308},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.1296341170091182},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.12378309410996735},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.1305444249883294},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.12999379797838628},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.12632924295030534},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.13185701298061758},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.13243032398167998},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.13216747203841805},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.13203144795261323},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.12637451803311706},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.13290249393321574},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.13249230908695608},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.132315909024328},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.13295703497715294},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.13272218697238714},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.1384420859394595},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.1328193290391937},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.13954223797190934},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.13979117292910814},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.13895338994916528},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.14179203496314585},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.14013325097039342},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.13445434800814837},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.1405532129574567},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.1363693499006331},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.14372524106875062},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.1439504709560424},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.14506065100431442},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.1379968849942088},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.13682874804362655},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1450432789279148},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.13761299895122647},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.14437269791960716},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.14580935798585415},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.14671656000427902},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.14893923606723547},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.14821670798119158},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.14814867195673287},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1405109049519524},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.14826983807142824},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1491902939742431},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1423833869630471},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.14996503898873925},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.14966931706294417},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.14187186409253627},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.1489674790063873},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.15052221796941012},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.13813448196742684},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.14560759195592254},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.1373717699898407},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.14669288601726294},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.13763322809245437},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.14509601797908545},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.14877241500653327},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.13849701802246273},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.14797999698203057},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.14700240304227918},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.14496524399146438},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.15231437096372247},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.14662817504722625},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1546230569947511},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15461820899508893},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.1471476840088144},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.15570318198297173},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.15439139609225094},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.14693015802185982},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.15558894898276776},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.14749224693514407},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.1571215270087123},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.14765299705322832},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.15723630192223936},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.15694389608688653},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.1584627500269562},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.14858691801782697},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1587328229798004},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.15827767504379153},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.14964758697897196},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.1595225869677961},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.16115404991433024},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.16093773604370654},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.16071839700452983},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.16174039302859455},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.15078445803374052},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.1611654800362885},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.16222789301536977},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.1637194239301607},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.16360445995815098},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.1539139850065112},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.16386689897626638},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.15333357197232544},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.16514118597842753},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.15474925306625664},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.16541944502387196},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.1627571190474555},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.1731199180940166},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.1727521630236879},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.17422729393001646},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.16242626297753304},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.17435220698826015},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.16342700901441276},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.17437619203701615},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.16376860602758825},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.17222589091397822},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.18304876890033484},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.17281954793725163},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.1845698149409145},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.17326674202922732},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.17358473595231771},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.18537985696457326},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.1766381609486416},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.18897575291339308},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.17656980291940272},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.18851612601429224},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.17701085598673671},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.18958157300949097},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.19056210306007415},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.1779338780324906},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.1896045160247013},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.1802065660012886},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.192850471008569},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.18175912299193442},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.19426261400803924},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.1815304959891364},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.1818574359640479},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.19508751295506954},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.1844307859428227},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.19890962191857398},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.18550623103510588},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.19769212091341615},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.1862896829843521},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.19887803308665752},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.18708136910572648},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.19967927690595388},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.19971334200818092},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.19950599397998303},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.18886230199132115},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.20337699493393302},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.1893988320371136},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.2035478629404679},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.1900639230152592},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.20358492294326425},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.20432771800551564},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.20498727797530591},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.20471324806567281},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18855484807863832},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.18833845399785787},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.20428595994599164},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.19002199498936534},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.20564029598608613},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.18959847500082105},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.2047056930605322},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.20775392896030098},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.1926694830181077},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.19352634099777788},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.20735322090331465},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.19317262095864862},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.20921813999302685},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.2124386669602245},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.2020364940399304},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.2023636350641027},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.20287270401604474},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.2044544949894771},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.21614631300326437},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.2046007370809093},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.20466273301281035},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.20612035098019987},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.21758103906176984},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.22702524403575808},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.21315111604053527},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.2271009060787037},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.21914339298382401},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.2321837960043922},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.21990865108091384},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.2197807519696653},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.22351765201892704},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.2359845059690997},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.22476619109511375},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.23744187399279326},{"candidates":16,"kind":"choice","padded_tokens":3392,"seconds":0.22473136498592794},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.22710950206965208},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.24271042097825557},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.22850469406694174},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.23069186392240226},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.23061878699809313},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.2326261509442702},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.23236523498781025},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.23410821601282805},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.23546412400901318},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.23684734397102147},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.2381680179387331},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.2380914359819144},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.2392455379012972},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.24352026102133095},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.25905444601085037},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.24281631805934012},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.24738385796081275},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.24636324390303344},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.24748936505056918},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.2485430499073118},{"candidates":16,"kind":"choice","padded_tokens":3824,"seconds":0.25092930102255195},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.2500870400108397},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.27220732904970646},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.2545977830886841},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.26050182606559247},{"candidates":16,"kind":"choice","padded_tokens":3952,"seconds":0.2617401659954339},{"candidates":16,"kind":"choice","padded_tokens":3984,"seconds":0.2661120999837294},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.2658472260227427},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.26553838001564145},{"candidates":16,"kind":"choice","padded_tokens":4064,"seconds":0.2693321160040796},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.2710888530127704},{"candidates":16,"kind":"choice","padded_tokens":4128,"seconds":0.284181694034487},{"candidates":16,"kind":"choice","padded_tokens":4128,"seconds":0.3048733089817688},{"candidates":16,"kind":"choice","padded_tokens":4176,"seconds":0.29035697295330465},{"candidates":16,"kind":"choice","padded_tokens":4224,"seconds":0.29062369000166655},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.28509769309312105},{"candidates":16,"kind":"choice","padded_tokens":4272,"seconds":0.2878117050277069},{"candidates":16,"kind":"choice","padded_tokens":4304,"seconds":0.28703890298493207},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.28777900396380574},{"candidates":16,"kind":"choice","padded_tokens":4384,"seconds":0.2917394720716402},{"candidates":16,"kind":"choice","padded_tokens":4416,"seconds":0.2936593620106578},{"candidates":16,"kind":"choice","padded_tokens":4464,"seconds":0.2937660519964993},{"candidates":16,"kind":"choice","padded_tokens":4528,"seconds":0.305456196074374},{"candidates":16,"kind":"choice","padded_tokens":4544,"seconds":0.3356039089849219},{"candidates":16,"kind":"choice","padded_tokens":4576,"seconds":0.3111635820241645},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.3119134099688381},{"candidates":16,"kind":"choice","padded_tokens":4640,"seconds":0.3224271710496396},{"candidates":16,"kind":"choice","padded_tokens":4720,"seconds":0.32316553802229464},{"candidates":16,"kind":"choice","padded_tokens":4736,"seconds":0.32337037997785956},{"candidates":16,"kind":"choice","padded_tokens":4816,"seconds":0.33352687396109104},{"candidates":16,"kind":"choice","padded_tokens":4896,"seconds":0.3363147838972509},{"candidates":16,"kind":"choice","padded_tokens":4944,"seconds":0.32425150310155004},{"candidates":16,"kind":"choice","padded_tokens":5008,"seconds":0.32603174401447177},{"candidates":16,"kind":"choice","padded_tokens":5104,"seconds":0.3291627940488979},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.3624801990808919},{"candidates":16,"kind":"choice","padded_tokens":5328,"seconds":0.37063370808027685},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.37967181496787816},{"candidates":16,"kind":"choice","padded_tokens":5584,"seconds":0.37961697299033403},{"candidates":16,"kind":"choice","padded_tokens":5728,"seconds":0.3894385629100725},{"candidates":16,"kind":"choice","padded_tokens":5904,"seconds":0.39648633799515665},{"candidates":16,"kind":"choice","padded_tokens":6256,"seconds":0.42086518998257816},{"candidates":16,"kind":"choice","padded_tokens":6544,"seconds":0.45887877396307886},{"candidates":16,"kind":"choice","padded_tokens":7104,"seconds":0.4816511300159618},{"candidates":16,"kind":"choice","padded_tokens":7936,"seconds":0.5607748739421368},{"candidates":14,"kind":"choice","padded_tokens":7980,"seconds":0.5826781990472227},{"candidates":10,"kind":"choice","padded_tokens":6850,"seconds":0.5033715239260346},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.3213192339753732}],"chat_template":false,"data_loading_seconds":0.07094222400337458,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251813,"gpu":1,"id":"c14378062c3b5a69b3d0d96d","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_load_seconds":1.8751625180011615,"model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":361821120,"peak_gpu_bytes":1845263360,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":71.08455222600605,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.42201834862385323,"scoring_seconds":66.69843208498787,"seconds":69.28700043901335,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu1"}