{"acc_norm":0.6314984709480123,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.7425076452599388,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.2813723039580509},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.08605234394781291},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.08824147598352283},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.09305990301072598},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.09380945900920779},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.09473711904138327},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.09828333300538361},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.09895885200239718},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.10150107298977673},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11044563306495547},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.10517156799323857},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.10512618394568563},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.10828648693859577},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.10831782501190901},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.10897082299925387},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.109390364959836},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11573383398354053},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1201799160335213},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12021989095956087},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11951787199359387},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11710765690077096},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11660314700566232},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.13889259193092585},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.13784181303344667},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13952694495674223},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.14086401893291622},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.14118189108558},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.14060937392059714},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.140967761981301},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.14664150297176093},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.14633049303665757},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1480534760048613},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14970113907475024},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.14932474691886455},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1500505399890244},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.14818667504005134},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1482670339755714},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14830632705707103},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.15288174990564585},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1529974560253322},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.15403362805955112},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.15326376201119274},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.15246698702685535},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.15299339802004397},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.15380595298483968},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.15538544906303287},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.1554155049379915},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.15518542600329965},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.1544405159074813},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.15487838396802545},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.1551531150471419},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.15654009208083153},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1565994129050523},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1570956880459562},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.15596411807928234},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.15568011905997992},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.16902683791704476},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.17035628703888506},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.17145512602292},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.17106090998277068},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.16972883697599173},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.1695301269646734},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.17190613702405244},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.17331064702011645},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.17328326497226954},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.17506356502417475},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.17430529103148729},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.17476618103682995},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.17723442404530942},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.17707027809228748},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.17881153500638902},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.17698405706323683},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.17680675606243312},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.1772319480078295},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1761702389921993},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1750994420144707},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.17619515501428396},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.17534828605130315},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.17680233798455447},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.17651334893889725},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.17664602701552212},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.17666812799870968},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.18143006693571806},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.18049781990703195},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.18034427298698574},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.18098969699349254},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.18117118498776108},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.18167030601762235},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.18185968010220677},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.18216798407956958},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.18252730392850935},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.18193878699094057},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.1819118760759011},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.18216514796949923},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.18468035908881575},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1864615910453722},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.18867973797023296},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.1873137499205768},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.18641832005232573},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.18727803404908627},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.18727724207565188},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.19012896995991468},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1898443589452654},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.18915384204592556},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.18883953499607742},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.19313032703939825},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.1959086679853499},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.1942488420754671},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.19494004198350012},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.19248047505971044},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.19527701009064913},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.19635586207732558},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.19680398795753717},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.1971800789469853},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.1958354520611465},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.19853977498132735},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.19782825803849846},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.19892313901800662},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.19815779302734882},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.19818930106703192},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1982033969834447},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.1989155448973179},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.20144086505752057},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.19994688895531},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.1993199889548123},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.20028394693508744},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.20122315897606313},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.20232161798048764},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.200970466947183},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.2016621270449832},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.20338255597744137},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.20364234095904976},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.20397340808995068},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.20347519894130528},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.20167488104198128},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.2035691939527169},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.2040609010728076},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.21205601003021002},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.21417198597919196},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.21063044003676623},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.21292665298096836},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.2141954880207777},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.21219237393233925},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.21199133002664894},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.2149866259424016},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.21467727806884795},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.21451180009171367},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.21404660202097148},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.21477642399258912},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.21557571401353925},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.21867072500754148},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.2175997180165723},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.2180254029808566},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.21883687598165125},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.22051417408511043},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.21954476600512862},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.21910329209640622},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.22044780000578612},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.22089650703128427},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.22036488493904471},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.22100027999840677},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.2211845429847017},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.2204621669370681},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.2318788570119068},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.23245634301565588},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.23311632394324988},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.23262404603883624},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.23419680004008114},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.23273586493451148},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.2347471360117197},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.2348380850162357},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.23585947894025594},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.2333589270710945},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.23694892204366624},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.23740327905397862},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.23851131496485323},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2373786229873076},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.2381871900288388},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.24102190707344562},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.24166078900452703},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.23917864006944},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.24054313404485583},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.24141094204969704},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.24282586108893156},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.24283781391568482},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.24363736400846392},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.26639999297913164},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.26600703003350645},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.2662992449477315},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.2679664350580424},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.2670913920737803},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.26737231598235667},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.26872784399893135},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.26922195602674037},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.2691805879585445},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.2696261389646679},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.27277912897989154},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.27264452795498073},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.2720510809449479},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.2731654089875519},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.2753964599687606},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.2740771099925041},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.27335868997033685},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.2763774390332401},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.2758915030863136},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.2746443869546056},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.2752578320214525},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.2762210170039907},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.27890887996181846},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.27917634905315936},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.28030817001126707},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.28249768295791},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.2809634720906615},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.28082707698922604},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.283462070976384},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.2836628759978339},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.2832635110244155},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.2810822029132396},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.2833513739751652},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.28138643002603203},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.28385979298036546},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.28216379997320473},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.2837169370613992},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.28502309194300324},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.28193713701330125},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.2855581399053335},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.2861321000382304},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.284998566028662},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.2855838480172679},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.2973245830507949},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.29819807200692594},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.297532991040498},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.29916513501666486},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.29925282893236727},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.30003220203798264},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.2996899139834568},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.29992628400214016},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.29982935404405},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.30055560800246894},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.2974161528982222},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.2920390859944746},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.2940064240247011},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.2953295720508322},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.29264676896855235},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.2948445479851216},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.29753228998743},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.2976025310344994},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.29772139189299196},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.30341589206364006},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.3049453239655122},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.30302276904694736},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.3044159170240164},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.30450904997996986},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.3045559979509562},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.30436127504799515},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.30701223807409406},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.3049079339252785},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.30589840200264007},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.30737072601914406},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.30756612203549594},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.30918435798957944},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.31048254994675517},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.30947611399460584},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.3117457749322057},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.3108092399779707},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.311872060992755},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.321859015035443},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.32313444395549595},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.32354700309224427},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.3241326150018722},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.32409697910770774},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.3246385679813102},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.32597798702772707},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.3267071560258046},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.32518354593776166},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.3330755119677633},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.3347263609757647},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.33198277407791466},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.3357356230262667},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.3371372469700873},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.3351018310058862},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.3373075040290132},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.3369478939566761},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.33825014205649495},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.3411715399706736},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.3422285510459915},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.3396353949792683},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.34165504097472876},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.3420955130131915},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.3422805379377678},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.3429230059264228},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.3428665209794417},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.3430910900933668},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.3475740399444476},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.3474807860329747},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.3503024170640856},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.34911991201806813},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.3508149639237672},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.34947467409074306},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.35289339895825833},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.3543069150764495},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.3522372560109943},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.3554038910660893},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.3545800249557942},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.35444866900797933},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.35575383401010185},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.35578640399035066},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.3540270630037412},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.3597162930527702},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.36039589100982994},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.3593987609492615},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.361263868981041},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.3609555750153959},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.3612787659512833},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.36227303103078157},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.3633403709391132},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.36986951797734946},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.3687502299435437},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.37153267895337194},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.3713230879511684},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.37073267705272883},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.3674865129869431},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.3673780319513753},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.36745135695673525},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.37194958596955985},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.37248640693724155},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.3928225409472361},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.395883799996227},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.3958373130299151},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.39659373299218714},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.39779406203888357},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.39815864199772477},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.40021163993515074},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.4003903230186552},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.40039686497766525},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.4171754450071603},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.4121634679613635},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.4178004009881988},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.417593183927238},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.424854175071232},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.4289472680538893},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.42286654794588685},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.4294468390289694},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.43086911202408373},{"candidates":16,"kind":"choice","padded_tokens":3360,"seconds":0.43660697201266885},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.43550533801317215},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.43615010008215904},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.437224774970673},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.43590340996161103},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.4405303280800581},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.4390449980273843},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.44703379599377513},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.4443327789194882},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.44947752403095365},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.44837981602177024},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.45058389590121806},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.4558723090449348},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.45804762502666563},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.4585796770406887},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.46032900002319366},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.46019436803180724},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.4665314060403034},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.46409389993641526},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.4747564339777455},{"candidates":16,"kind":"choice","padded_tokens":3760,"seconds":0.47359997802414},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.47425808501429856},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.47757493099197745},{"candidates":16,"kind":"choice","padded_tokens":3824,"seconds":0.4800563369644806},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.4795788569608703},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.4862478740978986},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.4876478350488469},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.49641413206700236},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.49999471998307854},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.4976583730895072},{"candidates":16,"kind":"choice","padded_tokens":3984,"seconds":0.5016228349413723},{"candidates":16,"kind":"choice","padded_tokens":4016,"seconds":0.500210641999729},{"candidates":16,"kind":"choice","padded_tokens":4048,"seconds":0.5082725659012794},{"candidates":16,"kind":"choice","padded_tokens":4080,"seconds":0.5067137790611014},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.5119201489724219},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.5331702870316803},{"candidates":16,"kind":"choice","padded_tokens":4176,"seconds":0.541872555972077},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.5424399330513552},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.5352756729116663},{"candidates":16,"kind":"choice","padded_tokens":4256,"seconds":0.5358579490566626},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.5412868830608204},{"candidates":16,"kind":"choice","padded_tokens":4368,"seconds":0.5537131760502234},{"candidates":16,"kind":"choice","padded_tokens":4432,"seconds":0.5542196609312668},{"candidates":16,"kind":"choice","padded_tokens":4448,"seconds":0.5554190970724449},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.5712137820664793},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.5877049349946901},{"candidates":16,"kind":"choice","padded_tokens":4624,"seconds":0.5923372840043157},{"candidates":16,"kind":"choice","padded_tokens":4640,"seconds":0.5932958310004324},{"candidates":16,"kind":"choice","padded_tokens":4720,"seconds":0.6040523800766096},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.6095906289992854},{"candidates":16,"kind":"choice","padded_tokens":4848,"seconds":0.6117852909956127},{"candidates":16,"kind":"choice","padded_tokens":4896,"seconds":0.62019471405074},{"candidates":16,"kind":"choice","padded_tokens":4976,"seconds":0.6280462950235233},{"candidates":16,"kind":"choice","padded_tokens":5040,"seconds":0.6305977739393711},{"candidates":16,"kind":"choice","padded_tokens":5136,"seconds":0.6650280309841037},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.6681849070591852},{"candidates":16,"kind":"choice","padded_tokens":5280,"seconds":0.6814523190259933},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.7014480950310826},{"candidates":16,"kind":"choice","padded_tokens":5680,"seconds":0.7066739830188453},{"candidates":16,"kind":"choice","padded_tokens":6064,"seconds":0.7471354079898447},{"candidates":16,"kind":"choice","padded_tokens":6288,"seconds":0.7987943830667064},{"candidates":16,"kind":"choice","padded_tokens":6752,"seconds":0.8577674289699644},{"candidates":16,"kind":"choice","padded_tokens":7744,"seconds":0.984633422922343},{"candidates":14,"kind":"choice","padded_tokens":7854,"seconds":0.9964341500308365},{"candidates":11,"kind":"choice","padded_tokens":8030,"seconds":1.015414344961755},{"candidates":3,"kind":"choice","padded_tokens":3072,"seconds":0.37991983897518367}],"chat_template":false,"data_loading_seconds":0.07252137106843293,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_common_core.py":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-qwen35-boolq-lm-eval-v2","finished":1792249320,"gpu":1,"id":"43c0538c00e25eee8470f774","loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen3.5-0.8B-Base","model_load_seconds":2.6611542949685827,"model_revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":852985920,"peak_gpu_bytes":4670398976,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"qwen35-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"BoolQ / full sub-1B common core (qwen35)","parent_task":"reading","protocol_version":"codesota-common-core-qwen35-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core_qwen35.py","runtime_backend":"qwen35","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":127.71608892502263,"measurement_version":"gpu-slot-v1"},"runner_sha256":"eb01abebdaa64a9d8003d37b2006b198582b99f61377a6fe6d0645db15934307","scope":"087725fb28ea8991a3f6bacbedcecfe584a0305a5e4d876496d753c3718af752","score":0.7425076452599388,"scoring_seconds":122.61964568693656,"seconds":126.00197083002422,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"qwen35-boolq-common-core-full","text_parameters":752393024,"tf32":false,"truncated_candidates":2,"vision_parameters":100592896,"worker":"white-gpu1"}