{"acc_norm":0.3506825938566553,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.3387372013651877,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.3699197300011292},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.14134315203409642},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13578293402679265},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13461116794496775},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13336805999279022},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1375374449416995},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1380134830251336},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13838465500157326},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1394937529694289},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13597300800029188},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1375921170692891},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13945943000726402},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13787243992555887},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1355817390140146},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13698956300504506},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13793592795263976},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1385854691034183},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1397673530736938},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.14299548405688256},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.14058765396475792},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13984724297188222},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.14228752302005887},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.14476770802866668},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1453813730040565},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14250955695752054},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14065199391916394},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14142972405534238},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14021901600062847},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1404772880487144},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1426930190064013},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.141472763963975},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14209418196696788},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13875042798463255},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13785654993262142},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1377433988964185},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14036089088767767},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14116334705613554},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14316301606595516},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13950136804487556},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14068199903704226},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13924804504495114},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1393546840408817},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1401836599688977},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14668743801303208},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14157782006077468},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13598282600287348},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1369866180466488},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1381071270443499},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13860556692816317},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14423105807509273},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14394620701204985},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14213267504237592},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14181307901162654},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14147733303252608},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14192099997308105},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14208385301753879},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14090754999779165},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1443660989170894},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14051683107391},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14012411900330335},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1395659779664129},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14053944393526763},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14210663607809693},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1418315829942003},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14229929505381733},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14600942400284111},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14124348596669734},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1416920330375433},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1428701300173998},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14031855098437518},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13724948791787028},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13743140804581344},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1451758099719882},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1433209099341184},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13724518998060375},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.137153048068285},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13941622897982597},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14418150600977242},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14172759000211954},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14073759294115007},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14313977304846048},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14288680104073137},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14282434503547847},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1417836940381676},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1422314289957285},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14286440901923925},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14208402391523123},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14221590000670403},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13666042999830097},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14030012802686542},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1423872490413487},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14292325894348323},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14147199201397598},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13854887103661895},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14060305198654532},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14204487099777907},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14342222001869231},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14491949207149446},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14401102799456567},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14148494694381952},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14446715801022947},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14169035910163075},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1401546458946541},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14182569202966988},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14073458698112518},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14033067401032895},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14091030496638268},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1415237399050966},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14375707402359694},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14473534806165844},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14210753794759512},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14218797802459449},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1416297770338133},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14215098798740655},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14229704102035612},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14251512801274657},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14418940001633018},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14178079809062183},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14313343004323542},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14191693300381303},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14265733293723315},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14083893201313913},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14290177903603762},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14239755901508033},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13797294802498072},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13946269499137998},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13903504807967693},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13872849592007697},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1379263399867341},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1379180250223726},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1361976460320875},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1357760609826073},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14266708097420633},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13760883803479373},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14282636798452586},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14232710702344775},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14124198397621512},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14197661296930164},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1420284400228411},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14059757196810097},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14342758001293987},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.137951196054928},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13792393601033837},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13728437409736216},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13773362094070762},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13856288802344352},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13819912890903652},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1392957540228963},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13837971596512944},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13701592292636633},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13644649100024253},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13714858004823327},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13745395897421986},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13749892404302955},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13715242699254304},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13942819100338966},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13960902800317854},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13718678196892142},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13831877300981432},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13741697103250772},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13610460294876248},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1364054341102019},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13708407001104206},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13623071799520403},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1366846349555999},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13541967596393079},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13457967003341764},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1354593699797988},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1405152779771015},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.137070675031282},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13449964998289943},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13918043905869126},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14037518703844398},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13802261999808252},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13647083600517362},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13737498095724732},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13528063602279872},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13564172096084803},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13793851097580045},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13635052007157356},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1356526119634509},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13580457400530577},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13656649400945753},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1342946080258116},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13668324204627424},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13592018000781536},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13749971496872604},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1394463850883767},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14097509603016078},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13571988500189036},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13471348898019642},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1344091520877555},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13450254604686052},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13620164396706969},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13568507099989802},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13479239598382264},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1370442359475419},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13723131408914924},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13764935405924916},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1392518930369988},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1356026369612664},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13604694593232125},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1359371510334313},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13922096509486437},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1370448460802436},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1348921119933948},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13616030698176473},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13662254891823977},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13500262901652604},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14234727493021637},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13591699302196503},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1360519150039181},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13711443694774061},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1363134429557249},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1381754350150004},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1368447740096599},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13771580799948424},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13743697805330157},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13844113994855434},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13993471499998122},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13618151703849435},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1365285529755056},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13726652995683253},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14006063004489988},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13916470005642623},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13560628390405327},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.135327504016459},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13692438194993883},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13509251596406102},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13986267207656056},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.135935127036646},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13691744895186275},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13527334306854755},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13676940300501883},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13535190001130104},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1356513580540195},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13681753200944513},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13542923401109874},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13549078803043813},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13606281497050077},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13709616207052022},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1352608100278303},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13577518903184682},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13582501199562103},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13743244903162122},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13581191701814532},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1351314689964056},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13663249695673585},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13935230998322368},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13771100901067257},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14042732398957014},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13632540497928858},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13747399696148932},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1364914949517697},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.136196844978258},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13601743010804057},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13755555904936045},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1360330699244514},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1365466370480135},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13635929906740785},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1391865200130269},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13626860897056758},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13818653603084385},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13580700801685452},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13845097902230918},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13717894698493183},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13843471906147897},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13712756102904677},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13596982194576412},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13559537404216826},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.13904746097978204},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13830661005340517},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13875162997283041},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.14178030705079436},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.14202627597842366},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14012248592916876},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1384359309449792},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1397115490399301},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14223041699733585},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1408890859456733},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.13912801095284522},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14130290795583278},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.14106779801659286},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.14033857907634228},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14036918606143445},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.1400385299930349},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14313871099147946},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.14189859700854868},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1399831660091877},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14022575900889933},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14064130291808397},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.14261134702246636},{"candidates":15,"kind":"choice","padded_tokens":2610,"seconds":0.16042258206289262}],"chat_template":false,"data_loading_seconds":0.06240190006792545,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249237,"gpu":0,"id":"8a8b014089999e1d5f493a4c","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_load_seconds":2.007599369972013,"model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":894092288,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":45.0970214480767,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.3387372013651877,"scoring_seconds":41.264393112040125,"seconds":43.453665363020264,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu0"}