{"acc_norm":0.2773037542662116,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.2593856655290102,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.13430251204408705},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.012594706960953772},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.012050091987475753},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.013131157960742712},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.013436075998470187},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.012858399073593318},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.0127977259689942},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.012796964962035418},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.01227502094116062},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.011674089939333498},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.011662618955597281},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.011828057002276182},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.011639625998213887},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.012422927073203027},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.012650852091610432},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.012676098034717143},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.012710843933746219},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.01763718097936362},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.013132329913787544},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.012948325951583683},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.01292633602861315},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.012993450975045562},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.014137033023871481},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.014231609064154327},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.014075848972424865},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.013948560925200582},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.018883895012550056},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.01387531601358205},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.01402275008149445},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.01411351899150759},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.014118649065494537},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.014243441983126104},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.014164574909955263},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.01414008904248476},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0197373169939965},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.014933307887986302},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.013965612975880504},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.014086800045333803},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.014516480965539813},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014612799976021051},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014451449969783425},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014221310964785516},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.01955473597627133},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014474482974037528},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014399082050658762},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014414660981856287},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014422075008042157},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.014456278993748128},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.014961550012230873},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.014741179998964071},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.020059478003531694},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.01569077093154192},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.014597792993299663},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.01481951610185206},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.014629532000981271},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.01476811000611633},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.014645301038399339},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.01495347497984767},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.019969830987975},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015604369924403727},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015417081071063876},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015395189984701574},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015594591037370265},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015609970083460212},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.01569478795863688},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.021569473086856306},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.016792826005257666},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.01517879602033645},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015114646055735648},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015413022949360311},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015301765059120953},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015431336942128837},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.015184876974672079},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.02112888207193464},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.015407953993417323},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.015052280039526522},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.015236373990774155},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.015260918997228146},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.015244557987898588},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.01525676203891635},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.021049103001132607},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.016371730947867036},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.015038253972306848},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.015153538901358843},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.016154115088284016},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.016092139994725585},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.016356661915779114},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.016508044907823205},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.022286219988018274},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.017231684061698616},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.016310765990056098},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0162689290009439},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.016545474994927645},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.01637452596332878},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.016731751966290176},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.022782135056331754},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.017525751958601177},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.016392720048315823},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.01666626997757703},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.016659728018566966},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.016625683987513185},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.01736017398070544},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.021881194901652634},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.016968623967841268},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.01688409608323127},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.01721224805805832},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.017278460902161896},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.01753458008170128},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.022840303950943053},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.01756912295240909},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.016938677988946438},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.01723325694911182},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.017432559048756957},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.01724939700216055},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.01728908100631088},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.023439010023139417},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.017909058020450175},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.017595072044059634},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.01785319298505783},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.017857732018455863},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.01809399190824479},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.01789641403593123},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.023129452019929886},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.017746213939972222},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.017547734081745148},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.017804161994718015},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.018013742985203862},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.017763265990652144},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.02303677995223552},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0180549300275743},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.017437146976590157},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.018193879048340023},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.01767267705872655},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.01796760701108724},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0177930009085685},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.023880343069322407},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.018458702019415796},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.017511776997707784},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.017486699973233044},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.01770651899278164},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0181861340533942},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.023478833958506584},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.01857752399519086},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.017878649989143014},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.01862434099894017},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.018724758992902935},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.019077486940659583},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.024323528981767595},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.018524324987083673},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.018992487923242152},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.018848768901079893},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.018932876992039382},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.018719809013418853},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.0187634410103783},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.024180943961255252},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.018641945091076195},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.01885796699207276},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.01949325203895569},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.019808189012110233},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.020114380051381886},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.025388966081663966},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.019951416994445026},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.020281943026930094},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.02014454605523497},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.020048397011123598},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.020259540993720293},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.025787909980863333},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.019889701041392982},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.019932882045395672},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.020522902021184564},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.020852957968600094},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.026353194029070437},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.02112204907462001},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.021079509984701872},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.021006253082305193},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.02077494200784713},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.02072949695866555},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.027549655991606414},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.02174865803681314},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.020477837999351323},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.021264796028845012},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.021328734001144767},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.02571686706505716},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.021410688059404492},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.020749714924022555},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.020815616939216852},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.02084270806517452},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.021087925997562706},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.026715259067714214},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.02184247295372188},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.020484509994275868},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.02075483405496925},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.021523036994040012},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.027195575996302068},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.02160964801441878},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.021615759003907442},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.021900631953030825},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.022195070050656796},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.021997713018208742},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.02721868793014437},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.02178918302524835},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.021647698944434524},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.021886826027184725},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.02183119102846831},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.027446282911114395},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.02210896904580295},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.021744700032286346},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.021853723912499845},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.02200510597322136},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.022191814961843193},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.02758567302953452},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.021964068990200758},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.02189544099383056},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.022918539005331695},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.023027081973850727},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.02966845699120313},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.021898437989875674},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.02257992804516107},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.023080270970240235},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.02281017799396068},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.027696129051037133},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.022589375963434577},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.022541305981576443},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.023092404007911682},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.023075120989233255},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.027585583040490746},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.02281320304609835},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.023222206044010818},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.023248223937116563},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.023067658068612218},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.029208107967860997},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.023585813003592193},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.022445878013968468},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.023331729928031564},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.02357951202429831},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.029310539015568793},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.02503214089665562},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.02337612200062722},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.02397677197586745},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.023968717083334923},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.02985724899917841},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.023797789006493986},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.02357632596977055},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.025334245059639215},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.025951795978471637},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.03317108901683241},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.025061103980988264},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.02604388806503266},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.027082363958470523},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.032711060950532556},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.027571696904487908},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.026968703023158014},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.02778804104309529},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.03354532702360302},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.027799310977570713},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.027727467007935047},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.02933782001491636},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.028991764993406832},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.03261446102987975},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.028524082037620246},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.029106528963893652},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.029251188971102238},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.03579180606175214},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.02879802300594747},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.029005030053667724},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.029186348081566393},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.0353174009360373},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.027916629915125668},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.028301596990786493},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.03018171398434788},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.037114351987838745},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.03144119295757264},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.03084747609682381},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.03747159894555807},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.03191501600667834},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.03170572698581964},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.03278894501272589},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.038627714035101235},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.03236641804687679},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.032351559959352016},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.03359028103295714},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.03994760604109615},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.033633671002462506},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.03421168902423233},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.041945821954868734},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.03619182202965021},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.03740452299825847},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.046111791976727545},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.03955110698007047},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.045337065937928855},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.05101546598598361},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.04998136800713837},{"candidates":15,"kind":"choice","padded_tokens":2625,"seconds":0.07690137904137373}],"chat_template":false,"data_loading_seconds":0.04906679200939834,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249263,"gpu":0,"id":"06eac1bd5eb9dc5a7f34bd72","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_load_seconds":1.7823046809062362,"model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":134515008,"peak_gpu_bytes":764762112,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":10.394792075036094,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.2593856655290102,"scoring_seconds":6.666798416990787,"seconds":8.709941439912654,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu0"}