{"acc_norm":0.3387372013651877,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.30716723549488056,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.07674525794573128},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.026262435014359653},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.023210352985188365},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.022992989048361778},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.021961465012282133},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.023564474075101316},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.02837660803925246},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.02317560906521976},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022742741974070668},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.023081674007698894},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.022670096019282937},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02912446204572916},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02285025198943913},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.021590552991256118},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.022290789056569338},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.026591970003210008},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.03376801195554435},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.028958874056115746},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02707559196278453},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02746861393097788},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.033970409887842834},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02758885908406228},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.027792589040473104},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02782805496826768},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.028610012959688902},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.03549745702184737},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.027903506066650152},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03026438900269568},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.029629183001816273},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.035595159977674484},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03298171597998589},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.042461442993953824},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.04355280799791217},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.030918347998522222},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03068277903366834},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.031062155961990356},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.037853541085496545},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02865168103016913},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.028930359985679388},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.036785228992812335},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03090435196645558},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0291024309117347},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0304440229665488},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03592680802103132},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.033338551060296595},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03266265103593469},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03154046798590571},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.039236029027961195},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.030775461927987635},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03200433298479766},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03146391606424004},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.036607616930268705},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03217284695710987},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.032021344057284296},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03919739695265889},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03118330193683505},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03352087095845491},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03242951596621424},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.04047347710002214},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.032453560968860984},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03599925304297358},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03938422491773963},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03262444899883121},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.031640985049307346},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03246604395098984},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.040166724007576704},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.031915387022309005},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03322143293917179},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03981843509245664},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.034279935993254185},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03235839295666665},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03298544301651418},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03878192196134478},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.032648504013195634},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.033411616925150156},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03437022492289543},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.040966124972328544},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03260010399390012},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.036633283947594464},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03991491498891264},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0330345940310508},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03437634604051709},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03898719500284642},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03418656310532242},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.036385984043590724},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03535112401004881},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04337966593448073},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03954789007548243},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04170613503083587},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.049844452063553035},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.036660124897025526},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.042534781037829816},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04831314703915268},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03876232507172972},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04771555191837251},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.040428883978165686},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03915877500548959},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04723757004830986},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03874030406586826},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04243047605268657},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.050002034986391664},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.038607387105003},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0401434910017997},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04622139502316713},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03988039004616439},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04333664593286812},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.047865902073681355},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.037873147055506706},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04147213906981051},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04869486892130226},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04142892896197736},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04234435595571995},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.05039967701304704},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.040684671024791896},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04828244901727885},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.039875481044873595},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.043383843963965774},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04988589906133711},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.040224150987342},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04480299004353583},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05394369608256966},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.041636344976723194},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04459053394384682},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04859866900369525},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04429870005697012},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05387502908706665},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04459256795234978},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04628925200086087},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.05003777297679335},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.044321712921373546},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.05355053301900625},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04322139103896916},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.046633123070932925},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.05211720091756433},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04227603797335178},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04869277402758598},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.05111050291452557},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04877906502224505},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.054661624948494136},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04362983303144574},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04672050604131073},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0518444610061124},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.047104391967877746},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.05310643499251455},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04623604309745133},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04842790006659925},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.052906492026522756},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04595445701852441},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05974597705062479},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04674797807820141},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.057566871982999146},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04952460597269237},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05193298601079732},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05932915909215808},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.05042937200050801},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.06119491800200194},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.050765709951519966},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.05991225701291114},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.04765980795491487},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.05206948099657893},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.058163376059383154},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.052650464000180364},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.05692826199810952},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.051275501027703285},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.061650817980989814},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.045801512082107365},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.06100347195751965},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.050530971959233284},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.052161603001877666},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.058364289929158986},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05097877699881792},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.06318819592706859},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.05635003396309912},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.06365880405064672},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.05300473503302783},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.06830392603296787},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.0540205700090155},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.06265186599921435},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.05186678294558078},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.061739525059238076},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.05449121701531112},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.06165859301108867},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.05777353898156434},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.06760482199024409},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.05604147806297988},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.05637592205312103},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0614492119057104},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.05489379807841033},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.06196168903261423},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05797652690671384},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.0671353560173884},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.055220999056473374},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.06618937198072672},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05486463406123221},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06718642194755375},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06299123901408166},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06502941902726889},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06464314891491085},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06849655497353524},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.07108701602555811},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06263101589865983},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06860309396870434},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05897772405296564},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.0689910469809547},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05971193395089358},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.0698938809800893},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.05861234199255705},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.07271038403268903},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.05916050495579839},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.06755797495134175},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.060859994031488895},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.07093632500618696},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.061892629018984735},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.06811330001801252},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06145012506749481},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06812264700420201},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06024993699975312},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06603261001873761},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06283289403654635},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.0644304429879412},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.06159031600691378},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.06545633496716619},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.061320994049310684},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06717455002944916},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06690498697571456},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.059937073034234345},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.06612097506877035},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.061192784924060106},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.06887912901584059},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.06829806498717517},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.07600898400414735},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.0696386150084436},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.07211627496872097},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.07911039900500327},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.07036432903259993},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.0827257800847292},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.06926944595761597},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.07682684995234013},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.07964378304313868},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.06926703196950257},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.07962678093463182},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.07065764500293881},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.07663254800718278},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.08205073093995452},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.06773590703960508},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.08297904208302498},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.07439336099196225},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.08107914996799082},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.08623210899531841},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.07880088093224913},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.0816227529430762},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.08607635798398405},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.07526819303166121},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.08339534804690629},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.08553283498622477},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.08044032799080014},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.08244445500895381},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.08211557299364358},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.07320370303932577},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.08615127904340625},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.08565202704630792},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.07450270594563335},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.08221843396313488},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.08247242798097432},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.08007975609507412},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.0918844590196386},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.0922663320088759},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.08383141201920807},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.09090470196679235},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.09081437508575618},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.09807141788769513},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.08608629694208503},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.09264461696147919},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.0980622700881213},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.09313221706543118},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.08962094900198281},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.09823634498752654},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.10114593105390668},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.09631175606045872},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.09761752106714994},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.09168617997784168},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.10337916598655283},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.10474250803235918},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.11744155001360923},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.11311483103781939},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.11078020802233368},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.11753246909938753},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12114707997534424},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.12035418103914708},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14208189002238214},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.13962354592513293},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.16068199509754777},{"candidates":15,"kind":"choice","padded_tokens":2670,"seconds":0.21512349997647107}],"chat_template":false,"data_loading_seconds":0.0475026749772951,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249361,"gpu":0,"id":"1d5a6e1c6de9a839eb75ee0c","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B-Instruct","model_load_seconds":2.1011930770473555,"model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":494032768,"peak_gpu_bytes":2634867712,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":20.724927515955642,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.30716723549488056,"scoring_seconds":16.661885487032123,"seconds":19.02325914904941,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu0"}