{"acc_norm":0.34044368600682595,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.3174061433447099,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.08833120297640562},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.01845074794255197},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.01887219399213791},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.020023760967887938},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.021082906052470207},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.020121012930758297},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.020293725072406232},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.020260883029550314},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.021452796063385904},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.020296169095672667},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.020651621976867318},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02061992301605642},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.020694472012110054},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.023513838998042047},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02415296097751707},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.024103178991936147},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02410184603650123},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.023563381982967257},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.023537834989838302},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.023654671967960894},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02383013896178454},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02384815295226872},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.024877984076738358},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.02533705998212099},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.025355514022521675},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.025037341052666306},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.02520575502421707},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.025125956046395004},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.025501276017166674},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.025320829008705914},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.025670912000350654},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.025960072060115635},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02551186503842473},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.025438679032959044},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.025474725989624858},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.025501525960862637},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02571290999185294},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.027202017954550683},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.028892069007270038},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0297746550058946},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.02911011502146721},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.02828473597764969},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.02819603099487722},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.028013710980303586},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.02782133303117007},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.027994015021249652},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.027965620975010097},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.028089553001336753},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.027988343965262175},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.027971341973170638},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028150024940259755},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028585466905497015},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028391005005687475},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028186594019643962},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.02805452700704336},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028342243982478976},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028240513987839222},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028411472914740443},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028465923969633877},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.027769576059654355},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.02814922307152301},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.02819967793766409},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.028009123052470386},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.028280679020099342},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.02835952490568161},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.028046861989423633},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.028117425041273236},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.027907732990570366},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.028121192008256912},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.02832870907150209},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.029301802976988256},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.02961052895989269},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.029253012035042048},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.028723074006848037},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.02971787902060896},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.029343339963816106},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.029075100901536644},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.028768478077836335},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.029203559970483184},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.02875110600143671},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.028589675086550415},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.028706654091365635},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.028403569012880325},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.028619640972465277},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.030328526976518333},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.031053910963237286},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.030951829976402223},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.031451081042177975},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.031209110049530864},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0307811819948256},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03134982299525291},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.030791281023994088},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.031052808044478297},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03138907591346651},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03144511999562383},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03140505601186305},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03140471503138542},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03152447799220681},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03150315803941339},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03155583702027798},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.031596772954799235},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03161785297561437},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03286618005950004},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03293247404508293},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03220043901819736},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.032769319019280374},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.032222259091213346},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03142118500545621},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03333770902827382},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.032476904918439686},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0321094790706411},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03219909605104476},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.032280709012411535},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03215038497000933},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03208680707029998},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.03202420996967703},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.03320887906011194},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.032684201025404036},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.03222701896447688},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.032617405988276005},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.032990942010656},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.03231324895750731},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.033007083111442626},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.032532247016206384},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0326174070360139},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.033054852043278515},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.033236130024306476},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0327266399981454},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.033379987929947674},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03351742494851351},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.032877601915970445},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.03359241399448365},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.03443149907980114},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0340440560830757},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.03341987309977412},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.033914183964952826},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.033691879943944514},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.033190374029800296},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.032990713021717966},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0340996899176389},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.03389407694339752},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.033490734989754856},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.034023829037323594},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.03379570401739329},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.0380787099711597},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.039740249048918486},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.03970354003831744},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.03931243100669235},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.03891678305808455},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.03984627698082477},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.03918400197289884},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.03952991694677621},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.03929617104586214},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.039248191053047776},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.03971958998590708},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.039399513974785805},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.040328326052986085},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.03782972600311041},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.03950485005043447},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.038916744058951735},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.03853928006719798},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.03875155607238412},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.03770009393338114},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.041114161955192685},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.04107616993132979},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.03861078305635601},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.03897981101181358},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.03996821399778128},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.039390236022882164},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.040940087055787444},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04228059703018516},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04148167697712779},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04138244199566543},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.042119215009734035},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04184519499540329},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04174311400856823},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04073896200861782},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04221243993379176},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.041644281009212136},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04223379900213331},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04311369999777526},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04301568795926869},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.04122377606108785},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.04418979701586068},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0428719389019534},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.04256447602529079},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.042059113970026374},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.04220722895115614},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.040944764972664416},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.04236961202695966},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.041522211977280676},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.04100239300169051},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.044513150001876056},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.045591992093250155},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.045185443945229053},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.04451860999688506},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.04498865699861199},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.045690956991165876},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.045349851017817855},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.04535574093461037},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.045483460067771375},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.04532059596385807},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.04538978496566415},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.044595203013159335},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.046981352963484824},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.04664416296873242},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.046180269098840654},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.04619983502198011},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.04502174898516387},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.04695767897646874},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.04637906898278743},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.045957062975503504},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.045346172992140055},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.04789225209970027},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.04730405402369797},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.04689360794145614},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.045494299032725394},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.047395264031365514},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.0473699769936502},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.04724355204962194},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.04599994304589927},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.04802612098865211},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.04749426897615194},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.04761556605808437},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.04789341299328953},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.04624598100781441},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.04952806199435145},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.048484275001101196},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.048667207011021674},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.05069867498241365},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.05338727904018015},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.05291112093254924},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.05109434307087213},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.054020880023017526},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.05243693699594587},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.05103520199190825},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.05314812297001481},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.05267183494288474},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.053420219919644296},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.051764722098596394},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.05390374199487269},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.05329981492832303},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.05334124295040965},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.05362548294942826},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.05606108496431261},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.05453636194579303},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.053231808939017355},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.05597783997654915},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.05559699994046241},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.05334541096817702},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.056930697057396173},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.05534202395938337},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.055980203906074166},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.05443130503408611},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.05745278007816523},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.0579695540945977},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.06157278397586197},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.05967000499367714},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.056807246059179306},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.059609321993775666},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.05971598101314157},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.06187792192213237},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.06011470395606011},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.06335906393360347},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.06179494806565344},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.061939336941577494},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.05906604800838977},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.05924510199110955},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.058938720962032676},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.06712012807838619},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.06720044696703553},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.06700052495580167},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.06708574399817735},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.06702414900064468},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.06723915098700672},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.06805858795996755},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.06973154796287417},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.0713802429381758},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.07173499395139515},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.07269070704933256},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.07426268793642521},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.0733016359154135},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.07565203891135752},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.0751737569225952},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.07398796605411917},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.08353546995203942},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.0853704719338566},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.08822031610179693},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.08954455598723143},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.10182360396720469},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.10442059708293527},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.11435589496977627},{"candidates":15,"kind":"choice","padded_tokens":2625,"seconds":0.16092428693082184}],"chat_template":false,"data_loading_seconds":0.06535516702570021,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249827,"gpu":1,"id":"fbd3fd393b595f36e656a1c4","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-360M-Instruct","model_load_seconds":2.2507320620352402,"model_revision":"a10cc1512eabd3dde888204e902eca88bddb4951","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":361821120,"peak_gpu_bytes":1683770368,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":16.590056234039366,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.3174061433447099,"scoring_seconds":12.364513890002854,"seconds":14.890956026036292,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu1"}