{"acc_norm":0.5093856655290102,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.26360904902685434},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.05060120695270598},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05108982906676829},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.051286443951539695},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05233064200729132},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05225523200351745},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.053789412952028215},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05248786509037018},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.052584445918910205},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05313807795755565},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05234835599549115},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.052743021049536765},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0570484499912709},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.056950237951241434},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.056936832959763706},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0579544099746272},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05905121599789709},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05879109003581107},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.060113425948657095},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.058050279039889574},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.057816001935862005},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05765501200221479},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05791104899253696},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05863464903086424},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05811221501789987},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05865140992682427},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058507901965640485},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.060016706003807485},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05863449804019183},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06003800604958087},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.058967620017938316},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05973797698970884},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06154595804400742},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.059474895941093564},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.060930489911697805},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05897566501516849},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06496667698957026},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06524174998048693},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06386298895813525},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06375877396203578},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06392251001670957},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06364735693205148},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06350612302776426},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06344284501392394},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06372285808902234},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06579173600766808},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06648809497710317},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0672522900858894},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06732830195687711},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06656883493997157},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06558282591868192},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06407483399379998},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0639077820815146},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06601772690191865},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06950063293334097},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0664122230373323},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06676782702561468},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06681002397090197},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06679808301851153},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06765811599325389},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06741100596264005},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06567625002935529},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06824234605301172},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06805950496345758},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06807511392980814},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06702917290385813},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06514040101319551},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06940420297905803},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06648835504893214},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06708077993243933},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06715870508924127},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06817052199039608},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.067810088978149},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06752486794721335},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06794336799066514},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06952331494539976},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06800941098481417},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06878307508304715},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07037295994814485},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06831565196625888},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07145788299385458},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07213066809345037},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07223059399984777},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07252303906716406},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07178725604899228},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07288748899009079},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07127027295064181},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07129008998163044},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07187731505837291},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07142206595744938},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07255070109385997},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0724152879556641},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0720907230861485},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07408397004473954},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07246497098822147},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07311485300306231},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07409034203737974},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07398822205141187},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07215494301635772},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07387803692836314},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.072024469031021},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07607058493886143},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07445800793357193},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07463911606464535},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07035670906770974},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0717677609063685},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07457766204606742},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07244067604187876},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07260210695676506},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07265942404046655},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07367717195302248},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0742397599387914},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07351656199898571},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07286277203820646},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07721447909716517},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0759617619914934},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07602082297671586},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07635556592140347},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07625722407829016},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07755223801359534},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07675503089558333},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07807845994830132},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07713322702329606},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07583192002493888},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07613463397137821},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07914594002068043},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0786082079866901},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07663557806517929},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08057819202076644},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07835251092910767},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07873740897048265},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07838241697754711},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07756958995014429},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07716769096441567},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07915289397351444},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07883851695805788},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07779779692646116},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07761058700270951},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07917406305205077},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07811404601670802},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07700799300801009},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07989879499655217},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08502790005877614},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08453100291080773},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08341105503495783},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08707789308391511},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08774094004184008},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.0879138020100072},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08909317199140787},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08399458299390972},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08313180494587868},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08474029297940433},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08728554006665945},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08730002690572292},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08739703800529242},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08514847501646727},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08885449694935232},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08945529710035771},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08633718092460185},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08837087405845523},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08794472902081907},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08751370606478304},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08882668404839933},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08698437898419797},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08817497896961868},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08406950207427144},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08529463689774275},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09013242903165519},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08808292692992836},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08693449606653303},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.0895952379796654},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.08946511498652399},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0924392509041354},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.093349608941935},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.08639378705993295},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09033220098353922},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09197360300458968},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09230219596065581},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08664852194488049},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0869741189526394},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0868327560601756},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08902599604334682},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09455521695781499},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09685496601741761},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09250724804587662},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09121943591162562},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08753515500575304},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08733877900522202},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08725052501540631},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09537847305182368},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09822770592290908},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09595072991214693},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09525997110176831},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09660379798151553},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09675416897516698},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09812169801443815},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09739811904728413},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09584951098076999},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09926723502576351},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10067019204143435},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09885328297968954},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09899003803730011},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09450562496203929},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09507536794990301},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09345982503145933},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09409950801637024},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09730204998049885},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09888098505325615},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09860374801792204},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09712956903968006},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09509198903106153},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10254342400003225},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10313172300811857},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10561918094754219},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10118598397821188},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10242840996943414},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10434573597740382},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10475214396137744},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10580484697129577},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10317325999494642},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10121447604615241},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10338601691182703},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10485437395982444},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10629422997590154},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10305279400199652},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10223576100543141},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.10878030501771718},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.10908756905701011},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.10712851607240736},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.10899137996602803},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.10530186898540705},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.10641006496734917},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.10916324995923787},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.10777313797734678},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1100164009258151},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11167674709577113},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11395939497742802},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11260275403037667},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.10800743394065648},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11103646398987621},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.10884346393868327},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11510350799653679},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11148181394673884},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11080411996226758},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11450655502267182},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11229620291851461},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1105784890241921},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11515953298658133},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11198850895743817},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11784120299853384},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11597986205015332},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.114344491972588},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11900294001679868},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12128075899090618},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11822907696478069},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12381224904675037},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12218163802754134},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11935581802390516},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.119887680048123},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12872984097339213},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12293915101327002},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1268210100242868},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12287916906643659},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1281428060028702},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1277390030445531},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12985393695998937},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12896628095768392},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.12817566702142358},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13035701506305486},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.15399600996170193},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15561363706365228},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1511256869416684},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.14700406999327242},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15099238802213222},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1526808269554749},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1670689390739426},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.16734721802640706},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.16640489106066525},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.16845525498501956},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.16387619508896023},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.1672501569846645},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.17296588700264692},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.17085007100831717},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.1724198079900816},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.17033233609981835},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.17862277594394982},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.19732944201678038},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.19042181107215583},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.19189309398643672},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.21350103104487062},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.21516597596928477},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2377812119666487},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.321765375090763}],"chat_template":false,"data_loading_seconds":0.06315458798781037,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212102,"gpu":1,"id":"0e0d5b30a00095f1910c3513","images_supplied":false,"loaded_class":"Qwen3_5ForCausalLM","loader_ignored_auxiliary_patterns":["^mtp.*","^model.visual.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"Mapika/decider-0.8b","model_load_seconds":3.2070934480288997,"model_revision":"a0a01d6f8135298f400a8c856b355793012ae971","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":752393024,"peak_gpu_bytes":4111305728,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":32.45979036309291,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.46757679180887374,"scoring_seconds":27.303209768026136,"seconds":30.76483834197279,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":0,"worker":"white-gpu1"}