{"api_version":"codesota.v1","data":{"method":"Same pinned model, dataset scope and GPU; warmed inference only. Data/model loading and Docker testing excluded. Choice: median of 3 repeats; generation: one run. Numeric changes are possible even with greedy decoding.","production_policy":"Choice batch 16, math batch 8. Code generation remains serial because one 0.5B unit-test outcome changed. Media and embeddings keep existing runners.","reports":[{"comparisons":[{"batch_size":4,"batched_repeats":[0.2971665080403909,0.28036247799173,0.2772137160645798],"batched_seconds":0.28036247799173,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.047359466552734375,"peak_gpu_bytes":1047984128,"scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","serial_repeats":[0.600220178021118,0.6144361699698493,0.5958431039471179],"serial_seconds":0.600220178021118,"speedup":2.140871996568753,"task":"reading"},{"batch_size":8,"batched_repeats":[0.25134951004292816,0.24548107490409166,0.24949766602367163],"batched_seconds":0.24949766602367163,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.019006967544555664,"peak_gpu_bytes":1090581504,"scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","serial_repeats":[0.600220178021118,0.6144361699698493,0.5958431039471179],"serial_seconds":0.600220178021118,"speedup":2.4057146008098163,"task":"reading"},{"batch_size":16,"batched_repeats":[0.2640327640110627,0.2595339029794559,0.2600505769951269],"batched_seconds":0.2600505769951269,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.02636098861694336,"peak_gpu_bytes":1175579648,"scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","serial_repeats":[0.600220178021118,0.6144361699698493,0.5958431039471179],"serial_seconds":0.600220178021118,"speedup":2.3080901606012034,"task":"reading"},{"batch_size":4,"batched_repeats":[0.34159845497924834,0.3548655279446393,0.3366321820067242],"batched_seconds":0.34159845497924834,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.005686283111572266,"peak_gpu_bytes":1088720384,"scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","serial_repeats":[1.1285778289893642,1.124889801023528,1.117881710990332],"serial_seconds":1.124889801023528,"speedup":3.293017824368859,"task":"commonsense"},{"batch_size":8,"batched_repeats":[0.21036017301958054,0.21072633704170585,0.21184177696704865],"batched_seconds":0.21072633704170585,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.0069122314453125,"peak_gpu_bytes":1131270656,"scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","serial_repeats":[1.1285778289893642,1.124889801023528,1.117881710990332],"serial_seconds":1.124889801023528,"speedup":5.338154768954655,"task":"commonsense"},{"batch_size":16,"batched_repeats":[0.16562068497296423,0.16379654197953641,0.16374776093289256],"batched_seconds":0.16379654197953641,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.007098674774169922,"peak_gpu_bytes":1216372736,"scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","serial_repeats":[1.1285778289893642,1.124889801023528,1.117881710990332],"serial_seconds":1.124889801023528,"speedup":6.8676040863186465,"task":"commonsense"},{"batch_size":4,"batched_repeats":[0.16647919500246644,0.169218824012205,0.170905630919151],"batched_seconds":0.169218824012205,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.042963504791259766,"peak_gpu_bytes":1012788224,"scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","serial_repeats":[0.5583822580520064,0.5459399229148403,0.5491064289817587],"serial_seconds":0.5491064289817587,"speedup":3.244948853575262,"task":"sentiment"},{"batch_size":8,"batched_repeats":[0.10019821999594569,0.0923683779546991,0.09657264896668494],"batched_seconds":0.09657264896668494,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.04221677780151367,"peak_gpu_bytes":1020902912,"scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","serial_repeats":[0.5583822580520064,0.5459399229148403,0.5491064289817587],"serial_seconds":0.5491064289817587,"speedup":5.685941463314174,"task":"sentiment"},{"batch_size":16,"batched_repeats":[0.06832152104470879,0.065673129982315,0.06924992194399238],"batched_seconds":0.06832152104470879,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.029059410095214844,"peak_gpu_bytes":1036140544,"scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","serial_repeats":[0.5583822580520064,0.5459399229148403,0.5491064289817587],"serial_seconds":0.5491064289817587,"speedup":8.037093152865111,"task":"sentiment"},{"batch_size":4,"batched_repeats":[0.2345536849461496,0.23058623599354178,0.34950518095865846],"batched_seconds":0.2345536849461496,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.033130645751953125,"peak_gpu_bytes":1013556224,"scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","serial_repeats":[0.8538763679098338,0.8507431350881234,0.8288339640712366],"serial_seconds":0.8507431350881234,"speedup":3.627072136101561,"task":"entailment"},{"batch_size":8,"batched_repeats":[0.13751603895798326,0.13804111792705953,0.1356655169511214],"batched_seconds":0.13751603895798326,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.08371353149414062,"peak_gpu_bytes":1019535360,"scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","serial_repeats":[0.8538763679098338,0.8507431350881234,0.8288339640712366],"serial_seconds":0.8507431350881234,"speedup":6.186501164042836,"task":"entailment"},{"batch_size":16,"batched_repeats":[0.1062843919498846,0.10566993593238294,0.10544044803828001],"batched_seconds":0.10566993593238294,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.055899620056152344,"peak_gpu_bytes":1035193344,"scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","serial_repeats":[0.8538763679098338,0.8507431350881234,0.8288339640712366],"serial_seconds":0.8507431350881234,"speedup":8.050947770352627,"task":"entailment"},{"batch_size":4,"batched_repeats":[0.3399381880881265,0.33929265406914055,0.34753246104810387],"batched_seconds":0.3399381880881265,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.05374336242675781,"peak_gpu_bytes":1066841600,"scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","serial_repeats":[1.1347573839593679,1.1321275099180639,1.1261092749191448],"serial_seconds":1.1321275099180639,"speedup":3.3303922583259995,"task":"arc-challenge-choice"},{"batch_size":8,"batched_repeats":[0.22515708999708295,0.22777703602332622,0.23066113505046815],"batched_seconds":0.22777703602332622,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.015370845794677734,"peak_gpu_bytes":1098745344,"scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","serial_repeats":[1.1347573839593679,1.1321275099180639,1.1261092749191448],"serial_seconds":1.1321275099180639,"speedup":4.9703320830029805,"task":"arc-challenge-choice"},{"batch_size":16,"batched_repeats":[0.19077692099381238,0.1884768509771675,0.1931891089770943],"batched_seconds":0.19077692099381238,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.03254365921020508,"peak_gpu_bytes":1161676288,"scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","serial_repeats":[1.1347573839593679,1.1321275099180639,1.1261092749191448],"serial_seconds":1.1321275099180639,"speedup":5.93430014500959,"task":"arc-challenge-choice"},{"batch_size":4,"batched_repeats":[0.3121457149973139,0.3071422229986638,0.3080282249720767],"batched_seconds":0.3080282249720767,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.05489826202392578,"peak_gpu_bytes":1030667776,"scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","serial_repeats":[1.0915181819582358,1.0965651560109109,1.0927511820336804],"serial_seconds":1.0927511820336804,"speedup":3.5475683507014337,"task":"openbookqa-main-choice"},{"batch_size":8,"batched_repeats":[0.17987539898604155,0.1821843560319394,0.18135019892361015],"batched_seconds":0.18135019892361015,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.029590606689453125,"peak_gpu_bytes":1043742720,"scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","serial_repeats":[1.0915181819582358,1.0965651560109109,1.0927511820336804],"serial_seconds":1.0927511820336804,"speedup":6.025640934058077,"task":"openbookqa-main-choice"},{"batch_size":16,"batched_repeats":[0.11729059799108654,0.11620235804002732,0.11819219007156789],"batched_seconds":0.11729059799108654,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.03309154510498047,"peak_gpu_bytes":1070492160,"scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","serial_repeats":[1.0915181819582358,1.0965651560109109,1.0927511820336804],"serial_seconds":1.0927511820336804,"speedup":9.316613613963531,"task":"openbookqa-main-choice"},{"batch_size":4,"batched_score":0.03125,"batched_seconds":17.86346282402519,"changed_item_scores":0,"changed_predictions":3,"examples":32,"kind":"generation","peak_gpu_bytes":1033307648,"repeats":1,"scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","serial_score":0.03125,"serial_seconds":31.58672497002408,"speedup":1.76823078935972,"task":"math"},{"batch_size":8,"batched_score":0.03125,"batched_seconds":11.093595852027647,"changed_item_scores":0,"changed_predictions":0,"examples":32,"kind":"generation","peak_gpu_bytes":1058638336,"repeats":1,"scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","serial_score":0.03125,"serial_seconds":31.58672497002408,"speedup":2.847293645031315,"task":"math"},{"batch_size":4,"batched_score":0.59375,"batched_seconds":22.598658623057418,"changed_item_scores":1,"changed_predictions":3,"examples":32,"kind":"generation","peak_gpu_bytes":1037457408,"repeats":1,"scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","serial_score":0.625,"serial_seconds":53.38210642198101,"speedup":2.3621803095655967,"task":"code-generation"},{"batch_size":8,"batched_score":0.59375,"batched_seconds":11.159352226997726,"changed_item_scores":1,"changed_predictions":3,"examples":32,"kind":"generation","peak_gpu_bytes":1070066176,"repeats":1,"scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","serial_score":0.625,"serial_seconds":53.38210642198101,"speedup":4.783620530664328,"task":"code-generation"}],"config":{"choice_batch_sizes":[4,8,16],"choice_repeats":3,"choice_tasks":["reading","commonsense","sentiment","entailment","arc-challenge-choice","openbookqa-main-choice"],"generation_batch_sizes":[4,8],"generation_tasks":["math","code-generation"],"seed":42},"gpu":"NVIDIA GeForce RTX 3090","legacy_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","measured_at":1792197459,"model":"Qwen/Qwen2.5-0.5B","model_load_seconds":0.9620922919129953,"physical_gpu":"0","revision":"060db6499f32faf8b98477b0a26969ef7d8b9987","runner_sha256":"e4c8e9487ef46d6c7358d927a6536cd118c53f66e8e432964cae0946de70480b","torch":"2.8.0+cu128","transformers":"4.56.2"},{"comparisons":[{"batch_size":4,"batched_repeats":[0.7088887150166556,0.6886495479848236,0.6891941240755841],"batched_seconds":0.6891941240755841,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.03110504150390625,"peak_gpu_bytes":3174513664,"scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","serial_repeats":[1.0232102070003748,1.0176695550326258,1.0186945659806952],"serial_seconds":1.0186945659806952,"speedup":1.4780952570468733,"task":"reading"},{"batch_size":8,"batched_repeats":[0.660398209001869,0.6605109899537638,0.6614233909640461],"batched_seconds":0.6605109899537638,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.02793264389038086,"peak_gpu_bytes":3252205568,"scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","serial_repeats":[1.0232102070003748,1.0176695550326258,1.0186945659806952],"serial_seconds":1.0186945659806952,"speedup":1.5422825380271181,"task":"reading"},{"batch_size":16,"batched_repeats":[0.6625273310346529,0.6618705959990621,0.6625410560518503],"batched_seconds":0.6625273310346529,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.025635719299316406,"peak_gpu_bytes":3407441920,"scope":"3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8","serial_repeats":[1.0232102070003748,1.0176695550326258,1.0186945659806952],"serial_seconds":1.0186945659806952,"speedup":1.5375887427765804,"task":"reading"},{"batch_size":4,"batched_repeats":[0.5235468610189855,0.5253849279833958,0.5220861460547894],"batched_seconds":0.5235468610189855,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.0060443878173828125,"peak_gpu_bytes":3181053440,"scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","serial_repeats":[1.2752673069480807,1.2805055560311303,1.2750125910388306],"serial_seconds":1.2752673069480807,"speedup":2.4358226586747413,"task":"commonsense"},{"batch_size":8,"batched_repeats":[0.4411652380367741,0.4425054669845849,0.4417891720077023],"batched_seconds":0.4417891720077023,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.008848190307617188,"peak_gpu_bytes":3223603712,"scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","serial_repeats":[1.2752673069480807,1.2805055560311303,1.2750125910388306],"serial_seconds":1.2752673069480807,"speedup":2.8865970190094368,"task":"commonsense"},{"batch_size":16,"batched_repeats":[0.38581330596935004,0.38615964201744646,0.38853677501901984],"batched_seconds":0.38615964201744646,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.005952358245849609,"peak_gpu_bytes":3308705792,"scope":"702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062","serial_repeats":[1.2752673069480807,1.2805055560311303,1.2750125910388306],"serial_seconds":1.2752673069480807,"speedup":3.3024354908907463,"task":"commonsense"},{"batch_size":4,"batched_repeats":[0.23924480401910841,0.23198114801198244,0.2303108519408852],"batched_seconds":0.23198114801198244,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.054396986961364746,"peak_gpu_bytes":3110803456,"scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","serial_repeats":[0.6380494689801708,0.6465045380173251,0.6480961060151458],"serial_seconds":0.6465045380173251,"speedup":2.7868839496558206,"task":"sentiment"},{"batch_size":8,"batched_repeats":[0.18559291190467775,0.18671607796568424,0.3003705239389092],"batched_seconds":0.18671607796568424,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.02863633632659912,"peak_gpu_bytes":3126555136,"scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","serial_repeats":[0.6380494689801708,0.6465045380173251,0.6480961060151458],"serial_seconds":0.6465045380173251,"speedup":3.4625006323030387,"task":"sentiment"},{"batch_size":16,"batched_repeats":[0.1597348980139941,0.1593990409746766,0.16007488104514778],"batched_seconds":0.1597348980139941,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.04548025131225586,"peak_gpu_bytes":3150947328,"scope":"e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0","serial_repeats":[0.6380494689801708,0.6465045380173251,0.6480961060151458],"serial_seconds":0.6465045380173251,"speedup":4.047359381421372,"task":"sentiment"},{"batch_size":4,"batched_repeats":[0.36132709903176874,0.3600469919620082,0.3582041549962014],"batched_seconds":0.3600469919620082,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.08685684204101562,"peak_gpu_bytes":3111495680,"scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","serial_repeats":[0.9548074660124257,0.967170984018594,0.9650672299321741],"serial_seconds":0.9650672299321741,"speedup":2.680392425092131,"task":"entailment"},{"batch_size":8,"batched_repeats":[0.3057265429524705,0.30174276407342404,0.3020788700086996],"batched_seconds":0.3020788700086996,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.11722755432128906,"peak_gpu_bytes":3125683200,"scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","serial_repeats":[0.9548074660124257,0.967170984018594,0.9650672299321741],"serial_seconds":0.9650672299321741,"speedup":3.194752515806157,"task":"entailment"},{"batch_size":16,"batched_repeats":[0.25027340301312506,0.2514021770330146,0.25329436699394137],"batched_seconds":0.2514021770330146,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.1078042984008789,"peak_gpu_bytes":3149303808,"scope":"38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7","serial_repeats":[0.9548074660124257,0.967170984018594,0.9650672299321741],"serial_seconds":0.9650672299321741,"speedup":3.838738555575196,"task":"entailment"},{"batch_size":4,"batched_repeats":[0.6274732939200476,0.6249694839352742,0.6240577240241691],"batched_seconds":0.6249694839352742,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.012441158294677734,"peak_gpu_bytes":3158866432,"scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","serial_repeats":[1.3201875739032403,1.3202161670196801,1.3258234449895099],"serial_seconds":1.3202161670196801,"speedup":2.112449009040592,"task":"arc-challenge-choice"},{"batch_size":8,"batched_repeats":[0.5227666050195694,0.5243469929555431,0.5235125969629735],"batched_seconds":0.5235125969629735,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.010687828063964844,"peak_gpu_bytes":3190478848,"scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","serial_repeats":[1.3201875739032403,1.3202161670196801,1.3258234449895099],"serial_seconds":1.3202161670196801,"speedup":2.521842214836055,"task":"arc-challenge-choice"},{"batch_size":16,"batched_repeats":[0.47089706792030483,0.47099284594878554,0.47230031504295766],"batched_seconds":0.47099284594878554,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.014543533325195312,"peak_gpu_bytes":3253701120,"scope":"78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06","serial_repeats":[1.3201875739032403,1.3202161670196801,1.3258234449895099],"serial_seconds":1.3202161670196801,"speedup":2.803049299740822,"task":"arc-challenge-choice"},{"batch_size":4,"batched_repeats":[0.39738868502900004,0.39117859199177474,0.3917699650628492],"batched_seconds":0.3917699650628492,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.032810211181640625,"peak_gpu_bytes":3122692608,"scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","serial_repeats":[1.2713205959880725,1.2904391420306638,1.274097544956021],"serial_seconds":1.274097544956021,"speedup":3.252157282530899,"task":"openbookqa-main-choice"},{"batch_size":8,"batched_repeats":[0.2992883659899235,0.29547362099401653,0.29501281201373786],"batched_seconds":0.29547362099401653,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.02657318115234375,"peak_gpu_bytes":3135767552,"scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","serial_repeats":[1.2713205959880725,1.2904391420306638,1.274097544956021],"serial_seconds":1.274097544956021,"speedup":4.312051751590448,"task":"openbookqa-main-choice"},{"batch_size":16,"batched_repeats":[0.2519514620071277,0.2523090390022844,0.2523421309888363],"batched_seconds":0.2523090390022844,"changed_predictions":0,"examples":32,"kind":"choice","max_logprob_delta":0.022623062133789062,"peak_gpu_bytes":3162516992,"scope":"4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79","serial_repeats":[1.2713205959880725,1.2904391420306638,1.274097544956021],"serial_seconds":1.274097544956021,"speedup":5.049749901922798,"task":"openbookqa-main-choice"},{"batch_size":4,"batched_score":0.09375,"batched_seconds":20.89458508696407,"changed_item_scores":0,"changed_predictions":1,"examples":32,"kind":"generation","peak_gpu_bytes":3147105792,"repeats":1,"scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","serial_score":0.09375,"serial_seconds":47.12332120502833,"speedup":2.255288679286966,"task":"math"},{"batch_size":8,"batched_score":0.09375,"batched_seconds":12.950173959019594,"changed_item_scores":0,"changed_predictions":0,"examples":32,"kind":"generation","peak_gpu_bytes":3223840256,"repeats":1,"scope":"8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306","serial_score":0.09375,"serial_seconds":47.12332120502833,"speedup":3.63881762161254,"task":"math"},{"batch_size":4,"batched_score":0.75,"batched_seconds":25.80460851197131,"changed_item_scores":0,"changed_predictions":2,"examples":32,"kind":"generation","peak_gpu_bytes":3161031680,"repeats":1,"scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","serial_score":0.75,"serial_seconds":58.91738116601482,"speedup":2.283211587522507,"task":"code-generation"},{"batch_size":8,"batched_score":0.75,"batched_seconds":13.047834396013059,"changed_item_scores":0,"changed_predictions":2,"examples":32,"kind":"generation","peak_gpu_bytes":3225998848,"repeats":1,"scope":"e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00","serial_score":0.75,"serial_seconds":58.91738116601482,"speedup":4.515491182507484,"task":"code-generation"}],"config":{"choice_batch_sizes":[4,8,16],"choice_repeats":3,"choice_tasks":["reading","commonsense","sentiment","entailment","arc-challenge-choice","openbookqa-main-choice"],"generation_batch_sizes":[4,8],"generation_tasks":["math","code-generation"],"seed":42},"gpu":"NVIDIA GeForce RTX 3090","legacy_sha256":"a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e","measured_at":1792197460,"model":"Qwen/Qwen2.5-1.5B","model_load_seconds":1.326716179959476,"physical_gpu":"1","revision":"8faed761d45a263340a0528343f099c05c9a4323","runner_sha256":"e4c8e9487ef46d6c7358d927a6536cd118c53f66e8e432964cae0946de70480b","torch":"2.8.0+cu128","transformers":"4.56.2"}]}}