{
  "method": "Same pinned model, dataset scope and GPU; warmed inference only. Data/model loading and Docker testing excluded. Choice: median of 3 repeats; generation: one run. Numeric changes are possible even with greedy decoding.",
  "production_policy": "Choice batch 16, math batch 8. Code generation remains serial because one 0.5B unit-test outcome changed. Media and embeddings keep existing runners.",
  "reports": [
    {
      "model": "Qwen/Qwen2.5-0.5B",
      "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987",
      "gpu": "NVIDIA GeForce RTX 3090",
      "physical_gpu": "0",
      "torch": "2.8.0+cu128",
      "transformers": "4.56.2",
      "config": {
        "choice_tasks": [
          "reading",
          "commonsense",
          "sentiment",
          "entailment",
          "arc-challenge-choice",
          "openbookqa-main-choice"
        ],
        "generation_tasks": [
          "math",
          "code-generation"
        ],
        "choice_batch_sizes": [
          4,
          8,
          16
        ],
        "generation_batch_sizes": [
          4,
          8
        ],
        "choice_repeats": 3,
        "seed": 42
      },
      "model_load_seconds": 0.9620922919129953,
      "runner_sha256": "e4c8e9487ef46d6c7358d927a6536cd118c53f66e8e432964cae0946de70480b",
      "legacy_sha256": "a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e",
      "measured_at": 1792197459,
      "comparisons": [
        {
          "task": "reading",
          "kind": "choice",
          "scope": "3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 0.600220178021118,
          "batched_seconds": 0.28036247799173,
          "serial_repeats": [
            0.600220178021118,
            0.6144361699698493,
            0.5958431039471179
          ],
          "batched_repeats": [
            0.2971665080403909,
            0.28036247799173,
            0.2772137160645798
          ],
          "speedup": 2.140871996568753,
          "changed_predictions": 0,
          "max_logprob_delta": 0.047359466552734375,
          "peak_gpu_bytes": 1047984128
        },
        {
          "task": "reading",
          "kind": "choice",
          "scope": "3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 0.600220178021118,
          "batched_seconds": 0.24949766602367163,
          "serial_repeats": [
            0.600220178021118,
            0.6144361699698493,
            0.5958431039471179
          ],
          "batched_repeats": [
            0.25134951004292816,
            0.24548107490409166,
            0.24949766602367163
          ],
          "speedup": 2.4057146008098163,
          "changed_predictions": 0,
          "max_logprob_delta": 0.019006967544555664,
          "peak_gpu_bytes": 1090581504
        },
        {
          "task": "reading",
          "kind": "choice",
          "scope": "3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 0.600220178021118,
          "batched_seconds": 0.2600505769951269,
          "serial_repeats": [
            0.600220178021118,
            0.6144361699698493,
            0.5958431039471179
          ],
          "batched_repeats": [
            0.2640327640110627,
            0.2595339029794559,
            0.2600505769951269
          ],
          "speedup": 2.3080901606012034,
          "changed_predictions": 0,
          "max_logprob_delta": 0.02636098861694336,
          "peak_gpu_bytes": 1175579648
        },
        {
          "task": "commonsense",
          "kind": "choice",
          "scope": "702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 1.124889801023528,
          "batched_seconds": 0.34159845497924834,
          "serial_repeats": [
            1.1285778289893642,
            1.124889801023528,
            1.117881710990332
          ],
          "batched_repeats": [
            0.34159845497924834,
            0.3548655279446393,
            0.3366321820067242
          ],
          "speedup": 3.293017824368859,
          "changed_predictions": 0,
          "max_logprob_delta": 0.005686283111572266,
          "peak_gpu_bytes": 1088720384
        },
        {
          "task": "commonsense",
          "kind": "choice",
          "scope": "702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 1.124889801023528,
          "batched_seconds": 0.21072633704170585,
          "serial_repeats": [
            1.1285778289893642,
            1.124889801023528,
            1.117881710990332
          ],
          "batched_repeats": [
            0.21036017301958054,
            0.21072633704170585,
            0.21184177696704865
          ],
          "speedup": 5.338154768954655,
          "changed_predictions": 0,
          "max_logprob_delta": 0.0069122314453125,
          "peak_gpu_bytes": 1131270656
        },
        {
          "task": "commonsense",
          "kind": "choice",
          "scope": "702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 1.124889801023528,
          "batched_seconds": 0.16379654197953641,
          "serial_repeats": [
            1.1285778289893642,
            1.124889801023528,
            1.117881710990332
          ],
          "batched_repeats": [
            0.16562068497296423,
            0.16379654197953641,
            0.16374776093289256
          ],
          "speedup": 6.8676040863186465,
          "changed_predictions": 0,
          "max_logprob_delta": 0.007098674774169922,
          "peak_gpu_bytes": 1216372736
        },
        {
          "task": "sentiment",
          "kind": "choice",
          "scope": "e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 0.5491064289817587,
          "batched_seconds": 0.169218824012205,
          "serial_repeats": [
            0.5583822580520064,
            0.5459399229148403,
            0.5491064289817587
          ],
          "batched_repeats": [
            0.16647919500246644,
            0.169218824012205,
            0.170905630919151
          ],
          "speedup": 3.244948853575262,
          "changed_predictions": 0,
          "max_logprob_delta": 0.042963504791259766,
          "peak_gpu_bytes": 1012788224
        },
        {
          "task": "sentiment",
          "kind": "choice",
          "scope": "e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 0.5491064289817587,
          "batched_seconds": 0.09657264896668494,
          "serial_repeats": [
            0.5583822580520064,
            0.5459399229148403,
            0.5491064289817587
          ],
          "batched_repeats": [
            0.10019821999594569,
            0.0923683779546991,
            0.09657264896668494
          ],
          "speedup": 5.685941463314174,
          "changed_predictions": 0,
          "max_logprob_delta": 0.04221677780151367,
          "peak_gpu_bytes": 1020902912
        },
        {
          "task": "sentiment",
          "kind": "choice",
          "scope": "e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 0.5491064289817587,
          "batched_seconds": 0.06832152104470879,
          "serial_repeats": [
            0.5583822580520064,
            0.5459399229148403,
            0.5491064289817587
          ],
          "batched_repeats": [
            0.06832152104470879,
            0.065673129982315,
            0.06924992194399238
          ],
          "speedup": 8.037093152865111,
          "changed_predictions": 0,
          "max_logprob_delta": 0.029059410095214844,
          "peak_gpu_bytes": 1036140544
        },
        {
          "task": "entailment",
          "kind": "choice",
          "scope": "38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 0.8507431350881234,
          "batched_seconds": 0.2345536849461496,
          "serial_repeats": [
            0.8538763679098338,
            0.8507431350881234,
            0.8288339640712366
          ],
          "batched_repeats": [
            0.2345536849461496,
            0.23058623599354178,
            0.34950518095865846
          ],
          "speedup": 3.627072136101561,
          "changed_predictions": 0,
          "max_logprob_delta": 0.033130645751953125,
          "peak_gpu_bytes": 1013556224
        },
        {
          "task": "entailment",
          "kind": "choice",
          "scope": "38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 0.8507431350881234,
          "batched_seconds": 0.13751603895798326,
          "serial_repeats": [
            0.8538763679098338,
            0.8507431350881234,
            0.8288339640712366
          ],
          "batched_repeats": [
            0.13751603895798326,
            0.13804111792705953,
            0.1356655169511214
          ],
          "speedup": 6.186501164042836,
          "changed_predictions": 0,
          "max_logprob_delta": 0.08371353149414062,
          "peak_gpu_bytes": 1019535360
        },
        {
          "task": "entailment",
          "kind": "choice",
          "scope": "38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 0.8507431350881234,
          "batched_seconds": 0.10566993593238294,
          "serial_repeats": [
            0.8538763679098338,
            0.8507431350881234,
            0.8288339640712366
          ],
          "batched_repeats": [
            0.1062843919498846,
            0.10566993593238294,
            0.10544044803828001
          ],
          "speedup": 8.050947770352627,
          "changed_predictions": 0,
          "max_logprob_delta": 0.055899620056152344,
          "peak_gpu_bytes": 1035193344
        },
        {
          "task": "arc-challenge-choice",
          "kind": "choice",
          "scope": "78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 1.1321275099180639,
          "batched_seconds": 0.3399381880881265,
          "serial_repeats": [
            1.1347573839593679,
            1.1321275099180639,
            1.1261092749191448
          ],
          "batched_repeats": [
            0.3399381880881265,
            0.33929265406914055,
            0.34753246104810387
          ],
          "speedup": 3.3303922583259995,
          "changed_predictions": 0,
          "max_logprob_delta": 0.05374336242675781,
          "peak_gpu_bytes": 1066841600
        },
        {
          "task": "arc-challenge-choice",
          "kind": "choice",
          "scope": "78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 1.1321275099180639,
          "batched_seconds": 0.22777703602332622,
          "serial_repeats": [
            1.1347573839593679,
            1.1321275099180639,
            1.1261092749191448
          ],
          "batched_repeats": [
            0.22515708999708295,
            0.22777703602332622,
            0.23066113505046815
          ],
          "speedup": 4.9703320830029805,
          "changed_predictions": 0,
          "max_logprob_delta": 0.015370845794677734,
          "peak_gpu_bytes": 1098745344
        },
        {
          "task": "arc-challenge-choice",
          "kind": "choice",
          "scope": "78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 1.1321275099180639,
          "batched_seconds": 0.19077692099381238,
          "serial_repeats": [
            1.1347573839593679,
            1.1321275099180639,
            1.1261092749191448
          ],
          "batched_repeats": [
            0.19077692099381238,
            0.1884768509771675,
            0.1931891089770943
          ],
          "speedup": 5.93430014500959,
          "changed_predictions": 0,
          "max_logprob_delta": 0.03254365921020508,
          "peak_gpu_bytes": 1161676288
        },
        {
          "task": "openbookqa-main-choice",
          "kind": "choice",
          "scope": "4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 1.0927511820336804,
          "batched_seconds": 0.3080282249720767,
          "serial_repeats": [
            1.0915181819582358,
            1.0965651560109109,
            1.0927511820336804
          ],
          "batched_repeats": [
            0.3121457149973139,
            0.3071422229986638,
            0.3080282249720767
          ],
          "speedup": 3.5475683507014337,
          "changed_predictions": 0,
          "max_logprob_delta": 0.05489826202392578,
          "peak_gpu_bytes": 1030667776
        },
        {
          "task": "openbookqa-main-choice",
          "kind": "choice",
          "scope": "4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 1.0927511820336804,
          "batched_seconds": 0.18135019892361015,
          "serial_repeats": [
            1.0915181819582358,
            1.0965651560109109,
            1.0927511820336804
          ],
          "batched_repeats": [
            0.17987539898604155,
            0.1821843560319394,
            0.18135019892361015
          ],
          "speedup": 6.025640934058077,
          "changed_predictions": 0,
          "max_logprob_delta": 0.029590606689453125,
          "peak_gpu_bytes": 1043742720
        },
        {
          "task": "openbookqa-main-choice",
          "kind": "choice",
          "scope": "4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 1.0927511820336804,
          "batched_seconds": 0.11729059799108654,
          "serial_repeats": [
            1.0915181819582358,
            1.0965651560109109,
            1.0927511820336804
          ],
          "batched_repeats": [
            0.11729059799108654,
            0.11620235804002732,
            0.11819219007156789
          ],
          "speedup": 9.316613613963531,
          "changed_predictions": 0,
          "max_logprob_delta": 0.03309154510498047,
          "peak_gpu_bytes": 1070492160
        },
        {
          "task": "math",
          "kind": "generation",
          "scope": "8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 31.58672497002408,
          "batched_seconds": 17.86346282402519,
          "speedup": 1.76823078935972,
          "changed_predictions": 3,
          "peak_gpu_bytes": 1033307648,
          "repeats": 1,
          "changed_item_scores": 0,
          "serial_score": 0.03125,
          "batched_score": 0.03125
        },
        {
          "task": "math",
          "kind": "generation",
          "scope": "8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 31.58672497002408,
          "batched_seconds": 11.093595852027647,
          "speedup": 2.847293645031315,
          "changed_predictions": 0,
          "peak_gpu_bytes": 1058638336,
          "repeats": 1,
          "changed_item_scores": 0,
          "serial_score": 0.03125,
          "batched_score": 0.03125
        },
        {
          "task": "code-generation",
          "kind": "generation",
          "scope": "e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 53.38210642198101,
          "batched_seconds": 22.598658623057418,
          "speedup": 2.3621803095655967,
          "changed_predictions": 3,
          "peak_gpu_bytes": 1037457408,
          "repeats": 1,
          "changed_item_scores": 1,
          "serial_score": 0.625,
          "batched_score": 0.59375
        },
        {
          "task": "code-generation",
          "kind": "generation",
          "scope": "e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 53.38210642198101,
          "batched_seconds": 11.159352226997726,
          "speedup": 4.783620530664328,
          "changed_predictions": 3,
          "peak_gpu_bytes": 1070066176,
          "repeats": 1,
          "changed_item_scores": 1,
          "serial_score": 0.625,
          "batched_score": 0.59375
        }
      ]
    },
    {
      "model": "Qwen/Qwen2.5-1.5B",
      "revision": "8faed761d45a263340a0528343f099c05c9a4323",
      "gpu": "NVIDIA GeForce RTX 3090",
      "physical_gpu": "1",
      "torch": "2.8.0+cu128",
      "transformers": "4.56.2",
      "config": {
        "choice_tasks": [
          "reading",
          "commonsense",
          "sentiment",
          "entailment",
          "arc-challenge-choice",
          "openbookqa-main-choice"
        ],
        "generation_tasks": [
          "math",
          "code-generation"
        ],
        "choice_batch_sizes": [
          4,
          8,
          16
        ],
        "generation_batch_sizes": [
          4,
          8
        ],
        "choice_repeats": 3,
        "seed": 42
      },
      "model_load_seconds": 1.326716179959476,
      "runner_sha256": "e4c8e9487ef46d6c7358d927a6536cd118c53f66e8e432964cae0946de70480b",
      "legacy_sha256": "a71a6c0fe0c3cc3f1694659595be1112cfc36eb190be5ff6e0a3380105415b3e",
      "measured_at": 1792197460,
      "comparisons": [
        {
          "task": "reading",
          "kind": "choice",
          "scope": "3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 1.0186945659806952,
          "batched_seconds": 0.6891941240755841,
          "serial_repeats": [
            1.0232102070003748,
            1.0176695550326258,
            1.0186945659806952
          ],
          "batched_repeats": [
            0.7088887150166556,
            0.6886495479848236,
            0.6891941240755841
          ],
          "speedup": 1.4780952570468733,
          "changed_predictions": 0,
          "max_logprob_delta": 0.03110504150390625,
          "peak_gpu_bytes": 3174513664
        },
        {
          "task": "reading",
          "kind": "choice",
          "scope": "3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 1.0186945659806952,
          "batched_seconds": 0.6605109899537638,
          "serial_repeats": [
            1.0232102070003748,
            1.0176695550326258,
            1.0186945659806952
          ],
          "batched_repeats": [
            0.660398209001869,
            0.6605109899537638,
            0.6614233909640461
          ],
          "speedup": 1.5422825380271181,
          "changed_predictions": 0,
          "max_logprob_delta": 0.02793264389038086,
          "peak_gpu_bytes": 3252205568
        },
        {
          "task": "reading",
          "kind": "choice",
          "scope": "3cb0127874969f63a3acc3b00d4f41283f02f4501883a6bfd431009004858ce8",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 1.0186945659806952,
          "batched_seconds": 0.6625273310346529,
          "serial_repeats": [
            1.0232102070003748,
            1.0176695550326258,
            1.0186945659806952
          ],
          "batched_repeats": [
            0.6625273310346529,
            0.6618705959990621,
            0.6625410560518503
          ],
          "speedup": 1.5375887427765804,
          "changed_predictions": 0,
          "max_logprob_delta": 0.025635719299316406,
          "peak_gpu_bytes": 3407441920
        },
        {
          "task": "commonsense",
          "kind": "choice",
          "scope": "702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 1.2752673069480807,
          "batched_seconds": 0.5235468610189855,
          "serial_repeats": [
            1.2752673069480807,
            1.2805055560311303,
            1.2750125910388306
          ],
          "batched_repeats": [
            0.5235468610189855,
            0.5253849279833958,
            0.5220861460547894
          ],
          "speedup": 2.4358226586747413,
          "changed_predictions": 0,
          "max_logprob_delta": 0.0060443878173828125,
          "peak_gpu_bytes": 3181053440
        },
        {
          "task": "commonsense",
          "kind": "choice",
          "scope": "702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 1.2752673069480807,
          "batched_seconds": 0.4417891720077023,
          "serial_repeats": [
            1.2752673069480807,
            1.2805055560311303,
            1.2750125910388306
          ],
          "batched_repeats": [
            0.4411652380367741,
            0.4425054669845849,
            0.4417891720077023
          ],
          "speedup": 2.8865970190094368,
          "changed_predictions": 0,
          "max_logprob_delta": 0.008848190307617188,
          "peak_gpu_bytes": 3223603712
        },
        {
          "task": "commonsense",
          "kind": "choice",
          "scope": "702861d3926d39f93e04807e1243549fc078b7e844a41ed6c60a871ed97bb062",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 1.2752673069480807,
          "batched_seconds": 0.38615964201744646,
          "serial_repeats": [
            1.2752673069480807,
            1.2805055560311303,
            1.2750125910388306
          ],
          "batched_repeats": [
            0.38581330596935004,
            0.38615964201744646,
            0.38853677501901984
          ],
          "speedup": 3.3024354908907463,
          "changed_predictions": 0,
          "max_logprob_delta": 0.005952358245849609,
          "peak_gpu_bytes": 3308705792
        },
        {
          "task": "sentiment",
          "kind": "choice",
          "scope": "e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 0.6465045380173251,
          "batched_seconds": 0.23198114801198244,
          "serial_repeats": [
            0.6380494689801708,
            0.6465045380173251,
            0.6480961060151458
          ],
          "batched_repeats": [
            0.23924480401910841,
            0.23198114801198244,
            0.2303108519408852
          ],
          "speedup": 2.7868839496558206,
          "changed_predictions": 0,
          "max_logprob_delta": 0.054396986961364746,
          "peak_gpu_bytes": 3110803456
        },
        {
          "task": "sentiment",
          "kind": "choice",
          "scope": "e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 0.6465045380173251,
          "batched_seconds": 0.18671607796568424,
          "serial_repeats": [
            0.6380494689801708,
            0.6465045380173251,
            0.6480961060151458
          ],
          "batched_repeats": [
            0.18559291190467775,
            0.18671607796568424,
            0.3003705239389092
          ],
          "speedup": 3.4625006323030387,
          "changed_predictions": 0,
          "max_logprob_delta": 0.02863633632659912,
          "peak_gpu_bytes": 3126555136
        },
        {
          "task": "sentiment",
          "kind": "choice",
          "scope": "e1843d5c329a05094d05157b826e49ecaf8d2d146c41eb15d1287a350638c5b0",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 0.6465045380173251,
          "batched_seconds": 0.1597348980139941,
          "serial_repeats": [
            0.6380494689801708,
            0.6465045380173251,
            0.6480961060151458
          ],
          "batched_repeats": [
            0.1597348980139941,
            0.1593990409746766,
            0.16007488104514778
          ],
          "speedup": 4.047359381421372,
          "changed_predictions": 0,
          "max_logprob_delta": 0.04548025131225586,
          "peak_gpu_bytes": 3150947328
        },
        {
          "task": "entailment",
          "kind": "choice",
          "scope": "38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 0.9650672299321741,
          "batched_seconds": 0.3600469919620082,
          "serial_repeats": [
            0.9548074660124257,
            0.967170984018594,
            0.9650672299321741
          ],
          "batched_repeats": [
            0.36132709903176874,
            0.3600469919620082,
            0.3582041549962014
          ],
          "speedup": 2.680392425092131,
          "changed_predictions": 0,
          "max_logprob_delta": 0.08685684204101562,
          "peak_gpu_bytes": 3111495680
        },
        {
          "task": "entailment",
          "kind": "choice",
          "scope": "38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 0.9650672299321741,
          "batched_seconds": 0.3020788700086996,
          "serial_repeats": [
            0.9548074660124257,
            0.967170984018594,
            0.9650672299321741
          ],
          "batched_repeats": [
            0.3057265429524705,
            0.30174276407342404,
            0.3020788700086996
          ],
          "speedup": 3.194752515806157,
          "changed_predictions": 0,
          "max_logprob_delta": 0.11722755432128906,
          "peak_gpu_bytes": 3125683200
        },
        {
          "task": "entailment",
          "kind": "choice",
          "scope": "38bbd002f19fa172682f143fb1299a09e00b6693591f1fb8d9d938db1f26f0b7",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 0.9650672299321741,
          "batched_seconds": 0.2514021770330146,
          "serial_repeats": [
            0.9548074660124257,
            0.967170984018594,
            0.9650672299321741
          ],
          "batched_repeats": [
            0.25027340301312506,
            0.2514021770330146,
            0.25329436699394137
          ],
          "speedup": 3.838738555575196,
          "changed_predictions": 0,
          "max_logprob_delta": 0.1078042984008789,
          "peak_gpu_bytes": 3149303808
        },
        {
          "task": "arc-challenge-choice",
          "kind": "choice",
          "scope": "78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 1.3202161670196801,
          "batched_seconds": 0.6249694839352742,
          "serial_repeats": [
            1.3201875739032403,
            1.3202161670196801,
            1.3258234449895099
          ],
          "batched_repeats": [
            0.6274732939200476,
            0.6249694839352742,
            0.6240577240241691
          ],
          "speedup": 2.112449009040592,
          "changed_predictions": 0,
          "max_logprob_delta": 0.012441158294677734,
          "peak_gpu_bytes": 3158866432
        },
        {
          "task": "arc-challenge-choice",
          "kind": "choice",
          "scope": "78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 1.3202161670196801,
          "batched_seconds": 0.5235125969629735,
          "serial_repeats": [
            1.3201875739032403,
            1.3202161670196801,
            1.3258234449895099
          ],
          "batched_repeats": [
            0.5227666050195694,
            0.5243469929555431,
            0.5235125969629735
          ],
          "speedup": 2.521842214836055,
          "changed_predictions": 0,
          "max_logprob_delta": 0.010687828063964844,
          "peak_gpu_bytes": 3190478848
        },
        {
          "task": "arc-challenge-choice",
          "kind": "choice",
          "scope": "78d7cecafa6a6800233d22492171ba6acc9a8b644ca0ab6cc7013271bb6e4b06",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 1.3202161670196801,
          "batched_seconds": 0.47099284594878554,
          "serial_repeats": [
            1.3201875739032403,
            1.3202161670196801,
            1.3258234449895099
          ],
          "batched_repeats": [
            0.47089706792030483,
            0.47099284594878554,
            0.47230031504295766
          ],
          "speedup": 2.803049299740822,
          "changed_predictions": 0,
          "max_logprob_delta": 0.014543533325195312,
          "peak_gpu_bytes": 3253701120
        },
        {
          "task": "openbookqa-main-choice",
          "kind": "choice",
          "scope": "4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 1.274097544956021,
          "batched_seconds": 0.3917699650628492,
          "serial_repeats": [
            1.2713205959880725,
            1.2904391420306638,
            1.274097544956021
          ],
          "batched_repeats": [
            0.39738868502900004,
            0.39117859199177474,
            0.3917699650628492
          ],
          "speedup": 3.252157282530899,
          "changed_predictions": 0,
          "max_logprob_delta": 0.032810211181640625,
          "peak_gpu_bytes": 3122692608
        },
        {
          "task": "openbookqa-main-choice",
          "kind": "choice",
          "scope": "4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 1.274097544956021,
          "batched_seconds": 0.29547362099401653,
          "serial_repeats": [
            1.2713205959880725,
            1.2904391420306638,
            1.274097544956021
          ],
          "batched_repeats": [
            0.2992883659899235,
            0.29547362099401653,
            0.29501281201373786
          ],
          "speedup": 4.312051751590448,
          "changed_predictions": 0,
          "max_logprob_delta": 0.02657318115234375,
          "peak_gpu_bytes": 3135767552
        },
        {
          "task": "openbookqa-main-choice",
          "kind": "choice",
          "scope": "4be725c6398a1c6edfd89751b16ad115044ec5c7273cb39dd1c5c0eb2dfd2c79",
          "examples": 32,
          "batch_size": 16,
          "serial_seconds": 1.274097544956021,
          "batched_seconds": 0.2523090390022844,
          "serial_repeats": [
            1.2713205959880725,
            1.2904391420306638,
            1.274097544956021
          ],
          "batched_repeats": [
            0.2519514620071277,
            0.2523090390022844,
            0.2523421309888363
          ],
          "speedup": 5.049749901922798,
          "changed_predictions": 0,
          "max_logprob_delta": 0.022623062133789062,
          "peak_gpu_bytes": 3162516992
        },
        {
          "task": "math",
          "kind": "generation",
          "scope": "8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 47.12332120502833,
          "batched_seconds": 20.89458508696407,
          "speedup": 2.255288679286966,
          "changed_predictions": 1,
          "peak_gpu_bytes": 3147105792,
          "repeats": 1,
          "changed_item_scores": 0,
          "serial_score": 0.09375,
          "batched_score": 0.09375
        },
        {
          "task": "math",
          "kind": "generation",
          "scope": "8e8d6a0bf6bf7cfab1c3af7809751d1e4b8244fb749a0cdb846b091b82226306",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 47.12332120502833,
          "batched_seconds": 12.950173959019594,
          "speedup": 3.63881762161254,
          "changed_predictions": 0,
          "peak_gpu_bytes": 3223840256,
          "repeats": 1,
          "changed_item_scores": 0,
          "serial_score": 0.09375,
          "batched_score": 0.09375
        },
        {
          "task": "code-generation",
          "kind": "generation",
          "scope": "e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00",
          "examples": 32,
          "batch_size": 4,
          "serial_seconds": 58.91738116601482,
          "batched_seconds": 25.80460851197131,
          "speedup": 2.283211587522507,
          "changed_predictions": 2,
          "peak_gpu_bytes": 3161031680,
          "repeats": 1,
          "changed_item_scores": 0,
          "serial_score": 0.75,
          "batched_score": 0.75
        },
        {
          "task": "code-generation",
          "kind": "generation",
          "scope": "e63ceb841bb76e0a85ce2b63991c18e85c6722dbc86117c37504a28658774a00",
          "examples": 32,
          "batch_size": 8,
          "serial_seconds": 58.91738116601482,
          "batched_seconds": 13.047834396013059,
          "speedup": 4.515491182507484,
          "changed_predictions": 2,
          "peak_gpu_bytes": 3225998848,
          "repeats": 1,
          "changed_item_scores": 0,
          "serial_score": 0.75,
          "batched_score": 0.75
        }
      ]
    }
  ]
}