Inspect the measurement.

Download evidence

Example-level outputs

All 32 scored examples. Ground truth and model predictions are preserved in the downloadable JSONL.

{"id":"HumanEval/0","prediction":"    for i in range(len(numbers)):\n        for j in range(i + 1, len(numbers)):\n            if abs(numbers[i] - numbers[j]) < threshold:\n                return True\n    return False","reference":"passes hidden unit tests","score":1.0,"seconds":0.7532083919504657}
{"id":"HumanEval/1","prediction":"    # TODO: Write your code here\n    pass\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.7002589710755274}
{"id":"HumanEval/2","prediction":"    return number - int(number)\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.6875531850382686}
{"id":"HumanEval/3","prediction":"    balance = 0\n    for op in operations:\n        balance += op\n        if balance < 0:\n            return True\n    return False\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":0.7303831819444895}
{"id":"HumanEval/4","prediction":"    # TODO: Implement this function\n    # raise NotImplementedError(\"Implement this function\")\n    return 0.0\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.7023668320616707}
{"id":"HumanEval/5","prediction":"    if len(numbers) == 0:\n        return []\n    result = [numbers[0]]\n    for i in range(1, len(numbers)):\n        result.append(numbers[i])\n        result.append(delimeter)\n    result.append(numbers[-1])\n    return result\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":0.9365633979905397}
{"id":"HumanEval/6","prediction":"    return [len(group) for group in paren_string.split(' ') if group]\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.6998633430339396}
{"id":"HumanEval/7","prediction":"    return [s for s in strings if substring in s]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.704184201080352}
{"id":"HumanEval/8","prediction":"    sum = 0\n    product = 1\n    for number in numbers:\n        sum += number\n        product *= number\n    return sum, product","reference":"passes hidden unit tests","score":1.0,"seconds":0.6058823349885643}
{"id":"HumanEval/9","prediction":"    return [max(numbers[:i + 1]) for i in range(len(numbers))]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.6859911219216883}
{"id":"HumanEval/10","prediction":"    if is_palindrome(string):\n        return string\n    else:\n        return string + string[::-1][:-1][::-1]\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":0.9568753009662032}
{"id":"HumanEval/11","prediction":"    return ''.join('1' if a[i] != b[i] else '0' for i in range(len(a)))\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":1.5441035820404068}
{"id":"HumanEval/12","prediction":"    if not strings:\n        return None\n    return max(strings, key=len)\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":0.5931679189670831}
{"id":"HumanEval/13","prediction":"    while b != 0:\n        a, b = b, a % b\n    return a\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.6898770589614287}
{"id":"HumanEval/14","prediction":"    return [string[:i] for i in range(1, len(string) + 1)]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.6890219929628074}
{"id":"HumanEval/15","prediction":"    return ' '.join(str(i) for i in range(n + 1))\n","reference":"passes hidden unit tests","score":1.0,"seconds":0.8891033630352467}
{"id":"HumanEval/16","prediction":"    return len(set(string.lower()))\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.6739798500202596}
{"id":"HumanEval/17","prediction":"    pass\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.765880418010056}
{"id":"HumanEval/18","prediction":"    count = 0\n    for i in range(len(string) - len(substring) + 1):\n        if string[i:i+len(substring)] == substring:\n            count += 1\n    return count\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.7225845500361174}
{"id":"HumanEval/19","prediction":"    numbers = numbers.split()\n    numbers.sort()\n    return ' '.join(numbers)\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.7122694610152394}
{"id":"HumanEval/20","prediction":"    if len(numbers) < 2:\n        raise ValueError(\"List must contain at least two elements\")\n\n    sorted_numbers = sorted(numbers)\n    closest_pair = (sorted_numbers[0], sorted_numbers[1])\n    for i in range(1, len(sorted_numbers) - 1):\n        if abs(sorted_numbers[i] - sorted_numbers[i + 1]) < abs(closest_pair[0] - closest_pair[1]):\n            closest_pair = (sorted_numbers[i], sorted_numbers[i + 1])\n    return closest_pair\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":1.4775514989160001}
{"id":"HumanEval/21","prediction":"    min_val = min(numbers)\n    max_val = max(numbers)\n    return [(x - min_val) / (max_val - min_val) for x in numbers]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.7526494920020923}
{"id":"HumanEval/22","prediction":"    return [value for value in values if isinstance(value, int)]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":0.5652509209467098}
{"id":"HumanEval/23","prediction":"    if string == '':\n        return 0\n    else:\n        return 1 + strlen(string[1:])\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.7049601989565417}
{"id":"HumanEval/24","prediction":"    for i in range(n-1, 0, -1):\n        if n % i == 0:\n            return i\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.6976620780769736}
{"id":"HumanEval/25","prediction":"    factors = []\n    i = 2\n    while i * i <= n:\n        while n % i == 0:\n            factors.append(i)\n            n //= i\n        i += 1\n    if n > 1:\n        factors.append(n)\n    return factors\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":1.0488179990788922}
{"id":"HumanEval/26","prediction":"    return list(set(numbers))\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.702776446007192}
{"id":"HumanEval/27","prediction":"    return string.swapcase()\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.68776166299358}
{"id":"HumanEval/28","prediction":"    return ''.join(strings)\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.7075182700064033}
{"id":"HumanEval/29","prediction":"    return [s for s in strings if s.startswith(prefix)]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.7356261829845607}
{"id":"HumanEval/30","prediction":"    return [x for x in l if x > 0]\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.714401858043857}
{"id":"HumanEval/31","prediction":"    \"*** YOUR CODE HERE ***\"\n    if n == 1:\n        return False\n    for i in range(2, n):\n        if n % i == 0:\n            return False\n    return True\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.700851465924643}