Inspect the measurement.

Download evidence

Example-level outputs

All 32 scored examples. Ground truth and model predictions are preserved in the downloadable JSONL.

{"id":"HumanEval/0","prediction":"    for i in range(len(numbers) - 1):\n        for j in range(i + 1, len(numbers)):\n            if abs(numbers[i] - numbers[j]) < threshold:\n                return True\n    return False\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":0.8881170509848744}
{"id":"HumanEval/1","prediction":"    stack = []\n    result = []\n    for char in paren_string:\n        if char == '(':\n            stack.append(char)\n        elif char == ')':\n            if not stack:\n                result.append(char)\n            elif stack[-1] == '(':\n                stack.pop()\n            else:\n                result.append(char)\n    return result\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.187736082007177}
{"id":"HumanEval/2","prediction":"    return number - int(number)\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.190397324040532}
{"id":"HumanEval/3","prediction":"    balance = 0\n    for op in operations:\n        balance += op\n        if balance < 0:\n            return True\n    return False\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.1958132659783587}
{"id":"HumanEval/4","prediction":"    return sum(abs(x - mean(numbers)) for x in numbers) / len(numbers)\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.1909280550898984}
{"id":"HumanEval/5","prediction":"    return numbers + [delimeter] * (len(numbers) - 1)\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":0.5166685340227559}
{"id":"HumanEval/6","prediction":"    stack = []\n    for char in paren_string:\n        if char == '(':\n            stack.append(char)\n        elif char == ')':\n            if len(stack) == 0:\n                return []\n            else:\n                stack.pop()\n    return [len(stack)]\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.2008176998933777}
{"id":"HumanEval/7","prediction":"    return [s for s in strings if substring in s]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2704596479889005}
{"id":"HumanEval/8","prediction":"    sum = 0\n    product = 1\n    for number in numbers:\n        sum += number\n        product *= number\n    return sum, product\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":0.6515998139511794}
{"id":"HumanEval/9","prediction":"    maxes = [numbers[0]]\n    for i in range(1, len(numbers)):\n        maxes.append(max(maxes[-1], numbers[i]))\n    return maxes\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.2023643240099773}
{"id":"HumanEval/10","prediction":"    # Find the longest postfix of supplied string that is a palindrome.\n    # This is the longest palindrome that can be formed from the supplied string.\n    # We can use the is_palindrome function to check if a string is a palindrome.\n    # We can use the string slicing syntax to get the longest postfix of the supplied string.\n    # We can use the string slicing syntax to get the reverse of the substring.\n    # We can use the string slicing syntax to get the substring before the longest postfix.\n    # We can use the string slicing syntax to get the substring after the longest postfix.\n    # We can use the string slicing syntax to get the reverse of the substring after the longest postfix.\n    # We can use the string slicing syntax to get the substring before the longest postfix.\n    # We can use the string slicing syntax to get the substring after the longest postfix.\n    # We can use the string slicing syntax to get the reverse of the substring after the longest postfix.\n    # We can use the string slicing syntax to get the substring before the longest postfix.\n    # We can use the string slicing syntax to get the substring after the longest postfix.\n    # We can use the string slicing syntax to get the reverse of the substring after the longest postfix.\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.198593942099251}
{"id":"HumanEval/11","prediction":"    return ''.join([str(int(a[i]) ^ int(b[i])) for i in range(len(a))])\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.266123039997183}
{"id":"HumanEval/12","prediction":"    if not strings:\n        return None\n    return max(strings, key=len)\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":0.5149509010370821}
{"id":"HumanEval/13","prediction":"    while b != 0:\n        a, b = b, a % b\n    return a\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.206816336954944}
{"id":"HumanEval/14","prediction":"    prefixes = []\n    for i in range(len(string)):\n        prefixes.append(string[:i + 1])\n    return prefixes\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2083780110115185}
{"id":"HumanEval/15","prediction":"    return ' '.join(str(i) for i in range(n+1))\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2066792419645935}
{"id":"HumanEval/16","prediction":"    return len(set(string.lower()))\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.1984309480758384}
{"id":"HumanEval/17","prediction":"    notes = []\n    for note in music_string:\n        if note == 'o':\n            notes.append(4)\n        elif note == 'o|':\n            notes.append(2)\n        elif note == '.|':\n            notes.append(1)\n        else:\n            notes.append(0)\n    return notes\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.206559928948991}
{"id":"HumanEval/18","prediction":"    count = 0\n    for i in range(len(string) - len(substring) + 1):\n        if string[i:i+len(substring)] == substring:\n            count += 1\n    return count\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.194610002101399}
{"id":"HumanEval/19","prediction":"    numbers = numbers.split()\n    numbers = [int(num) for num in numbers]\n    numbers.sort()\n    return ' '.join([str(num) for num in numbers])\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.1921270909952}
{"id":"HumanEval/20","prediction":"    if len(numbers) < 2:\n        raise ValueError(\"List must contain at least two numbers\")\n    numbers.sort()\n    return numbers[0], numbers[-1]\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":0.653485169983469}
{"id":"HumanEval/21","prediction":"    return [x / max(numbers) for x in numbers]\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.279065289068967}
{"id":"HumanEval/22","prediction":"    return [value for value in values if isinstance(value, int)]","reference":"passes hidden unit tests","score":1.0,"seconds":0.34864249103702605}
{"id":"HumanEval/23","prediction":"    return len(string)\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2201722369063646}
{"id":"HumanEval/24","prediction":"    for i in range(2, n):\n        if n % i == 0:\n            return i\n    return n\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":2.2038853620178998}
{"id":"HumanEval/25","prediction":"    factors = []\n    while n % 2 == 0:\n        factors.append(2)\n        n //= 2\n    for i in range(3, int(n**0.5) + 1, 2):\n        while n % i == 0:\n            factors.append(i)\n            n //= i\n    if n > 2:\n        factors.append(n)\n    return factors\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2427150530274957}
{"id":"HumanEval/26","prediction":"    seen = set()\n    result = []\n    for number in numbers:\n        if number not in seen:\n            seen.add(number)\n            result.append(number)\n    return result\n\n","reference":"passes hidden unit tests","score":0.0,"seconds":0.6610672910464928}
{"id":"HumanEval/27","prediction":"    return string.swapcase()\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.209409393952228}
{"id":"HumanEval/28","prediction":"    return ''.join(strings)\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2046410409966484}
{"id":"HumanEval/29","prediction":"    return [s for s in strings if s.startswith(prefix)]\n\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2008757579606026}
{"id":"HumanEval/30","prediction":"    return [x for x in l if x > 0]\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.267637345008552}
{"id":"HumanEval/31","prediction":"    if n < 2:\n        return False\n    for i in range(2, int(n**0.5) + 1):\n        if n % i == 0:\n            return False\n    return True\n","reference":"passes hidden unit tests","score":1.0,"seconds":2.2074035829864442}