"""Full source-pinned HumanEval, serial greedy completion and isolated hidden tests. Native Slayer models use a reviewed frozen loader and explicit autoregression. HF models use a serial cached generate call. Neither path executes remote Python. """ from __future__ import annotations import hashlib,importlib.metadata,json,os,platform,re,secrets,selectors,subprocess,sys,time,tempfile from pathlib import Path ROOT=Path(__file__).resolve().parent DEPENDENCIES={'sub1b_native.py': 'ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7', 'sub1b_slayer139.py': 'f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326', 'sub1b_gollem.py': 'b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7', 'sub1b_balanced.py': 'f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b', 'humaneval-systems.json': '1ec4c8db6d55ce5d03aa99e1910844139f24dc4c6146e312cdd2eaeed81da271'} CONTEXT=1024 MAX_NEW_TOKENS=256 PROMPT_LIMIT=CONTEXT-MAX_NEW_TOKENS def code_test(program,image): """Execute candidate + hidden checks inside a networkless disposable container.""" name='codesota-eval-'+secrets.token_hex(8) cmd=['docker','run','--name',name,'--rm','--network=none','--read-only','--cap-drop=ALL','--security-opt=no-new-privileges','--pids-limit=32','--memory=128m','--cpus=1','--user=65534:65534','--tmpfs=/tmp:rw,noexec,nosuid,size=16m','-i',image,'python','-I','-'] marker='CODESOTA_CHECKS_PASSED_'+secrets.token_hex(32) # A zero exit alone is insufficient: generated code could call sys.exit(0). # This unpredictable marker is emitted only after all appended tests return. guarded='import signal\nsignal.alarm(4)\n'+program+'\n__import__("os").write(1, '+repr(('\n'+marker+'\n').encode())+')\n' p=None;input_file=tempfile.TemporaryFile() try: input_file.write(guarded.encode());input_file.seek(0) p=subprocess.Popen(cmd,stdin=input_file,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) captured=bytearray();deadline=time.monotonic()+12 with selectors.DefaultSelector() as selector: selector.register(p.stdout,selectors.EVENT_READ) while selector.get_map(): if time.monotonic()>deadline:return False for key,_ in selector.select(timeout=.2): chunk=os.read(key.fileobj.fileno(),65536) if not chunk:selector.unregister(key.fileobj);continue if len(captured)+len(chunk)>1_000_000:return False captured.extend(chunk) remaining=max(.01,deadline-time.monotonic());code=p.wait(timeout=remaining) return code==0 and marker.encode() in captured.splitlines() except (subprocess.TimeoutExpired,BrokenPipeError):return False finally: if p is not None and p.poll() is None: p.kill();p.wait(timeout=5) if p is not None and p.stdout is not None:p.stdout.close() input_file.close() subprocess.run(['docker','rm','--force',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=15) def extract_completion(raw): # Same completion-only extraction as the earlier CodeSOTA code pilot. # No repair, markdown stripping, retry or reference solution is applied. return re.split(r'\n(?=def |class |if __name__|print\(|#)',raw)[0] def program_for(row,completion): return row['prompt']+completion+'\n'+row['test']+f"\ncheck({row['entry_point']})\n" def native_greedy(model,prompt_ids,eos_id,device,max_new_tokens=MAX_NEW_TOKENS): import torch if not prompt_ids or len(prompt_ids)+max_new_tokens>CONTEXT:raise ValueError('Invalid native generation context budget') ids=torch.tensor([prompt_ids],dtype=torch.long,device=device);generated=[];reason='length' with torch.inference_mode(): for _ in range(max_new_tokens): mask=torch.ones_like(ids);positions=torch.arange(ids.shape[1],device=device)[None,:] logits=model(ids,attention_mask=mask,position_ids=positions,use_cache=False,logits_to_keep=1).logits token=int(logits[0,-1].argmax());generated.append(token) if token==eos_id:reason='eos';break ids=torch.cat((ids,torch.tensor([[token]],dtype=torch.long,device=device)),dim=1) return generated,reason def verify_sources(): for name,digest in DEPENDENCIES.items(): if hashlib.sha256((ROOT/name).read_bytes()).hexdigest()!=digest:raise ValueError('Pinned dependency differs: '+name) def load_system(info,device): import torch if info['adapter']=='hf': from transformers import AutoTokenizer,AutoModelForCausalLM tokenizer=AutoTokenizer.from_pretrained(info['id'],revision=info['revision'],trust_remote_code=False) model=AutoModelForCausalLM.from_pretrained(info['id'],revision=info['revision'],trust_remote_code=False,torch_dtype=torch.float32).to(device).eval() return tokenizer,model from sub1b_native import load from huggingface_hub import hf_hub_download from tokenizers import Tokenizer tokens,model=load(info) raw=Tokenizer.from_file(hf_hub_download(info['id'],'tokenizer.json',revision=info['revision'],token=False));raw.no_padding();raw.no_truncation() class NativeTokens: eos_token_id=tokens.eos_token_id def encode(self,text,add_special_tokens=False):return raw.encode(text,add_special_tokens=add_special_tokens).ids def decode(self,ids,skip_special_tokens=True):return raw.decode(ids,skip_special_tokens=skip_special_tokens) return NativeTokens(),model def generate(model,tokenizer,prompt,device,adapter): import torch full=tokenizer.encode(prompt,add_special_tokens=False);ids=full[-PROMPT_LIMIT:] if not ids:raise ValueError('Empty HumanEval prompt') if adapter=='hf': inputs=torch.tensor([ids],dtype=torch.long,device=device) with torch.inference_mode(): out=model.generate(input_ids=inputs,attention_mask=torch.ones_like(inputs),max_new_tokens=MAX_NEW_TOKENS,do_sample=False,use_cache=True,pad_token_id=tokenizer.eos_token_id,eos_token_id=tokenizer.eos_token_id) generated=out[0,len(ids):].tolist();reason='eos' if generated and generated[-1]==tokenizer.eos_token_id else 'length' else:generated,reason=native_greedy(model,ids,tokenizer.eos_token_id,device) return tokenizer.decode(generated,skip_special_tokens=True),dict(prompt_tokens=len(full),input_tokens=len(ids),input_truncated=len(full)>len(ids),generated_tokens=len(generated),stop_reason=reason) def evaluate(job,output): import numpy as np import torch from huggingface_hub import hf_hub_download import pyarrow.parquet as pq started=time.monotonic();t=job['task'];model_id=job['model'];revision=job['revision'] if (t.get('protocol_version'),t.get('adapter'),t.get('dataset'),t.get('dataset_revision'),t.get('split'),t.get('limit'))!=('codesota-humaneval-full-v2','code','openai/openai_humaneval','7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544','test',164):raise ValueError('Expected the pinned full HumanEval protocol') verify_sources();info=next((m for m in json.loads((ROOT/'humaneval-systems.json').read_text()) if m['id']==model_id and m['revision']==revision),None) if not info:raise ValueError('Unknown pinned HumanEval system') image=os.environ.get('CODESOTA_SANDBOX_IMAGE') if not image or '@sha256:' not in image:raise ValueError('Pinned Docker sandbox image required') subprocess.run(['docker','image','inspect',image],check=True,stdout=subprocess.DEVNULL) torch.manual_seed(42);np.random.seed(42);torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=False device='cuda:0' if torch.cuda.is_available() else 'cpu' path=hf_hub_download(t['dataset'],'openai_humaneval/test-00000-of-00001.parquet',repo_type='dataset',revision=t['dataset_revision']) rows=pq.read_table(path).to_pylist() if len(rows)!=164 or len({r['task_id'] for r in rows})!=164:raise ValueError('Expected all 164 distinct HumanEval problems') tokenizer,model=load_system(info,device);samples=[] for row in rows: torch.cuda.synchronize() if device.startswith('cuda') else None begin=time.monotonic();raw,details=generate(model,tokenizer,row['prompt'],device,info['adapter']) torch.cuda.synchronize() if device.startswith('cuda') else None generation_seconds=time.monotonic()-begin;completion=extract_completion(raw);program=program_for(row,completion) before=time.monotonic();passed=code_test(program,image);test_seconds=time.monotonic()-before samples.append(dict(id=row['task_id'],reference='passes original HumanEval hidden unit tests',prediction=completion,raw_completion=raw,score=float(passed),passed=passed,program_sha256=hashlib.sha256(program.encode()).hexdigest(),generation_seconds=generation_seconds,test_seconds=test_seconds,seconds=generation_seconds+test_seconds,**details)) print(json.dumps({'completed':len(samples),'total':164,'passed':sum(s['passed'] for s in samples)}),flush=True) duration=time.monotonic()-started;score=sum(s['passed'] for s in samples)/164 result=dict(score=score,n=164,parameters=sum(p.numel() for p in model.parameters()),model_revision=revision,dataset_revision=t['dataset_revision'],runner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),dependency_sources=DEPENDENCIES,seconds=duration,peak_gpu_bytes=torch.cuda.max_memory_allocated() if device.startswith('cuda') else 0,selection='full pinned HumanEval test split; all 164 distinct problems in source order',seed=42,device=device,protocol=t,measured_cost_usd=None,execution_profile='humaneval-full-v2',sandbox_image=image,generation=dict(strategy='greedy',samples_per_problem=1,max_new_tokens=MAX_NEW_TOKENS,total_context_limit=CONTEXT,prompt_token_limit=PROMPT_LIMIT,precision='float32',tf32=False,chat_template=False,add_bos=False,serial=True,hf_kv_cache=True,native_kv_cache=False),notes='Full 164-problem HumanEval; one greedy 256-token completion per problem; pass@1 is fraction passing original isolated unit tests. FP32; no chat template/BOS. Fixed top-level completion extraction; no retries or repair. Native generation recomputes context per token; HF uses a serial KV cache. Not HumanEval+ or an unrestricted-generation score.') packages={d.metadata['Name']:d.version for d in importlib.metadata.distributions()} output.parent.mkdir(parents=True,exist_ok=True);output.write_text(json.dumps(dict(result=result,samples=samples,environment=dict(python=sys.version,platform=platform.platform(),torch=torch.__version__,cuda=torch.version.cuda,gpu=torch.cuda.get_device_name() if device.startswith('cuda') else None,packages=packages)),allow_nan=False)) if __name__=='__main__':evaluate(json.loads(Path(sys.argv[1]).read_text()),Path(sys.argv[2]))