"""Source-pinned full ARC-Easy: zero-shot raw conditional likelihood accuracy. The original token-normalized pilot remains immutable. This distinct protocol uses the established Question/Answer prompt and reports acc_norm separately. """ from __future__ import annotations import hashlib, importlib.metadata, json, math, os, platform, re, subprocess, sys, time, secrets from concurrent.futures import ThreadPoolExecutor from pathlib import Path DEPENDENCIES={'sub1b_native.py': 'ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7', 'sub1b_slayer139.py': 'f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326', 'sub1b_gollem.py': 'b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7', 'sub1b_balanced.py': 'f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b', 'sub1b-models.json': 'e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd'} def code_test(program, image): """Execute candidate + hidden checks inside a networkless disposable container.""" name='codesota-eval-'+secrets.token_hex(8) cmd=['docker','run','--name',name,'--rm','--network=none','--read-only','--cap-drop=ALL','--security-opt=no-new-privileges','--pids-limit=32','--memory=128m','--cpus=1','--user=65534:65534','--tmpfs=/tmp:rw,noexec,nosuid,size=16m','-i',image,'python','-I','-'] try: # Docker has an in-container alarm as well as a host deadline. guarded='import signal\nsignal.alarm(4)\n'+program p=subprocess.run(cmd,input=guarded,text=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=12) return p.returncode==0 except subprocess.TimeoutExpired:return False finally: subprocess.run(['docker','rm','--force',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=15) def dataset_for(t): if t['adapter']=='choice': from huggingface_hub import hf_hub_download import pyarrow.parquet as pq file=hf_hub_download(t['dataset'],t['recipe']['data_file'],repo_type='dataset',revision=t['dataset_revision']) return (row for batch in pq.ParquetFile(file).iter_batches(batch_size=128) for row in batch.to_pylist()) if t['adapter']=='commonsense': # Read the pinned parquet directly: its newer HF feature metadata uses # List, which older datasets versions cannot parse. Arrow preserves row order. from huggingface_hub import hf_hub_download import pyarrow.parquet as pq file=hf_hub_download(t['dataset'],f"data/{t['split']}-00000-of-00001.parquet",repo_type='dataset',revision=t['dataset_revision']) return (row for batch in pq.ParquetFile(file).iter_batches(batch_size=128) for row in batch.to_pylist()) from datasets import load_dataset return load_dataset(t['dataset'],t['dataset_config'],split=t['split'],revision=t['dataset_revision'],streaming=True) def pack_lengths(lengths,batch_size=8,token_budget=8192): groups=[];group=[];longest=0 for i in sorted(range(len(lengths)),key=lambda i:(lengths[i],i)): length=lengths[i] if group and (len(group)>=batch_size or max(longest,length)*(len(group)+1)>token_budget): groups.append(group);group=[];longest=0 group.append(i);longest=max(longest,length) if group:groups.append(group) return groups def choice_mapping(row,recipe): def field(path): value=row for key in path.split('.'):value=value[key] return value question=field(recipe['question_field']);choices=field(recipe['choices_field']);answer=field(recipe['answer_field']) kind=recipe['answer_type'] target=int(answer) if kind=='index' else [str(x) for x in field(recipe['choice_labels_field'])].index(str(answer)) if kind=='label' else choices.index(str(answer)) return question,choices,target def batched_choice(model,tokenizer,problems,device,batch_size=16): import torch candidates=[];scores=[[None]*len(answers) for _,answers in problems];timings=[] for question,(prompt,answers) in enumerate(problems): prefix=tokenizer.encode(prompt,add_special_tokens=False)[-1800:] if not prefix:raise ValueError('Empty prompt tokens') for answer,text in enumerate(answers): suffix=tokenizer.encode(' '+text,add_special_tokens=False) if not suffix:raise ValueError('Empty answer tokens') if len(suffix)>512:raise ValueError('Answer exceeds sub-1B protocol budget') candidates.append((question,answer,prefix[-max(1,1024-len(suffix)):]+suffix,suffix)) for group in pack_lengths([len(c[2]) for c in candidates],batch_size): selected=[candidates[i] for i in group];width=max(len(c[2]) for c in selected) ids=torch.full((len(group),width),tokenizer.eos_token_id,dtype=torch.long,device=device);mask=torch.zeros_like(ids) for i,(_,_,tokens,_) in enumerate(selected):ids[i,-len(tokens):]=torch.tensor(tokens,device=device);mask[i,-len(tokens):]=1 positions=(mask.cumsum(-1)-1).clamp_min(0);keep=max(len(c[3]) for c in selected)+1 torch.cuda.synchronize() if device.startswith('cuda') else None begin=time.monotonic() with torch.inference_mode(): logits=model(ids,attention_mask=mask,position_ids=positions,use_cache=False,logits_to_keep=keep).logits values=[] for i,(_,_,_,suffix) in enumerate(selected): answer_logits=logits[i,-len(suffix)-1:-1].float();target=torch.tensor(suffix,device=device) values.append(torch.log_softmax(answer_logits,dim=-1).gather(1,target[:,None]).sum()) values=torch.stack(values).cpu().tolist() elapsed=time.monotonic()-begin for (question,answer,_,_),value in zip(selected,values):scores[question][answer]=value timings.append(dict(kind='choice',candidates=len(group),padded_tokens=len(group)*width,seconds=elapsed)) del logits return scores,timings def choice_decision(log_likelihood_sums,choices): """Raw accuracy and character-normalized accuracy are different decisions.""" if len(log_likelihood_sums)!=len(choices) or not choices:raise ValueError('Choice score shape differs') if any(not isinstance(c,str) or not c for c in choices):raise ValueError('Empty choice text') if any(not math.isfinite(v) for v in log_likelihood_sums):raise ValueError('Non-finite choice likelihood') pred=max(range(len(choices)),key=lambda i:log_likelihood_sums[i]) # lm-evaluation-harness acc_norm divides by characters, excluding the delimiter. normalized=[v/len(c) for v,c in zip(log_likelihood_sums,choices)] return pred,max(range(len(choices)),key=lambda i:normalized[i]) def batched_generate(model,tokenizer,prompts,device,max_tokens,batch_size=8): import torch encoded=[tokenizer(p,return_tensors=None,truncation=True,max_length=1800)['input_ids'] for p in prompts] predictions=[None]*len(prompts);timings=[] for group in pack_lengths([len(tokens)+max_tokens for tokens in encoded],batch_size,8192): width=max(len(encoded[i]) for i in group);ids=torch.full((len(group),width),tokenizer.eos_token_id,dtype=torch.long,device=device);mask=torch.zeros_like(ids) for row,i in enumerate(group):ids[row,-len(encoded[i]):]=torch.tensor(encoded[i],device=device);mask[row,-len(encoded[i]):]=1 torch.cuda.synchronize() if device.startswith('cuda') else None begin=time.monotonic() with torch.inference_mode():out=model.generate(input_ids=ids,attention_mask=mask,max_new_tokens=max_tokens,do_sample=False,pad_token_id=tokenizer.eos_token_id) texts=tokenizer.batch_decode(out[:,width:],skip_special_tokens=True) elapsed=time.monotonic()-begin for i,prediction in zip(group,texts):predictions[i]=prediction timings.append(dict(kind='generation',examples=group,seconds=elapsed,padded_input_tokens=width*len(group),generated_tokens=int(out[:,width:].numel()))) return predictions,timings def evaluate(job, output): import numpy as np import torch from datasets import load_dataset from transformers import AutoTokenizer, AutoModelForCausalLM from huggingface_hub import HfApi t=job['task'];adapter=t['adapter'];revision=job['revision'];model_id=job['model'] if t.get('protocol_version')!='codesota-sub1b-arc-easy-standard-v2' or t.get('dataset')!='allenai/ai2_arc' or t.get('dataset_config')!='ARC-Easy' or t.get('split')!='test' or t.get('limit')!=2376 or adapter!='choice':raise ValueError('This runner only supports the pinned full ARC-Easy standard protocol') torch.manual_seed(42);np.random.seed(42);torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=False device='cuda:0' if torch.cuda.is_available() else 'cpu' dataset=dataset_for(t) if adapter=='asr': from datasets import Audio dataset=dataset.cast_column('audio',Audio(sampling_rate=16000)) rows=[] for index,row in enumerate(dataset): if adapter=='entailment' and row['label']<0:continue rows.append((index,row)) if len(rows)==t['limit']:break if len(rows)!=t['limit']:raise ValueError('Dataset did not yield the declared sample count') samples=[];started=time.monotonic();special={'execution_profile':'sub1b-arc-easy-standard-v2','batch_policy':{'choice_candidates':16,'generation_examples':8,'padded_token_budget':8192,'code_test_workers':4},'batch_timings':[]};peak=0 def save(index,reference,prediction,score=None,**extra): samples.append(dict(id=str(index),reference=reference,prediction=prediction,score=score,**extra)) def load_causal(): for name,digest in DEPENDENCIES.items(): if hashlib.sha256((Path(__file__).parent/name).read_bytes()).hexdigest()!=digest:raise ValueError('Pinned native dependency differs: '+name) approved=next(m for m in json.loads((Path(__file__).parent/'sub1b-models.json').read_text()) if m['id']==model_id and m['revision']==revision) special['dependency_sources']=DEPENDENCIES if approved['adapter']!='hf': from sub1b_native import load return load(approved) tok=AutoTokenizer.from_pretrained(model_id,revision=revision,trust_remote_code=False) model=AutoModelForCausalLM.from_pretrained(model_id,revision=revision,trust_remote_code=False,torch_dtype=torch.float32).to(device).eval() return tok,model if adapter in ('reading','commonsense','sentiment','entailment','math','code','choice'): tokenizer,model=load_causal() if adapter=='code': image=os.environ.get('CODESOTA_SANDBOX_IMAGE') if not image or '@sha256:' not in image:raise ValueError('Pinned Docker sandbox image required') subprocess.run(['docker','image','inspect',image],check=True,stdout=subprocess.DEVNULL) special['sandbox_image']=image problems=[];generations=None if adapter in ('math','code'): prompts=[f"Question: {row['question']}\nSolve the problem. End with #### followed by the numerical answer.\nAnswer:" if adapter=='math' else row['prompt'] for _,row in rows] generations,batches=batched_generate(model,tokenizer,prompts,device,256);special['batch_timings'].extend(batches) else: for index,row in rows: if adapter=='reading':prompt=f"Passage: {row['passage']}\nQuestion: {row['question']}\nAnswer:";choices=['no','yes'];target=int(row['answer']) elif adapter=='commonsense':prompt=row['ctx'];choices=row['endings'];target=int(row['label']) elif adapter=='sentiment':prompt=f"Review: {row['sentence']}\nSentiment:";choices=['negative','positive'];target=int(row['label']) elif adapter=='entailment':prompt=f"Premise: {row['premise']}\nHypothesis: {row['hypothesis']}\nRelationship:";choices=['entailment','neutral','contradiction'];target=int(row['label']) else: question,choices,target=choice_mapping(row,t['recipe']);prompt='Question: '+question+'\nAnswer:' if any(len(tokenizer.encode(' '+c,add_special_tokens=False))>512 for c in choices):raise ValueError('Choice exceeds token budget') problems.append((index,prompt,choices,target)) all_scores,batches=batched_choice(model,tokenizer,[(p,c) for _,p,c,_ in problems],device);special['batch_timings'].extend(batches) for (index,_,choices,target),likelihoods in zip(problems,all_scores): pred,pred_norm=choice_decision(likelihoods,choices) lengths=[len(tokenizer.encode(' '+c,add_special_tokens=False)) for c in choices] save(index,target,pred,float(pred==target),choices=choices,log_likelihood_sum=likelihoods,answer_token_lengths=lengths,prediction_char_norm=pred_norm,score_char_norm=float(pred_norm==target)) special['acc_norm']=float(np.mean([s['score_char_norm'] for s in samples]));special['acc_norm_definition']='argmax summed conditional log likelihood / Python character length of answer text, excluding leading delimiter' special['scoring']='argmax raw summed conditional answer-token log likelihood; zero-shot Question/Answer prompt; no chat template or added BOS' if adapter=='math': for (index,row),raw in zip(rows,generations): matches=re.findall(r'####\s*([-+]?\d[\d,]*(?:\.\d+)?)',raw);guess=matches[-1].replace(',','') if matches else None;answer=row['answer'].split('####')[-1].strip().replace(',','') try:correct=float(guess)==float(answer) except (TypeError,ValueError):correct=False save(index,answer,raw,float(correct),extracted=guess) elif adapter=='code': completions=[re.split(r'\n(?=def |class |if __name__|print\(|#)',raw)[0] for raw in generations] programs=[row['prompt']+completion+'\n'+row['test']+f"\ncheck({row['entry_point']})\n" for (_,row),completion in zip(rows,completions)] before=time.monotonic() with ThreadPoolExecutor(max_workers=4) as pool:correct=list(pool.map(lambda program:code_test(program,special['sandbox_image']),programs)) special['code_test_seconds']=time.monotonic()-before for (index,row),completion,passed in zip(rows,completions,correct):save(row['task_id'],'passes hidden unit tests',completion,float(passed)) score=float(np.mean([s['score'] for s in samples])) elif adapter=='similarity': from sentence_transformers import SentenceTransformer from scipy.stats import spearmanr model=SentenceTransformer(model_id,revision=revision,device=device,trust_remote_code=False) left=model.encode([r['sentence1'] for _,r in rows],normalize_embeddings=True,convert_to_numpy=True) right=model.encode([r['sentence2'] for _,r in rows],normalize_embeddings=True,convert_to_numpy=True) pred=np.sum(left*right,axis=1) refs=[float(r['score']) for _,r in rows] score=float(spearmanr(pred,refs).statistic) for (i,r),p in zip(rows,pred):save(i,float(r['score']),float(p)) elif adapter=='ocr': from transformers import TrOCRProcessor, VisionEncoderDecoderModel from jiwer import cer processor=TrOCRProcessor.from_pretrained(model_id,revision=revision) model=VisionEncoderDecoderModel.from_pretrained(model_id,revision=revision).to(device).eval() refs=[];preds=[] for index,row in rows: image=row['image'].convert('RGB');ref=row['text'] pixels=processor(images=image,return_tensors='pt').pixel_values.to(device) with torch.inference_mode():out=model.generate(pixels,max_new_tokens=128,do_sample=False) pred=processor.batch_decode(out,skip_special_tokens=True)[0] # Case and punctuation preserved; strip only outer whitespace. ref=ref.strip();pred=pred.strip();refs.append(ref);preds.append(pred);save(index,ref,pred,cer(ref,pred)) score=float(cer(refs,preds)) elif adapter=='asr': from transformers import WhisperProcessor, WhisperForConditionalGeneration from jiwer import wer processor=WhisperProcessor.from_pretrained(model_id,revision=revision) model=WhisperForConditionalGeneration.from_pretrained(model_id,revision=revision).to(device).eval() from transformers.models.whisper.english_normalizer import BasicTextNormalizer normalize=BasicTextNormalizer();refs=[];preds=[] for index,row in rows: audio=row['audio'];inputs=processor(audio['array'],sampling_rate=audio['sampling_rate'],return_tensors='pt').input_features.to(device) with torch.inference_mode():out=model.generate(inputs,max_new_tokens=128,do_sample=False) pred=normalize(processor.batch_decode(out,skip_special_tokens=True)[0]);ref=normalize(row['text']) refs.append(ref);preds.append(pred);save(row.get('id',index),ref,pred,wer(ref,pred)) score=float(wer(refs,preds));special['normalizer']='transformers Whisper BasicTextNormalizer' elif adapter=='vision': from transformers import CLIPProcessor, CLIPModel labels=['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck'] prompts=[f'a photo of a {label}.' for label in labels] processor=CLIPProcessor.from_pretrained(model_id,revision=revision) model=CLIPModel.from_pretrained(model_id,revision=revision).to(device).eval() for index,row in rows: inputs=processor(text=prompts,images=row['img'].convert('RGB'),return_tensors='pt',padding=True).to(device) with torch.inference_mode(): pred=int(model(**inputs).logits_per_image[0].argmax()) save(index,int(row['label']),pred,float(pred==row['label'])) score=float(np.mean([s['score'] for s in samples]));special['class_prompts']=prompts;special['training_regime']='zero-shot CLIP; compare only within this protocol' else:raise ValueError('Unknown adapter') if torch.cuda.is_available():peak=torch.cuda.max_memory_allocated() duration=time.monotonic()-started packages={d.metadata['Name']:d.version for d in importlib.metadata.distributions()} result=dict(score=score,n=len(samples),model_revision=revision,dataset_revision=t['dataset_revision'],runner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),seconds=duration,peak_gpu_bytes=peak,selection='full pinned ARC-Easy test split; all 2376 examples in source order',seed=42,device=device,protocol=t,measured_cost_usd=None,notes='Full ARC-Easy test split (2376 examples); FP32, 1024-token input ceiling, raw summed answer-token log likelihood, zero-shot Question/Answer prompt. Character-normalized accuracy reported separately as acc_norm. Native masked-padding adapter uses unpadded row forwards. Distinct from the earlier token-normalized pilot.',**special) output.parent.mkdir(parents=True,exist_ok=True) output.write_text(json.dumps(dict(result=result,samples=samples,environment=dict(python=sys.version,platform=platform.platform(),torch=torch.__version__,cuda=torch.version.cuda,gpu=torch.cuda.get_device_name() if torch.cuda.is_available() else None,packages=packages)),allow_nan=False)) if __name__=='__main__':evaluate(json.loads(Path(sys.argv[1]).read_text()),Path(sys.argv[2]))