"""Full raw-likelihood ARC-Challenge for pinned stock Qwen3.5 text/VL checkpoints.

Runs in /opt/codesota-qwen35-venv, independently of frozen evaluator environments.
No images, chat templates, generated programs or custom model code are executed.
"""
from __future__ import annotations
import hashlib,importlib.metadata,importlib.util,json,os,platform,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parent
DEPENDENCIES={'evaluate_sub1b_standard.py': 'ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a', 'requirements-qwen35.txt': 'd41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa'}

def verify_sources():
    for name,digest in DEPENDENCIES.items():
        if hashlib.sha256((ROOT/name).read_bytes()).hexdigest()!=digest:raise ValueError('Pinned dependency differs: '+name)

def approved_class(config):
    kind=config.model_type;architectures=config.architectures or []
    expected={'qwen3_5':'Qwen3_5ForConditionalGeneration','qwen3_5_text':'Qwen3_5ForCausalLM'}
    if kind not in expected or architectures!=[expected[kind]]:raise ValueError('Unsupported checkpoint architecture; a real stock causal LM head is required')
    if getattr(config,'quantization_config',None) or getattr(config,'quantization',None):raise ValueError('Quantized runtime variants require a separate protocol')
    return expected[kind]

def unique_parameters(*modules):
    return sum({id(p):p.numel() for module in modules if module is not None for p in module.parameters()}.values())

def load_model(ident,revision,device):
    import torch
    from transformers import AutoConfig,AutoTokenizer,Qwen3_5ForCausalLM,Qwen3_5ForConditionalGeneration
    config=AutoConfig.from_pretrained(ident,revision=revision,trust_remote_code=False)
    name=approved_class(config);cls=Qwen3_5ForConditionalGeneration if name=='Qwen3_5ForConditionalGeneration' else Qwen3_5ForCausalLM
    tokenizer=AutoTokenizer.from_pretrained(ident,revision=revision,trust_remote_code=False)
    if tokenizer.eos_token_id is None:raise ValueError('Tokenizer has no defined EOS/padding ID')
    model,info=cls.from_pretrained(ident,revision=revision,config=config,trust_remote_code=False,dtype=torch.float32,attn_implementation='sdpa',output_loading_info=True)
    info={key:sorted(str(item) for item in info.get(key,[])) for key in ('missing_keys','unexpected_keys','mismatched_keys','error_msgs')}
    failures={key:value for key,value in info.items() if value}
    if failures:raise ValueError('Checkpoint did not load cleanly: '+json.dumps(failures)[:1500])
    if model.get_output_embeddings() is None:raise ValueError('Checkpoint has no language-model output head')
    count=unique_parameters(model)
    if not 0<count<1_000_000_000:raise ValueError('Loaded system exceeds the strict sub-1B unique-parameter cohort: '+str(count))
    text=getattr(model.model,'language_model',model.model);vision=getattr(model.model,'visual',None)
    measured=dict(parameters=count,text_parameters=unique_parameters(text,model.get_output_embeddings()),vision_parameters=unique_parameters(vision),embedding_head_tied=model.get_input_embeddings().weight is model.get_output_embeddings().weight)
    if getattr(config,'tie_word_embeddings',False) and not measured['embedding_head_tied']:raise ValueError('Configured shared embedding/output weights are not tied')
    model=model.to(device).eval()
    return tokenizer,model,dict(**measured,loaded_class=name,loading_info=info,loader_ignored_auxiliary_patterns=getattr(cls,'_keys_to_ignore_on_load_unexpected',[]))

def evaluate(job,output):
    import numpy as np
    import torch
    import transformers
    from evaluate_sub1b_standard import dataset_for,choice_mapping,batched_choice,choice_decision
    task=job['task'];model_id=job['model'];revision=job['revision'];started=time.monotonic()
    if (task.get('protocol_version'),task.get('dataset'),task.get('dataset_revision'),task.get('dataset_config'),task.get('split'),task.get('adapter'),task.get('metric'),task.get('limit'))!=('codesota-qwen35-arc-challenge-v1','allenai/ai2_arc','210d026faf9955653af8916fad021475a3f00453','ARC-Challenge','test','choice','accuracy',1172):raise ValueError('Expected the pinned full Qwen3.5 ARC-Challenge protocol')
    if not any(m['id']==model_id and m['revision']==revision for m in task['models']):raise ValueError('System revision is not pinned in this cohort')
    verify_sources();torch.manual_seed(42);np.random.seed(42);torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
    device='cuda:0' if torch.cuda.is_available() else 'cpu'
    rows=list(dataset_for(task))
    if len(rows)!=1172:raise ValueError('Expected all 1172 pinned ARC-Challenge test items')
    data_seconds=time.monotonic()-started;loaded=time.monotonic();tokenizer,model,details=load_model(model_id,revision,device);load_seconds=time.monotonic()-loaded
    problems=[];truncated=0
    for index,row in enumerate(rows):
        question,choices,target=choice_mapping(row,task['recipe']);prompt='Question: '+question+'\nAnswer:'
        prefix=tokenizer.encode(prompt,add_special_tokens=False)
        for choice in choices:
            suffix=tokenizer.encode(' '+choice,add_special_tokens=False)
            if not suffix or len(suffix)>512:raise ValueError('Choice violates the fixed token budget')
            truncated+=len(prefix)+len(suffix)>1024
        problems.append((index,prompt,choices,target))
    scoring=time.monotonic();scores,batches=batched_choice(model,tokenizer,[(p,c) for _,p,c,_ in problems],device,batch_size=16);scoring_seconds=time.monotonic()-scoring;samples=[]
    for (index,_,choices,target),values in zip(problems,scores,strict=True):
        pred,pred_norm=choice_decision(values,choices)
        samples.append(dict(id=str(index),reference=target,prediction=pred,score=float(pred==target),choices=choices,log_likelihood_sum=values,answer_token_lengths=[len(tokenizer.encode(' '+c,add_special_tokens=False)) for c in choices],prediction_char_norm=pred_norm,score_char_norm=float(pred_norm==target)))
    kernels={name:importlib.util.find_spec(name) is not None for name in ('causal_conv1d','fla','flash_attn','kernels')}
    result=dict(score=sum(s['score'] for s in samples)/1172,n=1172,acc_norm=sum(s['score_char_norm'] for s in samples)/1172,acc_norm_definition='argmax summed conditional likelihood / answer character length, excluding leading delimiter',model_revision=revision,dataset_revision=task['dataset_revision'],runner_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),dependency_sources=DEPENDENCIES,seconds=time.monotonic()-started,data_loading_seconds=data_seconds,model_load_seconds=load_seconds,scoring_seconds=scoring_seconds,peak_gpu_bytes=torch.cuda.max_memory_allocated() if device.startswith('cuda') else 0,selection='Full pinned ARC-Challenge test split; all 1172 source-order items',seed=42,device=device,protocol=task,execution_profile='qwen35-arc-challenge-v1',precision='float32',tf32=False,images_supplied=False,chat_template=False,batch_policy=dict(choice_candidates=16,padded_token_budget=8192,context=1024),batch_timings=batches,truncated_candidates=truncated,optional_kernel_packages=kernels,measured_cost_usd=None,notes='Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.',**details)
    packages={d.metadata['Name']:d.version for d in importlib.metadata.distributions()}
    output.parent.mkdir(parents=True,exist_ok=True);output.write_text(json.dumps(dict(result=result,samples=samples,environment=dict(python=sys.version,platform=platform.platform(),torch=torch.__version__,transformers=transformers.__version__,cuda=torch.version.cuda,gpu=torch.cuda.get_device_name() if device.startswith('cuda') else None,packages=packages)),allow_nan=False))

if __name__=='__main__':evaluate(json.loads(Path(sys.argv[1]).read_text()),Path(sys.argv[2]))