#!/usr/bin/env python3
"""Pinned HumanEval fetch, local Ollama measurement, and executable claim checks."""
import argparse, datetime as dt, gzip, hashlib, json, math, os, platform, random, re, resource, secrets, subprocess, sys, threading, time, urllib.request
from pathlib import Path
import psutil
ROOT=Path(__file__).resolve().parents[3]
BASE=ROOT/'paper/free-tokens-coding'
DATA=Path(os.environ.get('P29_DATA_OVERRIDE',str(ROOT/'assets/free-tokens-coding/HumanEval.jsonl.gz')))
RESULT=BASE/'results/measurement.json'
ROWS=BASE/'results/problems.jsonl'
SHA='b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef'
URL='https://raw.githubusercontent.com/openai/human-eval/6d43fb980f9fee3c892a914eda09951f772ad10d/data/HumanEval.jsonl.gz'
MODEL='qwen2.5-coder:1.5b'
SEED=29029
N=50
POLICY='(version 1) (allow default) (deny network*) (deny file-write*) (deny process-fork)'

def digest(p):return hashlib.sha256(p.read_bytes()).hexdigest()
def fetch(allow_download=True):
 DATA.parent.mkdir(parents=True,exist_ok=True)
 if not DATA.exists():
  if not allow_download:raise RuntimeError('ASSET_MISSING: HumanEval; run fetch.py')
  urllib.request.urlretrieve(URL,DATA)
 if digest(DATA)!=SHA:raise RuntimeError('ASSET_HASH_MISMATCH: HumanEval')
 print('HumanEval SHA-256 verified',SHA)
def problems(allow_download=True):
 fetch(allow_download); a=[json.loads(x) for x in gzip.open(DATA,'rt')]
 assert len(a)==164 and len(set(x['task_id'] for x in a))==164
 return [a[i] for i in sorted(random.Random(SEED).sample(range(len(a)),N))]
def proc_rss():
 total=0
 for p in psutil.process_iter(['name','cmdline','memory_info']):
  try:
   name=(p.info['name'] or '').lower();cmd=' '.join(p.info['cmdline'] or []).lower()
   if 'ollama' in name or (name=='llama-server' and 'sha256-29d8c98fa6b0' in cmd):total+=p.info['memory_info'].rss
  except (psutil.NoSuchProcess,psutil.AccessDenied):pass
 return total

def generate(problem):
 prompt='Complete this Python programming problem. Return only executable Python code implementing the requested function, with no markdown or explanation.\n\n'+problem['prompt']
 payload={'model':MODEL,'stream':False,'messages':[{'role':'user','content':prompt}],'options':{'temperature':0,'seed':SEED,'num_predict':256,'num_ctx':2048},'keep_alive':'30m'}
 samples=[];stop=threading.Event()
 def watch():
  while not stop.is_set():samples.append(proc_rss());stop.wait(.1)
 t=threading.Thread(target=watch,daemon=True);t.start();start=time.monotonic()
 try:
  req=urllib.request.Request('http://127.0.0.1:11434/api/chat',data=json.dumps(payload).encode(),headers={'Content-Type':'application/json'})
  with urllib.request.urlopen(req,timeout=180) as response:out=json.load(response)
 finally:stop.set();t.join()
 raw=out['message']['content'];code=raw.strip()
 match=re.search(r'```(?:python)?\s*\n(.*?)```',code,re.S|re.I)
 if match:code=match.group(1).strip()
 code=re.sub(r'^```(?:python)?\s*','',code).strip()
 return {'code':code,'raw':raw,'eval_count':out.get('eval_count',0),'eval_duration_ns':out.get('eval_duration',0),'prompt_eval_count':out.get('prompt_eval_count',0),'wall_s':round(time.monotonic()-start,3),'peak_ollama_rss_bytes':max(samples) if samples else None}

def test_code(code,p):
 # Generated code is untrusted. The macOS seatbelt denies writes and networking;
 # a child process also has CPU and file-size limits plus an eight-second wall timeout.
 marker='P29_TEST_COMPLETE_'+secrets.token_hex(16)
 script=code+'\n'+p['test']+'\ncheck('+p['entry_point']+')\nprint('+repr(marker)+')\n'
 def limits():
  resource.setrlimit(resource.RLIMIT_CPU,(5,5));resource.setrlimit(resource.RLIMIT_FSIZE,(0,0))
 try:
  cp=subprocess.run(['/usr/bin/sandbox-exec','-p',POLICY,sys.executable,'-c',script],capture_output=True,text=True,timeout=8,preexec_fn=limits,cwd='/private/tmp',env={'PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'})
  completed=cp.returncode==0 and cp.stdout.splitlines()[-1:]==[marker]
  return {'passed':completed,'status':'pass' if completed else 'fail','returncode':cp.returncode,'error_tail':cp.stderr[-240:]}
 except subprocess.TimeoutExpired:return {'passed':False,'status':'timeout','returncode':None,'error_tail':'8s timeout'}

def wilson(k,n):
 z=1.959963984540054;p=k/n;d=1+z*z/n;c=(p+z*z/(2*n))/d;h=z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/d
 return [round(c-h,4),round(c+h,4)]
def measure(run_id):
 from fetch import main as verify_assets
 verify_assets(allow_download=False)
 selected=problems(allow_download=False);ROWS.parent.mkdir(parents=True,exist_ok=True)
 rows=[]
 with ROWS.open('w') as f:
  for i,p in enumerate(selected,1):
   g=generate(p);test=test_code(g['code'],p);row={'task_id':p['task_id'],'sample_position':i,**g,**test}
   rows.append(row);f.write(json.dumps(row,sort_keys=True)+'\n');f.flush()
   print(f"{i}/{N} {p['task_id']} {test['status']} {g['eval_count']} tokens",flush=True)
 k=sum(x['passed'] for x in rows);tok=sum(x['eval_count'] for x in rows);secs=sum(x['eval_duration_ns'] for x in rows)/1e9
 result={'run_id':run_id,'date_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'model':MODEL,'model_digest':os.environ.get('P29_MODEL_DIGEST','unknown'),'quantization':'Ollama Q4_K_M registry artifact; model blob SHA-256 pinned','benchmark':'OpenAI HumanEval','benchmark_sha256':SHA,'seed':SEED,'sample_n':N,'population_n':164,'task_ids':[x['task_id'] for x in rows],'passed':k,'pass_rate':round(k/N,4),'pass_ci95_wilson':wilson(k,N),'output_tokens':tok,'eval_seconds':round(secs,4),'tokens_per_second':round(tok/secs,3),'peak_ollama_rss_bytes':max(x['peak_ollama_rss_bytes'] or 0 for x in rows),'daily_tokens_continuous_24h':round(tok/secs*86400),'daily_assumption':'24 h continuous generation at aggregate observed decode rate; excludes prompts, tests, idle time, electricity and wear','host':{'cpu':platform.processor(),'machine':platform.machine(),'ram_bytes':psutil.virtual_memory().total,'python':platform.python_version(),'ollama_version':os.environ.get('P29_OLLAMA_VERSION','unknown')},'protocol':{'temperature':0,'num_predict':256,'num_ctx':2048,'test_timeout_s':8,'sandbox':'macOS sandbox-exec deny network, file writes and process fork; randomized completion marker; CPU 5s, wall 8s'}}
 RESULT.write_text(json.dumps(result,indent=2,sort_keys=True)+'\n');print(json.dumps({k:result[k] for k in ['passed','sample_n','pass_rate','pass_ci95_wilson','tokens_per_second','peak_ollama_rss_bytes','daily_tokens_continuous_24h']},indent=2))
def checks():
 selected=problems(allow_download=False);r=json.loads(RESULT.read_text());rows=[json.loads(x) for x in ROWS.read_text().splitlines()]
 from fetch import main as verify_assets
 verify_assets(allow_download=False)
 for row,problem in zip(rows,selected):
  repeated=test_code(row['code'],problem)
  assert repeated['passed']==row['passed'],'TEST_REPLAY_MISMATCH:'+row['task_id']
 assert len(rows)==N and r['task_ids']==[x['task_id'] for x in rows]==[x['task_id'] for x in selected] and len(set(r['task_ids']))==N,'SAMPLE_MISMATCH'
 assert r['benchmark_sha256']==digest(DATA),'ASSET_HASH_MISMATCH'
 k=sum(x['passed'] for x in rows);tok=sum(x['eval_count'] for x in rows);secs=sum(x['eval_duration_ns'] for x in rows)/1e9
 claims=[('50 seeded, distinct public benchmark tasks were attempted',len(rows)==N and len(set(x['task_id'] for x in rows))==N,str(len(rows))+' tasks'),('Observed pass count and Wilson interval match individual tests',r['passed']==k and r['pass_ci95_wilson']==wilson(k,N),f'{k}/{N}; CI {wilson(k,N)}'),('Measured output throughput matches Ollama decode counters',abs(r['tokens_per_second']-tok/secs)<.001,f'{tok} tokens / {secs:.2f}s = {tok/secs:.3f} tok/s'),('Observed Ollama RSS peak matches sampled runs',r['peak_ollama_rss_bytes']==max(x['peak_ollama_rss_bytes'] or 0 for x in rows),str(r['peak_ollama_rss_bytes'])+' bytes'),('Continuous 24-hour free-token projection uses the measured decode rate',r['daily_tokens_continuous_24h']==round(tok/secs*86400),str(r['daily_tokens_continuous_24h'])+' output tokens/day')]
 source=json.loads((BASE/'evidence/hosted-source.json').read_text());aws=json.loads((BASE/'evidence/hosted-source-aws.json').read_text());claims.append(('CITED: public free allowances use completions and requests, not tokens',source['figure']==2000 and 'up to 2,000 code completions' in source['quote'] and aws['figure']==50 and '50 agentic requests per month' in aws['quote'],'GitHub up to 2,000 completions; AWS 50 agentic requests/month; read '+source['read_utc'][:10]))
 lines=['claim | outcome | actual check','--- | --- | ---']
 for i,(name,ok,value) in enumerate(claims,1):lines.append(f'{i}. {name} | {"PASS" if ok else "FAIL"} | {value}')
 table='\n'.join(lines)+'\n';(BASE/'evidence/check-table.md').write_text(table);print(table)
 return all(ok for _,ok,_ in claims)
if __name__=='__main__':
 a=argparse.ArgumentParser();a.add_argument('action',choices=['fetch','measure','checks']);a.add_argument('--run-id',default='29-free-tokens-20261010-02');x=a.parse_args()
 if x.action=='fetch':fetch()
 elif x.action=='measure':measure(x.run_id)
 else:sys.exit(0 if checks() else 1)