#!/usr/bin/env python3
"""Claim checks: verify pins, recompute every headline number from raw rows, and replay a seeded sample
of saved final programs through the same gate and sandboxed tests. Writes evidence/check-table.md."""
import json, random, sys, tempfile
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
import study, analyze  # noqa: E402

BASE = study.BASE
REPLAY_N = 49  # 7 per arm


def main():
    tasks = {t['id']: t for t in study.load_tasks()}  # verifies dataset hashes
    rows = [json.loads(x) for x in (BASE / 'results/runs.jsonl').read_text().splitlines()]
    saved = json.loads((BASE / 'results/summary.json').read_text())
    analyze.main()
    fresh = json.loads((BASE / 'results/summary.json').read_text())
    claims = []
    claims.append(('152 tasks present in all seven arms, no duplicates', fresh['tasks'] == 152 and len(rows) == 152 * 7 and len({(r['task'], r['arm']) for r in rows}) == len(rows), f"{fresh['tasks']} tasks, {len(rows)} rows"))
    srows = [json.loads(x) for x in (BASE / 'results/sensitivity-ts-jsstops.jsonl').read_text().splitlines()]
    claims.append(('Sensitivity arm (TypeScript, JS stop rule) covers the same 152 tasks once each', sorted(r['task'] for r in srows) == fresh['task_ids'], f"{len(srows)} rows"))
    claims.append(('Summary recomputes identically from raw rows', saved == fresh, 'summary.json byte-equivalent after recompute'))
    # Hidden tests must never reach the model. Diagnostics are the only channel back; replies are checked too.
    leak = 0
    for r in rows + srows:
        hidden = [l.strip() for l in tasks[r['task']][r['lang']]['tests'].splitlines() if len(l.strip()) > 25 and ('assert' in l or 'candidate(' in l or 'actual:' in l)]
        for a in r['attempts']:
            text = ' '.join(a['diagnostics']) + (a.get('reply') or '')
            leak += any(h in text for h in hidden)
    claims.append(('No hidden test line appears verbatim in any diagnostic fed back or any model reply', leak == 0, f'{leak} attempts with a verbatim hidden-test line'))
    rng = random.Random(study.SEED)
    mism = 0
    for arm in analyze.ARMS:
        for r in rng.sample([x for x in rows if x['arm'] == arm], REPLAY_N // 7):
            t = tasks[r['task']]; last = r['attempts'][-1]; lang = r['lang']
            prompt = study.go_imports(t[lang]['prompt'], last.get('reply') or '') if (lang == 'go' and last['k'] > 0) else t[lang]['prompt']
            prog = study.join(prompt, last['completion'], t[lang]['tests'])
            with tempfile.TemporaryDirectory(prefix='ckh-check-') as d:
                ok, _ = study.gate(arm, prog, d, solution=prompt + last['completion'])
                res = study.test(arm, prog, d)
            if ok != last['gate_ok'] or res['passed'] != last['test']['passed']:
                mism += 1
                print('REPLAY_MISMATCH', arm, r['task'], ok, last['gate_ok'], res['passed'], last['test']['passed'])
    claims.append((f'Seeded replay of {REPLAY_N} saved final programs reproduces gate and test outcomes', mism == 0, f'{mism} mismatches'))
    lines = ['| # | Claim | Outcome | Actual check |', '|---|---|---|---|']
    for i, (name, ok, value) in enumerate(claims, 1):
        lines.append(f'| {i} | {name} | {"PASS" if ok else "FAIL"} | {value} |')
    (BASE / 'evidence/check-table.md').write_text('\n'.join(lines) + '\n')
    print('\n'.join(lines))
    return all(ok for _, ok, _ in claims)


if __name__ == '__main__':
    sys.exit(0 if main() else 1)