"""Paired FLORES devtest token counts; no model inference or speed estimate."""
import argparse
import hashlib
import json
import platform
import random
import statistics
import sys
from pathlib import Path
from tokenizers import Tokenizer

ROOT = Path(__file__).resolve().parent
LANGS = {'English':'eng_Latn','Hindi':'hin_Deva','Bengali':'ben_Beng','Tamil':'tam_Taml','Telugu':'tel_Telu','Marathi':'mar_Deva'}
MODELS = ['GPT-2','mBERT','XLM-R','Qwen2.5','UMT5']
SEED = 28004


def interval(rows, other, seed):
    rng = random.Random(seed)
    vals = []
    n = len(rows)
    for _ in range(2000):
        ix = [rng.randrange(n) for _ in range(n)]
        vals.append(sum(rows[i] for i in ix) / sum(other[i] for i in ix))
    vals.sort()
    return [round(vals[49], 4), round(vals[1949], 4)]


def main():
    p=argparse.ArgumentParser();p.add_argument('--n',type=int,default=300);p.add_argument('--run-id',default='28-paper4-20261010-01');a=p.parse_args()
    manifest=json.loads((ROOT/'paper4-assets.json').read_text())
    for item in manifest:
        path=ROOT/'assets/paper4'/item['name']
        if not path.exists() or hashlib.sha256(path.read_bytes()).hexdigest()!=item['sha256']:
            raise RuntimeError('ASSET_HASH_MISMATCH:'+item['name'])
    texts={name:(ROOT/'assets/paper4/data'/f'{code}.devtest').read_text().splitlines() for name,code in LANGS.items()}
    assert {len(v) for v in texts.values()}=={1012}
    ids=sorted(random.Random(SEED).sample(range(1012),a.n))
    counts={};pieces={};unknown={};config={}
    for model in MODELS:
        token_file=ROOT/'assets/paper4/tokenizers'/f'{model}.json'
        raw=json.loads(token_file.read_text())
        assert raw.get('truncation') is None and raw.get('padding') is None
        family={'GPT-2':'byte BPE','mBERT':'WordPiece','XLM-R':'SentencePiece Unigram','Qwen2.5':'byte BPE','UMT5':'SentencePiece Unigram'}[model]
        tok=Tokenizer.from_file(str(token_file))
        marker=raw['model'].get('unk_token') or ('<unk>' if model in ('XLM-R','UMT5') else None)
        unknown_id=raw['model'].get('unk_id')
        if unknown_id is None and marker:
            unknown_id=tok.token_to_id(marker)
        config[model]={'family':family,'truncation':raw.get('truncation'),'padding':raw.get('padding'),'unknown_token':marker,'unknown_id':unknown_id}
        counts[model]={}
        unknown[model]={}
        for name, lines in texts.items():
            enc=tok.encode_batch([lines[i] for i in ids],add_special_tokens=False)
            counts[model][name]=[len(e.ids) for e in enc]
            unknown[model][name]=sum(token_id==unknown_id for e in enc for token_id in e.ids) if unknown_id is not None else 0
            if model in ('GPT-2','mBERT','Qwen2.5') and name in ('English','Hindi'):
                pieces.setdefault(model,{})[name]={'total_tokens':len(enc[0].ids),'tokens':enc[0].tokens[:5],'ids':enc[0].ids[:5]}
    stats={}
    for model in MODELS:
        stats[model]={}
        english=counts[model]['English']
        for j,(name,lines) in enumerate(texts.items()):
            vals=counts[model][name]
            chars=sum(len(lines[i]) for i in ids)
            ratio=sum(vals)/sum(english)
            stats[model][name]={'tokens_per_sentence':round(statistics.mean(vals),4),'ratio_to_english':round(ratio,4),'ratio_ci95':interval(vals,english,SEED+100*j+MODELS.index(model)),'tokens_per_character':round(sum(vals)/chars,4),'context_4096_sentences':int(4096//statistics.mean(vals))}
    out={'run_id':a.run_id,'seed':SEED,'sample_n':a.n,'population_n':1012,'sample_indices_zero_based':ids,'model_order':MODELS,'language_codes':LANGS,'counts':counts,'stats':stats,'pieces':pieces,'unknown_token_counts':unknown,'tokenizer_config':config,'context_tokens':4096,'bootstrap_resamples':2000,'bootstrap_unit':'aligned sentence row','python':sys.version.split()[0],'platform':platform.platform(),'machine':platform.machine(),'tokenizers_version':__import__('tokenizers').__version__,'asset_sha256':{x['name']:x['sha256'] for x in manifest}}
    dest=ROOT/'evidence/paper4';dest.mkdir(parents=True,exist_ok=True)
    (dest/'results.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n')
    print('measured',a.n,'aligned rows x',len(LANGS),'languages x',len(MODELS),'tokenizers')
    for model in MODELS:
        print(model,'Hindi/English',stats[model]['Hindi']['ratio_to_english'],stats[model]['Hindi']['ratio_ci95'])


if __name__=='__main__':main()