#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Reproduce the v1.1 editorial correction from archived CSV. No network calls.
Usage: python3 build_ai_crawler_benchmark.py --input INPUT_DIR --output OUTPUT_DIR
This preserves the historical classifier; it does not validate actual crawler access.
"""
import argparse
import csv
import hashlib
import html
import json
import shutil
import statistics
import zipfile
from collections import Counter
from decimal import Decimal, ROUND_HALF_UP
from pathlib import Path

VERSION = 'v1.1'
GROUP_COLORS = {'search': '#087f8c', 'user': '#7956a5', 'data': '#1e40af', 'control': '#b7791f', 'baseline': '#64748b'}
LABELS = {
 'en': {'rate':'Root-path restrictions by token', 'scope':'Historical parser subset; declared rules for / only', 'distribution':'Restricted AI-related tokens per domain', 'rank':'Root-path restrictions by Tranco rank', 'llms':'Restriction rate by heuristic llms.txt detection', 'detected':'llms.txt detected', 'not_detected':'Not detected', 'none':'0 tokens', 'restricted':'restricted', 'not_restricted':'not restricted', 'pair':'Declared root-path rules', 'sample':'domains', 'all':'all 12'},
 'zh': {'rate':'各标识的根路径限制比例', 'scope':'历史解析器子样本；仅表示 / 路径的声明规则', 'distribution':'每域名受限的 AI 相关标识数量', 'rank':'不同 Tranco 排名组的根路径限制比例', 'llms':'按 llms.txt 启发式检出分组的限制比例', 'detected':'检出 llms.txt', 'not_detected':'未检出', 'none':'0 个标识', 'restricted':'受限', 'not_restricted':'未判为受限', 'pair':'根路径声明规则对照', 'sample':'个域名', 'all':'全部 12 个'},
}

def digest(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()

def ratio(n, d):
    return {'numerator': n, 'denominator': d, 'percent': str((Decimal(n)*100/Decimal(d)).quantize(Decimal('.1'), rounding=ROUND_HALF_UP)) if d else None}

def summarize(rows, bots, snapshot):
    if len({r['domain'] for r in rows}) != len(rows):
        raise ValueError('Duplicate domains')
    keys = [b['key'] for b in bots]
    if len(keys) != 14 or len(set(keys)) != 14:
        raise ValueError('Expected 14 distinct historical tokens')
    if any(r[k] not in {'blocked','allowed','no_rule','unknown'} for r in rows for k in keys):
        raise ValueError('Unexpected verdict')
    subset = [r for r in rows if r['robots_parse_ok'] == '1']
    if any(r[k] == 'unknown' for r in subset for k in keys):
        raise ValueError('Historical parser subset contains an unknown verdict')
    ai_keys = [b['key'] for b in bots if b['group'] != 'baseline']
    restricted = lambda r, ks: any(r[k] == 'blocked' for k in ks)
    count = lambda rs, ks: sum(restricted(r, ks) for r in rs)
    metrics = {'version': VERSION, 'published': snapshot['published'], 'modified': snapshot['modified'], 'scope': 'archived_classifier_root_path_only',
       'source_sha256': snapshot['csv_sha256'], 'sample': {'total':len(rows),'columns':len(rows[0]),'historical_parser_subset':len(subset),'outside_subset':len(rows)-len(subset),'rank_min':min(int(r['rank']) for r in rows),'rank_max':max(int(r['rank']) for r in rows)},
       'ai_token_count':len(ai_keys),'all_token_count':len(keys), 'ai_any':ratio(count(subset,ai_keys),len(subset)),
       'legacy_all14_any':ratio(count(subset,keys),len(subset)), 'bots':[], 'pairs':[], 'rank_buckets':[]}
    for b in bots:
        metrics['bots'].append({**b, 'restricted':ratio(sum(r[b['key']]=='blocked' for r in subset),len(subset))})
    for training, search in [('gptbot','oai_searchbot'),('claudebot','claude_searchbot')]:
        cells=[]
        for first in (True,False):
            for second in (True,False):
                n=sum((r[training]=='blocked')==first and (r[search]=='blocked')==second for r in subset)
                cells.append({'first_restricted':first,'second_restricted':second,**ratio(n,len(subset))})
        only=next(c['numerator'] for c in cells if c['first_restricted'] and not c['second_restricted'])
        total=sum(c['numerator'] for c in cells if c['first_restricted'])
        metrics['pairs'].append({'first':training,'second':search,'first_name':next(b['name'] for b in bots if b['key']==training),'second_name':next(b['name'] for b in bots if b['key']==search),'cells':cells,'first_restricted_second_not':ratio(only,total)})
    for lo,hi,label in [(1,100,'1–100'),(101,500,'101–500'),(501,1000,'501–1000'),(1001,10**9,'1001+')]:
        sub=[r for r in subset if lo<=int(r['rank'])<=hi]
        metrics['rank_buckets'].append({'label':label,'n':len(sub),'ai_any':ratio(count(sub,ai_keys),len(sub)),'gptbot':ratio(sum(r['gptbot']=='blocked' for r in sub),len(sub)),'llms':ratio(sum(r['has_llms_txt']=='1' for r in sub),len(sub))})
    dist=Counter(sum(r[k]=='blocked' for k in ai_keys) for r in subset)
    metrics['distribution']=[{'label':label,'n':sum(n for k,n in dist.items() if lo<=k<=hi)} for lo,hi,label in [(0,0,'0'),(1,3,'1–3'),(4,7,'4–7'),(8,11,'8–11'),(12,12,'12')]]
    cross=[]
    for detected in (True,False):
        sub=[r for r in subset if (r['has_llms_txt']=='1')==detected]
        n=count(sub,ai_keys)
        cross.append({'detected':detected,'n':len(sub),'restricted':ratio(n,len(sub)),'not_restricted':len(sub)-n})
    metrics['llms']={'heuristic_detected_all':ratio(sum(r['has_llms_txt']=='1' for r in rows),len(rows)), 'comparable_subset':len(subset),'groups':cross,
        'legacy_all14_groups':[{'detected':d,'restricted':ratio(count([r for r in subset if (r['has_llms_txt']=='1')==d],keys),sum((r['has_llms_txt']=='1')==d for r in subset))} for d in (True,False)]}
    named=sum(any(r[k]=='blocked' and r[k+'_group']=='named' for k in ai_keys) for r in subset)
    metrics['ai_named']=named;metrics['ai_wildcard_only']=count(subset,ai_keys)-named
    metrics['legacy_all14_named']=sum(any(r[k]=='blocked' and r[k+'_group']=='named' for k in keys) for r in subset)
    metrics['legacy_all14_all_restricted']=sum(all(r[k]=='blocked' for k in keys) for r in subset)
    metrics['legacy_all14_none_restricted']=sum(not restricted(r,keys) for r in subset)
    metrics['operational']={'historical_redirects':ratio(sum(int(r['robots_redirects'])>0 for r in subset),len(subset)), 'historical_median_bytes':statistics.median(int(r['robots_bytes']) for r in subset)}
    if sum(b['n'] for b in metrics['rank_buckets']) != len(subset) or sum(b['n'] for b in metrics['distribution']) != len(subset):
        raise ValueError('Partition count does not match subset')
    if sum(g['n'] for g in cross) != len(subset) or sum(g['restricted']['numerator'] for g in cross) != metrics['ai_any']['numerator']:
        raise ValueError('Conditional-probability table does not reconcile')
    for pair in metrics['pairs']:
        if sum(c['numerator'] for c in pair['cells']) != len(subset):raise ValueError('Pair matrix does not reconcile')
    return metrics


def text(x,y,value,size=14,color='#334155',anchor='start',weight=400):
    return f'<text x="{x}" y="{y}" font-family="system-ui, sans-serif" font-size="{size}" fill="{color}" text-anchor="{anchor}" font-weight="{weight}">{html.escape(str(value))}</text>'

def svg_start(ident,title,subtitle,width,height):
    return [f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {width} {height}" role="img" aria-labelledby="{ident}-title {ident}-desc">',f'<title id="{ident}-title">{html.escape(title)}</title><desc id="{ident}-desc">{html.escape(subtitle)}</desc>','<rect width="100%" height="100%" fill="#fff"/>',text(24,32,title,18,'#123964',weight=700),text(24,58,subtitle,12)]

def bars(ident,title,subtitle,series):
    height=100+len(series)*43
    parts=svg_start(ident,title,subtitle,900,height)
    maximum=max((v for _,v,_,_ in series),default=1) or 1
    for i,(label,value,annotation,color) in enumerate(series):
        y=80+i*43; width=450*value/maximum
        parts += [text(24,y+19,label,13),f'<rect x="255" y="{y}" width="{width:.2f}" height="27" rx="3" fill="{color}"/>',text(255+width+10,y+19,annotation,12)]
    parts.append('</svg>');return '\n'.join(parts)+'\n'

def charts(metrics,locale):
    l=LABELS[locale];n=metrics['sample']['historical_parser_subset'];scope=f"n={n} · / · {VERSION}"
    result={}
    result['rates']=bars(f'{locale}-rates',l['rate'],scope,[(b['name'],float(b['restricted']['percent'] or 0),f"{b['restricted']['percent']}% ({b['restricted']['numerator']}/{n})",GROUP_COLORS[b['group']]) for b in sorted(metrics['bots'],key=lambda b:-b['restricted']['numerator'])])
    result['distribution']=bars(f'{locale}-distribution',l['distribution'],scope,[(d['label'],d['n'],str(d['n']),'#1e40af') for d in metrics['distribution']])
    result['rank']=bars(f'{locale}-rank',l['rank'],scope,[(b['label'],float(b['ai_any']['percent'] or 0),f"{b['ai_any']['percent']}% ({b['ai_any']['numerator']}/{b['n']})",'#087f8c') for b in metrics['rank_buckets']])
    result['llms']=bars(f'{locale}-llms',l['llms'],scope,[(l['detected' if g['detected'] else 'not_detected'],float(g['restricted']['percent'] or 0),f"{g['restricted']['percent']}% ({g['restricted']['numerator']}/{g['n']})",'#087f8c') for g in metrics['llms']['groups']])
    for index,pair in enumerate(metrics['pairs']):
        ident=f'{locale}-pair-{index}';name='openai' if index==0 else 'anthropic';r=pair['first_restricted_second_not']
        subtitle=f"{pair['first_name']} / {pair['second_name']} · {r['numerator']}/{r['denominator']} ({r['percent'] or '—'}%) · {scope}"
        parts=svg_start(ident,l['pair'],subtitle,900,420)
        for col,restricted in enumerate((True,False)):
            parts.append(text(390+col*225,102,l['restricted' if restricted else 'not_restricted'],13,anchor='middle',weight=600))
        for row,restricted in enumerate((True,False)):
            parts.append(text(263,170+row*115,l['restricted' if restricted else 'not_restricted'],13,anchor='end'))
        for c in pair['cells']:
            col=0 if c['second_restricted'] else 1;row=0 if c['first_restricted'] else 1;x=280+col*225;y=120+row*115
            parts += [f'<rect x="{x}" y="{y}" width="220" height="105" rx="6" fill="#e3edf9"/>',text(x+110,y+48,c['numerator'],27,'#123964',anchor='middle',weight=700),text(x+110,y+75,f"{c['percent']}%",13,anchor='middle')]
        parts += [text(24,385,pair['first_name']+' ↓  /  '+pair['second_name']+' →',13),'</svg>']
        result[name]='\n'.join(parts)+'\n'
    return result


def build(input_dir,output_dir):
    inp,out=Path(input_dir).resolve(),Path(output_dir).resolve()
    if inp==out or inp in out.parents:
        raise ValueError('Output must be separate from frozen input')
    snapshot=json.loads((inp/'snapshot.json').read_text())
    if digest(inp/'sites.csv')!=snapshot['csv_sha256']:raise ValueError('Frozen CSV hash mismatch')
    with (inp/'sites.csv').open(encoding='utf-8',newline='') as source:
        rows=list(csv.DictReader(source))
    bots=json.loads((inp/'bots.json').read_text())
    metrics=summarize(rows,bots,snapshot)
    out.mkdir(parents=True,exist_ok=True)
    for file in ('sites.csv','bots.json','snapshot.json','method-en.md','method-zh.md','review-decisions.csv','changelog.md','LICENSE-code.txt'):
        shutil.copyfile(inp/file,out/file)
    script=Path(__file__).resolve();shutil.copyfile(script,out/'build_ai_crawler_benchmark.py')
    (out/'metrics.json').write_text(json.dumps(metrics,ensure_ascii=False,indent=2)+'\n')
    for locale in ('en','zh'):
        target=out/'charts'/locale;target.mkdir(parents=True,exist_ok=True)
        for key,value in charts(metrics,locale).items():(target/f'{key}.svg').write_text(value)
    files={str(p.relative_to(out)):digest(p) for p in sorted(out.rglob('*')) if p.is_file() and p.name not in {'manifest.json','dataset.zip'}}
    manifest={'version':VERSION,'published':snapshot['published'],'modified':snapshot['modified'],'scope':metrics['scope'],'source_csv_sha256':snapshot['csv_sha256'],'files':files}
    (out/'manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n')
    with zipfile.ZipFile(out/'dataset.zip','w',compression=zipfile.ZIP_DEFLATED) as z:
        for p in sorted(out.rglob('*')):
            if p.is_file() and p.name!='dataset.zip':
                info=zipfile.ZipInfo(str(p.relative_to(out)),date_time=(2026,11,9,0,0,0));info.compress_type=zipfile.ZIP_DEFLATED
                z.writestr(info,p.read_bytes())
    return metrics

if __name__=='__main__':
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--input',required=True,type=Path);parser.add_argument('--output',required=True,type=Path)
    args=parser.parse_args();m=build(args.input,args.output)
    print(json.dumps({'version':m['version'],'sample':m['sample'],'ai_any':m['ai_any'],'legacy_all14_any':m['legacy_all14_any']},ensure_ascii=False))