#!/usr/bin/env python3 """v2.1 supplemental analysis of the frozen v2.0 ZIP. No network or paid APIs. python3 deepen.py --base-archive ../v2.0/dataset.zip --features markdown-features.json --reviews review.json --output ../rebuilt-v2.1 The provided CommonMark AST inventory can be independently re-extracted using extract_markdown.php and league/commonmark 2.8.2 (see method.md). """ import argparse from collections import Counter, defaultdict import csv from decimal import Decimal, ROUND_HALF_UP import hashlib import io import json from pathlib import Path import re import shutil from urllib.parse import urljoin, urlsplit import zipfile from policy import parse, verdict TRAINING_PAIR = ('gptbot', 'claudebot') SEARCH_TRIO = ('oai_searchbot', 'claude_searchbot', 'perplexitybot') BASELINES = ('googlebot', 'bingbot') LEGACY = ('claude-web', 'anthropic-ai') MODERN_CLAUDE = ('claudebot', 'claude-searchbot', 'claude-user') PROBE = '/__benchmark_scope_probe__' def sha(body): return hashlib.sha256(body).hexdigest() def dump(path, value): Path(path).write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2)+'\n', encoding='utf-8') def metric(n, d): return {'numerator': n, 'denominator': d, 'percent': str( (Decimal(n)*100/Decimal(d)).quantize(Decimal('.1'), rounding=ROUND_HALF_UP)) if d else None} def blocked(row, keys): return any(row[k+'_policy']=='disallow' for k in keys) def profile(row, ai_keys): if row['robots_analyzable']!='1': return 'outside_rule_denominator' if not blocked(row, ai_keys): return 'no_ai_disallow' if all(row[k+'_policy']=='disallow' for k in ai_keys): return 'all_studied_ai' training, search = blocked(row, TRAINING_PAIR), blocked(row, SEARCH_TRIO) if training and search: return 'training_pair_and_search' if training: return 'training_pair_without_search' if search: return 'search_without_training_pair' return 'other_ai_only' def policy_patterns(rows, bots): keys=[b['key'] for b in bots if b['group']!='baseline'] subset=[r for r in rows if r['robots_analyzable']=='1'] restricted=[r for r in subset if blocked(r,keys)] profiles=Counter(profile(r,keys) for r in subset) combinations=Counter(tuple(k for k in keys if r[k+'_policy']=='disallow') for r in restricted) matrix=[{'ai_restricted':a,'baseline_restricted':b,'count':sum( blocked(r,keys)==a and blocked(r,BASELINES)==b for r in subset)} for a,b in [(True,True),(True,False),(False,True),(False,False)]] named=sum(any(r[k+'_policy']=='disallow' and r[k+'_source']=='named' for k in keys) for r in restricted) all_ai=[r for r in subset if all(r[k+'_policy']=='disallow' for k in keys)] summary={'analyzable':len(subset), 'ai_restricted':len(restricted), 'profiles':dict(sorted(profiles.items())), 'ai_vs_baselines':matrix, 'ai_with_neither_baseline_restricted':metric(matrix[1]['count'],len(restricted)), 'named_restrictions':named,'wildcard_only_restrictions':len(restricted)-named, 'all_studied_ai':len(all_ai),'all_ai_neither_baseline':sum(not blocked(r,BASELINES) for r in all_ai), 'combinations':[{'tokens':list(tokens),'count':count} for tokens,count in sorted(combinations.items(),key=lambda item:(-item[1],item[0]))], 'training_pair':list(TRAINING_PAIR),'search_trio':list(SEARCH_TRIO)} domain_rows=[{'domain':r['domain'],'rank':r['rank'],'profile':profile(r,keys), 'baseline_disallow':int(blocked(r,BASELINES)) if r['robots_analyzable']=='1' else '', 'restricted_ai_tokens':';'.join(k for k in keys if r[k+'_policy']=='disallow'), 'robots_body_sha256':r['robots_body_sha256']} for r in rows] return summary,domain_rows def configuration_case(domain, groups, body_sha): old={token:verdict(groups,token) for token in LEGACY} modern={token:verdict(groups,token) for token in MODERN_CLAUDE} if not any(v['source']=='named' and v['policy']=='disallow' for v in old.values()) or modern['claudebot']['policy']=='disallow': return None root=modern['claudebot'];probe=verdict(groups,'claudebot',PROBE) selected=[g for g in groups if 'claudebot' in g['agents']] selected=selected or [g for g in groups if '*' in g['agents']] path_example=None for group in selected: for rule in group['rules']: path=rule['path'] if rule['kind']=='disallow' and path.startswith('/') and path!='/' and not any(c in path for c in '*$'): outcome=verdict(groups,'claudebot',path) if outcome['policy']=='disallow': path_example={'path':path,'outcome':outcome};break if path_example: break if root['policy']=='allow' and root['rule']['path']=='/$' and probe['policy']=='disallow': category='root_exception' elif root['source']=='named' and path_example: category='named_path_specific' else: category='wildcard_root_non_disallow' return {'domain':domain,'category':category,'body_sha256':body_sha,'legacy':old,'modern':modern, 'offline_probe':{'path':PROBE,'outcome':probe},'declared_path_example':path_example, 'interpretation':'Root rule difference; owner intent, actual requests and configuration error not established.'} def link_inventory(document, final_url, body): counts=Counter();targets=[] final_host=(urlsplit(final_url).hostname or '').lower() for link in document['links']: raw=link['url'].strip() if not raw: counts['empty_target']+=1;continue if raw.startswith('#'): counts['fragment_only']+=1;continue try: resolved=urljoin(final_url,raw);parts=urlsplit(resolved) if parts.scheme not in ('http','https') or not parts.hostname: counts['nonweb_target']+=1;continue targets.append(resolved);counts['web_links']+=1 host=parts.hostname.lower() # Host relationship only, not inferred registrable domain/ownership. same=host==final_host or host.endswith('.'+final_host) counts['same_host_or_subdomain' if same else 'other_host']+=1 if not urlsplit(raw).scheme and not raw.startswith('//'): counts['relative_web_links']+=1 if parts.path.lower().endswith('.md'): counts['md_suffix_links']+=1 except ValueError: counts['invalid_target']+=1 literal_urls=set(re.findall(r'https?://[^\s<>"\]]+',body.decode('utf-8-sig'))) embedded_schemes=sorted({url for url in literal_urls | {link['url'] for link in document['links']} if re.match(r'^https?://[^/?#]*https?://',url,re.I)}) h1=[h['text'] for h in document['headings'] if h['level']==1] h2=sum(h['level']==2 for h in document['headings']) if not document['links']: shape='literal_urls_without_markdown_links' if literal_urls else 'no_markdown_links_or_literal_http_urls' elif not counts['web_links']: shape='only_fragments_or_nonweb_links' elif not h2: shape='web_links_without_h2' else: shape='h2_and_web_links' return {'body_bytes':len(body),'h1_count':len(h1),'h1_title':h1[0] if h1 else '', 'first_h1_separator_only':int(bool(h1) and not any(c.isalnum() for c in h1[0])), 'h2_count':h2,'blockquote_present':int(document['has_blockquote']), 'markdown_links':len(document['links']),'web_links':counts['web_links'], 'unique_resolved_web_targets':len(set(targets)), 'same_host_or_subdomain_links':counts['same_host_or_subdomain'],'other_host_links':counts['other_host'], 'relative_web_links':counts['relative_web_links'],'md_suffix_links':counts['md_suffix_links'], 'fragment_only_links':counts['fragment_only'],'nonweb_links':counts['nonweb_target'], 'invalid_or_empty_targets':counts['invalid_target']+counts['empty_target'], 'literal_http_url_strings':len(literal_urls),'shape':shape, 'embedded_scheme_candidates':';'.join(embedded_schemes)} def write_csv(path, rows): if not rows: return with Path(path).open('w',encoding='utf-8',newline='') as file: writer=csv.DictWriter(file,fieldnames=list(rows[0]),lineterminator='\n') writer.writeheader();writer.writerows(rows) def analyze(base_archive, features_file, reviews_file, inputs_file): inputs=json.loads(inputs_file.read_text()) for name,path in [('base_archive_sha256',base_archive),('markdown_features_sha256',features_file), ('policy_sha256',Path(__file__).with_name('policy.py')),('reviews_sha256',reviews_file)]: if sha(path.read_bytes())!=inputs[name]: raise ValueError('Frozen input hash mismatch: '+name) features=json.loads(features_file.read_text()) if features['parser']!='league/commonmark' or features['version']!='2.8.2': raise ValueError('Unexpected Markdown parser version') with zipfile.ZipFile(base_archive) as archive: for name,key in [('sites.csv','base_sites_sha256'),('metrics.json','base_metrics_sha256'),('policy.py','policy_sha256')]: if sha(archive.read(name))!=inputs[key]: raise ValueError('Frozen archive member mismatch: '+name) rows=list(csv.DictReader(io.StringIO(archive.read('sites.csv').decode()))) metrics=json.loads(archive.read('metrics.json'));bots=json.loads(archive.read('bots.json')) records={r['domain']:r for r in json.loads(archive.read('records.json'))} if len({r['domain'] for r in rows})!=len(rows) or set(records)!={r['domain'] for r in rows}: raise ValueError('Duplicate/mismatched domains') patterns,domain_rows=policy_patterns(rows,bots) if patterns['analyzable']!=metrics['sample']['analyzable'] or patterns['ai_restricted']!=metrics['ai_any']['numerator']: raise ValueError('Base primary metric changed') feature_map={d['domain']:d for d in features['documents']} candidates={r['domain'] for r in rows if r['llms_observation']=='minimum_h1_candidate'} if set(feature_map)!=candidates or len(feature_map)!=len(features['documents']): raise ValueError('Incomplete/duplicate candidate AST inventory') cases=[];legacy_named=0;legacy_only=[];old_block=0;qualities=[] for row in rows: r=records[row['domain']] if row['robots_analyzable']=='1': body=archive.read('evidence/'+r['robots']['body_path']) if sha(body)!=row['robots_body_sha256']: raise ValueError('Robots body mismatch') groups,_=parse(body.decode('utf-8-sig')) agents={a for group in groups for a in group['agents']} if agents & set(LEGACY): legacy_named+=1 if not agents & set(MODERN_CLAUDE):legacy_only.append(row['domain']) if any(verdict(groups,t)['policy']=='disallow' and verdict(groups,t)['source']=='named' for t in LEGACY):old_block+=1 case=configuration_case(row['domain'],groups,row['robots_body_sha256']) if case:cases.append(case) if row['domain'] in candidates: body=archive.read('evidence/'+r['llms']['body_path']);d=feature_map[row['domain']] if sha(body)!=row['llms_body_sha256'] or d['body_sha256']!=row['llms_body_sha256']: raise ValueError('llms body/AST provenance mismatch') qualities.append({'domain':row['domain'],'final_url':row['llms_final_url'], 'body_sha256':row['llms_body_sha256']} | link_inventory(d,row['llms_final_url'],body)) body_groups=defaultdict(list);url_groups=defaultdict(list) for r in qualities: body_groups[r['body_sha256']].append(r['domain']);url_groups[r['final_url']].append(r['domain']) sizes=sorted(r['body_bytes'] for r in qualities);count=len(qualities) quality={'candidate_domains':count,'unique_bodies':len(body_groups),'unique_final_urls':len(url_groups), 'has_markdown_links':metric(sum(r['markdown_links']>0 for r in qualities),count), 'has_h2':metric(sum(r['h2_count']>0 for r in qualities),count), 'has_md_suffix_links':metric(sum(r['md_suffix_links']>0 for r in qualities),count), 'multiple_h1':sum(r['h1_count']>1 for r in qualities), 'first_h1_separator_only':sum(r['first_h1_separator_only'] for r in qualities), 'shapes':dict(sorted(Counter(r['shape'] for r in qualities).items())), 'median_bytes':(sizes[(count-1)//2]+sizes[count//2])/2 if count else None, 'p90_bytes_nearest_rank':sizes[(9*count+9)//10-1] if count else None, 'over_100_kib':sum(n>102400 for n in sizes), 'embedded_scheme_case_domains':[r['domain'] for r in qualities if r['embedded_scheme_candidates']], 'body_duplicate_groups':[v for v in body_groups.values() if len(v)>1], 'final_url_duplicate_groups':[v for v in url_groups.values() if len(v)>1]} reviews=json.loads(reviews_file.read_text()) quality_map={r['domain']:r for r in qualities} for review in reviews['llms_cases']: if review['domain'] not in quality_map or quality_map[review['domain']]['body_sha256']!=review['body_sha256']: raise ValueError('Review evidence mismatch') no_links={r['domain'] for r in qualities if r['markdown_links']==0} if not no_links.issubset({r['domain'] for r in reviews['llms_cases']}): raise ValueError('Missing no-link case review') insights={'version':'v2.1','analysis_date':'2026-11-10','base_version':'v2.0','window':metrics['window'], 'inputs':inputs,'patterns':patterns, 'configuration':{'legacy_named_domains':legacy_named,'legacy_only_named_domains':legacy_only, 'legacy_named_root_disallow':old_block,'case_count':len(cases), 'unique_case_bodies':len({c['body_sha256'] for c in cases}), 'categories':dict(sorted(Counter(c['category'] for c in cases).items())), 'cases':cases,'owner_intent_and_actual_access':'not_tested'}, 'llms_quality':quality,'review':reviews, 'limits':['post_hoc_descriptive_analysis','root_policies_not_whole_site_access', 'metadata_patterns_not_owner_intent','llms_structure_not_conformance_or_citation_effect', 'linked_resources_not_fetched','no_new_cohort_requests_or_paid_AI_calls']} return insights,domain_rows,qualities,cases def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-archive',required=True,type=Path) parser.add_argument('--features',required=True,type=Path) parser.add_argument('--reviews',required=True,type=Path) parser.add_argument('--inputs',type=Path,default=Path(__file__).with_name('inputs.json')) parser.add_argument('--output',required=True,type=Path) args=parser.parse_args() if args.output.exists():parser.error('output already exists; frozen analyses cannot be overwritten') insights,domains,qualities,cases=analyze(args.base_archive,args.features,args.reviews,args.inputs) args.output.mkdir(parents=True) dump(args.output/'insights.json',insights) write_csv(args.output/'policy-profiles.csv',domains) write_csv(args.output/'llms-quality.csv',qualities) write_csv(args.output/'configuration-cases.csv',[{ 'domain':c['domain'],'category':c['category'],'body_sha256':c['body_sha256'], 'legacy_named_root_disallow':';'.join(k for k,v in c['legacy'].items() if v['policy']=='disallow' and v['source']=='named'), 'claudebot_root_policy':c['modern']['claudebot']['policy'], 'claudebot_root_source':c['modern']['claudebot']['source'], 'offline_probe_path':c['offline_probe']['path'], 'offline_probe_policy':c['offline_probe']['outcome']['policy'], 'declared_path_example':(c['declared_path_example'] or {}).get('path','')} for c in cases]) for src,name in [(args.inputs,'inputs.json'),(args.features,'markdown-features.json'),(args.reviews,'review.json')]: shutil.copyfile(src,args.output/name) for name in ('deepen.py','policy.py'): shutil.copyfile(Path(__file__).with_name(name),args.output/name) helper=Path(__file__).with_name('extract_markdown.php.txt') if not helper.exists():helper=Path(__file__).with_name('extract_markdown.php') shutil.copyfile(helper,args.output/'extract_markdown.php.txt') dump(args.output/'manifest.json',{'version':'v2.1','files':{f.name:sha(f.read_bytes()) for f in sorted(args.output.iterdir()) if f.is_file()}}) print(json.dumps({'version':'v2.1','patterns':insights['patterns']['profiles'], 'config_cases':len(cases),'llms_shapes':insights['llms_quality']['shapes']})) if __name__=='__main__': main()