{ "analysis_date": "2026-11-10", "base_version": "v2.0", "configuration": { "case_count": 9, "cases": [ { "body_sha256": "5ed4645951de851a93c0d99a9ce9bf3c45059cbdc3efbc7299a978159e1eb045", "category": "root_exception", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 234, "path": "/auth/logout" }, "source": "wildcard" }, "path": "/auth/logout" }, "domain": "chatgpt.com", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 19, "path": "/" }, "source": "named" }, "claude-web": { "policy": "disallow", "rule": { "kind": "disallow", "line": 22, "path": "/" }, "source": "named" } }, "modern": { "claude-searchbot": { "policy": "allow", "rule": { "kind": "allow", "line": 52, "path": "/$" }, "source": "wildcard" }, "claude-user": { "policy": "allow", "rule": { "kind": "allow", "line": 52, "path": "/$" }, "source": "wildcard" }, "claudebot": { "policy": "allow", "rule": { "kind": "allow", "line": 52, "path": "/$" }, "source": "wildcard" } }, "offline_probe": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 231, "path": "/" }, "source": "wildcard" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "69e57dadcb6f535e960fcc201f1f8fcd998bc4423f0261186408274b1d8b86a6", "category": "wildcard_root_non_disallow", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 77, "path": "/ad/" }, "source": "wildcard" }, "path": "/ad/" }, "domain": "espn.com", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 19, "path": "/" }, "source": "named" }, "claude-web": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "modern": { "claude-searchbot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claude-user": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claudebot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "offline_probe": { "outcome": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "96ad98c97cd960d51288d2b2e3fa0bb8d5cceaebf58267a863164c000d8ed9b6", "category": "named_path_specific", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 830, "path": "/error" }, "source": "named" }, "path": "/error" }, "domain": "trustpilot.com", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 1028, "path": "/" }, "source": "named" }, "claude-web": { "policy": "disallow", "rule": { "kind": "disallow", "line": 1061, "path": "/" }, "source": "wildcard" } }, "modern": { "claude-searchbot": { "policy": "no_applicable_rule", "rule": null, "source": "named" }, "claude-user": { "policy": "no_applicable_rule", "rule": null, "source": "named" }, "claudebot": { "policy": "no_applicable_rule", "rule": null, "source": "named" } }, "offline_probe": { "outcome": { "policy": "no_applicable_rule", "rule": null, "source": "named" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "2c064ff50d1ae310d617dbc6022174c29092012c3fa3f15d1f5f951c50189408", "category": "wildcard_root_non_disallow", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 3, "path": "/ultimora/24ore/nazionale/news-dettaglio/3688417" }, "source": "wildcard" }, "path": "/ultimora/24ore/nazionale/news-dettaglio/3688417" }, "domain": "repubblica.it", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 80, "path": "/" }, "source": "named" }, "claude-web": { "policy": "allow", "rule": { "kind": "allow", "line": 2, "path": "/" }, "source": "wildcard" } }, "modern": { "claude-searchbot": { "policy": "allow", "rule": { "kind": "allow", "line": 2, "path": "/" }, "source": "wildcard" }, "claude-user": { "policy": "allow", "rule": { "kind": "allow", "line": 2, "path": "/" }, "source": "wildcard" }, "claudebot": { "policy": "allow", "rule": { "kind": "allow", "line": 2, "path": "/" }, "source": "wildcard" } }, "offline_probe": { "outcome": { "policy": "allow", "rule": { "kind": "allow", "line": 2, "path": "/" }, "source": "wildcard" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "79f37255ec52be82b752c45d26d4f5867a4a91777da13f49876f73b7cd5401bd", "category": "root_exception", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 270, "path": "/sign_in" }, "source": "named" }, "path": "/sign_in" }, "domain": "skroutz.gr", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 87, "path": "/" }, "source": "named" }, "claude-web": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "modern": { "claude-searchbot": { "policy": "allow", "rule": { "kind": "allow", "line": 274, "path": "/$" }, "source": "named" }, "claude-user": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claudebot": { "policy": "allow", "rule": { "kind": "allow", "line": 246, "path": "/$" }, "source": "named" } }, "offline_probe": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 245, "path": "/" }, "source": "named" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "0bff244799fb03d76c3cf02df056ca641d8e1134ca4fdfd3467266b29fd328a2", "category": "wildcard_root_non_disallow", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 47, "path": "/admin" }, "source": "wildcard" }, "path": "/admin" }, "domain": "e621.net", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 21, "path": "/" }, "source": "named" }, "claude-web": { "policy": "disallow", "rule": { "kind": "disallow", "line": 21, "path": "/" }, "source": "named" } }, "modern": { "claude-searchbot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claude-user": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claudebot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "offline_probe": { "outcome": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "74e6553e883d4e98a2303fe96f025c7d0c32f2e0c3225f6a8b8dcea9de986f58", "category": "wildcard_root_non_disallow", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 2, "path": "/ajaxlib/" }, "source": "wildcard" }, "path": "/ajaxlib/" }, "domain": "nikkansports.com", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 23, "path": "/" }, "source": "named" }, "claude-web": { "policy": "disallow", "rule": { "kind": "disallow", "line": 26, "path": "/" }, "source": "named" } }, "modern": { "claude-searchbot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claude-user": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claudebot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "offline_probe": { "outcome": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "2aa07b752b656c7172b286a134affbec581c33f63f283c15449e5f97bac64386", "category": "wildcard_root_non_disallow", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 3, "path": "/cgi-bin/" }, "source": "wildcard" }, "path": "/cgi-bin/" }, "domain": "ovh.com", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 224, "path": "/" }, "source": "named" }, "claude-web": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "modern": { "claude-searchbot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claude-user": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claudebot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "offline_probe": { "outcome": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "path": "/__benchmark_scope_probe__" } }, { "body_sha256": "ab375d40240abb6fa79532e2e29128fb812e5e0df5ebe2eb6a4caf3c60434920", "category": "wildcard_root_non_disallow", "declared_path_example": { "outcome": { "policy": "disallow", "rule": { "kind": "disallow", "line": 35, "path": "/format/" }, "source": "wildcard" }, "path": "/format/" }, "domain": "espn.com.ve", "interpretation": "Root rule difference; owner intent, actual requests and configuration error not established.", "legacy": { "anthropic-ai": { "policy": "disallow", "rule": { "kind": "disallow", "line": 19, "path": "/" }, "source": "named" }, "claude-web": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "modern": { "claude-searchbot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claude-user": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "claudebot": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" } }, "offline_probe": { "outcome": { "policy": "no_applicable_rule", "rule": null, "source": "wildcard" }, "path": "/__benchmark_scope_probe__" } } ], "categories": { "named_path_specific": 1, "root_exception": 2, "wildcard_root_non_disallow": 6 }, "legacy_named_domains": 161, "legacy_named_root_disallow": 122, "legacy_only_named_domains": [ "chatgpt.com", "espn.com", "repubblica.it", "amap.com", "e621.net", "nikkansports.com", "ovh.com", "espn.com.ve" ], "owner_intent_and_actual_access": "not_tested", "unique_case_bodies": 9 }, "inputs": { "analysis_version": "v2.1", "base_archive_sha256": "a51e435d138ebc8a104669ec013df8209961932697ab850c1d8b9c4c12878c2e", "base_metrics_sha256": "edbdec1742dc52d3e305b9e6fdb11c56163976561255cab6c9563a6ea147f1cc", "base_sites_sha256": "da47b5ae060c1a67f5131b9f40e413416f9d7a3f010456c5809316efca36dcd9", "base_version": "v2.0", "checked_at": "2026-11-10", "markdown_features_sha256": "7a4f0abea5a106a46cd69b737cfe3cd6a8f86d64649670606561606dd7fa7744", "policy_sha256": "1fe6a08d973c446df1853a8671dfc2dd1e24494f1e11975d03704e50a9a01f62", "reviews_sha256": "befa802618e6ad42d591c96273096220e7983cd286d60bd8b8ff166d6d185c54" }, "limits": [ "post_hoc_descriptive_analysis", "root_policies_not_whole_site_access", "metadata_patterns_not_owner_intent", "llms_structure_not_conformance_or_citation_effect", "linked_resources_not_fetched", "no_new_cohort_requests_or_paid_AI_calls" ], "llms_quality": { "body_duplicate_groups": [ [ "wordpress.com", "wp.com" ], [ "dailymail.com", "dailymail.co.uk" ], [ "intercom.io", "intercom.com" ], [ "lg.com", "lge.com" ], [ "one.one", "one.com" ] ], "candidate_domains": 156, "embedded_scheme_case_domains": [ "cursor.sh" ], "final_url_duplicate_groups": [ [ "wordpress.com", "wp.com" ], [ "dailymail.com", "dailymail.co.uk" ], [ "intercom.io", "intercom.com" ], [ "one.one", "one.com" ] ], "first_h1_separator_only": 1, "has_h2": { "denominator": 156, "numerator": 151, "percent": "96.8" }, "has_markdown_links": { "denominator": 156, "numerator": 144, "percent": "92.3" }, "has_md_suffix_links": { "denominator": 156, "numerator": 27, "percent": "17.3" }, "median_bytes": 9030.5, "multiple_h1": 13, "over_100_kib": 10, "p90_bytes_nearest_rank": 48384, "shapes": { "h2_and_web_links": 143, "literal_urls_without_markdown_links": 12, "web_links_without_h2": 1 }, "unique_bodies": 151, "unique_final_urls": 152 }, "patterns": { "ai_restricted": 249, "ai_vs_baselines": [ { "ai_restricted": true, "baseline_restricted": true, "count": 16 }, { "ai_restricted": true, "baseline_restricted": false, "count": 233 }, { "ai_restricted": false, "baseline_restricted": true, "count": 2 }, { "ai_restricted": false, "baseline_restricted": false, "count": 603 } ], "ai_with_neither_baseline_restricted": { "denominator": 249, "numerator": 233, "percent": "93.6" }, "all_ai_neither_baseline": 30, "all_studied_ai": 45, "analyzable": 854, "combinations": [ { "count": 45, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 31, "tokens": [ "gptbot", "google_extended", "claudebot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 18, "tokens": [ "bytespider" ] }, { "count": 13, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "bytespider", "ccbot" ] }, { "count": 7, "tokens": [ "ccbot" ] }, { "count": 6, "tokens": [ "bytespider", "ccbot" ] }, { "count": 6, "tokens": [ "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 6, "tokens": [ "gptbot" ] }, { "count": 6, "tokens": [ "gptbot", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 5, "tokens": [ "gptbot", "ccbot" ] }, { "count": 5, "tokens": [ "gptbot", "claudebot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 5, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 4, "tokens": [ "gptbot", "google_extended" ] }, { "count": 4, "tokens": [ "gptbot", "google_extended", "claudebot", "bytespider", "ccbot" ] }, { "count": 3, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 3, "tokens": [ "claudebot", "bytespider", "ccbot" ] }, { "count": 3, "tokens": [ "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "applebot_extended" ] }, { "count": 2, "tokens": [ "chatgpt_user", "claude_user", "perplexitybot", "perplexity_user", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "claude_user", "claudebot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "claudebot", "perplexitybot", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "google_extended", "claude_searchbot", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "gptbot", "claudebot", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "gptbot", "google_extended", "ccbot" ] }, { "count": 2, "tokens": [ "gptbot", "google_extended", "claudebot", "applebot_extended", "ccbot" ] }, { "count": 2, "tokens": [ "gptbot", "google_extended", "claudebot", "ccbot" ] }, { "count": 2, "tokens": [ "gptbot", "google_extended", "claudebot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "ccbot" ] }, { "count": 2, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claude_searchbot", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 2, "tokens": [ "perplexitybot" ] }, { "count": 1, "tokens": [ "chatgpt_user", "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "chatgpt_user", "gptbot", "ccbot" ] }, { "count": 1, "tokens": [ "chatgpt_user", "gptbot", "claudebot", "perplexitybot", "bytespider" ] }, { "count": 1, "tokens": [ "chatgpt_user", "gptbot", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "ccbot" ] }, { "count": 1, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "claude_searchbot", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "chatgpt_user", "gptbot", "google_extended", "claudebot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "claude_searchbot", "claude_user", "claudebot", "perplexity_user", "applebot_extended" ] }, { "count": 1, "tokens": [ "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "claude_searchbot", "claudebot", "bytespider" ] }, { "count": 1, "tokens": [ "claudebot" ] }, { "count": 1, "tokens": [ "claudebot", "bytespider" ] }, { "count": 1, "tokens": [ "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "google_extended" ] }, { "count": 1, "tokens": [ "google_extended", "applebot_extended" ] }, { "count": 1, "tokens": [ "google_extended", "claude_searchbot", "claude_user", "claudebot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "google_extended", "claudebot", "applebot_extended", "ccbot" ] }, { "count": 1, "tokens": [ "google_extended", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "google_extended", "perplexitybot", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "applebot_extended", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "claudebot", "applebot_extended" ] }, { "count": 1, "tokens": [ "gptbot", "claudebot", "applebot_extended", "bytespider" ] }, { "count": 1, "tokens": [ "gptbot", "claudebot", "bytespider" ] }, { "count": 1, "tokens": [ "gptbot", "claudebot", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "google_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "google_extended", "claudebot", "perplexitybot", "applebot_extended" ] }, { "count": 1, "tokens": [ "gptbot", "google_extended", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "google_extended", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "google_extended", "perplexitybot", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "gptbot", "perplexitybot", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "claude_searchbot", "claude_user", "claudebot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "applebot_extended", "bytespider" ] }, { "count": 1, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended" ] }, { "count": 1, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "ccbot" ] }, { "count": 1, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "oai_searchbot", "chatgpt_user", "gptbot", "google_extended", "claudebot", "perplexitybot", "perplexity_user", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "oai_searchbot", "gptbot", "google_extended", "claude_searchbot", "claude_user", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "oai_searchbot", "gptbot", "google_extended", "claude_searchbot", "claudebot", "perplexitybot", "applebot_extended", "bytespider", "ccbot" ] }, { "count": 1, "tokens": [ "perplexity_user" ] }, { "count": 1, "tokens": [ "perplexity_user", "applebot_extended", "bytespider" ] } ], "named_restrictions": 223, "profiles": { "all_studied_ai": 45, "no_ai_disallow": 605, "other_ai_only": 39, "search_without_training_pair": 8, "training_pair_and_search": 73, "training_pair_without_search": 84 }, "search_trio": [ "oai_searchbot", "claude_searchbot", "perplexitybot" ], "training_pair": [ "gptbot", "claudebot" ], "wildcard_only_restrictions": 26 }, "review": { "llms_cases": [ { "body_sha256": "3bd58e896fb53c2bd44ac796304791032c640687f165822fcc89a37c8caaf5ca", "category": "large_index", "domain": "unity3d.com", "note_en": "A 950,573-byte archived index with 2,478 parsed links. It illustrates body-size variation; token cost, usefulness and model handling were not measured.", "note_zh": "归档索引为 950,573 字节，解析出 2,478 个链接；用于展示文件体积差异，未测量 token 成本、实际效用或模型处理效果。" }, { "body_sha256": "236ce4c7a060cbbe51478eb011882a8fcb358a35e05d2ab24c461857017c4059", "category": "api_discovery", "domain": "name.com", "note_en": "Introduces API-discovery, documentation and agent-skill resources using literal URLs; no CommonMark hyperlinks were parsed. This is substantive discovery information, not an empty file.", "note_zh": "以裸 URL 提供 API 发现、文档和 agent skill 资源；未解析出 CommonMark 超链接，但并非空文件。" }, { "body_sha256": "252ecdbe75b42fcbd43219e933db4ce05bfe4fa62f3f82585cbab8b0a40cf81b", "category": "site_background", "domain": "dailymotion.com", "note_en": "Contains platform background, topical sections and contact/resource URLs in plain text. Statements about the business were not independently verified.", "note_zh": "包含平台背景、主题栏目及纯文本联系方式或资源 URL；未独立验证文件中的业务声明。" }, { "body_sha256": "ae4f97e050d97c2a8a3fe6703d9c785bd650ea6cb81dc949a72490f1dd3f92fd", "category": "bare_url_index", "domain": "cursor.sh", "note_en": "A documentation index expressed as bare URLs. One archived target concatenates two schemes (cursor.comhttps://cursor.com/changelog.md); this is a static repair lead, not a live broken-link test.", "note_zh": "以裸 URL 编排文档索引；归档中有一处 cursor.comhttps://cursor.com/changelog.md 拼接线索，仅作静态修复候选，未测试链接是否可达。" }, { "body_sha256": "5854f5f04eeec24cc0cd0396af36b1f8374013002ff6b2cbdd1a146d240c6424", "category": "api_reference", "domain": "porkbun.com", "note_en": "Contains a substantial API/agent reference with endpoint examples, code blocks and literal URLs. Zero parsed hyperlinks does not imply absent documentation.", "note_zh": "包含较完整的 API/agent 说明、端点示例、代码块和裸 URL；没有可解析超链接不等于缺少文档。" }, { "body_sha256": "9c7e33369f2c472196917ba94b727d65d71b1080d8935cbec3fab5bf9e41d53b", "category": "bare_url_resources", "domain": "flightradar24.com", "note_en": "Provides background and primary-resource URLs as plain text. Link destinations and factual claims were not fetched or verified.", "note_zh": "以纯文本提供背景及主要资源 URL；未访问或验证链接目标和事实声明。" }, { "body_sha256": "13fb36e78982a3944472873b1b2aafdbd00b22a127aa78911cfe0ce34fd990a9", "category": "localized_resource_map", "domain": "kwai.com", "note_en": "Lists localized paths, help-center patterns and resource URLs in plain text. Placeholder patterns are not confirmed page URLs.", "note_zh": "以纯文本列出多语言路径、帮助中心模式和资源 URL；带占位符的模式不视为已确认页面地址。" }, { "body_sha256": "1aaec91ffe2cc5f0e2cbb21e36c892bd21ecd34837ef3eefe697cf862cf63f3a", "category": "entity_url_patterns", "domain": "xing.com", "note_en": "Uses entity definitions, URL patterns and search parameters rather than a CommonMark link list. Pattern placeholders are not verified destinations.", "note_zh": "主要提供实体定义、URL 模式和搜索参数，而非 CommonMark 链接列表；占位符不是已验证的目标页面。" }, { "body_sha256": "d83aa406c02a3760a720a7ebba67ed5cd2608979de812ce34c768ae06e1dfe34", "category": "faq_and_resources", "domain": "moe.video", "note_en": "Provides company-description/FAQ prose and literal resource URLs. Publication is not evidence that a provider follows these instructions.", "note_zh": "包含公司介绍、问答及裸资源 URL；文件发布不代表供应商会遵循其中指引。" }, { "body_sha256": "42b55d0fe2d557f21dad31cfb1481427878c1f77771b90be91a82be1eca0fc5b", "category": "multilingual_guidance", "domain": "marktplaats.nl", "note_en": "Contains multilingual platform guidance, category resources and plain-text URLs; multiple H1 headings are a structural observation, not an automatic failure.", "note_zh": "包含多语言平台指引、分类资源与纯文本 URL；多个 H1 是结构观察，不自动判定失败。" }, { "body_sha256": "dd507481b4aa64bbdda354f3047e0b78432436875fac1cb9b8571ed39ef0dd59", "category": "content_usage_guidance", "domain": "jusbrasil.com.br", "note_en": "Uses YAML-like content/attribution guidance and URL patterns under an H1. This review does not certify legal effect or provider compliance.", "note_zh": "在 H1 下以类 YAML 结构描述内容使用、来源归属和 URL 模式；不认证其法律效力或供应商遵守情况。" }, { "body_sha256": "fc1d121bf65dd5cc7928cb0504e5687bf884cd090a3e6f5dbeefc0c49642235d", "category": "policy_comment_style", "domain": "eltiempo.es", "note_en": "Most lines start with # and express usage guidance; CommonMark interprets many as headings. This is policy-oriented text rather than a conventional navigation index; enforcement was not tested.", "note_zh": "大量行以 # 开头并表达使用指引，CommonMark 会将多行解析为标题；内容偏政策说明，不是常规导航索引，未测试执行效果。" }, { "body_sha256": "9f5a35d2bd6d864951e8e5a484d040f2cc1a1df0e768484952b759d702abf374", "category": "identity_guidance", "domain": "nustargame.com", "note_en": "Contains self-described identity and anti-impersonation guidance with literal URLs. Ownership, licensing and security assertions were not verified.", "note_zh": "包含自述身份及反仿冒指引和裸 URL；未验证所有权、许可或安全声明。" } ], "reviewed_at": "2026-11-10", "scope": "All 156 candidates inventoried automatically; targeted review of all 12 with zero CommonMark hyperlinks plus the largest-body example. Intro/headings/URL forms and relevant archived excerpts reviewed; not full conformance certification or independent validation of publisher claims." }, "version": "v2.1", "window": { "ended_at": "2026-11-10T01:18:54.938Z", "started_at": "2026-11-10T01:04:31.278Z" } }