{
  "ai_any": {
    "denominator": 854,
    "numerator": 249,
    "percent": "29.2"
  },
  "ai_named": 223,
  "ai_token_count": 12,
  "ai_wildcard_only": 26,
  "alias_sensitivity": {
    "conflicting_domains": [],
    "conflicting_final_urls": 0,
    "consistent_url_rate": {
      "denominator": 837,
      "numerator": 244,
      "percent": "29.2"
    },
    "domain_rate": {
      "denominator": 854,
      "numerator": 249,
      "percent": "29.2"
    },
    "grouping": "exact final robots URL; not companies",
    "groups": [
      {
        "domains": [
          "netflix.com",
          "netflix.net"
        ],
        "url": "https://www.netflix.com/robots.txt"
      },
      {
        "domains": [
          "pinterest.com",
          "pinimg.com"
        ],
        "url": "https://www.pinterest.com/robots.txt"
      },
      {
        "domains": [
          "zoom.us",
          "zoom.com"
        ],
        "url": "https://zoom.us/robots.txt"
      },
      {
        "domains": [
          "amazonvideo.com",
          "primevideo.com"
        ],
        "url": "https://www.primevideo.com/robots.txt"
      },
      {
        "domains": [
          "blogspot.com",
          "blogger.com"
        ],
        "url": "https://www.blogger.com/robots.txt"
      },
      {
        "domains": [
          "mozilla.org",
          "mozilla.com"
        ],
        "url": "https://www.mozilla.org/robots.txt"
      },
      {
        "domains": [
          "wordpress.com",
          "wp.com"
        ],
        "url": "https://wordpress.com/robots.txt"
      },
      {
        "domains": [
          "vk.com",
          "userapi.com"
        ],
        "url": "https://vk.com/robots.txt"
      },
      {
        "domains": [
          "discord.com",
          "discordapp.com"
        ],
        "url": "https://discord.com/robots.txt"
      },
      {
        "domains": [
          "xiaomi.com",
          "mi.com"
        ],
        "url": "https://www.mi.com/robots.txt"
      },
      {
        "domains": [
          "dailymail.com",
          "dailymail.co.uk"
        ],
        "url": "https://www.dailymail.com/robots.txt"
      },
      {
        "domains": [
          "intercom.io",
          "intercom.com"
        ],
        "url": "https://www.intercom.com/robots.txt"
      },
      {
        "domains": [
          "wpguardian.com",
          "wpguardian.io"
        ],
        "url": "https://wpguardian.com/robots.txt"
      },
      {
        "domains": [
          "networkadvertising.org",
          "thenai.org"
        ],
        "url": "https://thenai.org/robots.txt"
      },
      {
        "domains": [
          "huffpost.com",
          "huffingtonpost.com"
        ],
        "url": "https://www.huffpost.com/robots.txt"
      },
      {
        "domains": [
          "magnific.com",
          "freepik.com"
        ],
        "url": "https://www.magnific.com/robots.txt"
      },
      {
        "domains": [
          "one.one",
          "one.com"
        ],
        "url": "https://www.one.com/robots.txt"
      }
    ],
    "unique_final_urls": 837
  },
  "all_token_count": 14,
  "attempt_count": 3045,
  "body_kinds_success": {
    "decode_error": 1,
    "empty_or_comments": 9,
    "error_text": 1,
    "html_or_challenge": 47,
    "no_supported_rules": 3,
    "rules": 464,
    "rules_with_invalid_lines": 378,
    "truncated": 1,
    "unrecognized_text": 7
  },
  "bots": [
    {
      "group": "search",
      "key": "oai_searchbot",
      "kind": "crawler",
      "name": "OAI-SearchBot",
      "owner": "OpenAI",
      "purposes": [
        "search_index"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 78,
        "percent": "9.1"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "token": "oai-searchbot"
    },
    {
      "group": "user",
      "key": "chatgpt_user",
      "kind": "fetcher",
      "name": "ChatGPT-User",
      "owner": "OpenAI",
      "purposes": [
        "user_fetch"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 95,
        "percent": "11.1"
      },
      "robots_applicability": "user_dependent",
      "source_checked_at": "2026-11-09",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "token": "chatgpt-user"
    },
    {
      "group": "data",
      "key": "gptbot",
      "kind": "crawler",
      "name": "GPTBot",
      "owner": "OpenAI",
      "purposes": [
        "training"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 178,
        "percent": "20.8"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "token": "gptbot"
    },
    {
      "group": "baseline",
      "key": "googlebot",
      "kind": "crawler",
      "name": "Googlebot",
      "owner": "Google",
      "purposes": [
        "search_index"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 12,
        "percent": "1.4"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
      "token": "googlebot"
    },
    {
      "group": "control",
      "key": "google_extended",
      "kind": "control_token",
      "name": "Google-Extended",
      "owner": "Google",
      "purposes": [
        "training",
        "grounding"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 153,
        "percent": "17.9"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
      "token": "google-extended"
    },
    {
      "group": "search",
      "key": "claude_searchbot",
      "kind": "crawler",
      "name": "Claude-SearchBot",
      "owner": "Anthropic",
      "purposes": [
        "search_index"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 91,
        "percent": "10.7"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "token": "claude-searchbot"
    },
    {
      "group": "user",
      "key": "claude_user",
      "kind": "fetcher",
      "name": "Claude-User",
      "owner": "Anthropic",
      "purposes": [
        "user_fetch"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 91,
        "percent": "10.7"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "token": "claude-user"
    },
    {
      "group": "data",
      "key": "claudebot",
      "kind": "crawler",
      "name": "ClaudeBot",
      "owner": "Anthropic",
      "purposes": [
        "training"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 174,
        "percent": "20.4"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "token": "claudebot"
    },
    {
      "group": "search",
      "key": "perplexitybot",
      "kind": "crawler",
      "name": "PerplexityBot",
      "owner": "Perplexity AI",
      "purposes": [
        "search_index"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 120,
        "percent": "14.1"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "token": "perplexitybot"
    },
    {
      "group": "user",
      "key": "perplexity_user",
      "kind": "fetcher",
      "name": "Perplexity-User",
      "owner": "Perplexity AI",
      "purposes": [
        "user_fetch"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 99,
        "percent": "11.6"
      },
      "robots_applicability": "generally_not_applied",
      "source_checked_at": "2026-11-09",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "token": "perplexity-user"
    },
    {
      "group": "baseline",
      "key": "bingbot",
      "kind": "crawler",
      "name": "Bingbot",
      "owner": "Microsoft",
      "purposes": [
        "search_index"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 17,
        "percent": "2.0"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec",
      "token": "bingbot"
    },
    {
      "group": "control",
      "key": "applebot_extended",
      "kind": "control_token",
      "name": "Applebot-Extended",
      "owner": "Apple",
      "purposes": [
        "training_preference"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 151,
        "percent": "17.7"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": null,
      "source_url": "https://support.apple.com/en-us/119829",
      "token": "applebot-extended"
    },
    {
      "group": "data",
      "key": "bytespider",
      "kind": "crawler",
      "name": "Bytespider",
      "owner": "ByteDance / Doubao",
      "purposes": [
        "data_collection_unverified"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 200,
        "percent": "23.4"
      },
      "robots_applicability": "not_verified",
      "source_checked_at": null,
      "source_url": null,
      "token": "bytespider"
    },
    {
      "group": "data",
      "key": "ccbot",
      "kind": "crawler",
      "name": "CCBot",
      "owner": "Common Crawl",
      "purposes": [
        "open_dataset"
      ],
      "restricted": {
        "denominator": 854,
        "numerator": 202,
        "percent": "23.7"
      },
      "robots_applicability": "honored_declared",
      "source_checked_at": "2026-11-09",
      "source_url": "https://commoncrawl.org/ccbot",
      "token": "ccbot"
    }
  ],
  "collector_to_analysis_reclassifications": {
    "0xrpc.io": {
      "robots": {
        "body_kind": {
          "analyzed": "unrecognized_text",
          "collected": "no_supported_rules"
        }
      }
    },
    "abc.net.au": {
      "robots": {
        "groups": {
          "analyzed": 28,
          "collected": 27
        }
      }
    },
    "academia.edu": {
      "robots": {
        "groups": {
          "analyzed": 63,
          "collected": 62
        }
      }
    },
    "ad.nl": {
      "robots": {
        "groups": {
          "analyzed": 46,
          "collected": 45
        }
      }
    },
    "adblockplus.org": {
      "robots": {
        "groups": {
          "analyzed": 6,
          "collected": 3
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 27,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "adobe.com": {
      "robots": {
        "groups": {
          "analyzed": 7,
          "collected": 6
        }
      }
    },
    "agoda.com": {
      "robots": {
        "groups": {
          "analyzed": 9,
          "collected": 8
        }
      }
    },
    "airbnb.com": {
      "robots": {
        "groups": {
          "analyzed": 18,
          "collected": 17
        }
      }
    },
    "amap.com": {
      "robots": {
        "groups": {
          "analyzed": 49,
          "collected": 42
        }
      }
    },
    "amazon.ca": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.co.jp": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.co.uk": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.co.za": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.com": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.com.au": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.com.br": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.com.mx": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.de": {
      "robots": {
        "groups": {
          "analyzed": 102,
          "collected": 89
        }
      }
    },
    "amazon.es": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.fr": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazon.in": {
      "robots": {
        "groups": {
          "analyzed": 103,
          "collected": 90
        }
      }
    },
    "amazon.it": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 88
        }
      }
    },
    "amazonvideo.com": {
      "robots": {
        "groups": {
          "analyzed": 94,
          "collected": 81
        }
      }
    },
    "ancestry.com": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "aol.com": {
      "robots": {
        "groups": {
          "analyzed": 44,
          "collected": 41
        }
      }
    },
    "apa.org": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "apple.com": {
      "robots": {
        "groups": {
          "analyzed": 6,
          "collected": 3
        }
      }
    },
    "arabtnt.com": {
      "robots": {
        "groups": {
          "analyzed": 22,
          "collected": 18
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 7,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "arxiv.org": {
      "robots": {
        "groups": {
          "analyzed": 10,
          "collected": 8
        }
      }
    },
    "as.com": {
      "robots": {
        "groups": {
          "analyzed": 101,
          "collected": 95
        }
      }
    },
    "baidu.com": {
      "robots": {
        "groups": {
          "analyzed": 16,
          "collected": 10
        }
      }
    },
    "bfmtv.com": {
      "robots": {
        "groups": {
          "analyzed": 68,
          "collected": 63
        }
      }
    },
    "bloomberg.com": {
      "robots": {
        "groups": {
          "analyzed": 22,
          "collected": 20
        }
      }
    },
    "bmj.com": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 115,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "britannica.com": {
      "robots": {
        "groups": {
          "analyzed": 21,
          "collected": 19
        }
      }
    },
    "bt.dk": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 5,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 2,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 5,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 2,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "businessinsider.com": {
      "robots": {
        "groups": {
          "analyzed": 13,
          "collected": 12
        }
      }
    },
    "cbsnews.com": {
      "robots": {
        "groups": {
          "analyzed": 5,
          "collected": 4
        }
      }
    },
    "chatgpt.com": {
      "robots": {
        "groups": {
          "analyzed": 14,
          "collected": 12
        }
      }
    },
    "cloudinary.com": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "cookpad.com": {
      "robots": {
        "groups": {
          "analyzed": 26,
          "collected": 23
        }
      }
    },
    "cornell.edu": {
      "robots": {
        "groups": {
          "analyzed": 5,
          "collected": 1
        }
      }
    },
    "corriere.it": {
      "robots": {
        "groups": {
          "analyzed": 106,
          "collected": 87
        }
      }
    },
    "dagbladet.no": {
      "robots": {
        "groups": {
          "analyzed": 48,
          "collected": 43
        }
      }
    },
    "dell.com": {
      "robots": {
        "groups": {
          "analyzed": 9,
          "collected": 5
        }
      }
    },
    "digicert.com": {
      "robots": {
        "groups": {
          "analyzed": 23,
          "collected": 18
        }
      }
    },
    "diretta.it": {
      "robots": {
        "groups": {
          "analyzed": 17,
          "collected": 15
        }
      }
    },
    "dw.com": {
      "robots": {
        "groups": {
          "analyzed": 54,
          "collected": 47
        }
      }
    },
    "ea.com": {
      "robots": {
        "groups": {
          "analyzed": 8,
          "collected": 7
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 20,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 20,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 17,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "economist.com": {
      "robots": {
        "groups": {
          "analyzed": 22,
          "collected": 21
        }
      }
    },
    "elconfidencial.com": {
      "robots": {
        "groups": {
          "analyzed": 61,
          "collected": 56
        }
      }
    },
    "elpais.com": {
      "robots": {
        "groups": {
          "analyzed": 201,
          "collected": 175
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 60,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 42,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 39,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 36,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 204,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 201,
                "path": "/"
              },
              "source": "named"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 327,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 327,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 60,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 327,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 42,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 39,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 36,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 327,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 327,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 327,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 327,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 204,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 201,
                "path": "/"
              },
              "source": "named"
            }
          }
        }
      }
    },
    "elsevier.com": {
      "robots": {
        "groups": {
          "analyzed": 8,
          "collected": 1
        }
      }
    },
    "etsy.com": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        }
      }
    },
    "eventbrite.com": {
      "robots": {
        "groups": {
          "analyzed": 135,
          "collected": 75
        }
      }
    },
    "expireddomains.com": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "facebook.com": {
      "robots": {
        "groups": {
          "analyzed": 37,
          "collected": 36
        }
      }
    },
    "fb.com": {
      "robots": {
        "groups": {
          "analyzed": 37,
          "collected": 36
        }
      }
    },
    "figma.com": {
      "robots": {
        "groups": {
          "analyzed": 22,
          "collected": 21
        }
      }
    },
    "flashscore.com": {
      "robots": {
        "groups": {
          "analyzed": 17,
          "collected": 15
        }
      }
    },
    "flashscore.com.ng": {
      "robots": {
        "groups": {
          "analyzed": 17,
          "collected": 15
        }
      }
    },
    "flashscore.fr": {
      "robots": {
        "groups": {
          "analyzed": 17,
          "collected": 15
        }
      }
    },
    "flashscore.mobi": {
      "robots": {
        "groups": {
          "analyzed": 17,
          "collected": 15
        }
      }
    },
    "free.fr": {
      "robots": {
        "groups": {
          "analyzed": 15,
          "collected": 13
        }
      }
    },
    "gandi.net": {
      "robots": {
        "groups": {
          "analyzed": 212,
          "collected": 141
        }
      }
    },
    "gazzetta.it": {
      "robots": {
        "groups": {
          "analyzed": 15,
          "collected": 14
        }
      }
    },
    "gnu.org": {
      "robots": {
        "groups": {
          "analyzed": 27,
          "collected": 23
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 47,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "goal7.co": {
      "robots": {
        "analyzable": {
          "analyzed": false,
          "collected": true
        },
        "body_kind": {
          "analyzed": "unrecognized_text",
          "collected": "no_supported_rules"
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "bingbot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "bytespider": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "ccbot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "chatgpt_user": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "claude_searchbot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "claude_user": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "claudebot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "google_extended": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "googlebot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "gptbot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "oai_searchbot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "perplexity_user": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            },
            "perplexitybot": {
              "policy": "unknown",
              "rule": null,
              "source": "none"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "none"
            }
          }
        }
      }
    },
    "gsmarena.com": {
      "robots": {
        "groups": {
          "analyzed": 9,
          "collected": 8
        }
      }
    },
    "hbr.org": {
      "robots": {
        "groups": {
          "analyzed": 6,
          "collected": 4
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 36,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 38,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 34,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 36,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 38,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 34,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 30,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "healthline.com": {
      "robots": {
        "groups": {
          "analyzed": 165,
          "collected": 106
        }
      }
    },
    "hln.be": {
      "robots": {
        "groups": {
          "analyzed": 46,
          "collected": 45
        }
      }
    },
    "huffingtonpost.com": {
      "robots": {
        "groups": {
          "analyzed": 33,
          "collected": 31
        }
      }
    },
    "huffpost.com": {
      "robots": {
        "groups": {
          "analyzed": 33,
          "collected": 31
        }
      }
    },
    "hugedomains.com": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 35,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 35,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 35,
                "path": "/"
              },
              "source": "named"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 35,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 35,
                "path": "/"
              },
              "source": "named"
            }
          }
        }
      }
    },
    "ibm.com": {
      "robots": {
        "groups": {
          "analyzed": 10,
          "collected": 7
        }
      }
    },
    "ilfattoquotidiano.it": {
      "robots": {
        "groups": {
          "analyzed": 7,
          "collected": 3
        }
      }
    },
    "ilmeteo.it": {
      "robots": {
        "groups": {
          "analyzed": 8,
          "collected": 6
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "claude_searchbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "perplexity_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "claude_searchbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 11,
                "path": "/"
              },
              "source": "named"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 86,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "perplexity_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            }
          }
        }
      }
    },
    "iltalehti.fi": {
      "robots": {
        "groups": {
          "analyzed": 64,
          "collected": 54
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 21,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 34,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 37,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 40,
                "path": "/"
              },
              "source": "named"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 43,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 46,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 52,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 82,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 85,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 130,
                "path": "/"
              },
              "source": "named"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 146,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 149,
                "path": "/"
              },
              "source": "named"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 21,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 5,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 34,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 37,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 40,
                "path": "/"
              },
              "source": "named"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 43,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 46,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 52,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 82,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 5,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 85,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 130,
                "path": "/"
              },
              "source": "named"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 146,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 149,
                "path": "/"
              },
              "source": "named"
            }
          }
        }
      }
    },
    "indiamart.com": {
      "robots": {
        "groups": {
          "analyzed": 57,
          "collected": 42
        }
      }
    },
    "instagram.com": {
      "robots": {
        "groups": {
          "analyzed": 37,
          "collected": 36
        }
      }
    },
    "intel.com": {
      "robots": {
        "groups": {
          "analyzed": 9,
          "collected": 8
        }
      }
    },
    "intuit.com": {
      "robots": {
        "groups": {
          "analyzed": 16,
          "collected": 4
        }
      }
    },
    "issuu.com": {
      "robots": {
        "groups": {
          "analyzed": 47,
          "collected": 39
        }
      }
    },
    "jd.com": {
      "robots": {
        "groups": {
          "analyzed": 21,
          "collected": 18
        }
      }
    },
    "kicker.de": {
      "robots": {
        "groups": {
          "analyzed": 38,
          "collected": 35
        }
      }
    },
    "kompoz2.com": {
      "robots": {
        "groups": {
          "analyzed": 22,
          "collected": 18
        }
      }
    },
    "lemonde.fr": {
      "robots": {
        "groups": {
          "analyzed": 347,
          "collected": 274
        }
      }
    },
    "leroymerlin.es": {
      "robots": {
        "groups": {
          "analyzed": 149,
          "collected": 90
        }
      }
    },
    "letterboxd.com": {
      "robots": {
        "groups": {
          "analyzed": 25,
          "collected": 20
        }
      }
    },
    "linkedin.com": {
      "robots": {
        "groups": {
          "analyzed": 78,
          "collected": 71
        }
      }
    },
    "linktr.ee": {
      "robots": {
        "groups": {
          "analyzed": 60,
          "collected": 56
        }
      }
    },
    "livescore.cz": {
      "robots": {
        "groups": {
          "analyzed": 17,
          "collected": 15
        }
      }
    },
    "loc.gov": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "marca.com": {
      "robots": {
        "groups": {
          "analyzed": 109,
          "collected": 88
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 22,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 22,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 368,
                "path": "/"
              },
              "source": "named"
            }
          }
        }
      }
    },
    "markt.de": {
      "robots": {
        "groups": {
          "analyzed": 31,
          "collected": 26
        }
      }
    },
    "mbga.jp": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "meb.gov.tr": {
      "robots": {
        "groups": {
          "analyzed": 48,
          "collected": 41
        }
      }
    },
    "meethue.com": {
      "robots": {
        "groups": {
          "analyzed": 5,
          "collected": 2
        }
      }
    },
    "mobile.de": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "moloco.com": {
      "robots": {
        "groups": {
          "analyzed": 54,
          "collected": 38
        }
      }
    },
    "msn.com": {
      "robots": {
        "groups": {
          "analyzed": 11,
          "collected": 9
        }
      }
    },
    "n-tv.de": {
      "robots": {
        "groups": {
          "analyzed": 66,
          "collected": 62
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 33,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 60,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 63,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 117,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 132,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 33,
                "path": "/"
              },
              "source": "named"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 57,
                "path": "/"
              },
              "source": "named"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 60,
                "path": "/"
              },
              "source": "named"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 63,
                "path": "/"
              },
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 117,
                "path": "/"
              },
              "source": "named"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 132,
                "path": "/"
              },
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 12,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "name.com": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 93,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "nature.com": {
      "robots": {
        "groups": {
          "analyzed": 23,
          "collected": 22
        }
      }
    },
    "newsweek.com": {
      "robots": {
        "groups": {
          "analyzed": 28,
          "collected": 26
        }
      }
    },
    "nike.com": {
      "robots": {
        "groups": {
          "analyzed": 6,
          "collected": 3
        }
      }
    },
    "nintendo.com": {
      "robots": {
        "groups": {
          "analyzed": 26,
          "collected": 23
        }
      }
    },
    "nypost.com": {
      "robots": {
        "groups": {
          "analyzed": 66,
          "collected": 58
        }
      }
    },
    "nytimes.com": {
      "robots": {
        "groups": {
          "analyzed": 70,
          "collected": 65
        }
      }
    },
    "oecd.org": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "okdiario.com": {
      "robots": {
        "groups": {
          "analyzed": 28,
          "collected": 24
        }
      }
    },
    "onet.pl": {
      "robots": {
        "groups": {
          "analyzed": 15,
          "collected": 14
        }
      }
    },
    "oricon.co.jp": {
      "robots": {
        "groups": {
          "analyzed": 9,
          "collected": 8
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 58,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "ouest-france.fr": {
      "robots": {
        "groups": {
          "analyzed": 176,
          "collected": 154
        }
      }
    },
    "pikabu.ru": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "playstation.com": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "primevideo.com": {
      "robots": {
        "groups": {
          "analyzed": 94,
          "collected": 81
        }
      }
    },
    "publi24.ro": {
      "robots": {
        "groups": {
          "analyzed": 22,
          "collected": 20
        }
      }
    },
    "quora.com": {
      "robots": {
        "groups": {
          "analyzed": 29,
          "collected": 28
        }
      }
    },
    "redfin.com": {
      "robots": {
        "groups": {
          "analyzed": 12,
          "collected": 10
        }
      }
    },
    "repubblica.it": {
      "robots": {
        "groups": {
          "analyzed": 108,
          "collected": 89
        }
      }
    },
    "rightmove.co.uk": {
      "robots": {
        "groups": {
          "analyzed": 11,
          "collected": 10
        }
      }
    },
    "russianfood.com": {
      "robots": {
        "groups": {
          "analyzed": 12,
          "collected": 11
        }
      }
    },
    "sagepub.com": {
      "robots": {
        "groups": {
          "analyzed": 43,
          "collected": 42
        }
      }
    },
    "salesforce.com": {
      "robots": {
        "groups": {
          "analyzed": 9,
          "collected": 3
        }
      }
    },
    "scribd.com": {
      "robots": {
        "groups": {
          "analyzed": 9,
          "collected": 8
        }
      }
    },
    "sharethrough.com": {
      "robots": {
        "groups": {
          "analyzed": 33,
          "collected": 30
        }
      }
    },
    "sinoptik.ua": {
      "robots": {
        "groups": {
          "analyzed": 5,
          "collected": 4
        }
      }
    },
    "skroutz.gr": {
      "robots": {
        "groups": {
          "analyzed": 24,
          "collected": 23
        }
      }
    },
    "slideshare.net": {
      "robots": {
        "groups": {
          "analyzed": 11,
          "collected": 9
        }
      }
    },
    "snapchat.com": {
      "robots": {
        "groups": {
          "analyzed": 51,
          "collected": 44
        }
      }
    },
    "sotwe.com": {
      "robots": {
        "groups": {
          "analyzed": 566,
          "collected": 474
        }
      }
    },
    "spiegel.de": {
      "robots": {
        "groups": {
          "analyzed": 111,
          "collected": 102
        }
      }
    },
    "syosetu.com": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        }
      }
    },
    "tabelog.com": {
      "robots": {
        "groups": {
          "analyzed": 20,
          "collected": 19
        }
      }
    },
    "tabor.ru": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        }
      }
    },
    "tagesschau.de": {
      "robots": {
        "groups": {
          "analyzed": 30,
          "collected": 28
        }
      }
    },
    "ted.com": {
      "robots": {
        "groups": {
          "analyzed": 31,
          "collected": 30
        }
      }
    },
    "tenki.jp": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "theatlantic.com": {
      "robots": {
        "groups": {
          "analyzed": 44,
          "collected": 43
        }
      }
    },
    "theverge.com": {
      "robots": {
        "groups": {
          "analyzed": 106,
          "collected": 94
        }
      }
    },
    "threads.com": {
      "robots": {
        "groups": {
          "analyzed": 38,
          "collected": 37
        }
      }
    },
    "tjk.org": {
      "robots": {
        "groups": {
          "analyzed": 63,
          "collected": 61
        }
      }
    },
    "trustpilot.com": {
      "robots": {
        "groups": {
          "analyzed": 46,
          "collected": 45
        }
      }
    },
    "ultimate-guitar.com": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "unity3d.com": {
      "robots": {
        "groups": {
          "analyzed": 5,
          "collected": 4
        }
      }
    },
    "uol.com.br": {
      "robots": {
        "groups": {
          "analyzed": 50,
          "collected": 45
        }
      }
    },
    "usatoday.com": {
      "robots": {
        "groups": {
          "analyzed": 289,
          "collected": 249
        }
      }
    },
    "w3.org": {
      "robots": {
        "groups": {
          "analyzed": 2,
          "collected": 1
        }
      }
    },
    "washingtonpost.com": {
      "robots": {
        "groups": {
          "analyzed": 35,
          "collected": 31
        }
      }
    },
    "weather.com": {
      "robots": {
        "groups": {
          "analyzed": 26,
          "collected": 23
        }
      }
    },
    "who.int": {
      "robots": {
        "groups": {
          "analyzed": 527,
          "collected": 441
        }
      }
    },
    "wikipedia.org": {
      "robots": {
        "groups": {
          "analyzed": 33,
          "collected": 26
        }
      }
    },
    "wiktionary.org": {
      "robots": {
        "groups": {
          "analyzed": 33,
          "collected": 26
        }
      }
    },
    "wnacg.com": {
      "robots": {
        "groups": {
          "analyzed": 45,
          "collected": 44
        }
      }
    },
    "worldbank.org": {
      "robots": {
        "groups": {
          "analyzed": 2,
          "collected": 1
        }
      }
    },
    "www.gov.uk": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        }
      }
    },
    "xing.com": {
      "robots": {
        "groups": {
          "analyzed": 4,
          "collected": 3
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "google_extended": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "oai_searchbot": {
              "policy": "disallow",
              "rule": {
                "kind": "disallow",
                "line": 230,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "named"
            }
          }
        }
      }
    },
    "youtube.com": {
      "robots": {
        "groups": {
          "analyzed": 2,
          "collected": 1
        }
      }
    },
    "zebawy.com": {
      "robots": {
        "groups": {
          "analyzed": 3,
          "collected": 2
        },
        "policies": {
          "analyzed": {
            "applebot_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "no_applicable_rule",
              "rule": null,
              "source": "wildcard"
            }
          },
          "collected": {
            "applebot_extended": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bingbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "bytespider": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "ccbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "chatgpt_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_searchbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claude_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "claudebot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "google_extended": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "googlebot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "gptbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "oai_searchbot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexity_user": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            },
            "perplexitybot": {
              "policy": "allow",
              "rule": {
                "kind": "allow",
                "line": 8,
                "path": "/"
              },
              "source": "wildcard"
            }
          }
        }
      }
    },
    "zhihu.com": {
      "robots": {
        "groups": {
          "analyzed": 11,
          "collected": 10
        }
      }
    }
  },
  "dataset_published": "2026-11-10",
  "distribution": {
    "0": 605,
    "1": 38,
    "2": 17,
    "3": 12,
    "4": 13,
    "5": 22,
    "6": 40,
    "7": 13,
    "8": 6,
    "9": 12,
    "10": 6,
    "11": 25,
    "12": 45
  },
  "fetch_states": {
    "network_error": 6,
    "not_found": 43,
    "other_4xx": 39,
    "server_error": 1,
    "success": 911
  },
  "home_observations": {
    "challenge_review": 3,
    "content_candidate_unverified": 666,
    "http_status_review": 7,
    "login_candidate_review": 1,
    "non_html_review": 26,
    "off_domain_redirect_review": 115,
    "thin_review": 26,
    "unreachable_review": 8,
    "walled_review": 148
  },
  "integrity": {
    "bots_sha256": "961ba04b5aa712b4a799547f3be1142c3343c814bce7fa137819ed5d6d1ba6fd",
    "cohort_sha256": "e7e737058ade9fc5e7b3018c7a1a1968a9aee0040cea6b1bf4729d9a0010a473",
    "core_robots_llms_attempt_body_and_wire_hashes_verified": true,
    "public_home_evidence": "response metadata and supplied observation labels; original HTML retained locally"
  },
  "legacy_comparison": {
    "common_analyzable": 853,
    "common_historical_rate": {
      "denominator": 853,
      "numerator": 248,
      "percent": "29.1"
    },
    "common_new_rate": {
      "denominator": 853,
      "numerator": 248,
      "percent": "29.1"
    },
    "historical_analyzable": 891,
    "historical_rate": {
      "denominator": 891,
      "numerator": 261,
      "percent": "29.3"
    },
    "interpretation": "Both time and validation changed; not a causal policy-change estimate."
  },
  "llms_candidates_all_cohort": {
    "denominator": 1000,
    "numerator": 156,
    "percent": "15.6"
  },
  "llms_cross_tab": [
    {
      "ai_restricted": {
        "denominator": 154,
        "numerator": 22,
        "percent": "14.3"
      },
      "n": 154,
      "observation": "minimum_h1_candidate"
    },
    {
      "ai_restricted": {
        "denominator": 92,
        "numerator": 36,
        "percent": "39.1"
      },
      "n": 92,
      "observation": "observed_non_candidate"
    },
    {
      "ai_restricted": {
        "denominator": 532,
        "numerator": 152,
        "percent": "28.6"
      },
      "n": 532,
      "observation": "root_not_found"
    },
    {
      "ai_restricted": {
        "denominator": 76,
        "numerator": 39,
        "percent": "51.3"
      },
      "n": 76,
      "observation": "unknown"
    }
  ],
  "llms_observations": {
    "minimum_h1_candidate": 156,
    "observed_non_candidate": 148,
    "root_not_found": 573,
    "unknown": 123
  },
  "modified": "2026-11-10",
  "operational": {
    "formal_https_to_http_robots": 2,
    "median_complete_analyzable_bytes": 1248.0,
    "redirected_selected_robots": 638
  },
  "pairs": [
    {
      "cells": [
        {
          "count": 78,
          "first_restricted": true,
          "second_restricted": true
        },
        {
          "count": 100,
          "first_restricted": true,
          "second_restricted": false
        },
        {
          "count": 0,
          "first_restricted": false,
          "second_restricted": true
        },
        {
          "count": 676,
          "first_restricted": false,
          "second_restricted": false
        }
      ],
      "first": "gptbot",
      "first_restricted_second_not": {
        "denominator": 178,
        "numerator": 100,
        "percent": "56.2"
      },
      "second": "oai_searchbot"
    },
    {
      "cells": [
        {
          "count": 91,
          "first_restricted": true,
          "second_restricted": true
        },
        {
          "count": 83,
          "first_restricted": true,
          "second_restricted": false
        },
        {
          "count": 0,
          "first_restricted": false,
          "second_restricted": true
        },
        {
          "count": 680,
          "first_restricted": false,
          "second_restricted": false
        }
      ],
      "first": "claudebot",
      "first_restricted_second_not": {
        "denominator": 174,
        "numerator": 83,
        "percent": "47.7"
      },
      "second": "claude_searchbot"
    }
  ],
  "provenance": {
    "archive_member_timestamp": [
      2026,
      11,
      7,
      22,
      21,
      18
    ],
    "cohort_matches_original_rank": 1000,
    "cohort_sha256": "e7e737058ade9fc5e7b3018c7a1a1968a9aee0040cea6b1bf4729d9a0010a473",
    "historical_list_date": "2026-11-07",
    "historical_list_id": "94P72",
    "permanent_download_url": "https://tranco-list.eu/download/94P72/1000000",
    "pilot_selection": "25 preregistered anomaly/review cases + 5 lowest SHA256 domain strings. Pilot not used in prevalence estimates.",
    "published_v11_csv_sha256": "e11f05b0d3b4d1e472160a1100f9ee6153608c3c86ed56414b7df197d8cdffb4",
    "ranking_cache_sha256": "4ec943d5b0cb87afb9d9b427c3008797b3109ba930ed15777e2a3b3f6e4179f3",
    "ranking_csv_sha256": "2019dae9caab1d89c35529925d5502d25884e4e962c81868bd8ec9416e4c2776",
    "source": "Frozen original filtered cohort; all ranks match original archived ranking; permanent ID recovered by exact ranking CSV byte comparison."
  },
  "published": "2026-11-09",
  "rank_buckets": [
    {
      "ai_any": {
        "denominator": 40,
        "numerator": 16,
        "percent": "40.0"
      },
      "analyzable": 40,
      "label": "1–100"
    },
    {
      "ai_any": {
        "denominator": 171,
        "numerator": 55,
        "percent": "32.2"
      },
      "analyzable": 171,
      "label": "101–500"
    },
    {
      "ai_any": {
        "denominator": 249,
        "numerator": 74,
        "percent": "29.7"
      },
      "analyzable": 249,
      "label": "501–1000"
    },
    {
      "ai_any": {
        "denominator": 394,
        "numerator": 104,
        "percent": "26.4"
      },
      "analyzable": 394,
      "label": "1001–1726"
    }
  ],
  "reference": {
    "binary_sha256": "6a90ec90e578e4d7a10092dca35d57c2a1571335b3942ff230e45442fa32659c",
    "caveat": "Reference accepts some nonstandard syntax; agreement is not a universal conformance guarantee.",
    "cohort_sha256": "e7e737058ade9fc5e7b3018c7a1a1968a9aee0040cea6b1bf4729d9a0010a473",
    "commit": "22b355ff855419e6a3ff8ff09c0ad7fdb17116f9",
    "compared": 11956,
    "difference_review": {
      "cause": "Both bodies use Disallow: * without the leading slash required by the strict path grammar. Google leniently matches it; this analysis records an invalid line and no supported root prohibition for this named token.",
      "decision": "Keep strict slash-prefixed path matching; publish the two reference differences.",
      "domains": [
        "twitter.com",
        "x.com"
      ],
      "primary_ai_any_effect": "None. Both methods give 249 / 854 = 29.2% because other AI tokens are restricted at these domains.",
      "token": "google-extended"
    },
    "differences": [
      {
        "body_sha256": "b54dd4fefb8786b6441dc95ab04db1dbe4c6b9fcb5e7bb60d8381ab6e40989ff",
        "domain": "twitter.com",
        "local_disallow": false,
        "reference_returncode": 1,
        "token": "google-extended"
      },
      {
        "body_sha256": "b54dd4fefb8786b6441dc95ab04db1dbe4c6b9fcb5e7bb60d8381ab6e40989ff",
        "domain": "x.com",
        "local_disallow": false,
        "reference_returncode": 1,
        "token": "google-extended"
      }
    ],
    "implementation": "google/robotstxt robots-main",
    "reference_ai_any": {
      "denominator": 854,
      "numerator": 249,
      "percent": "29.2"
    },
    "reference_restricted_counts": {
      "applebot_extended": 151,
      "bingbot": 17,
      "bytespider": 200,
      "ccbot": 202,
      "chatgpt_user": 95,
      "claude_searchbot": 91,
      "claude_user": 91,
      "claudebot": 174,
      "google_extended": 155,
      "googlebot": 12,
      "gptbot": 178,
      "oai_searchbot": 78,
      "perplexity_user": 99,
      "perplexitybot": 120
    },
    "scope": "root verdict only, same archived body, all analyzable domains x 14 tokens",
    "source": "https://github.com/google/robotstxt/tree/22b355ff855419e6a3ff8ff09c0ad7fdb17116f9"
  },
  "request_count": 5443,
  "sample": {
    "analyzable": 854,
    "outside_subset": 146,
    "rank_max": 1726,
    "rank_min": 1,
    "total": 1000
  },
  "sample_review": [
    {
      "category": "community",
      "decision": "retained_in_frozen_cohort",
      "domain": "reddit.com",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/reddit.com/home-1.body",
      "home_body_sha256": "13d97e7f161990de458e50d541d51ecfb2358e928d61ee5e7bf4b3b00e32725c",
      "home_final_url": "https://www.reddit.com/",
      "home_status": 200,
      "home_title": "Reddit",
      "note_en": "The archived homepage presents the Reddit community platform; retained as a domain, not a separate-company claim.",
      "note_zh": "归档主页展示 Reddit 社区平台；按域名保留，不视为独立企业归属证明。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:05:03.428Z"
    },
    {
      "category": "moved_landing",
      "decision": "retained_in_frozen_cohort",
      "domain": "registrar-servers.com",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/registrar-servers.com/home-1.body",
      "home_body_sha256": "df1455593e2bea7e8814ebf705f9375e46f875cdb5de4809a1415b1f0c09c818",
      "home_final_url": "https://registrar-servers.com/",
      "home_status": 200,
      "home_title": "Home - registrar-servers.com",
      "note_en": "The homepage is a moved-site notice linking to Namecheap. This response does not establish a content publication or a DNS-service policy.",
      "note_zh": "主页是迁移提示，链接到 Namecheap；该响应不能证明它是内容出版站或 DNS 服务政策。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:05:09.853Z"
    },
    {
      "category": "script_gate",
      "decision": "retained_in_frozen_cohort",
      "domain": "skybridge.click",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/skybridge.click/home-1.body",
      "home_body_sha256": "3d8974be4a9dfd434d54b016f616a08817a0026ddcb1055fbc127da997af1e35",
      "home_final_url": "https://skybridge.click/",
      "home_status": 200,
      "home_title": "ООО \"СМП-74\"",
      "note_en": "The title names SMP-74, but visible content asks for JavaScript. Business category remains unresolved.",
      "note_zh": "标题出现 SMP-74，但可见内容要求启用 JavaScript，业务类别保持未解决。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:05:16.250Z"
    },
    {
      "category": "dns_landing",
      "decision": "retained_in_frozen_cohort",
      "domain": "netcraze.io",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/netcraze.io/home-1.body",
      "home_body_sha256": "b99dc2b71a9d0ab68c6e104ddedaee37b853166eb8d677e7fe581f693395019e",
      "home_final_url": "https://netcraze.io/",
      "home_status": 200,
      "home_title": "CrazeDNS",
      "note_en": "A thin landing page uses the CrazeDNS title. Service purpose and company identity are not verified from this response.",
      "note_zh": "简薄落地页使用 CrazeDNS 标题，仅凭该响应无法确认服务用途或企业归属。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:07:28.298Z"
    },
    {
      "category": "advertising",
      "decision": "retained_in_frozen_cohort",
      "domain": "sharethrough.com",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/sharethrough.com/home-1.body",
      "home_body_sha256": "2ae2e04938d7d9a1a28ab4989eaf571a81134c071443474b8a7a6abc915657d4",
      "home_final_url": "https://www.sharethrough.com/",
      "home_status": 200,
      "home_title": "Sharethrough — Human-Centric Programmatic Advertising",
      "note_en": "Homepage title and readable navigation describe programmatic advertising solutions. Retained; no exclusion based on its robots policy.",
      "note_zh": "主页标题及可读导航介绍程序化广告服务；继续保留，不按 robots 政策排除。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:09:25.514Z"
    },
    {
      "category": "blank",
      "decision": "retained_in_frozen_cohort",
      "domain": "hihonorcloud.com",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/hihonorcloud.com/home-1.body",
      "home_body_sha256": "d3de40af5499cd03f4f8d80a90fc686408146a9583d3fa8d2ddff4afb4988387",
      "home_final_url": "https://hihonorcloud.com/",
      "home_status": 200,
      "home_title": "",
      "note_en": "Homepage provides no readable title or substantive visible content in this response; category remains unresolved.",
      "note_zh": "本次主页响应没有可读标题或实质可见内容，类别保持未解决。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:09:47.077Z"
    },
    {
      "category": "walled",
      "decision": "retained_in_frozen_cohort",
      "domain": "adform.net",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/adform.net/home-1.body",
      "home_body_sha256": "32f2fa940d4b4fe19aca1e53a24e5aac29c57b7c5ee78588325b87f1b649c864",
      "home_final_url": "https://adform.net/",
      "home_status": 403,
      "home_title": "403 Forbidden",
      "note_en": "Homepage returns 403. Its name alone is not used to assign a business category.",
      "note_zh": "主页返回 403，不仅凭域名名称判断业务类别。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:10:44.401Z"
    },
    {
      "category": "marketing_redirect",
      "decision": "retained_in_frozen_cohort",
      "domain": "mailchi.mp",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/mailchi.mp/home-1.body",
      "home_body_sha256": "b2d01f0b41fb366f10dff609369cb728ddac85a69b500dc9db29f522dd8de0f7",
      "home_final_url": "https://mailchimp.com/",
      "home_status": 200,
      "home_title": "Email & SMS Marketing Platform | Mailchimp",
      "note_en": "Homepage redirects to the Mailchimp marketing landing page, so this is not an independent content-publication observation.",
      "note_zh": "主页跳转到 Mailchimp 营销落地页，不作为独立内容出版网站的证据。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:11:55.837Z"
    },
    {
      "category": "walled",
      "decision": "retained_in_frozen_cohort",
      "domain": "btloader.com",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/btloader.com/home-1.body",
      "home_body_sha256": "c029d1c560ecb35bbb632f9f7102907b41a289e258055d477f92e21884f9ab79",
      "home_final_url": "https://btloader.com/",
      "home_status": 403,
      "home_title": "",
      "note_en": "Homepage returns 403; business category remains unresolved.",
      "note_zh": "主页返回 403，业务类别保持未解决。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:11:58.293Z"
    },
    {
      "category": "community",
      "decision": "retained_in_frozen_cohort",
      "domain": "stackexchange.com",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/stackexchange.com/home-1.body",
      "home_body_sha256": "e6bfa7694def774536fe464ae9d7050ea5323e8b0b4cdd6bb03c02e53b15f195",
      "home_final_url": "https://stackexchange.com/",
      "home_status": 200,
      "home_title": "Hot Questions - Stack Exchange",
      "note_en": "Homepage presents the Stack Exchange questions network. Homepage content and robots retrieval outcome are separate evidence.",
      "note_zh": "主页展示 Stack Exchange 问答网络；主页内容和 robots 请求结果分别记录。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:12:34.460Z"
    },
    {
      "category": "link_management",
      "decision": "retained_in_frozen_cohort",
      "domain": "aka.ms",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/aka.ms/home-1.body",
      "home_body_sha256": "97bd04c9f4b6d973bbc89a39824aafb2d2465b64fbdc5d83ce8368c1fb477869",
      "home_final_url": "https://witty-sand-06796de1e.2.azurestaticapps.net/links?host=1",
      "home_status": 200,
      "home_title": "aka.ms Link Management",
      "note_en": "Homepage formally redirects to an Azure-hosted link-management landing page. It is not company deduplication evidence.",
      "note_zh": "主页正式跳转到 Azure 托管的链接管理落地页，不作为企业去重证据。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:17:17.640Z"
    },
    {
      "category": "racing",
      "decision": "retained_in_frozen_cohort",
      "domain": "keirin.jp",
      "historical_case": "historical_all14_restricted",
      "home_body_local_path": "full-02/bodies/keirin.jp/home-1.body",
      "home_body_sha256": "3d96d91800017fedea5d70ccead50661a1920b93fafd2c3e125fcbea2b385c27",
      "home_final_url": "https://keirin.jp:443/pc/top",
      "home_status": 200,
      "home_title": "トップページ｜KEIRIN.JP",
      "note_en": "Homepage presents racing information and results under KEIRIN.JP. It stays in the frozen cohort.",
      "note_zh": "主页在 KEIRIN.JP 品牌下展示竞赛资讯与结果，继续保留在固定队列中。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:18:19.363Z"
    },
    {
      "category": "lottery",
      "decision": "retained_in_frozen_cohort",
      "domain": "xsmn.mobi",
      "historical_case": "replacement_missing_audit",
      "home_body_local_path": "full-02/bodies/xsmn.mobi/home-1.body",
      "home_body_sha256": "03769a12f4eb5dec8254b4f2d4b8f00967ccc9830f3c12ae80c195d9bcdf47e5",
      "home_final_url": "https://xsmn.mobi/",
      "home_status": 200,
      "home_title": "XSMN - SXMN - Xổ số miền Nam - KQXSMN - XSKTMN - XSNM hôm nay",
      "note_en": "Homepage title and readable result sections identify a lottery-results site. This completes a previously missing audit.",
      "note_zh": "主页标题和可读结果栏目指向彩票结果网站，补齐此前缺失的审计。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:18:39.547Z"
    },
    {
      "category": "lottery",
      "decision": "retained_in_frozen_cohort",
      "domain": "minhngoc.net.vn",
      "historical_case": "replacement_missing_audit",
      "home_body_local_path": "full-02/bodies/minhngoc.net.vn/home-1.body",
      "home_body_sha256": "91f390a5c9f614275b59092e197ff11d7ddd41191631bd603b741c9ac8f966f6",
      "home_final_url": "https://www.minhngoc.net.vn/",
      "home_status": 200,
      "home_title": "Xổ số Minh Ngọc™ Kết Quả Xổ Số Trực Tiếp NHANH NHẤT",
      "note_en": "Homepage identifies lottery results. A missing robots endpoint does not invalidate the homepage observation.",
      "note_zh": "主页展示彩票结果；robots 端点不存在不等于主页观察无效。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:18:44.662Z"
    },
    {
      "category": "login",
      "decision": "retained_in_frozen_cohort",
      "domain": "kinnser.net",
      "historical_case": "replacement_missing_audit",
      "home_body_local_path": "full-02/bodies/kinnser.net/home-1.body",
      "home_body_sha256": "6bdf73f33ea3588b2fe10091b547dc1cef845d56a7b33591261e61c35528e90f",
      "home_final_url": "https://kinnser.net/login.cfm",
      "home_status": 200,
      "home_title": "Login | WellSky HHH",
      "note_en": "Homepage redirects to login.cfm and displays the WellSky HHH login title. It is a login portal, retained in the domain cohort.",
      "note_zh": "主页跳转至 login.cfm，标题为 WellSky HHH 登录页；标为登录门户，仍保留在域名队列。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:18:39.526Z"
    },
    {
      "category": "sports",
      "decision": "retained_in_frozen_cohort",
      "domain": "espn.com.ve",
      "historical_case": "replacement_missing_audit",
      "home_body_local_path": "full-02/bodies/espn.com.ve/home-1.body",
      "home_body_sha256": "89d83bc4df3105c01c662f6c55c5d0976323cfbad0508428fe22e801ece4c605",
      "home_final_url": "https://www.espn.com.ve/",
      "home_status": 200,
      "home_title": "ESPN Venezuela - Lo Último del Mundo Deportivo",
      "note_en": "Homepage identifies the ESPN Venezuela sports publication, completing the fourth previously missing audit.",
      "note_zh": "主页展示 ESPN Venezuela 体育媒体，补齐第四个此前缺失的审计。",
      "review_scope": "targeted_case_audit_not_representative_category_distribution",
      "reviewed_at": "2026-11-10",
      "source_attempt_started_at": "2026-11-10T01:18:42.270Z"
    }
  ],
  "scope": "declared root path policies, fixed filtered domain cohort",
  "signals": {
    "__usage_ai-training": {
      "domains_with_syntactic_record": [
        "facebook.com",
        "instagram.com",
        "twitter.com",
        "linkedin.com",
        "x.com",
        "whatsapp.com",
        "imdb.com",
        "threads.com",
        "fb.com",
        "cookpad.com"
      ],
      "provider_effect": "not_evaluated",
      "string_present_success_responses": 10,
      "syntactic_record_analyzable": 10
    },
    "anthropic-ai": {
      "domains_with_syntactic_record": [
        "linkedin.com",
        "tiktok.com",
        "wordpress.org",
        "yahoo.com",
        "vimeo.com",
        "msn.com",
        "chatgpt.com",
        "android.com",
        "tumblr.com",
        "snapchat.com",
        "nytimes.com",
        "ubuntu.com",
        "cnn.com",
        "forter.com",
        "ebay.com",
        "soundcloud.com",
        "theguardian.com",
        "bbc.com",
        "forbes.com",
        "amplitude.com",
        "bbc.co.uk",
        "nature.com",
        "canva.com",
        "espn.com",
        "trustpilot.com",
        "bloomberg.com",
        "washingtonpost.com",
        "aol.com",
        "indeed.com",
        "intel.com",
        "cnbc.com",
        "businessinsider.com",
        "statista.com",
        "behance.net",
        "ft.com",
        "sohu.com",
        "uol.com.br",
        "squarespace.com",
        "dailymail.com",
        "sophos.com",
        "usatoday.com",
        "sagepub.com",
        "techcrunch.com",
        "telegraph.co.uk",
        "conviva.com",
        "jotform.com",
        "ring.com",
        "yelp.com",
        "kick.com",
        "lemonde.fr",
        "dailymail.co.uk",
        "gumgum.com",
        "cookielaw.org",
        "spiegel.de",
        "cnet.com",
        "bestbuy.com",
        "apnews.com",
        "airbnb.com",
        "people.com",
        "ted.com",
        "ebay.de",
        "theverge.com",
        "ebay.co.uk",
        "repubblica.it",
        "corriere.it",
        "huffpost.com",
        "pixiv.net",
        "themoviedb.org",
        "ouest-france.fr",
        "theatlantic.com",
        "healthline.com",
        "bfmtv.com",
        "investopedia.com",
        "marca.com",
        "nypost.com",
        "cookpad.com",
        "finn.no",
        "skroutz.gr",
        "jimdo.com",
        "leboncoin.fr",
        "elconfidencial.com",
        "nu.nl",
        "theconversation.com",
        "aftonbladet.se",
        "ad.nl",
        "fmkorea.com",
        "vg.no",
        "ay267.com",
        "spankbang.com",
        "amap.com",
        "furaffinity.net",
        "abc.net.au",
        "hln.be",
        "kicker.de",
        "litnet.com",
        "economist.com",
        "actu.fr",
        "oxylabs.io",
        "psu.edu",
        "aljazeera.com",
        "e621.net",
        "hai8g.com",
        "dagbladet.no",
        "bbb.org",
        "nikkansports.com",
        "dw.com",
        "ovh.com",
        "iltalehti.fi",
        "nesine.com",
        "meb.gov.tr",
        "toyhou.se",
        "cardmarket.com",
        "huffingtonpost.com",
        "kargo.com",
        "ekstrabladet.dk",
        "newsweek.com",
        "omg10.com",
        "ebay.it",
        "joyclub.de",
        "bblaa.com",
        "pussyspace.com",
        "goodreturns.in",
        "shahvani.com",
        "myreadingmanga.info",
        "theqoo.net",
        "spankbang.party",
        "crn77.com",
        "tjk.org",
        "nustargame.com",
        "wnacg.com",
        "twkan.com",
        "dogdrip.net",
        "darkero.com",
        "publi24.ro",
        "ey43.com",
        "nya2.com",
        "arabx.cam",
        "manhwaweb.com",
        "eff.org",
        "espn.com.ve"
      ],
      "provider_effect": "not_evaluated",
      "string_present_success_responses": 141,
      "syntactic_record_analyzable": 140
    },
    "claude-web": {
      "domains_with_syntactic_record": [
        "linkedin.com",
        "amazon.com",
        "yahoo.com",
        "vimeo.com",
        "msn.com",
        "ui.com",
        "chatgpt.com",
        "snapchat.com",
        "nytimes.com",
        "ubuntu.com",
        "kaspersky.com",
        "cnn.com",
        "soundcloud.com",
        "bbc.com",
        "bbc.co.uk",
        "linktr.ee",
        "nature.com",
        "canva.com",
        "amazon.co.uk",
        "checkpoint.com",
        "amazon.de",
        "bloomberg.com",
        "washingtonpost.com",
        "aol.com",
        "trendmicro.com",
        "cnbc.com",
        "scribd.com",
        "amazon.co.jp",
        "statista.com",
        "behance.net",
        "ft.com",
        "slideshare.net",
        "sohu.com",
        "uol.com.br",
        "dailymail.com",
        "nintendo.com",
        "usatoday.com",
        "techcrunch.com",
        "telegraph.co.uk",
        "amazon.fr",
        "amazon.ca",
        "jotform.com",
        "amazon.in",
        "yelp.com",
        "kick.com",
        "lemonde.fr",
        "amazon.es",
        "dailymail.co.uk",
        "cookielaw.org",
        "amazon.it",
        "spiegel.de",
        "cnet.com",
        "bestbuy.com",
        "apnews.com",
        "people.com",
        "ted.com",
        "theverge.com",
        "amazon.com.br",
        "corriere.it",
        "amazon.com.au",
        "amazon.co.za",
        "huffpost.com",
        "themoviedb.org",
        "amazon.com.mx",
        "ouest-france.fr",
        "theatlantic.com",
        "bfmtv.com",
        "investopedia.com",
        "marca.com",
        "nypost.com",
        "cookpad.com",
        "jimdo.com",
        "nu.nl",
        "theconversation.com",
        "aftonbladet.se",
        "ad.nl",
        "vg.no",
        "ay267.com",
        "spankbang.com",
        "amap.com",
        "furaffinity.net",
        "abc.net.au",
        "hln.be",
        "kicker.de",
        "litnet.com",
        "economist.com",
        "aljazeera.com",
        "e621.net",
        "hai8g.com",
        "dagbladet.no",
        "nikkansports.com",
        "dw.com",
        "iltalehti.fi",
        "meb.gov.tr",
        "toyhou.se",
        "cardmarket.com",
        "huffingtonpost.com",
        "ekstrabladet.dk",
        "newsweek.com",
        "omg10.com",
        "joyclub.de",
        "bblaa.com",
        "pussyspace.com",
        "shahvani.com",
        "myreadingmanga.info",
        "theqoo.net",
        "spankbang.party",
        "crn77.com",
        "tjk.org",
        "wnacg.com",
        "twkan.com",
        "dogdrip.net",
        "darkero.com",
        "publi24.ro",
        "ey43.com",
        "nya2.com",
        "manhwaweb.com"
      ],
      "provider_effect": "not_evaluated",
      "string_present_success_responses": 117,
      "syntactic_record_analyzable": 117
    },
    "content-signal": {
      "domains_with_syntactic_record": [
        "forter.com",
        "amplitude.com",
        "linktr.ee",
        "centinel-analytica.com",
        "launchpad.net",
        "weibo.com",
        "nvidia.com",
        "trendmicro.com",
        "intercom.io",
        "supabase.co",
        "patreon.com",
        "corriere.it",
        "twilio.com",
        "theatlantic.com",
        "usercentrics.eu",
        "att.com",
        "intercom.com",
        "vinted.fr",
        "oxylabs.io",
        "psu.edu",
        "dagbladet.no",
        "bbb.org",
        "otomoto.pl",
        "therapservices.net",
        "imgsrc.ru"
      ],
      "provider_effect": "not_evaluated",
      "string_present_success_responses": 51,
      "syntactic_record_analyzable": 25
    }
  },
  "version": "v2.0",
  "window": {
    "ended_at": "2026-11-10T01:18:54.938Z",
    "started_at": "2026-11-10T01:04:31.278Z"
  }
}