[
  {
    "group": "search",
    "key": "oai_searchbot",
    "kind": "crawler",
    "name": "OAI-SearchBot",
    "owner": "OpenAI",
    "purposes": [
      "search_index"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://developers.openai.com/api/docs/bots",
    "token": "oai-searchbot"
  },
  {
    "group": "user",
    "key": "chatgpt_user",
    "kind": "fetcher",
    "name": "ChatGPT-User",
    "owner": "OpenAI",
    "purposes": [
      "user_fetch"
    ],
    "robots_applicability": "user_dependent",
    "source_checked_at": "2026-11-09",
    "source_url": "https://developers.openai.com/api/docs/bots",
    "token": "chatgpt-user"
  },
  {
    "group": "data",
    "key": "gptbot",
    "kind": "crawler",
    "name": "GPTBot",
    "owner": "OpenAI",
    "purposes": [
      "training"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://developers.openai.com/api/docs/bots",
    "token": "gptbot"
  },
  {
    "group": "baseline",
    "key": "googlebot",
    "kind": "crawler",
    "name": "Googlebot",
    "owner": "Google",
    "purposes": [
      "search_index"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
    "token": "googlebot"
  },
  {
    "group": "control",
    "key": "google_extended",
    "kind": "control_token",
    "name": "Google-Extended",
    "owner": "Google",
    "purposes": [
      "training",
      "grounding"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
    "token": "google-extended"
  },
  {
    "group": "search",
    "key": "claude_searchbot",
    "kind": "crawler",
    "name": "Claude-SearchBot",
    "owner": "Anthropic",
    "purposes": [
      "search_index"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
    "token": "claude-searchbot"
  },
  {
    "group": "user",
    "key": "claude_user",
    "kind": "fetcher",
    "name": "Claude-User",
    "owner": "Anthropic",
    "purposes": [
      "user_fetch"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
    "token": "claude-user"
  },
  {
    "group": "data",
    "key": "claudebot",
    "kind": "crawler",
    "name": "ClaudeBot",
    "owner": "Anthropic",
    "purposes": [
      "training"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
    "token": "claudebot"
  },
  {
    "group": "search",
    "key": "perplexitybot",
    "kind": "crawler",
    "name": "PerplexityBot",
    "owner": "Perplexity AI",
    "purposes": [
      "search_index"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
    "token": "perplexitybot"
  },
  {
    "group": "user",
    "key": "perplexity_user",
    "kind": "fetcher",
    "name": "Perplexity-User",
    "owner": "Perplexity AI",
    "purposes": [
      "user_fetch"
    ],
    "robots_applicability": "generally_not_applied",
    "source_checked_at": "2026-11-09",
    "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
    "token": "perplexity-user"
  },
  {
    "group": "baseline",
    "key": "bingbot",
    "kind": "crawler",
    "name": "Bingbot",
    "owner": "Microsoft",
    "purposes": [
      "search_index"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec",
    "token": "bingbot"
  },
  {
    "group": "control",
    "key": "applebot_extended",
    "kind": "control_token",
    "name": "Applebot-Extended",
    "owner": "Apple",
    "purposes": [
      "training_preference"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": null,
    "source_url": "https://support.apple.com/en-us/119829",
    "token": "applebot-extended"
  },
  {
    "group": "data",
    "key": "bytespider",
    "kind": "crawler",
    "name": "Bytespider",
    "owner": "ByteDance / Doubao",
    "purposes": [
      "data_collection_unverified"
    ],
    "robots_applicability": "not_verified",
    "source_checked_at": null,
    "source_url": null,
    "token": "bytespider"
  },
  {
    "group": "data",
    "key": "ccbot",
    "kind": "crawler",
    "name": "CCBot",
    "owner": "Common Crawl",
    "purposes": [
      "open_dataset"
    ],
    "robots_applicability": "honored_declared",
    "source_checked_at": "2026-11-09",
    "source_url": "https://commoncrawl.org/ccbot",
    "token": "ccbot"
  }
]