{
  "version": "v1.1",
  "published": "2026-11-09",
  "modified": "2026-11-09",
  "scope": "archived_classifier_root_path_only",
  "source_sha256": "e11f05b0d3b4d1e472160a1100f9ee6153608c3c86ed56414b7df197d8cdffb4",
  "sample": {
    "total": 1000,
    "columns": 64,
    "historical_parser_subset": 891,
    "outside_subset": 109,
    "rank_min": 1,
    "rank_max": 1726
  },
  "ai_token_count": 12,
  "all_token_count": 14,
  "ai_any": {
    "numerator": 261,
    "denominator": 891,
    "percent": "29.3"
  },
  "legacy_all14_any": {
    "numerator": 263,
    "denominator": 891,
    "percent": "29.5"
  },
  "bots": [
    {
      "key": "oai_searchbot",
      "name": "OAI-SearchBot",
      "token": "oai-searchbot",
      "owner": "OpenAI",
      "kind": "crawler",
      "group": "search",
      "purposes": [
        "search_index"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 84,
        "denominator": 891,
        "percent": "9.4"
      }
    },
    {
      "key": "chatgpt_user",
      "name": "ChatGPT-User",
      "token": "chatgpt-user",
      "owner": "OpenAI",
      "kind": "fetcher",
      "group": "user",
      "purposes": [
        "user_fetch"
      ],
      "robots_applicability": "user_dependent",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 105,
        "denominator": 891,
        "percent": "11.8"
      }
    },
    {
      "key": "gptbot",
      "name": "GPTBot",
      "token": "gptbot",
      "owner": "OpenAI",
      "kind": "crawler",
      "group": "data",
      "purposes": [
        "training"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 190,
        "denominator": 891,
        "percent": "21.3"
      }
    },
    {
      "key": "googlebot",
      "name": "Googlebot",
      "token": "googlebot",
      "owner": "Google",
      "kind": "crawler",
      "group": "baseline",
      "purposes": [
        "search_index"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 13,
        "denominator": 891,
        "percent": "1.5"
      }
    },
    {
      "key": "google_extended",
      "name": "Google-Extended",
      "token": "google-extended",
      "owner": "Google",
      "kind": "control_token",
      "group": "control",
      "purposes": [
        "training",
        "grounding"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 161,
        "denominator": 891,
        "percent": "18.1"
      }
    },
    {
      "key": "claude_searchbot",
      "name": "Claude-SearchBot",
      "token": "claude-searchbot",
      "owner": "Anthropic",
      "kind": "crawler",
      "group": "search",
      "purposes": [
        "search_index"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 96,
        "denominator": 891,
        "percent": "10.8"
      }
    },
    {
      "key": "claude_user",
      "name": "Claude-User",
      "token": "claude-user",
      "owner": "Anthropic",
      "kind": "fetcher",
      "group": "user",
      "purposes": [
        "user_fetch"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 99,
        "denominator": 891,
        "percent": "11.1"
      }
    },
    {
      "key": "claudebot",
      "name": "ClaudeBot",
      "token": "claudebot",
      "owner": "Anthropic",
      "kind": "crawler",
      "group": "data",
      "purposes": [
        "training"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 185,
        "denominator": 891,
        "percent": "20.8"
      }
    },
    {
      "key": "perplexitybot",
      "name": "PerplexityBot",
      "token": "perplexitybot",
      "owner": "Perplexity AI",
      "kind": "crawler",
      "group": "search",
      "purposes": [
        "search_index"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 130,
        "denominator": 891,
        "percent": "14.6"
      }
    },
    {
      "key": "perplexity_user",
      "name": "Perplexity-User",
      "token": "perplexity-user",
      "owner": "Perplexity AI",
      "kind": "fetcher",
      "group": "user",
      "purposes": [
        "user_fetch"
      ],
      "robots_applicability": "generally_not_applied",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 106,
        "denominator": 891,
        "percent": "11.9"
      }
    },
    {
      "key": "bingbot",
      "name": "Bingbot",
      "token": "bingbot",
      "owner": "Microsoft",
      "kind": "crawler",
      "group": "baseline",
      "purposes": [
        "search_index"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 18,
        "denominator": 891,
        "percent": "2.0"
      }
    },
    {
      "key": "applebot_extended",
      "name": "Applebot-Extended",
      "token": "applebot-extended",
      "owner": "Apple",
      "kind": "control_token",
      "group": "control",
      "purposes": [
        "training_preference"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://support.apple.com/en-us/119829",
      "source_checked_at": null,
      "restricted": {
        "numerator": 156,
        "denominator": 891,
        "percent": "17.5"
      }
    },
    {
      "key": "bytespider",
      "name": "Bytespider",
      "token": "bytespider",
      "owner": "ByteDance / Doubao",
      "kind": "crawler",
      "group": "data",
      "purposes": [
        "data_collection_unverified"
      ],
      "robots_applicability": "not_verified",
      "source_url": null,
      "source_checked_at": null,
      "restricted": {
        "numerator": 208,
        "denominator": 891,
        "percent": "23.3"
      }
    },
    {
      "key": "ccbot",
      "name": "CCBot",
      "token": "ccbot",
      "owner": "Common Crawl",
      "kind": "crawler",
      "group": "data",
      "purposes": [
        "open_dataset"
      ],
      "robots_applicability": "honored_declared",
      "source_url": "https://commoncrawl.org/ccbot",
      "source_checked_at": "2026-11-09",
      "restricted": {
        "numerator": 211,
        "denominator": 891,
        "percent": "23.7"
      }
    }
  ],
  "pairs": [
    {
      "first": "gptbot",
      "second": "oai_searchbot",
      "first_name": "GPTBot",
      "second_name": "OAI-SearchBot",
      "cells": [
        {
          "first_restricted": true,
          "second_restricted": true,
          "numerator": 84,
          "denominator": 891,
          "percent": "9.4"
        },
        {
          "first_restricted": true,
          "second_restricted": false,
          "numerator": 106,
          "denominator": 891,
          "percent": "11.9"
        },
        {
          "first_restricted": false,
          "second_restricted": true,
          "numerator": 0,
          "denominator": 891,
          "percent": "0.0"
        },
        {
          "first_restricted": false,
          "second_restricted": false,
          "numerator": 701,
          "denominator": 891,
          "percent": "78.7"
        }
      ],
      "first_restricted_second_not": {
        "numerator": 106,
        "denominator": 190,
        "percent": "55.8"
      }
    },
    {
      "first": "claudebot",
      "second": "claude_searchbot",
      "first_name": "ClaudeBot",
      "second_name": "Claude-SearchBot",
      "cells": [
        {
          "first_restricted": true,
          "second_restricted": true,
          "numerator": 96,
          "denominator": 891,
          "percent": "10.8"
        },
        {
          "first_restricted": true,
          "second_restricted": false,
          "numerator": 89,
          "denominator": 891,
          "percent": "10.0"
        },
        {
          "first_restricted": false,
          "second_restricted": true,
          "numerator": 0,
          "denominator": 891,
          "percent": "0.0"
        },
        {
          "first_restricted": false,
          "second_restricted": false,
          "numerator": 706,
          "denominator": 891,
          "percent": "79.2"
        }
      ],
      "first_restricted_second_not": {
        "numerator": 89,
        "denominator": 185,
        "percent": "48.1"
      }
    }
  ],
  "rank_buckets": [
    {
      "label": "1–100",
      "n": 41,
      "ai_any": {
        "numerator": 16,
        "denominator": 41,
        "percent": "39.0"
      },
      "gptbot": {
        "numerator": 12,
        "denominator": 41,
        "percent": "29.3"
      },
      "llms": {
        "numerator": 7,
        "denominator": 41,
        "percent": "17.1"
      }
    },
    {
      "label": "101–500",
      "n": 173,
      "ai_any": {
        "numerator": 54,
        "denominator": 173,
        "percent": "31.2"
      },
      "gptbot": {
        "numerator": 35,
        "denominator": 173,
        "percent": "20.2"
      },
      "llms": {
        "numerator": 47,
        "denominator": 173,
        "percent": "27.2"
      }
    },
    {
      "label": "501–1000",
      "n": 262,
      "ai_any": {
        "numerator": 80,
        "denominator": 262,
        "percent": "30.5"
      },
      "gptbot": {
        "numerator": 60,
        "denominator": 262,
        "percent": "22.9"
      },
      "llms": {
        "numerator": 60,
        "denominator": 262,
        "percent": "22.9"
      }
    },
    {
      "label": "1001+",
      "n": 415,
      "ai_any": {
        "numerator": 111,
        "denominator": 415,
        "percent": "26.7"
      },
      "gptbot": {
        "numerator": 83,
        "denominator": 415,
        "percent": "20.0"
      },
      "llms": {
        "numerator": 58,
        "denominator": 415,
        "percent": "14.0"
      }
    }
  ],
  "distribution": [
    {
      "label": "0",
      "n": 630
    },
    {
      "label": "1–3",
      "n": 67
    },
    {
      "label": "4–7",
      "n": 95
    },
    {
      "label": "8–11",
      "n": 49
    },
    {
      "label": "12",
      "n": 50
    }
  ],
  "llms": {
    "heuristic_detected_all": {
      "numerator": 172,
      "denominator": 1000,
      "percent": "17.2"
    },
    "comparable_subset": 891,
    "groups": [
      {
        "detected": true,
        "n": 172,
        "restricted": {
          "numerator": 23,
          "denominator": 172,
          "percent": "13.4"
        },
        "not_restricted": 149
      },
      {
        "detected": false,
        "n": 719,
        "restricted": {
          "numerator": 238,
          "denominator": 719,
          "percent": "33.1"
        },
        "not_restricted": 481
      }
    ],
    "legacy_all14_groups": [
      {
        "detected": true,
        "restricted": {
          "numerator": 23,
          "denominator": 172,
          "percent": "13.4"
        }
      },
      {
        "detected": false,
        "restricted": {
          "numerator": 240,
          "denominator": 719,
          "percent": "33.4"
        }
      }
    ]
  },
  "ai_named": 235,
  "ai_wildcard_only": 26,
  "legacy_all14_named": 238,
  "legacy_all14_all_restricted": 12,
  "legacy_all14_none_restricted": 628,
  "operational": {
    "historical_redirects": {
      "numerator": 583,
      "denominator": 891,
      "percent": "65.4"
    },
    "historical_median_bytes": 1289
  }
}