{ "study": "AEOeye AI Crawler Access Study", "startedAt": "2026-10-03T05:43:43.894Z", "finishedAt": "2026-10-03T05:46:35.278Z", "userAgent": "Mozilla/5.0 (compatible; AEOeyeResearch/1.0; +https://aeoeye.com/bot)", "sample": "Convenience sample of well-known brands chosen by AEOeye (SaaS, AI-visibility vendors incl. aeoeye.com, e-commerce & consumer brands) plus two contrast groups (news publishers; UGC/review sites). Not a random sample of the web. Full list: scripts/research/ai-crawler-study-domains.json", "method": [ "GET https:///robots.txt (redirects followed; www retried if the apex was unreachable). Parsed with the same parser AEOeye's SEO audit uses (robots-parser matching: longest match, * wildcards, $ anchors, Allow wins ties).", "A bot 'blocks' = its effective rules disallow the homepage path '/'. Partial blocks (e.g. Disallow: /checkout) are not counted as blocks. A bot is 'named' when robots.txt has a user-agent group for its exact product token.", "4xx robots.txt = no restrictions (RFC 9309). 5xx / unreachable robots.txt = excluded from the denominators (robotsUnavailable).", "llms.txt counts only when /llms.txt returns 200, is not HTML, stays on /llms.txt after redirects and starts with a Markdown H1. Fetched only where robots.txt allowed our user agent. 401/403/429/5xx/timeouts are 'unknown' (blocked or erroring) and excluded from the llms.txt denominator.", "Homepage fetched only where robots.txt allowed our user agent. JSON-LD @type values collected from every application/ld+json block, including @graph. 'Challenged' = 403/429 or a bot-challenge page.", "Politeness: requests per domain sequential with a 1-second gap; 8 domains at a time; 15-second timeout; self-identifying user agent." ], "botNotes": { "training": "GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Bytespider, meta-externalagent — crawl or control use of content for model training. Blocking them does not remove a site from ChatGPT search, Claude search or Google AI Overviews.", "retrieval": "OAI-SearchBot (ChatGPT search), ChatGPT-User (user-triggered fetches), Claude-SearchBot / Claude-User, PerplexityBot / Perplexity-User — fetch pages to answer and cite in real time.", "search": "Googlebot and Bingbot — classic search crawlers; Google AI Overviews / AI Mode draw on Googlebot's index, ChatGPT search and Copilot partly on Bing's.", "sources": [ "https://developers.openai.com/api/docs/bots", "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "https://docs.perplexity.ai/guides/bots", "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "https://support.apple.com/en-us/119829" ] }, "summary": { "all": { "domains": 391, "robotsUsable": 384, "robotsMissing": 33, "robotsUnavailable": 7, "blocksPct": { "GPTBot": 9.1, "ClaudeBot": 11.5, "Google-Extended": 8.1, "Applebot-Extended": 10.9, "CCBot": 11.5, "Bytespider": 14.6, "meta-externalagent": 9.4, "OAI-SearchBot": 4.2, "ChatGPT-User": 5.2, "Claude-SearchBot": 6.3, "Claude-User": 6.8, "PerplexityBot": 9.1, "Perplexity-User": 6.8, "Googlebot": 0.3, "Bingbot": 0.3 }, "namesPct": { "GPTBot": 23.4, "ClaudeBot": 23.7, "Google-Extended": 19, "Applebot-Extended": 15.1, "CCBot": 19.3, "Bytespider": 18.2, "meta-externalagent": 15.1, "OAI-SearchBot": 18.5, "ChatGPT-User": 19.3, "Claude-SearchBot": 12, "Claude-User": 13, "PerplexityBot": 22.9, "Perplexity-User": 11.5, "Googlebot": 13, "Bingbot": 10.7 }, "blocksAnyTrainingPct": 16.7, "blocksAnyRetrievalPct": 9.6, "blocksGPTBotButAllowsOAISearchBotPct": 4.9, "blocksAllForStarPct": 1.8, "contentSignalsPct": 9.4, "contentSignalAiTrainNoPct": 4.2, "sitemapDirectivePct": 84.6, "llmsChecked": 330, "llmsBlockedOrError": 46, "llmsTxtPct": 52.4, "llmsSoft404Pct": 2.7, "homeTried": 375, "homeChallengedPct": 16, "homesParsed": 312, "jsonLdAnyPct": 74.7, "organizationPct": 70.2, "webSitePct": 45.5, "faqPagePct": 10.9, "productOrSoftwarePct": 24, "sameAsPct": 60.9, "metaNoAiPct": 0 }, "aeo-vendors": { "domains": 43, "robotsUsable": 43, "robotsMissing": 2, "robotsUnavailable": 0, "blocksPct": { "GPTBot": 0, "ClaudeBot": 0, "Google-Extended": 0, "Applebot-Extended": 0, "CCBot": 0, "Bytespider": 0, "meta-externalagent": 0, "OAI-SearchBot": 0, "ChatGPT-User": 0, "Claude-SearchBot": 0, "Claude-User": 0, "PerplexityBot": 0, "Perplexity-User": 0, "Googlebot": 0, "Bingbot": 0 }, "namesPct": { "GPTBot": 25.6, "ClaudeBot": 20.9, "Google-Extended": 23.3, "Applebot-Extended": 14, "CCBot": 20.9, "Bytespider": 7, "meta-externalagent": 7, "OAI-SearchBot": 14, "ChatGPT-User": 18.6, "Claude-SearchBot": 7, "Claude-User": 7, "PerplexityBot": 23.3, "Perplexity-User": 9.3, "Googlebot": 18.6, "Bingbot": 14 }, "blocksAnyTrainingPct": 0, "blocksAnyRetrievalPct": 0, "blocksGPTBotButAllowsOAISearchBotPct": 0, "blocksAllForStarPct": 0, "contentSignalsPct": 2.3, "contentSignalAiTrainNoPct": 0, "sitemapDirectivePct": 90.7, "llmsChecked": 41, "llmsBlockedOrError": 2, "llmsTxtPct": 65.9, "llmsSoft404Pct": 4.9, "homeTried": 43, "homeChallengedPct": 2.3, "homesParsed": 42, "jsonLdAnyPct": 73.8, "organizationPct": 69, "webSitePct": 57.1, "faqPagePct": 42.9, "productOrSoftwarePct": 54.8, "sameAsPct": 54.8, "metaNoAiPct": 0 }, "ecommerce-consumer": { "domains": 95, "robotsUsable": 90, "robotsMissing": 16, "robotsUnavailable": 5, "blocksPct": { "GPTBot": 5.6, "ClaudeBot": 5.6, "Google-Extended": 4.4, "Applebot-Extended": 5.6, "CCBot": 5.6, "Bytespider": 6.7, "meta-externalagent": 5.6, "OAI-SearchBot": 1.1, "ChatGPT-User": 1.1, "Claude-SearchBot": 2.2, "Claude-User": 2.2, "PerplexityBot": 4.4, "Perplexity-User": 2.2, "Googlebot": 0, "Bingbot": 0 }, "namesPct": { "GPTBot": 17.8, "ClaudeBot": 16.7, "Google-Extended": 14.4, "Applebot-Extended": 6.7, "CCBot": 12.2, "Bytespider": 14.4, "meta-externalagent": 14.4, "OAI-SearchBot": 20, "ChatGPT-User": 21.1, "Claude-SearchBot": 10, "Claude-User": 13.3, "PerplexityBot": 20, "Perplexity-User": 10, "Googlebot": 20, "Bingbot": 13.3 }, "blocksAnyTrainingPct": 10, "blocksAnyRetrievalPct": 4.4, "blocksGPTBotButAllowsOAISearchBotPct": 4.4, "blocksAllForStarPct": 1.1, "contentSignalsPct": 1.1, "contentSignalAiTrainNoPct": 1.1, "sitemapDirectivePct": 75.6, "llmsChecked": 70, "llmsBlockedOrError": 19, "llmsTxtPct": 47.1, "llmsSoft404Pct": 4.3, "homeTried": 88, "homeChallengedPct": 27.3, "homesParsed": 61, "jsonLdAnyPct": 63.9, "organizationPct": 57.4, "webSitePct": 32.8, "faqPagePct": 4.9, "productOrSoftwarePct": 3.3, "sameAsPct": 45.9, "metaNoAiPct": 0 }, "publishers": { "domains": 39, "robotsUsable": 38, "robotsMissing": 1, "robotsUnavailable": 1, "blocksPct": { "GPTBot": 52.6, "ClaudeBot": 78.9, "Google-Extended": 52.6, "Applebot-Extended": 73.7, "CCBot": 76.3, "Bytespider": 78.9, "meta-externalagent": 63.2, "OAI-SearchBot": 28.9, "ChatGPT-User": 36.8, "Claude-SearchBot": 50, "Claude-User": 52.6, "PerplexityBot": 68.4, "Perplexity-User": 52.6, "Googlebot": 0, "Bingbot": 0 }, "namesPct": { "GPTBot": 60.5, "ClaudeBot": 73.7, "Google-Extended": 50, "Applebot-Extended": 71.1, "CCBot": 71.1, "Bytespider": 73.7, "meta-externalagent": 57.9, "OAI-SearchBot": 42.1, "ChatGPT-User": 47.4, "Claude-SearchBot": 50, "Claude-User": 52.6, "PerplexityBot": 63.2, "Perplexity-User": 47.4, "Googlebot": 21.1, "Bingbot": 7.9 }, "blocksAnyTrainingPct": 84.2, "blocksAnyRetrievalPct": 71.1, "blocksGPTBotButAllowsOAISearchBotPct": 23.7, "blocksAllForStarPct": 5.3, "contentSignalsPct": 2.6, "contentSignalAiTrainNoPct": 2.6, "sitemapDirectivePct": 97.4, "llmsChecked": 29, "llmsBlockedOrError": 7, "llmsTxtPct": 3.4, "llmsSoft404Pct": 3.4, "homeTried": 35, "homeChallengedPct": 22.9, "homesParsed": 27, "jsonLdAnyPct": 88.9, "organizationPct": 88.9, "webSitePct": 66.7, "faqPagePct": 0, "productOrSoftwarePct": 3.7, "sameAsPct": 59.3, "metaNoAiPct": 0 }, "saas": { "domains": 194, "robotsUsable": 193, "robotsMissing": 11, "robotsUnavailable": 1, "blocksPct": { "GPTBot": 2.1, "ClaudeBot": 1.6, "Google-Extended": 1, "Applebot-Extended": 1, "CCBot": 2.6, "Bytespider": 4.7, "meta-externalagent": 1.6, "OAI-SearchBot": 0.5, "ChatGPT-User": 0.5, "Claude-SearchBot": 0, "Claude-User": 0, "PerplexityBot": 0.5, "Perplexity-User": 0, "Googlebot": 0, "Bingbot": 0 }, "namesPct": { "GPTBot": 17.1, "ClaudeBot": 16.6, "Google-Extended": 12.4, "Applebot-Extended": 7.3, "CCBot": 10.9, "Bytespider": 8.8, "meta-externalagent": 7.8, "OAI-SearchBot": 13.5, "ChatGPT-User": 12.4, "Claude-SearchBot": 5.7, "Claude-User": 5.7, "PerplexityBot": 16.1, "Perplexity-User": 5.7, "Googlebot": 6.2, "Bingbot": 7.8 }, "blocksAnyTrainingPct": 6.2, "blocksAnyRetrievalPct": 0.5, "blocksGPTBotButAllowsOAISearchBotPct": 1.6, "blocksAllForStarPct": 0, "contentSignalsPct": 17.1, "contentSignalAiTrainNoPct": 7.3, "sitemapDirectivePct": 88.1, "llmsChecked": 180, "llmsBlockedOrError": 13, "llmsTxtPct": 60, "llmsSoft404Pct": 1.7, "homeTried": 193, "homeChallengedPct": 8.8, "homesParsed": 176, "jsonLdAnyPct": 77.3, "organizationPct": 73.3, "webSitePct": 44.3, "faqPagePct": 7.4, "productOrSoftwarePct": 27.8, "sameAsPct": 68.8, "metaNoAiPct": 0 }, "ugc-review": { "domains": 20, "robotsUsable": 20, "robotsMissing": 3, "robotsUnavailable": 0, "blocksPct": { "GPTBot": 30, "ClaudeBot": 30, "Google-Extended": 25, "Applebot-Extended": 35, "CCBot": 25, "Bytespider": 55, "meta-externalagent": 20, "OAI-SearchBot": 15, "ChatGPT-User": 20, "Claude-SearchBot": 15, "Claude-User": 20, "PerplexityBot": 20, "Perplexity-User": 20, "Googlebot": 5, "Bingbot": 5 }, "namesPct": { "GPTBot": 35, "ClaudeBot": 35, "Google-Extended": 35, "Applebot-Extended": 25, "CCBot": 30, "Bytespider": 45, "meta-externalagent": 25, "OAI-SearchBot": 25, "ChatGPT-User": 25, "Claude-SearchBot": 20, "Claude-User": 20, "PerplexityBot": 25, "Perplexity-User": 10, "Googlebot": 20, "Bingbot": 25 }, "blocksAnyTrainingPct": 55, "blocksAnyRetrievalPct": 25, "blocksGPTBotButAllowsOAISearchBotPct": 15, "blocksAllForStarPct": 20, "contentSignalsPct": 0, "contentSignalAiTrainNoPct": 0, "sitemapDirectivePct": 55, "llmsChecked": 10, "llmsBlockedOrError": 5, "llmsTxtPct": 40, "llmsSoft404Pct": 0, "homeTried": 16, "homeChallengedPct": 62.5, "homesParsed": 6, "jsonLdAnyPct": 50, "organizationPct": 33.3, "webSitePct": 33.3, "faqPagePct": 0, "productOrSoftwarePct": 0, "sameAsPct": 33.3, "metaNoAiPct": 0 } } }