{ "collected": "2026-08-06T19:43:01+00:00", "method": "One GET to https:///robots.txt per site. Parsed with Scout's own RFC 9309 robots parser (longest-match, wildcard and $-anchor support). A site is counted as blocking a crawler when that crawler is disallowed from '/'. Sites whose robots.txt is unreachable are excluded from percentages and reported separately.", "summary": { "n": 98, "attempted": 110, "unreachable": 3, "no_robots_txt": 9, "mentions_any_ai_bot": 42, "blocking_any_citation_bot": 29, "blocking_any_training_bot": 37, "blocking_a_search_engine": 4, "has_sitemap_directive": 79, "uses_content_signal": 8, "by_bot": { "GPTBot": { "blocked": 24, "pct": 24.5, "owner": "OpenAI", "purpose": "training" }, "OAI-SearchBot": { "blocked": 11, "pct": 11.2, "owner": "OpenAI", "purpose": "search index" }, "ChatGPT-User": { "blocked": 15, "pct": 15.3, "owner": "OpenAI", "purpose": "live fetch" }, "ClaudeBot": { "blocked": 32, "pct": 32.7, "owner": "Anthropic", "purpose": "training" }, "Claude-SearchBot": { "blocked": 20, "pct": 20.4, "owner": "Anthropic", "purpose": "search index" }, "Claude-User": { "blocked": 20, "pct": 20.4, "owner": "Anthropic", "purpose": "live fetch" }, "PerplexityBot": { "blocked": 25, "pct": 25.5, "owner": "Perplexity", "purpose": "search index" }, "Perplexity-User": { "blocked": 18, "pct": 18.4, "owner": "Perplexity", "purpose": "live fetch" }, "Google-Extended": { "blocked": 23, "pct": 23.5, "owner": "Google", "purpose": "Gemini grounding" }, "Applebot-Extended": { "blocked": 27, "pct": 27.6, "owner": "Apple", "purpose": "training" }, "Amazonbot": { "blocked": 23, "pct": 23.5, "owner": "Amazon", "purpose": "search index" }, "Bytespider": { "blocked": 31, "pct": 31.6, "owner": "ByteDance", "purpose": "training" }, "meta-externalagent": { "blocked": 27, "pct": 27.6, "owner": "Meta", "purpose": "training" }, "CCBot": { "blocked": 33, "pct": 33.7, "owner": "Common Crawl", "purpose": "open corpus" } }, "by_category": { "news": { "n": 19, "blocking_citation": 18, "pct": 94.7 }, "ecommerce": { "n": 15, "blocking_citation": 2, "pct": 13.3 }, "saas": { "n": 20, "blocking_citation": 1, "pct": 5.0 }, "seo-tools": { "n": 10, "blocking_citation": 0, "pct": 0.0 }, "local": { "n": 11, "blocking_citation": 1, "pct": 9.1 }, "reference": { "n": 14, "blocking_citation": 7, "pct": 50.0 }, "nonprofit": { "n": 9, "blocking_citation": 0, "pct": 0.0 } }, "most_blocked": [ [ "CCBot", { "blocked": 33, "pct": 33.7, "owner": "Common Crawl", "purpose": "open corpus" } ], [ "ClaudeBot", { "blocked": 32, "pct": 32.7, "owner": "Anthropic", "purpose": "training" } ], [ "Bytespider", { "blocked": 31, "pct": 31.6, "owner": "ByteDance", "purpose": "training" } ], [ "Applebot-Extended", { "blocked": 27, "pct": 27.6, "owner": "Apple", "purpose": "training" } ], [ "meta-externalagent", { "blocked": 27, "pct": 27.6, "owner": "Meta", "purpose": "training" } ] ] }, "records": [ { "host": "nytimes.com", "category": "news", "status": 200, "error": "", "bytes": 8259, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "theguardian.com", "category": "news", "status": 200, "error": "", "bytes": 2873, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "bbc.co.uk", "category": "news", "status": 200, "error": "", "bytes": 4907, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "reuters.com", "category": "news", "status": 200, "error": "", "bytes": 3797, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": true, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "apnews.com", "category": "news", "status": 200, "error": "", "bytes": 1238, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": true, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "CCBot", "ClaudeBot", "GPTBot" ], "blocks_search": [] }, { "host": "washingtonpost.com", "category": "news", "status": 200, "error": "", "bytes": 5783, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": false, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "wsj.com", "category": "news", "status": 200, "error": "", "bytes": 4309, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": true, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "bloomberg.com", "category": "news", "status": 200, "error": "", "bytes": 7048, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": true, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "ft.com", "category": "news", "status": 200, "error": "", "bytes": 1556, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "economist.com", "category": "news", "status": 200, "error": "", "bytes": 2313, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot" ], "blocks_search": [] }, { "host": "cnn.com", "category": "news", "status": 200, "error": "", "bytes": 3456, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot" ], "blocks_search": [] }, { "host": "forbes.com", "category": "news", "status": 200, "error": "", "bytes": 10719, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": false, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "theatlantic.com", "category": "news", "status": 200, "error": "", "bytes": 3649, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "search=yes, ai-input=no, ai-train=no", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": false }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [ "Applebot" ] }, { "host": "wired.com", "category": "news", "status": 200, "error": "", "bytes": 1994, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "arstechnica.com", "category": "news", "status": 200, "error": "", "bytes": 1912, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "techcrunch.com", "category": "news", "status": 200, "error": "", "bytes": 710, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Google-Extended" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot" ], "blocks_search": [] }, { "host": "theverge.com", "category": "news", "status": 200, "error": "", "bytes": 5650, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "vox.com", "category": "news", "status": 200, "error": "", "bytes": 5648, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "npr.org", "category": "news", "status": 0, "error": "TimeoutError: The read operation timed out", "bytes": 0, "reachable": false }, { "host": "time.com", "category": "news", "status": 200, "error": "", "bytes": 2326, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "amazon.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 7887, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "etsy.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 52775, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "ebay.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 18810, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": false, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "walmart.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 3584, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "target.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 3226, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "wayfair.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 3224, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "chewy.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 2513, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": false, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "allbirds.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 5495, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "gymshark.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 3656, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "warbyparker.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 1117, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": false, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "Bytespider", "CCBot" ], "blocks_search": [] }, { "host": "glossier.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 3632, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "patagonia.com", "category": "ecommerce", "status": 404, "error": "", "bytes": 10, "reachable": true, "has_robots": false, "served_with_status": 404, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "rei.com", "category": "ecommerce", "status": 403, "error": "", "bytes": 371, "reachable": true, "has_robots": false, "served_with_status": 403, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "nike.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 3503, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "ikea.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 4060, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "zara.com", "category": "ecommerce", "status": 403, "error": "", "bytes": 370, "reachable": true, "has_robots": false, "served_with_status": 403, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "asos.com", "category": "ecommerce", "status": 0, "error": "TimeoutError: The read operation timed out", "bytes": 0, "reachable": false }, { "host": "lush.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 8999, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "madewell.com", "category": "ecommerce", "status": 200, "error": "", "bytes": 2763, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "bombas.com", "category": "ecommerce", "status": 429, "error": "", "bytes": 33808, "reachable": true, "has_robots": false, "served_with_status": 429, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "stripe.com", "category": "saas", "status": 200, "error": "", "bytes": 643, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "notion.so", "category": "saas", "status": 200, "error": "", "bytes": 1136, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": false, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "figma.com", "category": "saas", "status": 200, "error": "", "bytes": 372698, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": false, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Google-Extended" ], "blocks_training_bots": [ "CCBot", "GPTBot" ], "blocks_search": [] }, { "host": "slack.com", "category": "saas", "status": 200, "error": "", "bytes": 520, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "zoom.us", "category": "saas", "status": 200, "error": "", "bytes": 2005, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "hubspot.com", "category": "saas", "status": 200, "error": "", "bytes": 12606, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "mailchimp.com", "category": "saas", "status": 200, "error": "", "bytes": 101, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "shopify.com", "category": "saas", "status": 200, "error": "", "bytes": 62, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "squarespace.com", "category": "saas", "status": 200, "error": "", "bytes": 2379, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "wix.com", "category": "saas", "status": 200, "error": "", "bytes": 2797, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "webflow.com", "category": "saas", "status": 200, "error": "", "bytes": 279, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "ai-train=yes, search=yes, ai-input=yes", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "canva.com", "category": "saas", "status": 200, "error": "", "bytes": 6324, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "airtable.com", "category": "saas", "status": 200, "error": "", "bytes": 506, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "asana.com", "category": "saas", "status": 200, "error": "", "bytes": 547, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "monday.com", "category": "saas", "status": 200, "error": "", "bytes": 2701, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "ai-train=yes, search=yes, ai-input=yes", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "intercom.com", "category": "saas", "status": 200, "error": "", "bytes": 297, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "ai-train=yes, search=yes, ai-input=yes", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "zendesk.com", "category": "saas", "status": 200, "error": "", "bytes": 1028, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "calendly.com", "category": "saas", "status": 200, "error": "", "bytes": 336, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "CCBot" ], "blocks_search": [] }, { "host": "linear.app", "category": "saas", "status": 200, "error": "", "bytes": 107, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "vercel.com", "category": "saas", "status": 200, "error": "", "bytes": 504, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "search=yes, ai-input=yes, ai-train=no", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "ahrefs.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 1006, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "semrush.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 3581, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "moz.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 1210, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "screamingfrog.co.uk", "category": "seo-tools", "status": 200, "error": "", "bytes": 1883, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "sitebulb.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 237, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "searchenginejournal.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 2020, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "searchengineland.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 170, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "backlinko.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 418, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "neilpatel.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 591, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "ai-train=yes, search=yes, ai-input=yes", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "seroundtable.com", "category": "seo-tools", "status": 200, "error": "", "bytes": 90, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "rotorooter.com", "category": "local", "status": 200, "error": "", "bytes": 575, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "servicemaster.com", "category": "local", "status": 200, "error": "", "bytes": 195, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "terminix.com", "category": "local", "status": 200, "error": "", "bytes": 2301, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "mrhandyman.com", "category": "local", "status": 200, "error": "", "bytes": 292, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "molymaid.com", "category": "local", "status": 0, "error": "ConnectionRefusedError: [Errno 61] Connection refused", "bytes": 0, "reachable": false }, { "host": "kumon.com", "category": "local", "status": 200, "error": "", "bytes": 1094, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "massageenvy.com", "category": "local", "status": 200, "error": "", "bytes": 87, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "supercuts.com", "category": "local", "status": 200, "error": "", "bytes": 3498, "reachable": true, "has_robots": false, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "jiffylube.com", "category": "local", "status": 200, "error": "", "bytes": 268, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "midas.com", "category": "local", "status": 200, "error": "", "bytes": 1183, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "hrblock.com", "category": "local", "status": 403, "error": "", "bytes": 383, "reachable": true, "has_robots": false, "served_with_status": 403, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "katzsdelicatessen.com", "category": "local", "status": 200, "error": "", "bytes": 1875, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "shakeshack.com", "category": "local", "status": 200, "error": "", "bytes": 3863, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "search=yes,ai-train=no,use=reference", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Google-Extended" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "chipotle.com", "category": "local", "status": 404, "error": "", "bytes": 43083, "reachable": true, "has_robots": false, "served_with_status": 404, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "sweetgreen.com", "category": "local", "status": 200, "error": "", "bytes": 62, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "wikipedia.org", "category": "reference", "status": 200, "error": "", "bytes": 28275, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "stackoverflow.com", "category": "reference", "status": 418, "error": "", "bytes": 113, "reachable": true, "has_robots": true, "served_with_status": 418, "content_signal": "search=no, ai-train=no", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": false, "Bingbot": false, "DuckDuckBot": false, "Applebot": false }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [ "Applebot", "Bingbot", "DuckDuckBot", "Googlebot" ] }, { "host": "github.com", "category": "reference", "status": 200, "error": "", "bytes": 2274, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "medium.com", "category": "reference", "status": 200, "error": "", "bytes": 884, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "substack.com", "category": "reference", "status": 200, "error": "", "bytes": 585, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "quora.com", "category": "reference", "status": 200, "error": "", "bytes": 18792, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": true, "Bytespider": false, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "ClaudeBot", "GPTBot" ], "blocks_search": [] }, { "host": "reddit.com", "category": "reference", "status": 200, "error": "", "bytes": 538, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": false, "Bingbot": false, "DuckDuckBot": false, "Applebot": false }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [ "Applebot", "Bingbot", "DuckDuckBot", "Googlebot" ] }, { "host": "imdb.com", "category": "reference", "status": 200, "error": "", "bytes": 1911, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "webmd.com", "category": "reference", "status": 200, "error": "", "bytes": 926, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "CCBot", "ClaudeBot", "GPTBot" ], "blocks_search": [] }, { "host": "mayoclinic.org", "category": "reference", "status": 403, "error": "", "bytes": 386, "reachable": true, "has_robots": false, "served_with_status": 403, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "healthline.com", "category": "reference", "status": 200, "error": "", "bytes": 8864, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "investopedia.com", "category": "reference", "status": 200, "error": "", "bytes": 3443, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "nerdwallet.com", "category": "reference", "status": 200, "error": "", "bytes": 1720, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": false, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Google-Extended" ], "blocks_training_bots": [], "blocks_search": [] }, { "host": "tripadvisor.com", "category": "reference", "status": 200, "error": "", "bytes": 25115, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [] }, { "host": "yelp.com", "category": "reference", "status": 200, "error": "", "bytes": 11825, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "PerplexityBot": false, "Perplexity-User": false, "Google-Extended": false, "Applebot-Extended": false, "Amazonbot": false, "Bytespider": false, "meta-externalagent": false, "CCBot": false }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": false, "Applebot": false }, "has_sitemap_directive": false, "mentions_any_ai_bot": true, "blocks_citation_bots": [ "Claude-SearchBot", "Google-Extended", "OAI-SearchBot", "PerplexityBot" ], "blocks_training_bots": [ "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "GPTBot", "meta-externalagent" ], "blocks_search": [ "Applebot", "DuckDuckBot" ] }, { "host": "khanacademy.org", "category": "nonprofit", "status": 200, "error": "", "bytes": 1129, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "GPTBot" ], "blocks_search": [] }, { "host": "coursera.org", "category": "nonprofit", "status": 200, "error": "", "bytes": 1291, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": false, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": true, "blocks_citation_bots": [], "blocks_training_bots": [ "meta-externalagent" ], "blocks_search": [] }, { "host": "edx.org", "category": "nonprofit", "status": 200, "error": "", "bytes": 1550, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "redcross.org", "category": "nonprofit", "status": 403, "error": "", "bytes": 384, "reachable": true, "has_robots": false, "served_with_status": 403, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "wwf.org.uk", "category": "nonprofit", "status": 200, "error": "", "bytes": 1073, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "unicef.org", "category": "nonprofit", "status": 200, "error": "", "bytes": 1945, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "doctorswithoutborders.org", "category": "nonprofit", "status": 200, "error": "", "bytes": 1667, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "habitat.org", "category": "nonprofit", "status": 200, "error": "", "bytes": 2189, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "mit.edu", "category": "nonprofit", "status": 200, "error": "", "bytes": 199, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": false, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] }, { "host": "stanford.edu", "category": "nonprofit", "status": 200, "error": "", "bytes": 197, "reachable": true, "has_robots": true, "served_with_status": 200, "content_signal": "", "ai_access": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "PerplexityBot": true, "Perplexity-User": true, "Google-Extended": true, "Applebot-Extended": true, "Amazonbot": true, "Bytespider": true, "meta-externalagent": true, "CCBot": true }, "search_access": { "Googlebot": true, "Bingbot": true, "DuckDuckBot": true, "Applebot": true }, "has_sitemap_directive": true, "mentions_any_ai_bot": false, "blocks_citation_bots": [], "blocks_training_bots": [], "blocks_search": [] } ] }