{ "schema_version": 1, "compiled": "2026-11-06", "revision": 9, "ai_crawlers": { "GPTBot": { "owner": "OpenAI", "purpose": "training", "impact": "Blocks OpenAI from training on your content. Does not affect whether ChatGPT can cite you live." }, "OAI-SearchBot": { "owner": "OpenAI", "purpose": "search index", "impact": "Blocks your pages from ChatGPT Search results entirely." }, "ChatGPT-User": { "owner": "OpenAI", "purpose": "live fetch", "impact": "Fetches a page when a ChatGPT user asks about it. OpenAI says that because a user starts these requests, robots.txt rules may not apply, so a block here may not stop ChatGPT reading your page on a user's behalf." }, "ClaudeBot": { "owner": "Anthropic", "purpose": "training", "impact": "Blocks Anthropic from training on your content." }, "Claude-SearchBot": { "owner": "Anthropic", "purpose": "search index", "impact": "Blocks your pages from Claude's search results." }, "Claude-User": { "owner": "Anthropic", "purpose": "live fetch", "impact": "Stops Claude fetching your page on a user's behalf." }, "PerplexityBot": { "owner": "Perplexity", "purpose": "search index", "impact": "Removes you from Perplexity's index and citations." }, "Perplexity-User": { "owner": "Perplexity", "purpose": "live fetch", "impact": "Fetches a page when a Perplexity user asks about it. Perplexity says that because a user requested the fetch, it generally ignores robots.txt rules, so a block here does not reliably stop it." }, "Google-Extended": { "owner": "Google", "purpose": "training / Gemini Apps grounding", "impact": "Blocks Gemini model training and grounding in Gemini Apps and Vertex AI. Does NOT affect AI Overviews, AI Mode or classic Google Search. Those follow your Googlebot rules." }, "Applebot-Extended": { "owner": "Apple", "purpose": "training", "impact": "Blocks Apple Intelligence training. Applebot (search) is separate." }, "Amazonbot": { "owner": "Amazon", "purpose": "search index", "impact": "Blocks Alexa and Amazon's answer surfaces." }, "Bytespider": { "owner": "ByteDance", "purpose": "training", "impact": "Blocks ByteDance/TikTok AI crawling." }, "meta-externalagent": { "owner": "Meta", "purpose": "training", "impact": "Blocks Meta AI training and grounding." }, "CCBot": { "owner": "Common Crawl", "purpose": "open corpus", "impact": "Removes you from Common Crawl, an input to many open models." }, "Meta-ExternalFetcher": { "owner": "Meta", "purpose": "live fetch", "impact": "Fetches a page when a Meta AI user asks for it. Meta says that because the user requested the fetch, it may bypass robots.txt, so a block here may not stop it." }, "MistralAI-Index": { "owner": "Mistral", "purpose": "search index", "impact": "Removes you from Le Chat's search index and citations." }, "MistralAI-Training": { "owner": "Mistral", "purpose": "training", "impact": "Blocks Mistral from training on your content." }, "GoogleOther": { "owner": "Google", "purpose": "research / product fetch", "impact": "Blocks Google's non-Search crawls. Does not affect Search." }, "Google-CloudVertexBot": { "owner": "Google", "purpose": "grounding", "impact": "Blocks Vertex AI grounding on your content for Google Cloud customers who ask it to index your site." }, "OAI-AdsBot": { "owner": "OpenAI", "purpose": "ads landing-page review", "impact": "Affects advertising on ChatGPT, not organic or AI-answer visibility. OpenAI's documentation says it only visits pages submitted as ads, to check they comply with their policies, and that what it collects is not used to train their foundation models. Blocking it is the right call for most sites and costs nothing in search; it matters only if you advertise on ChatGPT." }, "Meta-WebIndexer": { "owner": "Meta", "purpose": "search index", "impact": "Indexes pages to improve Meta AI's search results. Meta says allowing it helps Meta AI cite and link to your content in its responses." }, "Meta-ExternalAds": { "owner": "Meta", "purpose": "advertising", "impact": "Crawls for Meta's advertising and other business products. Meta's documentation does not tie it to Meta AI answers or citations." }, "ShapBot": { "owner": "Parallel Web Systems", "purpose": "search index", "impact": "Removes you from Parallel's search index. Parallel says restricting it may reduce your visibility through its web APIs and search features, and Anthropic lists Parallel as a web search provider for Claude." }, "Shap-User": { "owner": "Parallel Web Systems", "purpose": "live fetch", "impact": "Fetches a page on a user's behalf. Parallel says it is meant to make those requests visible rather than to be managed by robots.txt, so a block here may not stop it." } }, "web_vitals": { "LCP": { "good": 2500, "poor": 4000, "unit": "ms", "note": "Largest Contentful Paint. Field data only — a lab run is not this." }, "INP": { "good": 200, "poor": 500, "unit": "ms", "note": "Interaction to Next Paint. Replaced First Input Delay as a Core Web Vital in March 2024." }, "CLS": { "good": 0.1, "poor": 0.25, "unit": "", "note": "Cumulative Layout Shift. CrUX transmits this multiplied by 100." }, "FCP": { "good": 1800, "poor": 3000, "unit": "ms", "note": "First Contentful Paint. Diagnostic, not a Core Web Vital." }, "TTFB": { "good": 800, "poor": 1800, "unit": "ms", "note": "Time to First Byte. Diagnostic; a slow TTFB makes a good LCP unreachable." } }, "algorithm_notes": [ { "date": "2024-03-12", "title": "INP replaced FID as a Core Web Vital", "detail": "First Input Delay was retired. Interaction to Next Paint measures the full latency of the worst interaction rather than only the first, so pages that felt sluggish after load stopped passing by default." }, { "date": "2025-03-13", "title": "March 2025 core update", "detail": "Rolled out over 13 days and 21 hours from 13 March 2025." }, { "date": "2025-06-30", "title": "June 2025 core update", "detail": "Rolled out over 16 days and 18 hours from 30 June 2025." }, { "date": "2025-08-26", "title": "August 2025 spam update", "detail": "Rolled out over 26 days and 15 hours from 26 August 2025." }, { "date": "2025-12-11", "title": "December 2025 core update", "detail": "Rolled out over 18 days and 2 hours from 11 December 2025." }, { "date": "2026-02-05", "title": "February 2026 Discover update", "detail": "Rolled out over 21 days and 17 hours from 5 February 2026 — the longest rollout on the dashboard for either year." }, { "date": "2026-03-24", "title": "March 2026 spam update", "detail": "Rolled out over 19 hours and 30 minutes from 24 March 2026." }, { "date": "2026-03-27", "title": "March 2026 core update", "detail": "Rolled out over 12 days and 4 hours from 27 March 2026." }, { "date": "2026-03-31", "title": "Googlebot fetches at most 2MB of a page", "detail": "Googlebot fetches up to 2MB of any individual URL, including the HTTP headers, and passes that portion to indexing and to the Web Rendering Service as if it were the complete file. Bytes past the threshold are never fetched, rendered or indexed, and nothing reports it. PDFs get 64MB; a crawler that sets no limit defaults to 15MB. Referenced resources each have their own separate per-URL budget and do not count towards the parent page." }, { "date": "2026-04-13", "title": "Back-button hijacking added to the spam policies", "detail": "A new section of the malicious-practices spam policy covers 'back button hijacking' — manipulating history so a visitor pressing back cannot leave. It is a policy violation and can attract manual action." }, { "date": "2026-05-07", "title": "FAQ and HowTo rich results retired", "detail": "Google stopped showing FAQ and HowTo rich results in Search. The markup remains valid schema.org and is still useful for entity clarity, but it no longer produces a result feature." }, { "date": "2026-05-21", "title": "May 2026 core update", "detail": "Rolled out over 11 days and 21 hours from 21 May 2026." }, { "date": "2026-06-03", "title": "Search Console generative-AI control and reports", "detail": "Google began testing a Search Console toggle that decides whether a site appears in and helps ground AI Overviews, AI Mode and AI Overviews in Discover. Opting out forfeits traffic and impressions from those features and is not used as a ranking signal elsewhere. It leaves no external signal, so no third-party tool can detect it. Rolling out first to a subset of site owners in the UK." }, { "date": "2026-06-24", "title": "June 2026 spam update", "detail": "Rolled out over 2 days and 1 hour from 24 June 2026." }, { "date": "2026-07-01", "title": "AMP viewer and AMP Cache no longer used by Search", "detail": "Google Search now links users directly to the publisher's AMP host pages instead of routing through the AMP viewer or AMP Cache, and the documentation for the viewer, the cache and signed exchanges was removed. Publishers no longer need to update the AMP cache or configure signed exchanges. Google states AMP content continues to rank like any other web page, so this is a delivery change rather than a ranking one." }, { "date": "2026-08-18", "title": "August 2026 spam update", "detail": "Rolled out over 2 days and 16 hours from 18 August 2026." }, { "date": "2026-09-24", "title": "September 2026 spam update", "detail": "Started 24 September 2026. Google's ranking-updates page listed no duration when read on 5 October 2026." } ], "superseded": [ { "advice": "Block Google-Extended to stay out of AI Overviews", "why": "Google-Extended governs Gemini training and Vertex grounding. AI features in Google Search follow the Googlebot directives, so blocking Google-Extended does not remove you from AI Overviews.", "since": "2026-08-07" }, { "advice": "Add FAQPage markup to win FAQ rich results", "why": "Google retired FAQ rich results on 2026-05-07 and removed the documentation. The markup is still valid schema.org; the result feature no longer exists.", "since": "2026-05-07" }, { "advice": "Optimise for First Input Delay", "why": "FID was retired in March 2024. Measure INP instead.", "since": "2024-03-12" }, { "advice": "Blocking GPTBot keeps you out of ChatGPT", "why": "OpenAI split its crawlers: GPTBot collects training data, OAI-SearchBot builds the index ChatGPT Search answers from. Blocking GPTBot does not affect citation.", "since": "2024-08-01" }, { "advice": "Block anthropic-ai or Claude-Web to keep Anthropic out", "why": "Anthropic's crawler documentation lists exactly ClaudeBot, Claude-User and Claude-SearchBot. A rule under anthropic-ai or Claude-Web matches none of them, so the block never takes effect. 563 and 461 sites respectively in the Tranco top 10,000 still use those names.", "since": "2026-08-07" }, { "advice": "Block omgili or omgilibot to opt out of Webz.io", "why": "Webz.io replaced Omgilibot with the Webzio Duo — Webzio and Webzio-extended — in July 2024. Rules under the old names do nothing.", "since": "2026-08-07" }, { "advice": "Use Google-NotebookLM in robots.txt", "why": "Renamed to Google-GeminiNotebook. Google supports the old token only until August 2026, after which a rule under it stops applying.", "since": "2026-08-07" }, { "advice": "A version after the crawler name breaks a robots.txt rule", "why": "It does not. Parsers truncate the written User-agent value at the first character outside [a-zA-Z_-], so ChatGPT-User/2.0 is matched as ChatGPT-User and works. Scout briefly flagged this shape and would have been wrong on 89 sites.", "since": "2026-08-07" } ], "sources": { "ai_crawlers": "Each operator's published crawler documentation, re-read 2026-10-26 (Anthropic, Google, Meta, OpenAI, Perplexity) and 2026-11-06 (Parallel Web Systems: parallel.ai/parallel-web-systems-bots; Anthropic's subprocessor list at trust.anthropic.com/subprocessors names Parallel for web search), cross-checked against robots.txt directives observed in the Scout Index.", "web_vitals": "Google's published Core Web Vitals thresholds (web.dev).", "algorithm_notes": "Google Search Status Dashboard (developers.google.com/search/updates/ranking) and Search Central / blog.google announcements, each read on the date in the entry. Durations are Google's own. The ranking dashboard was re-read 2026-10-26; its newest entry is the September 2026 spam update." } }