{
  "licence": {
    "name": "CC BY 4.0",
    "url": "https://creativecommons.org/licenses/by/4.0/",
    "credit": "IndexHalo, https://www.indexhalo.com/data/"
  },
  "generatedFrom": "weekly readings of live robots.txt",
  "readings": [
    "2026-08-17",
    "2026-08-24",
    "2026-08-31",
    "2026-09-07",
    "2026-09-14",
    "2026-09-21",
    "2026-09-28",
    "2026-10-05",
    "2026-10-12",
    "2026-10-19",
    "2026-10-26",
    "2026-11-02",
    "2026-11-03"
  ],
  "changes": [
    {
      "date": "2026-11-03",
      "domain": "bbc.co.uk",
      "category": "news",
      "crawler": "Claude-SearchBot",
      "operator": "Anthropic",
      "change": "started blocking"
    },
    {
      "date": "2026-11-03",
      "domain": "bbc.co.uk",
      "category": "news",
      "crawler": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "change": "started blocking"
    },
    {
      "date": "2026-11-03",
      "domain": "researchgate.net",
      "category": "reference",
      "crawler": "Applebot-Extended",
      "operator": "Apple",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "cnbc.com",
      "category": "finance",
      "crawler": "Claude-SearchBot",
      "operator": "Anthropic",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "cnbc.com",
      "category": "finance",
      "crawler": "Perplexity-User",
      "operator": "Perplexity",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "fool.com",
      "category": "finance",
      "crawler": "CCBot",
      "operator": "Common Crawl",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "fool.com",
      "category": "finance",
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "fool.com",
      "category": "finance",
      "crawler": "GPTBot",
      "operator": "OpenAI",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "ikea.com",
      "category": "commerce",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "metacritic.com",
      "category": "media",
      "crawler": "Meta-ExternalAgent",
      "operator": "Meta",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "metacritic.com",
      "category": "media",
      "crawler": "Meta-ExternalFetcher",
      "operator": "Meta",
      "change": "started blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "newsweek.com",
      "category": "news",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "stopped blocking"
    },
    {
      "date": "2026-11-02",
      "domain": "newsweek.com",
      "category": "news",
      "crawler": "Perplexity-User",
      "operator": "Perplexity",
      "change": "started blocking"
    },
    {
      "date": "2026-10-26",
      "domain": "investopedia.com",
      "category": "finance",
      "crawler": "Diffbot",
      "operator": "Diffbot",
      "change": "started blocking"
    },
    {
      "date": "2026-10-26",
      "domain": "nbcnews.com",
      "category": "news",
      "crawler": "Meta-ExternalFetcher",
      "operator": "Meta",
      "change": "started blocking"
    },
    {
      "date": "2026-10-26",
      "domain": "nerdwallet.com",
      "category": "finance",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-26",
      "domain": "travelandleisure.com",
      "category": "travel",
      "crawler": "Diffbot",
      "operator": "Diffbot",
      "change": "started blocking"
    },
    {
      "date": "2026-10-26",
      "domain": "verywellhealth.com",
      "category": "health",
      "crawler": "Diffbot",
      "operator": "Diffbot",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "aliexpress.com",
      "category": "commerce",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "aliexpress.com",
      "category": "commerce",
      "crawler": "Bytespider",
      "operator": "ByteDance",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "aliexpress.com",
      "category": "commerce",
      "crawler": "CCBot",
      "operator": "Common Crawl",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "aliexpress.com",
      "category": "commerce",
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "aliexpress.com",
      "category": "commerce",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "aliexpress.com",
      "category": "commerce",
      "crawler": "GPTBot",
      "operator": "OpenAI",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "androidcentral.com",
      "category": "tech",
      "crawler": "Applebot-Extended",
      "operator": "Apple",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "cnn.com",
      "category": "news",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "howtogeek.com",
      "category": "tech",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "howtogeek.com",
      "category": "tech",
      "crawler": "ChatGPT-User",
      "operator": "OpenAI",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "howtogeek.com",
      "category": "tech",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "howtogeek.com",
      "category": "tech",
      "crawler": "Meta-ExternalAgent",
      "operator": "Meta",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "makeuseof.com",
      "category": "tech",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "makeuseof.com",
      "category": "tech",
      "crawler": "ChatGPT-User",
      "operator": "OpenAI",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "makeuseof.com",
      "category": "tech",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "makeuseof.com",
      "category": "tech",
      "crawler": "Meta-ExternalAgent",
      "operator": "Meta",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "pinterest.com",
      "category": "media",
      "crawler": "Meta-ExternalFetcher",
      "operator": "Meta",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "spiegel.de",
      "category": "news",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "techradar.com",
      "category": "tech",
      "crawler": "Applebot-Extended",
      "operator": "Apple",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "tomshardware.com",
      "category": "tech",
      "crawler": "Applebot-Extended",
      "operator": "Apple",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "xda-developers.com",
      "category": "tech",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "xda-developers.com",
      "category": "tech",
      "crawler": "ChatGPT-User",
      "operator": "OpenAI",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "xda-developers.com",
      "category": "tech",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "started blocking"
    },
    {
      "date": "2026-10-19",
      "domain": "xda-developers.com",
      "category": "tech",
      "crawler": "Meta-ExternalAgent",
      "operator": "Meta",
      "change": "started blocking"
    },
    {
      "date": "2026-10-12",
      "domain": "pexels.com",
      "category": "media",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-12",
      "domain": "pexels.com",
      "category": "media",
      "crawler": "PerplexityBot",
      "operator": "Perplexity",
      "change": "stopped blocking"
    },
    {
      "date": "2026-10-05",
      "domain": "figma.com",
      "category": "saas",
      "crawler": "Bytespider",
      "operator": "ByteDance",
      "change": "started blocking"
    },
    {
      "date": "2026-09-28",
      "domain": "airbnb.com",
      "category": "travel",
      "crawler": "AI2Bot",
      "operator": "Allen Institute for AI",
      "change": "started blocking"
    },
    {
      "date": "2026-09-28",
      "domain": "airbnb.com",
      "category": "travel",
      "crawler": "Applebot-Extended",
      "operator": "Apple",
      "change": "started blocking"
    },
    {
      "date": "2026-09-28",
      "domain": "airbnb.com",
      "category": "travel",
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "change": "started blocking"
    },
    {
      "date": "2026-09-28",
      "domain": "airbnb.com",
      "category": "travel",
      "crawler": "GPTBot",
      "operator": "OpenAI",
      "change": "started blocking"
    },
    {
      "date": "2026-09-14",
      "domain": "cntraveler.com",
      "category": "travel",
      "crawler": "YouBot",
      "operator": "You.com",
      "change": "started blocking"
    },
    {
      "date": "2026-09-14",
      "domain": "corriere.it",
      "category": "news",
      "crawler": "ChatGPT-User",
      "operator": "OpenAI",
      "change": "started blocking"
    },
    {
      "date": "2026-09-14",
      "domain": "github.com",
      "category": "developer",
      "crawler": "Bytespider",
      "operator": "ByteDance",
      "change": "started blocking"
    },
    {
      "date": "2026-09-14",
      "domain": "newyorker.com",
      "category": "news",
      "crawler": "YouBot",
      "operator": "You.com",
      "change": "started blocking"
    },
    {
      "date": "2026-09-14",
      "domain": "wired.com",
      "category": "tech",
      "crawler": "YouBot",
      "operator": "You.com",
      "change": "started blocking"
    },
    {
      "date": "2026-09-07",
      "domain": "amplitude.com",
      "category": "saas",
      "crawler": "Bytespider",
      "operator": "ByteDance",
      "change": "started blocking"
    },
    {
      "date": "2026-09-07",
      "domain": "straitstimes.com",
      "category": "news",
      "crawler": "Google-Extended",
      "operator": "Google",
      "change": "started blocking"
    },
    {
      "date": "2026-09-07",
      "domain": "webmd.com",
      "category": "health",
      "crawler": "ChatGPT-User",
      "operator": "OpenAI",
      "change": "stopped blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "Applebot-Extended",
      "operator": "Apple",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "Bytespider",
      "operator": "ByteDance",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "CCBot",
      "operator": "Common Crawl",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "cohere-ai",
      "operator": "Cohere",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "Diffbot",
      "operator": "Diffbot",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "ImagesiftBot",
      "operator": "ImageSift",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "Meta-ExternalAgent",
      "operator": "Meta",
      "change": "stopped blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "MistralAI-User",
      "operator": "Mistral AI",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "Omgilibot",
      "operator": "Webz.io",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "PerplexityBot",
      "operator": "Perplexity",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "fastcompany.com",
      "category": "finance",
      "crawler": "YouBot",
      "operator": "You.com",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "imdb.com",
      "category": "media",
      "crawler": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "change": "stopped blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "Applebot-Extended",
      "operator": "Apple",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "Bytespider",
      "operator": "ByteDance",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "CCBot",
      "operator": "Common Crawl",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "cohere-ai",
      "operator": "Cohere",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "Diffbot",
      "operator": "Diffbot",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "ImagesiftBot",
      "operator": "ImageSift",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "Meta-ExternalAgent",
      "operator": "Meta",
      "change": "stopped blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "MistralAI-User",
      "operator": "Mistral AI",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "Omgilibot",
      "operator": "Webz.io",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "PerplexityBot",
      "operator": "Perplexity",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "inc.com",
      "category": "finance",
      "crawler": "YouBot",
      "operator": "You.com",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "slate.com",
      "category": "news",
      "crawler": "Diffbot",
      "operator": "Diffbot",
      "change": "started blocking"
    },
    {
      "date": "2026-08-31",
      "domain": "slate.com",
      "category": "news",
      "crawler": "YouBot",
      "operator": "You.com",
      "change": "started blocking"
    },
    {
      "date": "2026-08-24",
      "domain": "metacritic.com",
      "category": "media",
      "crawler": "OAI-SearchBot",
      "operator": "OpenAI",
      "change": "started blocking"
    }
  ]
}