{ "licence": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/", "credit": "IndexHalo, https://www.indexhalo.com/data/" }, "generatedFrom": "live robots.txt", "sites": [ { "domain": "bbc.co.uk", "category": "news", "checkedAt": "2026-11-03T16:24:53.984Z", "accessPct": 33, "allowed": 10, "blocked": 20, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "theguardian.com", "category": "news", "checkedAt": "2026-11-03T16:24:53.899Z", "accessPct": 57, "allowed": 17, "blocked": 13, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "nytimes.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.109Z", "accessPct": 27, "allowed": 8, "blocked": 22, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "reuters.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.330Z", "accessPct": 17, "allowed": 5, "blocked": 25, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "allowed" } }, { "domain": "cnn.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.023Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ft.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.015Z", "accessPct": 57, "allowed": 17, "blocked": 13, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "bloomberg.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.279Z", "accessPct": 47, "allowed": 14, "blocked": 16, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wsj.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.122Z", "accessPct": 20, "allowed": 6, "blocked": 24, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "economist.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.732Z", "accessPct": 57, "allowed": 17, "blocked": 13, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "apnews.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.255Z", "accessPct": 67, "allowed": 20, "blocked": 10, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "aljazeera.com", "category": "news", "checkedAt": "2026-11-03T16:24:55.334Z", "accessPct": 80, "allowed": 24, "blocked": 6, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "independent.co.uk", "category": "news", "checkedAt": "2026-11-03T16:24:54.674Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "telegraph.co.uk", "category": "news", "checkedAt": "2026-11-03T16:24:54.959Z", "accessPct": 17, "allowed": 5, "blocked": 25, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "washingtonpost.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.844Z", "accessPct": 63, "allowed": 19, "blocked": 11, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "latimes.com", "category": "news", "checkedAt": "2026-11-03T16:24:55.338Z", "accessPct": 80, "allowed": 24, "blocked": 6, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "usatoday.com", "category": "news", "checkedAt": "2026-11-03T16:24:54.810Z", "accessPct": 10, "allowed": 3, "blocked": 27, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "cbsnews.com", "category": "news", "checkedAt": "2026-11-03T16:24:55.167Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "nbcnews.com", "category": "news", "checkedAt": "2026-11-03T16:24:56.406Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "abcnews.go.com", "category": "news", "checkedAt": "2026-11-03T16:24:55.267Z", "accessPct": 80, "allowed": 24, "blocked": 6, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sky.com", "category": "news", "checkedAt": "2026-11-03T16:24:56.548Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "dw.com", "category": "news", "checkedAt": "2026-11-03T16:24:57.336Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "france24.com", "category": "news", "checkedAt": "2026-11-03T16:24:56.987Z", "accessPct": 47, "allowed": 14, "blocked": 16, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "blocked", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "elpais.com", "category": "news", "checkedAt": "2026-11-03T16:24:56.215Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "allowed", "DuckAssistBot": "blocked", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "lemonde.fr", "category": "news", "checkedAt": "2026-11-03T16:24:57.542Z", "accessPct": 37, "allowed": 11, "blocked": 19, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "spiegel.de", "category": "news", "checkedAt": "2026-11-03T16:24:57.809Z", "accessPct": 37, "allowed": 11, "blocked": 19, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "zeit.de", "category": "news", "checkedAt": "2026-11-03T16:24:57.482Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "corriere.it", "category": "news", "checkedAt": "2026-11-03T16:24:59.200Z", "accessPct": 57, "allowed": 17, "blocked": 13, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "asahi.com", "category": "news", "checkedAt": "2026-11-03T16:24:57.974Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "scmp.com", "category": "news", "checkedAt": "2026-11-03T16:24:58.062Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "straitstimes.com", "category": "news", "checkedAt": "2026-11-03T16:24:58.886Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "smh.com.au", "category": "news", "checkedAt": "2026-11-03T16:24:59.018Z", "accessPct": 33, "allowed": 10, "blocked": 20, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "theage.com.au", "category": "news", "checkedAt": "2026-11-03T16:24:59.182Z", "accessPct": 33, "allowed": 10, "blocked": 20, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "globeandmail.com", "category": "news", "checkedAt": "2026-11-03T16:24:59.244Z", "accessPct": 13, "allowed": 4, "blocked": 26, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "irishtimes.com", "category": "news", "checkedAt": "2026-11-03T16:24:59.804Z", "accessPct": 93, "allowed": 28, "blocked": 2, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "thetimes.co.uk", "category": "news", "checkedAt": "2026-11-03T16:25:00.322Z", "accessPct": 17, "allowed": 5, "blocked": 25, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "axios.com", "category": "news", "checkedAt": "2026-11-03T16:24:59.667Z", "accessPct": 83, "allowed": 25, "blocked": 5, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "vox.com", "category": "news", "checkedAt": "2026-11-03T16:24:59.685Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "theatlantic.com", "category": "news", "checkedAt": "2026-11-03T16:25:00.047Z", "accessPct": 40, "allowed": 12, "blocked": 18, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "newyorker.com", "category": "news", "checkedAt": "2026-11-03T16:25:00.035Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "time.com", "category": "news", "checkedAt": "2026-11-03T16:24:59.869Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "newsweek.com", "category": "news", "checkedAt": "2026-11-03T16:25:00.102Z", "accessPct": 63, "allowed": 19, "blocked": 11, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "huffpost.com", "category": "news", "checkedAt": "2026-11-03T16:25:00.314Z", "accessPct": 33, "allowed": 10, "blocked": 20, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "buzzfeednews.com", "category": "news", "checkedAt": "2026-11-03T16:25:00.309Z", "accessPct": 33, "allowed": 10, "blocked": 20, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "slate.com", "category": "news", "checkedAt": "2026-11-03T16:25:00.295Z", "accessPct": 53, "allowed": 16, "blocked": 14, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "allowed" } }, { "domain": "techcrunch.com", "category": "tech", "checkedAt": "2026-11-03T16:25:00.315Z", "accessPct": 70, "allowed": 21, "blocked": 9, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "theverge.com", "category": "tech", "checkedAt": "2026-11-03T16:25:00.529Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "arstechnica.com", "category": "tech", "checkedAt": "2026-11-03T16:25:00.648Z", "accessPct": 27, "allowed": 8, "blocked": 22, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wired.com", "category": "tech", "checkedAt": "2026-11-03T16:25:00.857Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "engadget.com", "category": "tech", "checkedAt": "2026-11-03T16:25:00.675Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "zdnet.com", "category": "tech", "checkedAt": "2026-08-31T13:55:16.878Z", "accessPct": 14, "allowed": 4, "blocked": 24, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked" } }, { "domain": "venturebeat.com", "category": "tech", "checkedAt": "2026-11-03T16:25:00.982Z", "accessPct": 83, "allowed": 25, "blocked": 5, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "thenextweb.com", "category": "tech", "checkedAt": "2026-11-03T16:25:00.967Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "gizmodo.com", "category": "tech", "checkedAt": "2026-08-17T11:09:32.326Z", "accessPct": 68, "allowed": 19, "blocked": 9, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked" } }, { "domain": "mashable.com", "category": "tech", "checkedAt": "2026-11-03T16:25:01.204Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "tomshardware.com", "category": "tech", "checkedAt": "2026-11-03T16:25:01.549Z", "accessPct": 77, "allowed": 23, "blocked": 7, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "anandtech.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.114Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "macrumors.com", "category": "tech", "checkedAt": "2026-11-03T16:25:01.792Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "androidcentral.com", "category": "tech", "checkedAt": "2026-11-03T16:25:01.805Z", "accessPct": 77, "allowed": 23, "blocked": 7, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "xda-developers.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.190Z", "accessPct": 63, "allowed": 19, "blocked": 11, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "bleepingcomputer.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.210Z", "accessPct": 90, "allowed": 27, "blocked": 3, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "theregister.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.490Z", "accessPct": 20, "allowed": 6, "blocked": 24, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "hackernoon.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.052Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "slashdot.org", "category": "tech", "checkedAt": "2026-11-03T16:25:02.308Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "techradar.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.384Z", "accessPct": 77, "allowed": 23, "blocked": 7, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "digitaltrends.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.698Z", "accessPct": 93, "allowed": 28, "blocked": 2, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pcmag.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.796Z", "accessPct": 43, "allowed": 13, "blocked": 17, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "howtogeek.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.853Z", "accessPct": 63, "allowed": 19, "blocked": 11, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "makeuseof.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.886Z", "accessPct": 63, "allowed": 19, "blocked": 11, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "lifehacker.com", "category": "tech", "checkedAt": "2026-11-03T16:25:02.719Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "github.com", "category": "developer", "checkedAt": "2026-11-03T16:25:02.925Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "gitlab.com", "category": "developer", "checkedAt": "2026-11-03T16:25:02.982Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "developer.mozilla.org", "category": "developer", "checkedAt": "2026-11-03T16:25:02.857Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "w3schools.com", "category": "developer", "checkedAt": "2026-11-03T16:25:03.792Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "geeksforgeeks.org", "category": "developer", "checkedAt": "2026-11-03T16:25:03.259Z", "accessPct": 90, "allowed": 27, "blocked": 3, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "freecodecamp.org", "category": "developer", "checkedAt": "2026-11-03T16:25:03.373Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "dev.to", "category": "developer", "checkedAt": "2026-11-03T16:25:03.113Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "hashnode.com", "category": "developer", "checkedAt": "2026-11-03T16:25:03.335Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "digitalocean.com", "category": "developer", "checkedAt": "2026-11-03T16:25:03.391Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "smashingmagazine.com", "category": "developer", "checkedAt": "2026-11-03T16:25:03.781Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "css-tricks.com", "category": "developer", "checkedAt": "2026-11-03T16:25:03.412Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sitepoint.com", "category": "developer", "checkedAt": "2026-11-03T16:25:04.405Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "tutorialspoint.com", "category": "developer", "checkedAt": "2026-11-03T16:25:04.345Z", "accessPct": 90, "allowed": 27, "blocked": 3, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "codecademy.com", "category": "developer", "checkedAt": "2026-11-03T16:25:04.038Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "leetcode.com", "category": "developer", "checkedAt": "2026-11-03T16:25:04.386Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pypi.org", "category": "developer", "checkedAt": "2026-11-03T16:25:04.051Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "rubygems.org", "category": "developer", "checkedAt": "2026-11-03T16:25:04.205Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "packagist.org", "category": "developer", "checkedAt": "2026-11-03T16:25:04.350Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "crates.io", "category": "developer", "checkedAt": "2026-11-03T16:25:04.334Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "docker.com", "category": "developer", "checkedAt": "2026-11-03T16:25:04.655Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "kubernetes.io", "category": "developer", "checkedAt": "2026-11-03T16:25:04.675Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "python.org", "category": "developer", "checkedAt": "2026-11-03T16:25:04.639Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "nodejs.org", "category": "developer", "checkedAt": "2026-11-03T16:25:04.505Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "reactjs.org", "category": "developer", "checkedAt": "2026-11-03T16:25:05.244Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "vuejs.org", "category": "developer", "checkedAt": "2026-11-03T16:25:04.571Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "angular.io", "category": "developer", "checkedAt": "2026-11-03T16:25:04.805Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "svelte.dev", "category": "developer", "checkedAt": "2026-11-03T16:25:04.996Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "golang.org", "category": "developer", "checkedAt": "2026-11-03T16:25:05.759Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "php.net", "category": "developer", "checkedAt": "2026-11-03T16:25:05.334Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "postgresql.org", "category": "developer", "checkedAt": "2026-11-03T16:25:06.129Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "mysql.com", "category": "developer", "checkedAt": "2026-11-03T16:25:05.825Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "mongodb.com", "category": "developer", "checkedAt": "2026-11-03T16:25:06.359Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "redis.io", "category": "developer", "checkedAt": "2026-11-03T16:25:05.642Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sqlite.org", "category": "developer", "checkedAt": "2026-11-03T16:25:05.912Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "shopify.com", "category": "saas", "checkedAt": "2026-11-03T16:25:05.805Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "stripe.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.037Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "squarespace.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.052Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wix.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.166Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "webflow.com", "category": "saas", "checkedAt": "2026-08-31T13:55:20.693Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "hubspot.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.306Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "salesforce.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.728Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "atlassian.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.497Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "notion.so", "category": "saas", "checkedAt": "2026-11-03T16:25:06.510Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "slack.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.480Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "zoom.us", "category": "saas", "checkedAt": "2026-11-03T16:25:06.523Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "asana.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.682Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "monday.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.865Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "figma.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.311Z", "accessPct": 77, "allowed": 23, "blocked": 7, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "canva.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.895Z", "accessPct": 67, "allowed": 20, "blocked": 10, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "adobe.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.942Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "dropbox.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.907Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "box.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.942Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "airtable.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.989Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "clickup.com", "category": "saas", "checkedAt": "2026-11-03T16:25:06.971Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "linear.app", "category": "saas", "checkedAt": "2026-11-03T16:25:07.012Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "basecamp.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.333Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "trello.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.132Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "miro.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.339Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "loom.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.366Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "calendly.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.340Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "typeform.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.781Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "surveymonkey.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.891Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "mailchimp.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.517Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "klaviyo.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.717Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "intercom.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.787Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "zendesk.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.932Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "freshworks.com", "category": "saas", "checkedAt": "2026-11-03T16:25:07.995Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "drift.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.426Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "docusign.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.161Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "twilio.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.279Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sendgrid.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.290Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "segment.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.361Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "amplitude.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.466Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "mixpanel.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.411Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "datadog.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.552Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "newrelic.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.407Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pagerduty.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.996Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "rippling.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.954Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "deel.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.037Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "brex.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.124Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ramp.com", "category": "saas", "checkedAt": "2026-11-03T16:25:08.919Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "expensify.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.688Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "quickbooks.intuit.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.307Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "xero.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.615Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sage.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.721Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "zoho.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.628Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pipedrive.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.784Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "close.com", "category": "saas", "checkedAt": "2026-11-03T16:25:09.765Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "openai.com", "category": "ai", "checkedAt": "2026-11-03T16:25:09.710Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "anthropic.com", "category": "ai", "checkedAt": "2026-11-03T16:25:09.882Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "perplexity.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:09.901Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "mistral.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:09.857Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "cohere.com", "category": "ai", "checkedAt": "2026-11-03T16:25:09.837Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "huggingface.co", "category": "ai", "checkedAt": "2026-11-03T16:25:09.920Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "stability.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:09.914Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "runwayml.com", "category": "ai", "checkedAt": "2026-11-03T16:25:10.354Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "elevenlabs.io", "category": "ai", "checkedAt": "2026-11-03T16:25:10.114Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "replicate.com", "category": "ai", "checkedAt": "2026-11-03T16:25:10.389Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "together.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:10.381Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "groq.com", "category": "ai", "checkedAt": "2026-11-03T16:25:10.894Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "deepmind.google", "category": "ai", "checkedAt": "2026-11-03T16:25:10.474Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ai.meta.com", "category": "ai", "checkedAt": "2026-11-03T16:25:10.418Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "x.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:10.467Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "character.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:10.514Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "jasper.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:10.796Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "copy.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:11.085Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "writesonic.com", "category": "ai", "checkedAt": "2026-11-03T16:25:10.810Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "synthesia.io", "category": "ai", "checkedAt": "2026-11-03T16:25:11.033Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "descript.com", "category": "ai", "checkedAt": "2026-11-03T16:25:11.400Z", "accessPct": 93, "allowed": 28, "blocked": 2, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "otter.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:11.201Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fireflies.ai", "category": "ai", "checkedAt": "2026-11-03T16:25:11.026Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "gamma.app", "category": "ai", "checkedAt": "2026-08-17T11:09:38.697Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "cursor.com", "category": "ai", "checkedAt": "2026-11-03T16:25:11.524Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "codeium.com", "category": "ai", "checkedAt": "2026-11-03T16:25:11.889Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "tabnine.com", "category": "ai", "checkedAt": "2026-09-21T12:51:59.013Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "sourcegraph.com", "category": "ai", "checkedAt": "2026-11-03T16:25:11.723Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "semrush.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.206Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ahrefs.com", "category": "seo", "checkedAt": "2026-11-03T16:25:11.873Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "moz.com", "category": "seo", "checkedAt": "2026-11-03T16:25:11.892Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "similarweb.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.349Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "screamingfrog.co.uk", "category": "seo", "checkedAt": "2026-11-03T16:25:12.478Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "searchengineland.com", "category": "seo", "checkedAt": "2026-08-17T11:09:39.020Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "searchenginejournal.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.518Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "seroundtable.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.493Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "backlinko.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.559Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "neilpatel.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.568Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "yoast.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.771Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "conductor.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.267Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "brightedge.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.411Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "seranking.com", "category": "seo", "checkedAt": "2026-11-03T16:25:12.962Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "serpstat.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.399Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "spyfu.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.479Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "majestic.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.598Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sistrix.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.897Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "oncrawl.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.959Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "botify.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.957Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "lumar.io", "category": "seo", "checkedAt": "2026-11-03T16:25:13.617Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "contentkingapp.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.816Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "surferseo.com", "category": "seo", "checkedAt": "2026-11-03T16:25:13.708Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "clearscope.io", "category": "seo", "checkedAt": "2026-11-03T16:25:14.419Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "marketmuse.com", "category": "seo", "checkedAt": "2026-11-03T16:25:14.731Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "frase.io", "category": "seo", "checkedAt": "2026-11-03T16:25:14.394Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "cloudflare.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.075Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "vercel.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.081Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "netlify.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.092Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "railway.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.401Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fly.io", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.516Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "render.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.603Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "aws.amazon.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.535Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "cloud.google.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.560Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "azure.microsoft.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:14.571Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "linode.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:20.104Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "vultr.com", "category": "infrastructure", "checkedAt": "2026-08-17T11:09:40.786Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "ovhcloud.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:15.706Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "hetzner.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:15.973Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "namecheap.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:15.212Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "porkbun.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:15.168Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fastly.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:15.787Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "gandi.net", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:16.556Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "hover.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:16.041Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "dnsimple.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:16.065Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "netlify.app", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:16.224Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wpengine.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:16.111Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "kinsta.com", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:16.373Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pantheon.io", "category": "infrastructure", "checkedAt": "2026-11-03T16:25:16.213Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wikipedia.org", "category": "reference", "checkedAt": "2026-11-03T16:25:16.561Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "britannica.com", "category": "reference", "checkedAt": "2026-11-03T16:25:16.618Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "merriam-webster.com", "category": "reference", "checkedAt": "2026-11-03T16:25:16.620Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "dictionary.com", "category": "reference", "checkedAt": "2026-11-03T16:25:16.604Z", "accessPct": 90, "allowed": 27, "blocked": 3, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wiktionary.org", "category": "reference", "checkedAt": "2026-11-03T16:25:17.361Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "archive.org", "category": "reference", "checkedAt": "2026-11-03T16:25:16.826Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "gutenberg.org", "category": "reference", "checkedAt": "2026-11-03T16:25:17.300Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "arxiv.org", "category": "reference", "checkedAt": "2026-11-03T16:25:16.864Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "nature.com", "category": "reference", "checkedAt": "2026-11-03T16:25:17.321Z", "accessPct": 47, "allowed": 14, "blocked": 16, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sciencedirect.com", "category": "reference", "checkedAt": "2026-11-03T16:25:17.274Z", "accessPct": 13, "allowed": 4, "blocked": 26, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "jstor.org", "category": "reference", "checkedAt": "2026-11-03T16:25:17.486Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "springer.com", "category": "reference", "checkedAt": "2026-11-03T16:25:17.608Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pubmed.ncbi.nlm.nih.gov", "category": "reference", "checkedAt": "2026-11-03T16:25:17.538Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "plos.org", "category": "reference", "checkedAt": "2026-11-03T16:25:17.620Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "researchgate.net", "category": "reference", "checkedAt": "2026-11-03T16:25:17.550Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "academia.edu", "category": "reference", "checkedAt": "2026-11-03T16:25:18.155Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "ssrn.com", "category": "reference", "checkedAt": "2026-11-03T16:25:18.370Z", "accessPct": 90, "allowed": 27, "blocked": 3, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sciencemag.org", "category": "reference", "checkedAt": "2026-11-03T16:25:19.529Z", "accessPct": 83, "allowed": 25, "blocked": 5, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pnas.org", "category": "reference", "checkedAt": "2026-11-03T16:25:18.299Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "acm.org", "category": "reference", "checkedAt": "2026-11-03T16:25:18.278Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "mit.edu", "category": "reference", "checkedAt": "2026-11-03T16:25:19.243Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "stanford.edu", "category": "reference", "checkedAt": "2026-11-03T16:25:19.212Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "cam.ac.uk", "category": "reference", "checkedAt": "2026-11-03T16:25:19.864Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "berkeley.edu", "category": "reference", "checkedAt": "2026-11-03T16:25:20.281Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "cmu.edu", "category": "reference", "checkedAt": "2026-11-03T16:25:20.235Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "amazon.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:20.314Z", "accessPct": 20, "allowed": 6, "blocked": 24, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ebay.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:21.143Z", "accessPct": 73, "allowed": 22, "blocked": 8, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "walmart.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:21.134Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "target.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:21.127Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "bestbuy.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:30.259Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "costco.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:21.193Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "etsy.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:21.178Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "aliexpress.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:22.737Z", "accessPct": 80, "allowed": 24, "blocked": 6, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "alibaba.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:21.973Z", "accessPct": 93, "allowed": 28, "blocked": 2, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wayfair.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:22.070Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ikea.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:22.110Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "homedepot.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:22.130Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "zalando.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:21.709Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "nike.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:22.715Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ulta.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:23.499Z", "accessPct": 100, "allowed": 12, "blocked": 0, "notListed": 18, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "not listed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "allowed", "Amazonbot": "not listed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "chewy.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:23.313Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "instacart.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:23.300Z", "accessPct": 10, "allowed": 3, "blocked": 27, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "deliveroo.co.uk", "category": "commerce", "checkedAt": "2026-11-03T16:25:23.462Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "justeattakeaway.com", "category": "commerce", "checkedAt": "2026-11-03T16:25:23.558Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "investopedia.com", "category": "finance", "checkedAt": "2026-11-03T16:25:23.773Z", "accessPct": 27, "allowed": 8, "blocked": 22, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "forbes.com", "category": "finance", "checkedAt": "2026-11-03T16:25:23.811Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fool.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.362Z", "accessPct": 87, "allowed": 26, "blocked": 4, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "morningstar.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.321Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "marketwatch.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.323Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "barrons.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.396Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "cnbc.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.487Z", "accessPct": 43, "allowed": 13, "blocked": 17, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "businessinsider.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.597Z", "accessPct": 83, "allowed": 25, "blocked": 5, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fortune.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.603Z", "accessPct": 90, "allowed": 27, "blocked": 3, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fastcompany.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.966Z", "accessPct": 53, "allowed": 16, "blocked": 14, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "inc.com", "category": "finance", "checkedAt": "2026-11-03T16:25:24.955Z", "accessPct": 53, "allowed": 16, "blocked": 14, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "nerdwallet.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.115Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "bankrate.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.073Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "creditkarma.com", "category": "finance", "checkedAt": "2026-11-03T16:25:29.611Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "coinbase.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.149Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "binance.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.994Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "kraken.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.255Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "paypal.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.355Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wise.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.337Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "revolut.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.821Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "n26.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.671Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "monzo.com", "category": "finance", "checkedAt": "2026-11-03T16:25:25.709Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "starlingbank.com", "category": "finance", "checkedAt": "2026-11-03T16:25:26.476Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "chase.com", "category": "finance", "checkedAt": "2026-11-03T16:25:26.359Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "bankofamerica.com", "category": "finance", "checkedAt": "2026-11-03T16:25:31.125Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "wellsfargo.com", "category": "finance", "checkedAt": "2026-11-03T16:25:26.670Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "hsbc.com", "category": "finance", "checkedAt": "2026-11-03T16:25:28.116Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "barclays.co.uk", "category": "finance", "checkedAt": "2026-11-03T16:25:27.007Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "santander.com", "category": "finance", "checkedAt": "2026-11-03T16:25:27.368Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "gov.uk", "category": "government", "checkedAt": "2026-11-03T16:25:26.936Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "europa.eu", "category": "government", "checkedAt": "2026-11-03T16:25:27.423Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "who.int", "category": "government", "checkedAt": "2026-11-03T16:25:27.780Z", "accessPct": 97, "allowed": 0, "blocked": 1, "notListed": 29, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "blocked", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "worldbank.org", "category": "government", "checkedAt": "2026-11-03T16:25:27.827Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "oecd.org", "category": "government", "checkedAt": "2026-11-03T16:25:27.935Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "un.org", "category": "government", "checkedAt": "2026-11-03T16:25:28.708Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "imf.org", "category": "government", "checkedAt": "2026-10-05T14:44:36.852Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "nih.gov", "category": "government", "checkedAt": "2026-09-21T12:52:12.273Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "cdc.gov", "category": "government", "checkedAt": "2026-11-03T16:25:28.456Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fda.gov", "category": "government", "checkedAt": "2026-11-03T16:25:28.828Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "sec.gov", "category": "government", "checkedAt": "2026-11-03T16:25:28.769Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "irs.gov", "category": "government", "checkedAt": "2026-11-03T16:25:28.947Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "usa.gov", "category": "government", "checkedAt": "2026-11-03T16:25:29.096Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "canada.ca", "category": "government", "checkedAt": "2026-11-03T16:25:32.570Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "australia.gov.au", "category": "government", "checkedAt": "2026-11-03T16:25:30.264Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "bund.de", "category": "government", "checkedAt": "2026-11-03T16:25:31.109Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "service-public.fr", "category": "government", "checkedAt": "2026-11-03T16:25:31.509Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "belastingdienst.nl", "category": "government", "checkedAt": "2026-11-03T16:25:31.115Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "skatteverket.se", "category": "government", "checkedAt": "2026-11-03T16:25:31.527Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "healthline.com", "category": "health", "checkedAt": "2026-11-03T16:25:30.998Z", "accessPct": 70, "allowed": 21, "blocked": 9, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "webmd.com", "category": "health", "checkedAt": "2026-11-03T16:25:31.622Z", "accessPct": 90, "allowed": 27, "blocked": 3, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "nhs.uk", "category": "health", "checkedAt": "2026-11-03T16:25:32.585Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "clevelandclinic.org", "category": "health", "checkedAt": "2026-11-03T16:25:32.251Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "hopkinsmedicine.org", "category": "health", "checkedAt": "2026-11-03T16:25:32.239Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "medicalnewstoday.com", "category": "health", "checkedAt": "2026-11-03T16:25:32.273Z", "accessPct": 70, "allowed": 21, "blocked": 9, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "verywellhealth.com", "category": "health", "checkedAt": "2026-11-03T16:25:32.290Z", "accessPct": 27, "allowed": 8, "blocked": 22, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "drugs.com", "category": "health", "checkedAt": "2026-11-03T16:25:32.634Z", "accessPct": 63, "allowed": 19, "blocked": 11, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "goodrx.com", "category": "health", "checkedAt": "2026-11-03T16:25:32.865Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "teladoc.com", "category": "health", "checkedAt": "2026-11-03T16:25:32.882Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "headspace.com", "category": "health", "checkedAt": "2026-11-03T16:25:33.517Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "calm.com", "category": "health", "checkedAt": "2026-11-03T16:25:33.143Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "myfitnesspal.com", "category": "health", "checkedAt": "2026-11-03T16:25:33.331Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "blocked", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "strava.com", "category": "health", "checkedAt": "2026-11-03T16:25:33.307Z", "accessPct": 87, "allowed": 26, "blocked": 4, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "fitbit.com", "category": "health", "checkedAt": "2026-11-03T16:25:33.787Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ouraring.com", "category": "health", "checkedAt": "2026-11-03T16:25:33.246Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "booking.com", "category": "travel", "checkedAt": "2026-11-03T16:25:33.527Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "airbnb.com", "category": "travel", "checkedAt": "2026-11-03T16:25:34.055Z", "accessPct": 83, "allowed": 25, "blocked": 5, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "blocked", "Diffbot": "allowed", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "tripadvisor.com", "category": "travel", "checkedAt": "2026-11-03T16:25:33.982Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "expedia.com", "category": "travel", "checkedAt": "2026-11-03T16:25:33.918Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "kayak.com", "category": "travel", "checkedAt": "2026-11-03T16:25:34.173Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "skyscanner.net", "category": "travel", "checkedAt": "2026-11-03T16:25:34.648Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "vrbo.com", "category": "travel", "checkedAt": "2026-11-03T16:25:34.592Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "hostelworld.com", "category": "travel", "checkedAt": "2026-11-03T16:25:35.161Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "lonelyplanet.com", "category": "travel", "checkedAt": "2026-11-03T16:25:34.678Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "roughguides.com", "category": "travel", "checkedAt": "2026-11-03T16:25:34.784Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "cntraveler.com", "category": "travel", "checkedAt": "2026-11-03T16:25:34.595Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "travelandleisure.com", "category": "travel", "checkedAt": "2026-11-03T16:25:35.038Z", "accessPct": 27, "allowed": 8, "blocked": 22, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "afar.com", "category": "travel", "checkedAt": "2026-11-03T16:25:35.471Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "ryanair.com", "category": "travel", "checkedAt": "2026-11-03T16:25:35.049Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "britishairways.com", "category": "travel", "checkedAt": "2026-11-03T16:25:35.310Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "lufthansa.com", "category": "travel", "checkedAt": "2026-11-03T16:25:36.101Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "delta.com", "category": "travel", "checkedAt": "2026-11-03T16:25:36.840Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "trainline.com", "category": "travel", "checkedAt": "2026-11-03T16:25:36.202Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "omio.com", "category": "travel", "checkedAt": "2026-11-03T16:25:36.131Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "blocked", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "medium.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.179Z", "accessPct": 77, "allowed": 23, "blocked": 7, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "blocked", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "substack.com", "category": "media", "checkedAt": "2026-08-17T11:10:05.229Z", "accessPct": 100, "allowed": 28, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed" } }, { "domain": "reddit.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.367Z", "accessPct": 0, "allowed": 0, "blocked": 30, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "blocked", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "quora.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.564Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "linkedin.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.485Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "producthunt.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.626Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "youtube.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.661Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "vimeo.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.659Z", "accessPct": 83, "allowed": 25, "blocked": 5, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "twitch.tv", "category": "media", "checkedAt": "2026-11-03T16:25:37.021Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "spotify.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.935Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "soundcloud.com", "category": "media", "checkedAt": "2026-11-03T16:25:36.877Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "netflix.com", "category": "media", "checkedAt": "2026-11-03T16:25:37.204Z", "accessPct": 30, "allowed": 9, "blocked": 21, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "disneyplus.com", "category": "media", "checkedAt": "2026-11-03T16:25:37.445Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "hulu.com", "category": "media", "checkedAt": "2026-11-03T16:25:37.678Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "blocked", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "hbomax.com", "category": "media", "checkedAt": "2026-11-03T16:25:37.593Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "imdb.com", "category": "media", "checkedAt": "2026-11-03T16:25:37.721Z", "accessPct": 10, "allowed": 3, "blocked": 27, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "rottentomatoes.com", "category": "media", "checkedAt": "2026-11-03T16:25:38.032Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "metacritic.com", "category": "media", "checkedAt": "2026-11-03T16:25:38.368Z", "accessPct": 80, "allowed": 24, "blocked": 6, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "goodreads.com", "category": "media", "checkedAt": "2026-11-03T16:25:38.335Z", "accessPct": 93, "allowed": 28, "blocked": 2, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pinterest.com", "category": "media", "checkedAt": "2026-11-03T16:25:38.361Z", "accessPct": 7, "allowed": 2, "blocked": 28, "notListed": 0, "crawlers": { "OAI-SearchBot": "blocked", "GPTBot": "blocked", "ChatGPT-User": "blocked", "Claude-SearchBot": "blocked", "ClaudeBot": "blocked", "Claude-User": "blocked", "PerplexityBot": "blocked", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "allowed", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "blocked", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "flickr.com", "category": "media", "checkedAt": "2026-11-03T16:25:38.586Z", "accessPct": 23, "allowed": 7, "blocked": 23, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "blocked", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "blocked", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "blocked", "Google-CloudVertexBot": "blocked", "Meta-ExternalFetcher": "blocked", "MistralAI-User": "blocked", "cohere-ai": "blocked", "DuckAssistBot": "blocked", "YouBot": "blocked", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "blocked", "ElevenlabsBot": "blocked" } }, { "domain": "behance.net", "category": "media", "checkedAt": "2026-11-03T16:25:38.972Z", "accessPct": 60, "allowed": 18, "blocked": 12, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "blocked", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "blocked", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "blocked", "AI2Bot": "blocked", "Diffbot": "blocked", "ImagesiftBot": "blocked", "PetalBot": "allowed", "Omgilibot": "blocked", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "dribbble.com", "category": "media", "checkedAt": "2026-11-03T16:25:38.771Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "deviantart.com", "category": "media", "checkedAt": "2026-11-03T16:25:39.018Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "500px.com", "category": "media", "checkedAt": "2026-11-03T16:25:38.826Z", "accessPct": 100, "allowed": 0, "blocked": 0, "notListed": 30, "crawlers": { "OAI-SearchBot": "not listed", "GPTBot": "not listed", "ChatGPT-User": "not listed", "Claude-SearchBot": "not listed", "ClaudeBot": "not listed", "Claude-User": "not listed", "PerplexityBot": "not listed", "Perplexity-User": "not listed", "Google-Extended": "not listed", "Bingbot": "not listed", "Applebot-Extended": "not listed", "Meta-ExternalAgent": "not listed", "Amazonbot": "not listed", "Bytespider": "not listed", "CCBot": "not listed", "GoogleOther": "not listed", "Google-CloudVertexBot": "not listed", "Meta-ExternalFetcher": "not listed", "MistralAI-User": "not listed", "cohere-ai": "not listed", "DuckAssistBot": "not listed", "YouBot": "not listed", "Timpibot": "not listed", "AI2Bot": "not listed", "Diffbot": "not listed", "ImagesiftBot": "not listed", "PetalBot": "not listed", "Omgilibot": "not listed", "ElevenlabsCrawler": "not listed", "ElevenlabsBot": "not listed" } }, { "domain": "unsplash.com", "category": "media", "checkedAt": "2026-11-03T16:25:39.169Z", "accessPct": 100, "allowed": 30, "blocked": 0, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "pexels.com", "category": "media", "checkedAt": "2026-11-03T16:25:39.167Z", "accessPct": 83, "allowed": 25, "blocked": 5, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "blocked", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "blocked", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "blocked", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "blocked", "CCBot": "blocked", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "allowed", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } }, { "domain": "giphy.com", "category": "media", "checkedAt": "2026-11-03T16:25:39.103Z", "accessPct": 97, "allowed": 29, "blocked": 1, "notListed": 0, "crawlers": { "OAI-SearchBot": "allowed", "GPTBot": "allowed", "ChatGPT-User": "allowed", "Claude-SearchBot": "allowed", "ClaudeBot": "allowed", "Claude-User": "allowed", "PerplexityBot": "allowed", "Perplexity-User": "allowed", "Google-Extended": "allowed", "Bingbot": "allowed", "Applebot-Extended": "allowed", "Meta-ExternalAgent": "allowed", "Amazonbot": "allowed", "Bytespider": "allowed", "CCBot": "allowed", "GoogleOther": "allowed", "Google-CloudVertexBot": "allowed", "Meta-ExternalFetcher": "allowed", "MistralAI-User": "allowed", "cohere-ai": "allowed", "DuckAssistBot": "allowed", "YouBot": "allowed", "Timpibot": "allowed", "AI2Bot": "allowed", "Diffbot": "allowed", "ImagesiftBot": "allowed", "PetalBot": "blocked", "Omgilibot": "allowed", "ElevenlabsCrawler": "allowed", "ElevenlabsBot": "allowed" } } ] }