{"name":"AI Crawler Policy Tracker","description":"A living dataset tracking how leading news, publishing, retail, and technology sites configure robots.txt for major AI crawlers.","creator":{"name":"Anand Iyer","url":"https://www.anandriyer.com/about"},"canonical_url":"https://www.anandriyer.com/crawler-watch","scanned_at":"2026-07-27T06:01:30.224Z","data":{"scannedAt":"2026-07-27T06:01:30.224Z","domainsScanned":96,"reachable":94,"anyBlockCount":49,"crawlers":[{"ua":"GPTBot","name":"GPTBot","purpose":"OpenAI training crawler","blocked":32,"allowed":62},{"ua":"OAI-SearchBot","name":"OAI-SearchBot","purpose":"ChatGPT search index","blocked":18,"allowed":76},{"ua":"ChatGPT-User","name":"ChatGPT-User","purpose":"ChatGPT live browsing","blocked":22,"allowed":72},{"ua":"ClaudeBot","name":"ClaudeBot","purpose":"Anthropic training crawler","blocked":44,"allowed":50},{"ua":"Claude-Web","name":"Claude-Web","purpose":"Claude live browsing","blocked":29,"allowed":65},{"ua":"Google-Extended","name":"Google-Extended","purpose":"Gemini training / AI","blocked":31,"allowed":63},{"ua":"PerplexityBot","name":"PerplexityBot","purpose":"Perplexity index","blocked":34,"allowed":60},{"ua":"CCBot","name":"CCBot","purpose":"Common Crawl (feeds many models)","blocked":47,"allowed":47},{"ua":"Bytespider","name":"Bytespider","purpose":"ByteDance / TikTok AI","blocked":44,"allowed":50},{"ua":"Applebot-Extended","name":"Applebot-Extended","purpose":"Apple Intelligence","blocked":39,"allowed":55}],"sites":[{"domain":"nytimes.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"washingtonpost.com","category":"news","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Claude-Web","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"wsj.com","category":"news","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider"]},{"domain":"theguardian.com","category":"news","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"bbc.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"cnn.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"reuters.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider"]},{"domain":"apnews.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","Claude-Web","PerplexityBot","CCBot","Applebot-Extended"]},{"domain":"bloomberg.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"ft.com","category":"news","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"economist.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"forbes.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"businessinsider.com","category":"news","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","CCBot","Bytespider"]},{"domain":"npr.org","category":"news","reachable":false,"hasRobots":false,"blocked":[]},{"domain":"theatlantic.com","category":"news","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"vox.com","category":"news","reachable":true,"hasRobots":true,"blocked":["ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"axios.com","category":"news","reachable":true,"hasRobots":true,"blocked":["CCBot","Bytespider"]},{"domain":"politico.com","category":"news","reachable":true,"hasRobots":false,"blocked":[]},{"domain":"usatoday.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"nbcnews.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"cnbc.com","category":"news","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"time.com","category":"news","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"newyorker.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"wired.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"theverge.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":["ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"medium.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","Bytespider","Applebot-Extended"]},{"domain":"substack.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"vanityfair.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"rollingstone.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"nationalgeographic.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"gutenberg.org","category":"publishing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"poetryfoundation.org","category":"publishing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"getty.edu","category":"publishing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"goodreads.com","category":"publishing","reachable":true,"hasRobots":true,"blocked":["GPTBot","CCBot"]},{"domain":"wikipedia.org","category":"reference","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"britannica.com","category":"reference","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"investopedia.com","category":"reference","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"healthline.com","category":"reference","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"mayoclinic.org","category":"reference","reachable":true,"hasRobots":false,"blocked":[]},{"domain":"webmd.com","category":"reference","reachable":true,"hasRobots":true,"blocked":["GPTBot","ChatGPT-User","ClaudeBot","CCBot"]},{"domain":"nih.gov","category":"reference","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"khanacademy.org","category":"reference","reachable":true,"hasRobots":false,"blocked":[]},{"domain":"quora.com","category":"reference","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","PerplexityBot","Bytespider","Applebot-Extended"]},{"domain":"dictionary.com","category":"reference","reachable":true,"hasRobots":true,"blocked":["GPTBot","CCBot","Applebot-Extended"]},{"domain":"merriam-webster.com","category":"reference","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"wikihow.com","category":"reference","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"stackoverflow.com","category":"tech","reachable":true,"hasRobots":false,"blocked":[]},{"domain":"github.com","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"developer.mozilla.org","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"techcrunch.com","category":"tech","reachable":true,"hasRobots":true,"blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","CCBot","Bytespider","Applebot-Extended"]},{"domain":"arstechnica.com","category":"tech","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"engadget.com","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"gizmodo.com","category":"tech","reachable":true,"hasRobots":true,"blocked":["CCBot","Bytespider"]},{"domain":"hackernoon.com","category":"tech","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","Google-Extended","CCBot","Bytespider","Applebot-Extended"]},{"domain":"dev.to","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"freecodecamp.org","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"smashingmagazine.com","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"css-tricks.com","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"openai.com","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"anthropic.com","category":"tech","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"hubspot.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"moz.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"ahrefs.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"semrush.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"searchengineland.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"searchenginejournal.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"backlinko.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"neilpatel.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"contentmarketinginstitute.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","Claude-Web","CCBot","Bytespider","Applebot-Extended"]},{"domain":"mailchimp.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"buffer.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"hootsuite.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"wordstream.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"marketingprofs.com","category":"marketing","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","Google-Extended","CCBot","Bytespider","Applebot-Extended"]},{"domain":"amazon.com","category":"retail","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider"]},{"domain":"walmart.com","category":"retail","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"target.com","category":"retail","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"etsy.com","category":"retail","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"ebay.com","category":"retail","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"bestbuy.com","category":"retail","reachable":false,"hasRobots":false,"blocked":[]},{"domain":"shopify.com","category":"retail","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"wayfair.com","category":"retail","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"nike.com","category":"retail","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"ikea.com","category":"retail","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"yelp.com","category":"retail","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"tripadvisor.com","category":"retail","reachable":true,"hasRobots":true,"blocked":["GPTBot","ClaudeBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"reddit.com","category":"social","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"linkedin.com","category":"social","reachable":true,"hasRobots":false,"blocked":[]},{"domain":"pinterest.com","category":"social","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider"]},{"domain":"tumblr.com","category":"social","reachable":true,"hasRobots":true,"blocked":["ClaudeBot","Google-Extended","CCBot","Applebot-Extended"]},{"domain":"x.com","category":"social","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"facebook.com","category":"social","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"instagram.com","category":"social","reachable":true,"hasRobots":true,"blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","Google-Extended","PerplexityBot","CCBot","Bytespider","Applebot-Extended"]},{"domain":"twitch.tv","category":"social","reachable":true,"hasRobots":true,"blocked":[]},{"domain":"vimeo.com","category":"social","reachable":true,"hasRobots":true,"blocked":["CCBot","Bytespider"]},{"domain":"stackexchange.com","category":"social","reachable":true,"hasRobots":false,"blocked":[]}]}}