{ "name": "Skald crawler list", "schema": 1, "skald_version": "2.44.0", "generated": "2026-10-27", "url": "https://skaldanalytics.com/bots.json", "about": "https://skaldanalytics.com/ai-traffic-analytics/ai-crawlers/", "source": "https://github.com/spcaeo/skald", "matching": "Each pattern is a case-insensitive regular expression (Python re), searched anywhere in the request's user agent. The lists are asked in this order and the first that matches names the request: answer engines asked (a live fetch), answer engines training (their crawlers, search indexes among them), link previews, search engines. The SEO crawlers here are filed under bots, unnamed. A pattern of null is a robots.txt token: no request carries it.", "checking": "A named claim is checked against its owner: the owner's published address list (fetched once a day; a list older than 7 days never calls a request a fake), reverse DNS checked both ways (the name ends in the owner's domain and resolves back to the address; answers kept 7 days), or the owner's own network. A request proved not to be the owner's is filed under bots as \"claims to be \". \"not checked\": nothing to check against, so the claim stands. A Claude-User fetch from Claude Code runs on the developer's own machine and is never checked.", "purposes": { "training": "training crawl: reads pages to train a model; nobody is waiting on it", "live_fetch": "live fetch: a person just asked an assistant something, and it is reading the page to answer them", "search_index": "search index: crawls ahead of any question, for a search engine or an assistant's own search", "ads": "ads: checks the landing pages of ads", "seo_tool": "SEO tool: an SEO suite's crawler (links, rankings, site audits)", "other": "other: a tool or fetcher with another job" }, "counts": { "crawlers": 67, "matched_by_user_agent": 66, "robots_txt_tokens_only": 1, "ai": 40, "checked": 27, "by_purpose": { "training": 19, "live_fetch": 13, "search_index": 23, "ads": 3, "seo_tool": 6, "other": 3 }, "referrer_hosts": 25, "utm_sources": 10 }, "crawlers": [ { "id": "chatgpt-user", "name": "ChatGPT-User", "pattern": "ChatGPT-User", "owner": "OpenAI", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "openai-gptbot", "url": "https://openai.com/gptbot.json" }, { "name": "openai-searchbot", "url": "https://openai.com/searchbot.json" }, { "name": "openai-chatgpt-user", "url": "https://openai.com/chatgpt-user.json" }, { "name": "openai-adsbot", "url": "https://openai.com/adsbot.json" } ], "reverse_dns": [] } }, { "id": "claude-user", "name": "Claude-User", "pattern": "Claude-User", "owner": "Anthropic", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "anthropic", "url": "https://claude.com/crawling/bots.json" } ], "reverse_dns": [] } }, { "id": "perplexity-user", "name": "Perplexity-User", "pattern": "Perplexity-User", "owner": "Perplexity", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "perplexity-bot", "url": "https://www.perplexity.ai/perplexitybot.json" }, { "name": "perplexity-user", "url": "https://www.perplexity.ai/perplexity-user.json" } ], "reverse_dns": [] } }, { "id": "meta-externalfetcher", "name": "meta-externalfetcher", "pattern": "meta-externalfetcher", "owner": "Meta", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "owner's network" ], "lists": [ { "name": "meta", "url": "https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS32934" } ], "reverse_dns": [] } }, { "id": "gemini-user", "name": "Gemini-User", "pattern": "Gemini-User", "owner": "Google", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "bingpreview", "name": "BingPreview", "pattern": "BingPreview", "owner": "Microsoft", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "bing", "url": "https://www.bing.com/toolbox/bingbot.json" } ], "reverse_dns": [ "search.msn.com" ] } }, { "id": "mistralai-user", "name": "MistralAI-User", "pattern": "MistralAI-User", "owner": "Mistral", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "claude-web", "name": "Claude-Web", "pattern": "Claude-Web", "owner": "Anthropic", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "google-agent", "name": "Google-Agent", "pattern": "Google-Agent", "owner": "Google", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "googleagent-mariner", "name": "GoogleAgent-Mariner", "pattern": "GoogleAgent-Mariner", "owner": "Google", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "google-gemininotebook", "name": "Google-GeminiNotebook", "pattern": "Google-GeminiNotebook", "owner": "Google", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "google-notebooklm", "name": "Google-NotebookLM", "pattern": "Google-NotebookLM", "owner": "Google", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "google-common", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "google-special", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "google-user", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json" }, { "name": "google-user-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "reverse_dns": [ "googlebot.com", "google.com", "googleusercontent.com" ] } }, { "id": "amzn-user", "name": "Amzn-User", "pattern": "Amzn-User", "owner": "Amazon", "purpose": "live_fetch", "skald_files_as": "answer engines (asked)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "gptbot", "name": "GPTBot", "pattern": "GPTBot", "owner": "OpenAI", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "openai-gptbot", "url": "https://openai.com/gptbot.json" }, { "name": "openai-searchbot", "url": "https://openai.com/searchbot.json" }, { "name": "openai-chatgpt-user", "url": "https://openai.com/chatgpt-user.json" }, { "name": "openai-adsbot", "url": "https://openai.com/adsbot.json" } ], "reverse_dns": [] } }, { "id": "claudebot", "name": "ClaudeBot", "pattern": "ClaudeBot", "owner": "Anthropic", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "anthropic", "url": "https://claude.com/crawling/bots.json" } ], "reverse_dns": [] } }, { "id": "claude-searchbot", "name": "Claude-SearchBot", "pattern": "Claude-SearchBot", "owner": "Anthropic", "purpose": "search_index", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "anthropic", "url": "https://claude.com/crawling/bots.json" } ], "reverse_dns": [] } }, { "id": "ccbot", "name": "CCBot", "pattern": "CCBot", "owner": "Common Crawl", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "anthropic-ai", "name": "anthropic-ai", "pattern": "anthropic-ai", "owner": "Anthropic", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "anthropic", "url": "https://claude.com/crawling/bots.json" } ], "reverse_dns": [] } }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "pattern": "OAI-SearchBot", "owner": "OpenAI", "purpose": "search_index", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "openai-gptbot", "url": "https://openai.com/gptbot.json" }, { "name": "openai-searchbot", "url": "https://openai.com/searchbot.json" }, { "name": "openai-chatgpt-user", "url": "https://openai.com/chatgpt-user.json" }, { "name": "openai-adsbot", "url": "https://openai.com/adsbot.json" } ], "reverse_dns": [] } }, { "id": "perplexitybot", "name": "PerplexityBot", "pattern": "PerplexityBot", "owner": "Perplexity", "purpose": "search_index", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "perplexity-bot", "url": "https://www.perplexity.ai/perplexitybot.json" }, { "name": "perplexity-user", "url": "https://www.perplexity.ai/perplexity-user.json" } ], "reverse_dns": [] } }, { "id": "meta-externalagent", "name": "meta-externalagent", "pattern": "meta-externalagent", "owner": "Meta", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's network" ], "lists": [ { "name": "meta", "url": "https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS32934" } ], "reverse_dns": [] } }, { "id": "facebookbot", "name": "FacebookBot", "pattern": "FacebookBot", "owner": "Meta", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's network" ], "lists": [ { "name": "meta", "url": "https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS32934" } ], "reverse_dns": [] } }, { "id": "bytespider", "name": "Bytespider", "pattern": "Bytespider", "owner": "ByteDance", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "amazonbot", "name": "Amazonbot", "pattern": "Amazonbot", "owner": "Amazon", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "applebot-extended", "name": "Applebot-Extended", "pattern": "Applebot-Extended", "owner": "Apple", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": true, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "apple", "url": "https://search.developer.apple.com/applebot.json" } ], "reverse_dns": [ "applebot.apple.com" ] } }, { "id": "cohere-ai", "name": "cohere-ai", "pattern": "cohere-ai", "owner": "Cohere", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "diffbot", "name": "Diffbot", "pattern": "Diffbot", "owner": "Diffbot", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "omgili", "name": "omgili", "pattern": "omgili", "owner": "Webz.io", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "ai2bot", "name": "AI2Bot", "pattern": "AI2Bot", "owner": "Allen Institute for AI", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "timpibot", "name": "Timpibot", "pattern": "Timpibot", "owner": "Timpi", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "imagesiftbot", "name": "ImagesiftBot", "pattern": "ImagesiftBot", "owner": "ImageSift", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "youbot", "name": "YouBot", "pattern": "YouBot", "owner": "You.com", "purpose": "search_index", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "pangubot", "name": "PanguBot", "pattern": "PanguBot", "owner": "Huawei", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "kangaroo-bot", "name": "Kangaroo Bot", "pattern": "Kangaroo Bot", "owner": "Kangaroo LLM", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "duckassistbot", "name": "DuckAssistBot", "pattern": "DuckAssistBot", "owner": "DuckDuckGo", "purpose": "search_index", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "duckduckgo", "url": "https://duckduckgo.com/duckduckbot.json" } ], "reverse_dns": [] } }, { "id": "amzn-searchbot", "name": "Amzn-SearchBot", "pattern": "Amzn-SearchBot", "owner": "Amazon", "purpose": "search_index", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "google-cloudvertexbot", "name": "Google-CloudVertexBot", "pattern": "Google-CloudVertexBot", "owner": "Google", "purpose": "search_index", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "cohere-training-data-crawler", "name": "cohere-training-data-crawler", "pattern": "cohere-training-data-crawler", "owner": "Cohere", "purpose": "training", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "oai-adsbot", "name": "OAI-AdsBot", "pattern": "OAI-AdsBot", "owner": "OpenAI", "purpose": "ads", "skald_files_as": "answer engines (training)", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "openai-gptbot", "url": "https://openai.com/gptbot.json" }, { "name": "openai-searchbot", "url": "https://openai.com/searchbot.json" }, { "name": "openai-chatgpt-user", "url": "https://openai.com/chatgpt-user.json" }, { "name": "openai-adsbot", "url": "https://openai.com/adsbot.json" } ], "reverse_dns": [] } }, { "id": "google-extended", "name": "Google-Extended", "pattern": null, "owner": "Google", "purpose": "training", "skald_files_as": null, "robots_txt_only": true, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "googlebot", "name": "Googlebot", "pattern": "Googlebot", "owner": "Google", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "google-common", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "google-special", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "google-user", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json" }, { "name": "google-user-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "reverse_dns": [ "googlebot.com", "google.com", "googleusercontent.com" ] } }, { "id": "google-inspectiontool", "name": "Google-InspectionTool", "pattern": "Google-InspectionTool", "owner": "Google", "purpose": "other", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "google-common", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "google-special", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "google-user", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json" }, { "name": "google-user-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "reverse_dns": [ "googlebot.com", "google.com", "googleusercontent.com" ] } }, { "id": "googleother", "name": "GoogleOther", "pattern": "GoogleOther", "owner": "Google", "purpose": "other", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "google-common", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "google-special", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "google-user", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json" }, { "name": "google-user-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "reverse_dns": [ "googlebot.com", "google.com", "googleusercontent.com" ] } }, { "id": "apis-google", "name": "APIs-Google", "pattern": "APIs-Google", "owner": "Google", "purpose": "other", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "google-common", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "google-special", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "google-user", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json" }, { "name": "google-user-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "reverse_dns": [ "googlebot.com", "google.com", "googleusercontent.com" ] } }, { "id": "adsbot-google", "name": "AdsBot-Google", "pattern": "AdsBot-Google", "owner": "Google", "purpose": "ads", "skald_files_as": "search engines (ads check)", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "google-common", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "google-special", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "google-user", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json" }, { "name": "google-user-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "reverse_dns": [ "googlebot.com", "google.com", "googleusercontent.com" ] } }, { "id": "google-adwords-instant", "name": "Google-Adwords-Instant", "pattern": "Google-Adwords-Instant", "owner": "Google", "purpose": "ads", "skald_files_as": "search engines (ads check)", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "google-common", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "google-special", "url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "google-user", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json" }, { "name": "google-user-google", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "reverse_dns": [ "googlebot.com", "google.com", "googleusercontent.com" ] } }, { "id": "bingbot", "name": "bingbot", "pattern": "bingbot", "owner": "Microsoft", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "bing", "url": "https://www.bing.com/toolbox/bingbot.json" } ], "reverse_dns": [ "search.msn.com" ] } }, { "id": "slurp", "name": "Slurp", "pattern": "Slurp", "owner": "Yahoo", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "duckduckbot", "name": "DuckDuckBot", "pattern": "DuckDuckBot", "owner": "DuckDuckGo", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "owner's published list" ], "lists": [ { "name": "duckduckgo", "url": "https://duckduckgo.com/duckduckbot.json" } ], "reverse_dns": [] } }, { "id": "baiduspider", "name": "Baiduspider", "pattern": "Baiduspider", "owner": "Baidu", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "yandexbot", "name": "YandexBot", "pattern": "YandexBot", "owner": "Yandex", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "reverse DNS" ], "lists": [], "reverse_dns": [ "yandex.ru", "yandex.net", "yandex.com" ] } }, { "id": "exabot", "name": "Exabot", "pattern": "Exabot", "owner": "Exalead", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "applebot", "name": "Applebot", "pattern": "Applebot(?!-Extended)", "owner": "Apple", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "owner's published list", "reverse DNS" ], "lists": [ { "name": "apple", "url": "https://search.developer.apple.com/applebot.json" } ], "reverse_dns": [ "applebot.apple.com" ] } }, { "id": "sogou", "name": "Sogou", "pattern": "Sogou(?!MSE|MobileBrowser|Explorer)", "owner": "Sogou", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "seznambot", "name": "SeznamBot", "pattern": "SeznamBot", "owner": "Seznam.cz", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "yeti", "name": "Yeti", "pattern": "Yeti(?=/)", "owner": "Naver", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "coccocbot", "name": "coccocbot", "pattern": "coccocbot", "owner": "Coc Coc", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "qwantbot", "name": "Qwantbot", "pattern": "Qwantbot", "owner": "Qwant", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "qwantify", "name": "Qwantify", "pattern": "Qwantify", "owner": "Qwant", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "mojeekbot", "name": "MojeekBot", "pattern": "MojeekBot", "owner": "Mojeek", "purpose": "search_index", "skald_files_as": "search engines", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "ahrefsbot", "name": "AhrefsBot", "pattern": "AhrefsBot", "owner": "Ahrefs", "purpose": "seo_tool", "skald_files_as": "bots", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "semrushbot", "name": "SemrushBot", "pattern": "SemrushBot", "owner": "Semrush", "purpose": "seo_tool", "skald_files_as": "bots", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "mj12bot", "name": "MJ12bot", "pattern": "MJ12bot", "owner": "Majestic", "purpose": "seo_tool", "skald_files_as": "bots", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "dotbot", "name": "DotBot", "pattern": "DotBot", "owner": "Moz", "purpose": "seo_tool", "skald_files_as": "bots", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "blexbot", "name": "BLEXBot", "pattern": "BLEXBot", "owner": "WebMeUp", "purpose": "seo_tool", "skald_files_as": "bots", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "dataforseobot", "name": "DataForSeoBot", "pattern": "DataForSeoBot", "owner": "DataForSEO", "purpose": "seo_tool", "skald_files_as": "bots", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } }, { "id": "petalbot", "name": "PetalBot", "pattern": "PetalBot", "owner": "Huawei", "purpose": "search_index", "skald_files_as": "bots", "robots_txt_only": false, "check": { "methods": [ "not checked" ], "lists": [], "reverse_dns": [] } } ], "referrers": { "note": "A person who followed an assistant's citation, by the referrer's host (or a subdomain of it), or by the link's utm_source. Filed as people, tier cited.", "hosts": [ { "host": "chatgpt.com", "path_prefix": null, "assistant": "chatgpt.com" }, { "host": "chat.openai.com", "path_prefix": null, "assistant": "chatgpt.com" }, { "host": "openai.com", "path_prefix": null, "assistant": "chatgpt.com" }, { "host": "perplexity.ai", "path_prefix": null, "assistant": "perplexity.ai" }, { "host": "claude.ai", "path_prefix": null, "assistant": "claude.ai" }, { "host": "gemini.google.com", "path_prefix": null, "assistant": "gemini.google.com" }, { "host": "copilot.microsoft.com", "path_prefix": null, "assistant": "copilot.microsoft.com" }, { "host": "you.com", "path_prefix": null, "assistant": "you.com" }, { "host": "phind.com", "path_prefix": null, "assistant": "phind.com" }, { "host": "poe.com", "path_prefix": null, "assistant": "poe.com" }, { "host": "mistral.ai", "path_prefix": null, "assistant": "mistral.ai" }, { "host": "copilot.com", "path_prefix": null, "assistant": "copilot.microsoft.com" }, { "host": "meta.ai", "path_prefix": null, "assistant": "meta.ai" }, { "host": "chat.deepseek.com", "path_prefix": null, "assistant": "deepseek.com" }, { "host": "grok.com", "path_prefix": null, "assistant": "grok.com" }, { "host": "com.openai.chatgpt", "path_prefix": null, "assistant": "chatgpt.com" }, { "host": "ai.perplexity.app.android", "path_prefix": null, "assistant": "perplexity.ai" }, { "host": "com.anthropic.claude", "path_prefix": null, "assistant": "claude.ai" }, { "host": "com.google.android.apps.bard", "path_prefix": null, "assistant": "gemini.google.com" }, { "host": "com.microsoft.copilot", "path_prefix": null, "assistant": "copilot.microsoft.com" }, { "host": "com.deepseek.chat", "path_prefix": null, "assistant": "deepseek.com" }, { "host": "ai.x.grok", "path_prefix": null, "assistant": "grok.com" }, { "host": "com.facebook.stella", "path_prefix": null, "assistant": "meta.ai" }, { "host": "bing.com", "path_prefix": "/chat", "assistant": "copilot.microsoft.com" }, { "host": "x.com", "path_prefix": "/i/grok", "assistant": "grok.com" } ], "utm_sources": [ { "utm_source": "chatgpt", "assistant": "chatgpt.com" }, { "utm_source": "openai", "assistant": "chatgpt.com" }, { "utm_source": "perplexity", "assistant": "perplexity.ai" }, { "utm_source": "claude", "assistant": "claude.ai" }, { "utm_source": "gemini", "assistant": "gemini.google.com" }, { "utm_source": "copilot", "assistant": "copilot.microsoft.com" }, { "utm_source": "mistral", "assistant": "mistral.ai" }, { "utm_source": "deepseek", "assistant": "deepseek.com" }, { "utm_source": "grok", "assistant": "grok.com" }, { "utm_source": "meta.ai", "assistant": "meta.ai" } ] } }