# Every crawler, including the AI crawlers named below, follows the same rules: # the whole site is open except raw audio files and the newsletter reward download. # AI crawlers are named so they're explicitly welcome; a crawler that finds its own # name uses only that group, which is why they share this one rather than having their own. # (No blank lines inside this group: some crawlers treat a blank line as the end of the rules.) User-agent: * # # OpenAI: training, search, and ChatGPT browsing User-agent: GPTBot User-agent: OAI-SearchBot User-agent: ChatGPT-User # # Anthropic: training, search, and Claude browsing User-agent: ClaudeBot User-agent: Claude-SearchBot User-agent: Claude-User # # Perplexity: search and answer fetching User-agent: PerplexityBot User-agent: Perplexity-User # # Google (Gemini) and Apple (Apple Intelligence) AI use User-agent: Google-Extended User-agent: Applebot User-agent: Applebot-Extended # # Other AI assistants and datasets User-agent: Amazonbot User-agent: Meta-ExternalAgent User-agent: Meta-ExternalFetcher User-agent: DuckAssistBot User-agent: MistralAI-User User-agent: CCBot # # Block raw music file scraping Disallow: /*.mp3$ Disallow: /*.wav$ Disallow: /*.flac$ Disallow: /*.aiff$ Disallow: /*.m4a$ # # Keep unlisted downloads (newsletter reward) out of search results Disallow: /assets/downloads/ # # Allow lyric pages, album pages, and all structured content Allow: /lyrics/ Allow: /lyrics/* # # AI usage preferences (Content Signals, contentsignals.org): appear in search # and AI answers, but don't use this site's content to train AI models. Content-Signal: search=yes, ai-input=yes, ai-train=no # Direct AI crawlers to policy & indexing metadata Sitemap: https://www.darksoft.band/sitemap.xml # llms.txt: https://www.darksoft.band/llms.txt