robots.txt
The citationFriendly preset expanded into one group per known crawler. The Sitemap line is derived from site and base.
robots.txt
dist/robots.txt// astro.config.mjs
site: 'https://casoon.github.io',
base: '/astro-site-files/',
// site/src/site-files.ts
export const robots: RobotsOptions = {
// Search engines and AI assistants may read and cite the docs; training crawlers may not.
preset: 'citationFriendly',
};User-agent: * Allow: / User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: DuckDuckBot Allow: / User-agent: Applebot Allow: / User-agent: GPTBot Disallow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: OAI-AdsBot Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-User Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: Google-Extended Disallow: / User-agent: Google-CloudVertexBot Allow: / User-agent: CCBot Disallow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: YouBot Allow: / User-agent: DuckAssistBot Allow: / User-agent: MistralAI-Training Disallow: / User-agent: MistralAI-Index Allow: / User-agent: MistralAI-User Allow: / User-agent: GrokBot Disallow: / User-agent: xAI-Grok Allow: / User-agent: DeepSeekBot Disallow: / User-agent: KimiBot Disallow: / User-agent: Kimi-SearchBot Allow: / User-agent: Kimi-User Allow: / User-agent: QwenBot Disallow: / User-agent: TongyiBot Allow: / User-agent: Hunyuan Disallow: / User-agent: ERNIEBot Disallow: / User-agent: PanguBot Disallow: / User-agent: PetalBot Allow: / User-agent: cohere-training-data-crawler Disallow: / User-agent: cohere-ai Allow: / User-agent: Bytespider Disallow: / User-agent: TikTokSpider Disallow: / User-agent: DoubaoBot Allow: / User-agent: meta-externalagent Allow: / User-agent: meta-externalfetcher Allow: / User-agent: meta-webindexer Allow: / User-agent: Amazonbot Allow: / User-agent: bedrockbot Allow: / User-agent: Applebot-Extended Disallow: / User-agent: YandexAdditional User-agent: YandexAdditionalBot Disallow: / User-agent: AI2Bot User-agent: Ai2Bot-Dolma Disallow: / User-agent: ICC-Crawler Disallow: / User-agent: kagi-fetcher Allow: / User-agent: Diffbot Disallow: / User-agent: Omgilibot User-agent: omgili Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: Timpibot Disallow: / User-agent: FirecrawlAgent Disallow: / User-agent: Crawlspace Disallow: / User-agent: VelenPublicWebCrawler Disallow: / Sitemap: https://casoon.github.io/astro-site-files/sitemap.xml