robots.txt
The citationFriendly preset expanded into one group per known crawler. The Sitemap line is derived from site and base.
robots.txt
dist/robots.txt// astro.config.mjs
site: 'https://casoon.github.io',
base: '/astro-site-files/',
// site/src/site-files.ts
export const robots: RobotsOptions = {
// Search engines and AI assistants may read and cite the docs; training crawlers may not.
preset: 'citationFriendly',
};User-agent: * User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: DuckDuckBot Allow: / User-agent: GPTBot Disallow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: Google-Extended Disallow: / User-agent: CCBot Disallow: / User-agent: PerplexityBot Allow: / User-agent: YouBot Allow: / User-agent: Bytespider Disallow: / User-agent: meta-externalagent Allow: / User-agent: Amazonbot Allow: / User-agent: Applebot-Extended Disallow: / User-agent: Diffbot Disallow: / User-agent: Omgilibot Disallow: / Sitemap: https://casoon.github.io/astro-site-files/sitemap.xml