# All crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /custadmin/
Disallow: /neutralhost/
# Image optimizer + framework internals — not pages
Disallow: /_next/

# Allow markdown API for all agents
Allow: /api/md/

# AI agents — welcome, please use markdown versions for best results
# Append .md to any public URL or send Accept: text/markdown
# See /llms.txt for a full index
# Developer docs: /developers  ·  OpenAPI 3.1 spec: /openapi.json

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# Host
Host: https://www.aerix.uk

# Sitemaps
Sitemap: https://www.aerix.uk/sitemap.xml