User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: CCBot
Allow: /

# General robots rules
User-agent: *
# Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /404
Disallow: /fr/404
Disallow: /ru/404
Disallow: /ja/404
Disallow: /ko/404
Disallow: /zh/404
Disallow: /es/404
Disallow: /de/404
Disallow: /ar/404
Disallow: /it/404
Disallow: /en-GB/404
Disallow: /pt/404
Disallow: /hi/404
Disallow: /tr/404
Disallow: /pl/404
Disallow: /id/404
Disallow: /vi/404
Disallow: /uk/404
Host: https://intlayer.org
Sitemap: https://intlayer.org/sitemap.xml
Sitemap: https://app.intlayer.org/sitemap.xml
# Agentmap: https://intlayer.org/.well-known/ai-catalog.json