# LeilôAI - robots.txt
# Permite crawlers de motores de busca + AI assistants (ChatGPT, Claude,
# Perplexity, Gemini) para ranqueamento em respostas de IA.
# Bloqueia apenas SEO scrapers agressivos.

# ============ Crawlers PERMITIDOS ============
# Motores de busca tradicionais e social embeds
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: GoogleOther
User-agent: Google-InspectionTool
User-agent: AdsBot-Google
User-agent: bingbot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Applebot
User-agent: Slurp
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: WhatsApp
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: TelegramBot
User-agent: Slackbot
User-agent: Pinterest
User-agent: Bravebot
User-agent: ia_archiver

# AI assistants e LLM crawlers (permitidos pra ranqueamento em
# respostas de IA, descoberta via ChatGPT/Claude/Perplexity, etc.
# 2026-05-12: removidos da secao Disallow porque Claude no navegador
# respeita robots.txt e exibia 403 ao tentar abrir leiloai.com.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: DuckAssistBot
User-agent: MojeekBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /painel/
Disallow: /admin/

# ============ SCRAPERS BLOQUEADOS (SEO tools agressivos) ============
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: rogerbot
User-agent: Screaming Frog SEO Spider
User-agent: SEOkicks
User-agent: Barkrowler
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: serpstatbot
User-agent: SeekportBot
User-agent: zoominfobot
User-agent: Bytespider
Disallow: /

# ============ Default: permite com restricoes ============
User-agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /painel/
Disallow: /admin/
Crawl-delay: 5

Sitemap: https://leiloai.com/sitemap.xml
Sitemap: https://leiloai.com/news-sitemap.xml
Sitemap: https://leiloai.com/sitemap-iminente.xml