# Tax.Network Indexation Configuration
# Last updated: 2024-12-18
# Global Tax Information Powerhouse - 251 locations × 60 languages

# ===========================================
# SEARCH ENGINE CRAWLERS
# ===========================================

# All standard crawlers - full access
User-agent: *
Allow: /

# Protected areas
Disallow: /admin
Disallow: /admin/
Disallow: /account
Disallow: /account/
Disallow: /auth
Disallow: /auth/
Disallow: /api/
Disallow: /functions/

# User applications (require authentication)
Disallow: /chat
Disallow: /chat/
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites

# Registration and verification flows
Disallow: /expert/register
Disallow: /verify-opportunity

# OAuth callback - technical only
Disallow: /auth/oauth-callback

# Legacy routes - removed from app
Disallow: /articles
Disallow: /resources

# Prevent duplicate content from query parameters
Disallow: /*?preview=
Disallow: /*?tab=
Disallow: /*?page=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?locale=
Disallow: /*?utm_*

# Googlebot - prioritized crawling
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo Slurp
User-agent: Slurp
Allow: /
Crawl-delay: 1

# DuckDuckBot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Yandex
User-agent: Yandex
Allow: /
Crawl-delay: 2

# Baidu Spider
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# ===========================================
# AI/LLM CRAWLERS - Strategic Access for AIO
# ===========================================

# OpenAI GPTBot - Allow for AI visibility
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Google AI (Bard/Gemini training)
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Meta/Facebook AI
User-agent: FacebookBot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# ByteDance/TikTok
User-agent: Bytespider
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Apple Intelligence
User-agent: Applebot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# You.com AI Search
User-agent: YouBot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Meta External Agent (Llama models)
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Amazon Alexa/AI
User-agent: Amazonbot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# Microsoft Copilot
User-agent: Copilot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /auth
Disallow: /chat
Disallow: /submit-content
Disallow: /editor-dashboard
Disallow: /favorites
Disallow: /expert/register
Disallow: /verify-opportunity
Disallow: /articles
Disallow: /resources

# ===========================================
# STATIC ASSETS
# ===========================================
Allow: /logo-tax-network.svg
Allow: /robots.txt
Allow: /favicon.ico
Allow: /llms.txt
Allow: /locales/

# ===========================================
# SITEMAPS
# ===========================================
# Primary sitemap index - Tax.Network (global)
Sitemap: https://tax.network/sitemap.xml

# Primary sitemap index - Tributário (Brazil)
Sitemap: https://tributario.com.br/sitemap.xml

# Direct sitemap endpoints (backup — uses canonical domain)
Sitemap: https://tax.network/sitemap-index

# ===========================================
# AI/LLM GUIDANCE
# ===========================================
# See: https://tax.network/llms.txt
# Contains structured content guidance for AI crawlers
# Follows llmstxt.org specification