# robots.txt for Brickellsold.com - Premier Miami Real Estate Platform

# Default rule: allow all, block private/internal paths
User-Agent: *
Allow: /
Disallow: /auth/
Disallow: /en/auth/
Disallow: /es/auth/
Disallow: /ru/auth/
Disallow: /admin/
Disallow: /en/admin/
Disallow: /es/admin/
Disallow: /ru/admin/
Disallow: /api/
Disallow: /v1/
Disallow: /marketing/
Disallow: /en/marketing/
Disallow: /es/marketing/
Disallow: /ru/marketing/

# Ensure sitemaps are accessible
Allow: /sitemap.xml
Allow: /sitemaps/
Allow: /blog/sitemap_index.xml
Allow: /brickellsold_rss.xml

# AI search crawlers - allow public content
User-agent: OAI-SearchBot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: GPTBot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

# Social media crawlers - allow for rich previews
User-agent: LinkedInBot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: facebookexternalhit
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

User-agent: Twitterbot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/

# Known bad crawlers - block entirely
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Primary sitemap index — child sitemaps are nested within sitemap.xml
Sitemap: https://www.brickellsold.com/sitemap.xml
Sitemap: https://www.brickellsold.com/blog/sitemap_index.xml