# Xalery robots.txt - Hardened

# Block Google image crawler
User-agent: Googlebot-Image
Disallow: /images/
Disallow: /assets/
Disallow: /fonts/
Disallow: /placeholder.svg
Allow: /favicon.png

# Search engines
User-agent: Googlebot
Allow: /
Allow: /about
Allow: /pricing
Allow: /how-it-works
Allow: /faq
Allow: /contact
Allow: /features
Allow: /examples
Crawl-delay: 2

User-agent: Bingbot
Allow: /
Allow: /about
Allow: /pricing
Allow: /how-it-works
Allow: /faq
Allow: /contact
Crawl-delay: 2

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Block all other bots from sensitive areas
User-agent: *
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /onboarding
Disallow: /onboarding/
Disallow: /api/
Disallow: /auth
Disallow: /auth/
Disallow: /profile
Disallow: /profile/
Disallow: /seo
Disallow: /seo/
Disallow: /messages
Disallow: /messages/
Disallow: /pricing-dashboard
Disallow: /pricing-dashboard/
Disallow: /domain-settings
Disallow: /domain-settings/
Disallow: /booking-settings
Disallow: /booking-settings/
Disallow: /upload-images
Disallow: /upload-images/
Allow: /
Crawl-delay: 5

# Block aggressive scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /