# ROBOTS.TXT - Optimized for cost savings
# Block low-value bots that trigger ISR without SEO benefit

# ❌ BLOCK problematic crawlers (they cause ISR writes without value)
User-agent: Baiduspider
User-agent: Baidu
User-agent: Sogou
User-agent: Soso
User-agent: Yandex
User-agent: MJ12bot
User-agent: Semrush
User-agent: AhrefsBot
User-agent: Majestic
User-agent: PetalBot
User-agent: ByteSpider
User-agent: HTTrack
User-agent: SiteSucker
User-agent: Octoparse
Disallow: /

# ✅ ALLOW major search engines (provide SEO value)
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: facebookexternalhit
User-agent: Applebot
Allow: /

# ⚠️ For all other bots - strict crawl rate to prevent ISR spam
User-agent: *
Crawl-delay: 30
Request-rate: 1/30s
Allow: /

# Block unnecessary query parameters that duplicate content
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?tracking=
Disallow: /*?session=
Disallow: /api/
Disallow: /admin/
Disallow: /_next/

# Allow sitemaps for major search engines
Sitemap: https://healthmudraa.com/sitemap.xml
Sitemap: https://healthmudraa.com/api/sitemap-doctors.xml
Sitemap: https://healthmudraa.com/api/sitemap-hospitals.xml
Sitemap: https://healthmudraa.com/api/sitemap-treatments.xml
Sitemap: https://healthmudraa.com/api/sitemap-medicines.xml
Sitemap: https://healthmudraa.com/api/sitemap-conditions.xml

User-agent: Webz.io
Allow: /

User-agent: UiPath
Allow: /

# AI & LLM Crawlers - Explicitly allowed for HealthMudraa
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Allow:/static/
Allow: /blogs/

# Block aggressive crawlers
User-agent: PetalBot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: Majestic
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# Default rules for all other bots
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /static/
Allow:/static/
Allow: /blogs/
Allow: /

# ✅ Sitemaps - Comprehensive sitemap coverage
Sitemap: https://healthmudraa.com/api/sitemap-index.xml
Sitemap: https://healthmudraa.com/api/sitemap-main.xml
Sitemap: https://healthmudraa.com/api/sitemap-programmatic.xml
Sitemap: https://healthmudraa.com/api/sitemap-images.xml
Sitemap: https://healthmudraa.com/api/sitemap-blogs.xml

# Medical content sitemaps
Sitemap: https://healthmudraa.com/api/sitemap-medicines-index.xml
Sitemap: https://healthmudraa.com/api/sitemap-conditions-index.xml
Sitemap: https://healthmudraa.com/api/sitemap-packages-index.xml

# Professional listings sitemaps
Sitemap: https://healthmudraa.com/api/sitemap-doctors-index.xml
Sitemap: https://healthmudraa.com/api/sitemap-hospitals-index.xml
Sitemap: https://healthmudraa.com/api/sitemap-listings-index.xml

# Video content sitemaps
Sitemap: https://healthmudraa.com/api/sitemap-videos-index.xml

# City-specialty combination sitemaps (high SEO value)
Sitemap: https://healthmudraa.com/api/sitemap-city-dept/1.xml
Sitemap: https://healthmudraa.com/api/sitemap-city-treatments/1.xml

# Legacy sitemap for backward compatibility
Sitemap: https://healthmudraa.com/api/sitemap-diseases-index.xml

# ✅ Host declaration
Host: https://healthmudraa.com