# ----------------------------------------------------------------------
# ROBOTS.TXT FOR (Optimized for Fast Indexing)
# ----------------------------------------------------------------------

# 1. Google News Special Bot - Fast Indexing
User-agent: Googlebot-News
Allow: /
Disallow: /admin/
Disallow: /login/

# 2. General Google Bot
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /login/
Disallow: /search/

# 3. Adsense Bot - 
User-agent: Mediapartners-Google
Allow: /

# 4. Global rules for all other friendly bots
User-agent: *
Allow: /

# Block private directories to save crawl budget
Disallow: /admin/
Disallow: /login/
Disallow: /auth/
Disallow: /dashboard/
Disallow: /s/
Disallow: /uploads/reporter_ids/
Disallow: /uploads/contact_attachments/
Disallow: /uploads/notifications/
Disallow: /uploads/reporter_docs/

# Block dynamic search result pages
Disallow: /search/
Disallow: /*?q=*
Disallow: /*?search=*
Disallow: /*&sort=*

# 5. SEO Bots Block (Optional: Useful to keep competitors away)
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: / 

User-agent: SemrushBot
Disallow: /

# ----------------------------------------------------------------------
# ROCKET FUEL: SITEMAPS
# ----------------------------------------------------------------------
Sitemap: https://www.bollywoodwave.com/sitemap.xml
Sitemap: https://www.bollywoodwave.com/news-sitemap.xml