# Dream World Tours Robots.txt
# Updated: 2025-04-15

# Allow all web crawlers to index the website
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://www.dreamworldtours.in/sitemap.xml

# Important directories to index - main content areas
Allow: /tours/
Allow: /destinations/
Allow: /about/
Allow: /contact/
Allow: /reviews/
Allow: /corporate-travel/
Allow: /blog/
Allow: /group-tours/

# Disallow admin pages or any sensitive area
Disallow: /admin/
Disallow: /login/
Disallow: /dashboard/
Disallow: /api/
Disallow: /checkout/
Disallow: /user/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-json/
Disallow: /*.sql$
Disallow: /*.sql.gz$
Disallow: /*.log$
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.env
Disallow: /cdn-cgi/

# Performance optimizations for major crawlers
User-agent: Googlebot
Crawl-delay: 1

User-agent: Bingbot
Crawl-delay: 1

User-agent: Slurp
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Yandex
Allow: /

# Social media crawlers - ensure good previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Pinterestbot
Allow: /

# AI Crawlers - Control AI training on your content
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Claude-Web
Disallow: /