# ======================================================
# GLOBAL DEFAULT RULES (MOBILE-FIRST OPTIMIZED)
# ======================================================
User-agent: *
Allow: /
Disallow: /search
Disallow: /feeds
Disallow: /*?updated-max=
# Removed Disallow /*?m= to ensure USA Mobile Indexing works 100%

# ======================================================
# EXPLICITLY BLOCKED BOTS (SCRAPING & ABUSE)
# ======================================================
User-agent: Bytespider
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: CCBot
Disallow: /

# ======================================================
# PRIORITY: GOOGLE & ADSENSE (REVENUE PROTECTION)
# ======================================================
User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Google-Extended
Allow: /

# ======================================================
# AI / ANSWER ENGINES (ALLOW FOR AI-TRAFFIC)
# ======================================================
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: PerplexityBot
Allow: /

# ======================================================
# SOCIAL & PROFESSIONAL CRAWLERS
# ======================================================
User-agent: Twitterbot
Allow: /
User-agent: LinkedInBot
Allow: /
User-agent: Pinterestbot
Allow: /
User-agent: FacebookExternalHit
Allow: /

# ======================================================
# AI POLICY & SITEMAPS
# ======================================================
Allow: /llms.txt
Allow: /p/llms.html

Sitemap: https://www.pravinzende.co.in/sitemap.xml
Sitemap: https://www.pravinzende.co.in/sitemap-pages.xml
Sitemap: https://www.pravinzende.co.in/atom.xml?redirect=false&start-index=1&max-results=500