# UMVA Super App Platform - Robots.txt
# Enhanced SEO Configuration
# Last Updated: 2025-10-19

# Default Crawlers
User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /merchant/login
Disallow: /merchant/register
Disallow: /agent/login
Disallow: /agent/register
Disallow: /user/login
Disallow: /user/register
Disallow: /user/dashboard
Disallow: /user/password
Disallow: /password/
Disallow: /logout
Disallow: /verify/
Disallow: /otp-verification
Disallow: /api/
Disallow: /cron
Disallow: /webhook
Disallow: /ipn/
Disallow: /temp/
Disallow: /storage/
Disallow: /vendor/
Disallow: /core/
Disallow: /backups/
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*?*session
Disallow: /*?*token
Disallow: /*?*api_key
Disallow: /search?
Disallow: /*?page=
Disallow: /*&page=
Crawl-delay: 1

# Allow Important Pages
Allow: /api/documentation
Allow: /api/reference
Allow: /contact
Allow: /about
Allow: /tips/
Allow: /assets/
Allow: /sitemap.xml
Allow: /robots.txt

# Google Crawlers
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /user/
Disallow: /merchant/
Disallow: /agent/
Disallow: /api/
Crawl-delay: 0.5

User-agent: Googlebot-Image
Allow: /assets/images/
Disallow: /assets/images/user/
Disallow: /assets/images/merchant/
Disallow: /assets/images/agent/

User-agent: Googlebot-Mobile
Allow: /
Disallow: /admin/
Crawl-delay: 0.5

# Bing Crawler
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /user/
Crawl-delay: 1

User-agent: MSNBot
Allow: /
Disallow: /admin/
Crawl-delay: 1

# Yahoo Crawler
User-agent: Slurp
Allow: /
Disallow: /admin/
Crawl-delay: 1

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Crawl-delay: 1

# Baidu (Chinese Search Engine)
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Crawl-delay: 2

# Yandex (Russian Search Engine)
User-agent: YandexBot
Allow: /
Disallow: /admin/
Crawl-delay: 2

# Facebook Crawler (for Open Graph)
User-agent: facebookexternalhit
Allow: /
Disallow: /admin/
Crawl-delay: 0

# Twitter Crawler (for Twitter Cards)
User-agent: Twitterbot
Allow: /
Disallow: /admin/
Crawl-delay: 0

# LinkedIn Crawler
User-agent: LinkedInBot
Allow: /
Disallow: /admin/
Crawl-delay: 0

# Pinterest Crawler
User-agent: Pinterest
Allow: /
Disallow: /admin/
Crawl-delay: 1

# WhatsApp Crawler (for link previews)
User-agent: WhatsApp
Allow: /
Disallow: /admin/
Crawl-delay: 0

# Telegram Crawler (for link previews)
User-agent: TelegramBot
Allow: /
Disallow: /admin/
Crawl-delay: 0

# Slack Crawler (for link previews)
User-agent: Slackbot
Allow: /
Disallow: /admin/
Crawl-delay: 0

# Discord Crawler (for link previews)
User-agent: Discordbot
Allow: /
Disallow: /admin/
Crawl-delay: 0

# Block Bad Bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: rogerbot
Disallow: /

# Block AI Scrapers (Optional - uncomment if needed)
# User-agent: GPTBot
# Disallow: /

# User-agent: ChatGPT-User
# Disallow: /

# User-agent: CCBot
# Disallow: /

# User-agent: anthropic-ai
# Disallow: /

# User-agent: Claude-Web
# Disallow: /

# Block Download Bots
User-agent: wget
Disallow: /

User-agent: curl
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: ia_archiver
Disallow: /

# Sitemap Location
Sitemap: https://umva.net/sitemap.xml

# Additional Security
# Prevent directory listing
User-agent: *
Disallow: /*/

# Host (Optional - specify preferred domain)
# Host: https://umva.net