# Robots.txt for https://www.anubavam.com # Last updated: [Current Date] # Default crawler rules User-agent: * # === ALLOW IMPORTANT CONTENT === # Main sections Allow: / Allow: /services/ Allow: /products/ Allow: /about/ Allow: /contact/ Allow: /blog/ Allow: /case-studies/ Allow: /resources/ # Allow all media and assets Allow: /*.css Allow: /*.js Allow: /*.jpg Allow: /*.jpeg Allow: /*.png Allow: /*.gif Allow: /*.svg Allow: /*.webp Allow: /*.pdf Allow: /*.woff Allow: /*.woff2 # === DISALLOW SECTIONS === # Admin and user areas Disallow: /admin/ Disallow: /user/login Disallow: /user/register Disallow: /user/password Disallow: /user/logout Disallow: /node/add/ # System paths Disallow: /core/install/ Disallow: /core/modules/system/ Disallow: /profiles/testing/ Disallow: /comment/reply/ Disallow: /filter/tips # Duplicate content paths Disallow: /index.php Disallow: /*?*page= Disallow: /*&page= Disallow: /print/ Disallow: /printpdf/ # Files Disallow: /CHANGELOG.txt Disallow: /cron.php Disallow: /INSTALL.txt Disallow: /install.php Disallow: /LICENSE.txt Disallow: /MAINTAINERS.txt Disallow: /update.php Disallow: /UPGRADE.txt Disallow: /xmlrpc.php # Parameters (prevent duplicate content) Disallow: /*?*utm_ Disallow: /*?*ref= # Crawl-delay for responsible crawling Crawl-delay: 1 # Sitemap Sitemap: https://www.anubavam.com/sitemap.xml # === AI/LLM CRAWLERS (Explicit Allow for GEO) === User-agent: GPTBot Allow: / Crawl-delay: 1 User-agent: ChatGPT-User Allow: / User-agent: ClaudeBot Allow: / User-agent: CCBot Allow: / User-agent: PerplexityBot Allow: / User-agent: Google-Extended Allow: / User-agent: Bingbot Allow: / Crawl-delay: 1 # Block bad bots (optional) User-agent: MJ12bot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Crawl-delay: 10 User-agent: PerplexityBot Allow: / User-agent: Google-Extended Allow: / User-agent: Bingbot Allow: / Crawl-delay: 1 # Block bad bots (optional) User-agent: MJ12bot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Crawl-delay: 10