# robots.txt for DigitGrow # Default: allow all standard crawlers User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Amazonbot Allow: / User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / User-agent: FacebookBot Allow: / User-agent: * Allow: / Allow: /llms.txt Allow: /sitemap.xml Disallow: /admin Disallow: /auth # AI training crawlers: DISALLOW User-agent: GPTBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: CCBot Disallow: / User-agent: cohere-ai Disallow: / User-agent: Bytespider Disallow: / User-agent: Diffbot Disallow: / User-agent: omgili Disallow: / User-agent: AI2Bot Disallow: / User-agent: Timpibot Disallow: / User-agent: meta-externalagent Disallow: / # AI search / assistant / agent bots: ALLOW User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: ChatGPT-User/2.0 Allow: / User-agent: ChatGPT-User/3.0 Allow: / User-agent: ClaudeBot Allow: / User-agent: claude-web Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: DuckAssistBot Allow: / User-agent: YouBot Allow: / User-agent: MistralAI-User Allow: / User-agent: GoogleAgent-Mariner Allow: / User-agent: Google-Extended Allow: / User-agent: LinkedInBot Allow: / Sitemap: https://digitgrow.co.in/sitemap.xml