User-agent: *
Allow: /

# Allow all major AI crawlers and search engines for GenSEO optimization
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# AI-specific crawlers for business discovery
User-agent: SearchGPT
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: OpenAI-SearchBot
Allow: /

# NLWeb and MCP endpoints for AI agent integration
Allow: /api/nlweb/ask
Allow: /api/nlweb/mcp
Allow: /.well-known/nlweb.json

# Important pages for AI discovery
Allow: /
Allow: /blog
Allow: /contact
Allow: /pricing
Allow: /privacy-policy
Allow: /terms-of-service

# SEO files
Allow: /sitemap.xml
Allow: /feed.xml
Allow: /robots.txt

# Disallow admin and sensitive areas
Disallow: /api/admin
Disallow: /api/auth
Disallow: /api/memory
Disallow: /api/leads
Disallow: /admin

# Sitemap location
Sitemap: https://growwithbugle.ai/sitemap.xml

# NLWeb manifest for AI agents
# https://growwithbugle.ai/.well-known/nlweb.json

# Crawl delay for respectful crawling
Crawl-delay: 1

# Additional instructions for AI crawlers
# This website provides AI consultancy services for SMBs
# Key services: Operational AI, Strategic AI, NLWeb Implementation
# Contact: hello@growwithbugle.ai
# Booking: https://calendar.app.google/NWc1oRLj5pg2Qv9p6