# Robots.txt for Kordless
# Generated: 2025-08-14T03:22:15.433Z
# Contact: contact@kordless.ai

# Default access for all crawlers
User-agent: *
Allow: /
Crawl-delay: 1

# Disallow access to admin areas and development files
Disallow: /admin/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /.next/
Disallow: /scripts/
Disallow: /lib/private/
Disallow: /api/private/

# Search Engine Crawlers - Full access
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# AI Answer Engines - Allowed for business discovery
User-agent: Claude-Web
Allow: /
Crawl-delay: 2

User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

User-agent: YouBot
Allow: /
Crawl-delay: 2

User-agent: SearchGPT
Allow: /
Crawl-delay: 2

# AI Training Crawlers - Selective blocking
# Remove these blocks if you want to allow AI training on your content
User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Anthropic-AI
Allow: /
Crawl-delay: 5

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# SEO and Analytics Tools
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: MJ12bot
Allow: /
Crawl-delay: 10

# Aggressive or unwanted crawlers
User-agent: ia_archiver
Disallow: /

User-agent: ScrapyBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemaps
Sitemap: https://kordless.ai/sitemap.xml
Sitemap: https://www.kordless.ai/sitemap.xml

# Additional information
# Privacy Policy: https://kordless.ai/privacy
# Terms of Service: https://kordless.ai/terms
# Contact: contact@kordless.ai