# Turbify robots.txt

# Purpose:
# - Allow traditional search and social previews
# - Allow AI-assisted search/discovery where useful
# - Restrict AI model training / bulk ingestion
# - Preserve existing Turbify crawl controls
# - Point AI systems to curated company/product information

# AI/content usage posture
# search=yes: content may be used for search indexing and discovery
# ai-input=yes: content may be used for retrieval-grounded AI answers
# ai-train=no: content may not be used for AI model training or fine-tuning
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Global rules
User-agent: *
Allow: /
Disallow: /_xhr/
Disallow: /services/
Disallow: /servicecheck/
Disallow: /upgrade.html
Disallow: /tos/CA/privacy/

# Search engines — full access
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: bingbot
Allow: /

User-agent: Applebot
Allow: /

# Social crawlers — full access for previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Audit tools
User-agent: SiteAuditBot
Allow: /

# AI search / user-requested retrieval — allow
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# AI training / bulk ingestion — disallow
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

# LLM guidance file
LLMs: https://www.turbify.com/llms.txt

# Sitemaps
Sitemap: https://www.turbify.com/sitemap.xml
Sitemap: https://resources.turbify.com/sitemap_index.xml