# robots.txt — AI Labs Audit (https://ailabsaudit.com)
# Updated: 13 May 2026
# Optimized for AEO, GEO & AI crawler indexation
# ================================================
#
# AI TRAINING POLICY: ALLOWED on all public pages
# -------------------------------------------------
# Public pages (excluding /admin/, /client/, /audit/, /api/, /portal/, etc.)
# may be used for AI training, retrieval-augmented generation (RAG),
# answer engines (GEO/AEO) and citation. See:
#   • https://ailabsaudit.com/.well-known/ai.txt        (ai.txt v1.1.1)
#   • https://ailabsaudit.com/llms.txt                  (concise summary)
#   • https://ailabsaudit.com/llms-full.txt             (exhaustive details)
#
# Bots explicitly authorised below: GPTBot, ClaudeBot, Google-Extended,
# PerplexityBot, CCBot (Common Crawl), Applebot-Extended, Bytespider,
# Meta-ExternalAgent, Cohere, AI2Bot, and 70+ others.
# ================================================

# ========================================
# SEARCH ENGINE CRAWLERS
# ========================================

# Google — no Crawl-delay (Google ignores it)
User-agent: Googlebot
User-agent: Googlebot-Image
Allow: /
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Disallow: /api/
Disallow: /payments/
Disallow: /credits/
Disallow: /subscription/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /tmp/
Disallow: /private/

# Bing — important for ChatGPT Search (87% citations SearchGPT viennent de Bing)
# Pas de Crawl-delay : Bing alimente ChatGPT Search, plus on est crawles vite
# meilleure est la fraicheur des reponses IA.
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /credits/
Disallow: /subscription/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /tmp/
Disallow: /private/

# Yandex
User-agent: YandexBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/

# ========================================
# AI CRAWLERS — CONTROLLED ACCESS
# ========================================
# Allow AI models to crawl public content
# for better answers about AI visibility audits,
# AEO, GEO, LLMO and AI optimization

# OpenAI — ChatGPT, ChatGPT Search & Agents
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: Operator
User-agent: ChatGPT Agent
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Anthropic — Claude, Claude Search & Retrieval
# Note 2026-05 : Claude-Web et anthropic-ai retirés (bots dépréciés par Anthropic).
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Google — Gemini / AI training & Agents
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-CloudVertexBot
User-agent: GoogleAgent-Mariner
User-agent: Google-Agent
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Meta AI (Facebook / Instagram / WhatsApp / Llama)
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: meta-webindexer
User-agent: FacebookBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# DeepSeek AI
User-agent: DeepSeekBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Apple — Siri / Spotlight / Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/

# Amazon — Alexa, Amazon Q, Agents & Search
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: NovaAct
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Microsoft — CopilotBot, MicrosoftPreview & Azure AI Search
User-agent: CopilotBot
User-agent: MicrosoftPreview
User-agent: AzureAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Cohere AI
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cohere-Command
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# You.com
User-agent: YouBot
User-agent: YouChat
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Mistral AI
User-agent: MistralBot
User-agent: MistralAI-User
User-agent: Mistral-User
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# OpenAI — ChatGPT browsing agent
User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Perplexity — user-initiated retrieval
User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# xAI — Grok
User-agent: GrokBot
User-agent: xAI-Grok
User-agent: Grok-DeepSearch
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Brave Search AI
User-agent: BraveBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Google — Gemini Deep Research & NotebookLM
User-agent: Gemini-Deep-Research
User-agent: Google-NotebookLM
User-agent: NotebookLM
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Phind — developer AI search
User-agent: PhindBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Groq
User-agent: GroqBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Quora / Poe
User-agent: PoeBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Kagi
User-agent: KagiBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Andi
User-agent: AndiBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# iAsk
User-agent: iaskspider
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Character.AI
User-agent: Character.AI
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Jasper AI
User-agent: JasperBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Writesonic
User-agent: Writesonic
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Copy.ai
User-agent: CopyAI
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# WebPilot
User-agent: WebPilot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Together AI
User-agent: TogetherBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Replicate
User-agent: ReplicateBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Seekr
User-agent: Seekr
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Neeva / Snowflake
User-agent: NeevaBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Jina AI
User-agent: Jina
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# AI2 / Allen AI
User-agent: AI2Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# HuggingFace
User-agent: HuggingFace
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Diffbot
User-agent: Diffbot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Firecrawl
User-agent: Firecrawl
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# NICT Japan
User-agent: ICC-Crawler
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Kangaroo
User-agent: kangaroo bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# ========================================
# CHINESE & ASIAN AI CRAWLERS
# ========================================

# Baidu — Ernie Bot / Wenxin
User-agent: Baiduspider
User-agent: Baiduspider-render
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# ByteDance — Doubao / Skylark / TikTok AI
User-agent: BytedanceBot
User-agent: Bytespider
User-agent: Doubao-Bot
User-agent: Coze-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 3

# Alibaba — Qwen / Tongyi Qianwen / Yisou
User-agent: YisouSpider
User-agent: Alibaba-Bot
User-agent: Qwen-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Qihoo 360
User-agent: 360Spider
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Sogou / Tencent — HunYuan
User-agent: Sogou
User-agent: SogouSpider
User-agent: TencentBot
User-agent: HunYuan-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# iFlytek — SparkDesk / Xinghuo
User-agent: SparkBot
User-agent: iFlytek-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Zhipu AI — ChatGLM / GLM
User-agent: Zhipu-Bot
User-agent: ChatGLM-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# MiniMax
User-agent: MiniMax-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Moonshot AI — Kimi Chat
User-agent: Moonshot-Bot
User-agent: Kimi-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# 01.AI — Yi LLM
User-agent: 01-AI-Bot
User-agent: Yi-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# StepFun — Step-1 / Step-2
User-agent: StepFun-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# SenseTime — SenseChat
User-agent: SenseChat-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Samsung — Bixby
User-agent: SamsungBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Naver — Clova / LINE AI (Korea)
User-agent: NaverBot
User-agent: Yeti
User-agent: CloverBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Reka AI (Singapore)
User-agent: Reka-Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# AI21 Labs — Jamba / Jurassic (Israel)
User-agent: AI21Bot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Inflection AI — Pi
User-agent: InflectionBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Stability AI
User-agent: StabilityBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Huawei — PetalBot
User-agent: PetalBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Manus AI — autonomous agent
User-agent: Manus-User
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# Common Crawl — alimente les datasets de training (C4, RedPajama, Pile, etc.)
# Autorisé pour le training : voir AI TRAINING POLICY en haut + ai.txt.
User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /api/
Disallow: /payments/
Disallow: /login
Disallow: /private/
Disallow: /tmp/
Disallow: /prompts/
Disallow: /user/
Disallow: /subscription/
Disallow: /credits/
Crawl-delay: 2

# ========================================
# SOCIAL MEDIA BOTS (link previews)
# ========================================

User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Discordbot
User-agent: Slackbot
User-agent: Slack-ImgProxy
User-agent: facebookexternalhit
Allow: /

# ========================================
# BLOCKED BOTS (bandwidth / scraping)
# ========================================

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: Megaindex
User-agent: SerpstatBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: Scrapy
User-agent: magpie-crawler
Disallow: /

# ========================================
# DEFAULT RULES (all other bots)
# ========================================

User-agent: *
Allow: /
Allow: /api/aqa/
Allow: /api/v1/docs
Disallow: /admin/
Disallow: /client/
Disallow: /audit/
Disallow: /audits
Disallow: /api/
Disallow: /payments/
Disallow: /credits/
Disallow: /subscription/
Disallow: /user/
Disallow: /prompts/
Disallow: /login
Disallow: /logout
Disallow: /pdf/
Disallow: /my-credits
Disallow: /profil
Disallow: /profile
Disallow: /portal/
Disallow: /agency/
Disallow: /consumption/
Disallow: /account/
Disallow: /dashboard
Disallow: /provider-dashboard
Disallow: /agent-dashboard
Disallow: /test/
Disallow: /automation/
Disallow: /invite/
Disallow: /connexion
Disallow: /inscription
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email/
Disallow: /unsubscribe/
Disallow: /u/
Disallow: /f/
Disallow: /tmp/
Disallow: /private/
Disallow: /backup/
Disallow: /.git/
Disallow: /.env
Disallow: /config/
Disallow: /migrations/
# Bloque les JSON privés (data dumps internes), MAIS autorise le manifest PWA
Allow: /manifest.json
Allow: /static/manifest.json
Disallow: /static/data/*.json$
Disallow: /api/*.json$
Disallow: /*?session=
Disallow: /*?token=
Crawl-delay: 1

# ========================================
# SITEMAPS — index + sous-sitemaps annoncés explicitement (Bing les utilise)
# ========================================

Sitemap: https://ailabsaudit.com/sitemap.xml
Sitemap: https://ailabsaudit.com/sitemap-pages.xml
Sitemap: https://ailabsaudit.com/sitemap-blog.xml
Sitemap: https://ailabsaudit.com/sitemap-glossaire.xml
Sitemap: https://ailabsaudit.com/sitemap-vitrine.xml
Sitemap: https://ailabsaudit.com/sitemap-videos.xml

# AI-optimized content files
# See llms.txt for a concise summary, llms-full.txt for exhaustive details
# https://ailabsaudit.com/llms.txt
# https://ailabsaudit.com/llms-full.txt
# https://ailabsaudit.com/.well-known/ai.txt

# ========================================
# CONTENT OVERVIEW
# ========================================
# Languages: FR, EN, ES, DE
# Indexable sections:
#   Landing        / (FR) + /en (EN) + /es (ES) + /de (DE)
#   About          /a-propos (FR) + /about (EN) + /acerca-de (ES) + /ueber-uns (DE)
#   Agencies       /agences (FR) + /agencies (EN) + /agencias (ES) + /agenturen (DE)
#   Platform       /plateforme (FR) + /platform (EN) + /plataforma (ES) + /plattform (DE)
#   Pricing        /tarifs (FR) + /pricing (EN) + /precios (ES) + /preise (DE)
#   Nouveautes     /nouveautes (FR) + /updates (EN) + /novedades (ES) + /neuigkeiten (DE)
#   Blog           /blog/fr/ + /blog/en/ + /blog/es/ + /blog/de/ — 59 articles x 4 langs
#   Glossary       /glossaire/fr/ + /glossaire/en/ + /glossaire/es/ + /glossaire/de/ — 135 terms x 4 langs
#   Tutorial Hub   /tutoriel/fr (FR) + /tutorial/en (EN) + /tutorial/es (ES) + /tutorial/de (DE)
#   Tutorial Pages /tutoriel/fr/<slug> + /tutorial/en/<slug> — 13 tutorials x 4 langs
#   Videos         /videos/fr (FR) + /videos/en (EN) + /videos/es (ES) + /videos/de (DE)
#   AQA Hub        /aqa-hub + /aqa-validator
#   Vitrine        /vitrine/<slug> — 2 client showcase pages (dynamic, with hreflang)
#   API Docs       /api/v1/docs (Swagger UI, 105 endpoints)
#   Contact        /contact
#   Register       /register (canonical, /partner-request 301 redirect)
#   FAQ            /faq
#   Legal          /mentions-legales + /politique-confidentialite + /cgu
#   Plans          /plans/agence-plus
#   Total:         ~850+ multilingual URLs
#
# Last updated: 17 April 2026