# pdflean.com robots.txt
# Policy: all AI crawlers are allowed (search/assistant AND training/data alike);
# only non-AI commercial scraping is denied (paid-database SEO bots, data brokers,
# and scrapers that ignore boundaries). Private paths (account, checkout, admin,
# API) stay closed to every crawler.
# Content signals (self-hosted here; Cloudflare managed robots.txt is NOT used):
# search=yes = may be indexed for search results; ai-input=yes = may be used as live
# input for AI answers; ai-train=yes = may be used for model training or fine-tuning.
# All three are yes, matching the policy above.

# ---------- Denied: non-AI commercial scraping ----------
User-agent: ShapBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DotBot
User-agent: omgili
Disallow: /

# ---------- Welcome: AI search & assistants ----------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Applebot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /account
Disallow: /account/*
Disallow: /checkout
Disallow: /admin/*
Disallow: /api/*
Allow: /

# ---------- Allowed: AI training & data ----------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Diffbot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: Google-CloudVertexBot
User-agent: DeepSeekBot
User-agent: Google-Extended
User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /account
Disallow: /account/*
Disallow: /checkout
Disallow: /admin/*
Disallow: /api/*
Allow: /

# ---------- Welcome: traditional search ----------
User-agent: Googlebot
User-agent: bingbot
User-agent: YandexBot
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: Bravebot
User-agent: MojeekBot
User-agent: PetalBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /account
Disallow: /account/*
Disallow: /checkout
Disallow: /admin/*
Disallow: /api/*
Allow: /

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /account
Disallow: /account/*
Disallow: /checkout
Disallow: /admin/*
Disallow: /api/*
Allow: /

Sitemap: https://pdflean.com/sitemap-index.xml
Sitemap: https://pdflean.com/sitemap-en.xml
Sitemap: https://pdflean.com/sitemap-zh.xml
Sitemap: https://pdflean.com/sitemap-fr.xml
Sitemap: https://pdflean.com/sitemap-de.xml
Sitemap: https://pdflean.com/sitemap-es.xml
Sitemap: https://pdflean.com/sitemap-pt.xml
Sitemap: https://pdflean.com/sitemap-it.xml
Sitemap: https://pdflean.com/sitemap-nl.xml
Sitemap: https://pdflean.com/sitemap-pl.xml
Sitemap: https://pdflean.com/sitemap-sv.xml
Sitemap: https://pdflean.com/sitemap-ru.xml
Sitemap: https://pdflean.com/sitemap-tr.xml
Sitemap: https://pdflean.com/sitemap-zh-TW.xml
Sitemap: https://pdflean.com/sitemap-ja.xml
Sitemap: https://pdflean.com/sitemap-ko.xml
Sitemap: https://pdflean.com/sitemap-hi.xml
Sitemap: https://pdflean.com/sitemap-id.xml
Sitemap: https://pdflean.com/sitemap-vi.xml
Sitemap: https://pdflean.com/sitemap-th.xml
Sitemap: https://pdflean.com/sitemap-ms.xml
Sitemap: https://pdflean.com/sitemap-uk.xml
Sitemap: https://pdflean.com/sitemap-ar.xml