# robots.txt for https://uat.nodejstuts.com
#
# robots.txt is ADVISORY (RFC 9309). It states our crawling preferences to
# well-behaved crawlers; it is not access control, authentication or a licence.
# Private areas are protected by server-side authorization regardless of this file.
# A user-agent token is a claim, not verified identity.
#
# Content signals (Cloudflare Content Signals Policy convention):
#   search: building a search index and showing links/snippets
#   ai-input: using content as input to an AI answer (retrieval, grounding)
#   ai-train: training or fine-tuning AI models
# Our policy for owned editorial: Content-Signal: search=yes, ai-input=yes, ai-train=no
# Content signals express preferences; they do not grant a licence.
# Licensing for bulk, training or commercial machine access: https://uat.nodejstuts.com/licensing

# Googlebot. Operator: Google; purpose: search indexing
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /me/
Disallow: /me$
Disallow: /sponsor
Disallow: /preview
Disallow: /ask
Disallow: /go/

# Bingbot. Operator: Microsoft; purpose: search indexing
User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /me/
Disallow: /me$
Disallow: /sponsor
Disallow: /preview
Disallow: /ask
Disallow: /go/

# Google-Extended. Operator: Google; purpose: AI training / bulk collection
# Training reserved; licensing available.
User-agent: Google-Extended
Disallow: /

# OAI-SearchBot. Operator: OpenAI; purpose: AI search
# Search discovery permitted.
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /me/
Disallow: /me$
Disallow: /sponsor
Disallow: /preview
Disallow: /ask
Disallow: /go/

# ChatGPT-User. Operator: OpenAI; purpose: user-directed retrieval
User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /me/
Disallow: /me$
Disallow: /sponsor
Disallow: /preview
Disallow: /ask
Disallow: /go/

# GPTBot. Operator: OpenAI; purpose: AI training / bulk collection
# Training reserved; licensing available.
User-agent: GPTBot
Disallow: /

# ClaudeBot. Operator: Anthropic; purpose: AI training / bulk collection
# Training reserved; licensing available.
User-agent: ClaudeBot
Disallow: /

# Claude-SearchBot. Operator: Anthropic; purpose: AI search
User-agent: Claude-SearchBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /me/
Disallow: /me$
Disallow: /sponsor
Disallow: /preview
Disallow: /ask
Disallow: /go/

# PerplexityBot. Operator: Perplexity; purpose: AI search
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /me/
Disallow: /me$
Disallow: /sponsor
Disallow: /preview
Disallow: /ask
Disallow: /go/

# CCBot. Operator: Common Crawl; purpose: archiving
# Bulk collection reserved.
User-agent: CCBot
Disallow: /

# Bytespider. Operator: ByteDance; purpose: AI training / bulk collection
User-agent: Bytespider
Disallow: /

# Default group: every crawler not named above.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /me/
Disallow: /me$
Disallow: /sponsor
Disallow: /preview
Disallow: /ask
Disallow: /go/

Sitemap: https://uat.nodejstuts.com/sitemap.xml