# NOTICE: The collection of content and other data on this
# site through automated means, including any device, tool,
# or process designed to data mine or scrape content, is
# prohibited except (1) for the purpose of search engine indexing or
# artificial intelligence retrieval augmented generation or (2) with express
# written permission from this site's operator.

# To request permission to license our intellectual
# property and/or other materials, please contact this
# site's operator directly.

User-agent: *
Allow: /

# ——— OPENAI ———
# Search (shows my webpages as links inside ChatGPT search). NOT used for model training.
User-agent: OAI-SearchBot
Allow: /

# User-driven browsing from ChatGPT and Custom GPTs. Acts after a human click.
User-agent: ChatGPT-User
User-agent: ChatGPT-User/2.0
Allow: /

# Model-training crawler. Opt-out here if I don't want content in GPT-4o or GPT-5.
User-agent: GPTBot
Disallow: /private/          # example private folder
Allow: /                     # everything else

# ——— ANTHROPIC (Claude) ———
User-agent: anthropic-ai      # bulk model training
Allow: /
User-agent: ClaudeBot         # chat citation fetch
User-agent: claude-web        # web-focused crawl
Allow: /

# PERPLEXITY
User-agent: PerplexityBot     # index builder
Allow: /
User-agent: Perplexity-User   # human-triggered visit
Allow: /

# GOOGLE (Gemini)
User-agent: Google-Extended
Allow: /

# MICROSOFT (Bing / Copilot)
User-agent: BingBot
Allow: /

# AMAZON
User-agent: Amazonbot
Allow: /

# APPLE
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# META
User-agent: FacebookBot
User-agent: meta-externalagent
Allow: /

# LINKEDIN
User-agent: LinkedInBot
Allow: /

# BYTEDANCE
User-agent: Bytespider
Allow: /

# DUCKDUCKGO
User-agent: DuckAssistBot
Allow: /

# COHERE
User-agent: cohere-ai
Allow: /

# ALLEN INSTITUTE / COMMON CRAWL / OTHER RESEARCH
User-agent: AI2Bot
User-agent: CCBot
User-agent: Diffbot
User-agent: omgili
Allow: /

# EMERGING SEARCH START-UPS
User-agent: TimpiBot
User-agent: YouBot
Allow: /

# Sitemaps
Sitemap: https://devopsservicespro.com/sitemap.xml

# LLM Discovery Files
# See: https://llmstxt.org/
# llms.txt: https://devopsservicespro.com/llms.txt
# llms-full.txt: https://devopsservicespro.com/llms-full.txt

# Block admin and sensitive areas
Disallow: /admin/

# Crawl delay (optional - helps with server load)
Crawl-delay: 1