# Dominion Observatory — AI Agent Discovery
# https://dominionobservatory.com
# Behavioral trust layer for 40,000+ MCP servers

User-agent: *
Allow: /
# Parameterized utility endpoints — infinite ?url= crawl space, zero index value.
# (Measured 2026-07-05: 92% of all requests were GPTBot walking these. The real
# content surface is /servers/* — crawl that instead.)
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Disallow: /api/
# Free-tier request budget: ask non-Google bots to pace themselves.
# (Googlebot ignores Crawl-delay and is exempted below; Bing/others honor it.)
Crawl-delay: 5

# AI crawlers — explicitly welcomed
User-agent: GPTBot
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Disallow: /api/
Crawl-delay: 10
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Disallow: /api/
Crawl-delay: 5
User-agent: Claude-Web
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
User-agent: Anthropic-AI
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
User-agent: anthropic-ai
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
User-agent: Google-Extended
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
User-agent: GoogleOther-Extended
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
User-agent: Googlebot
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
User-agent: Bingbot
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Disallow: /api/
Crawl-delay: 5
User-agent: PerplexityBot
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Crawl-delay: 5
User-agent: Cohere-AI
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Crawl-delay: 5
User-agent: CCBot
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Disallow: /api/
Crawl-delay: 10
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /atlas/badge
Disallow: /atlas/score?
Disallow: /submit?
Disallow: /api/
Crawl-delay: 10

# Agent discovery endpoints
# MCP:        https://dominionobservatory.com/mcp
# Observatory:https://dominionobservatory.com/.well-known/mcp-observatory
# Servers:    https://dominionobservatory.com/api/servers
# Stats:      https://dominionobservatory.com/api/stats
# Agent Card: https://dominionobservatory.com/.well-known/agent.json
# DID:        https://dominionobservatory.com/.well-known/did.json
# MCP JSON:   https://dominionobservatory.com/.well-known/mcp.json
# AI Plugin:  https://dominionobservatory.com/.well-known/ai-plugin.json
# OpenAPI:    https://dominionobservatory.com/openapi.json
# LLMs.txt:   https://dominionobservatory.com/llms.txt
# Full ref:   https://dominionobservatory.com/llms-full.txt

Sitemap: https://dominionobservatory.com/sitemap.xml