# Domain Availability API - Production
# https://domscan.net

User-agent: *
Allow: /

# Allow public documentation and machine-readable discovery
Allow: /mcp-domain-checker
Allow: /mcp/server-card
Allow: /.well-known/agents.md
Allow: /.well-known/api-catalog
Allow: /.well-known/ai-catalog.json
Allow: /.well-known/oauth-protected-resource
Allow: /.well-known/oauth-authorization-server

# Allow glossary pages that start with "auth" (before disallow rules)
Allow: /glossary/auth
Allow: /*/glossary/auth

# Disallow API endpoints and private interfaces from indexing
Disallow: /v1/
Disallow: /v2/
Disallow: /api/
Disallow: /oauth/
Disallow: /webhooks/
Disallow: /*/v1/
Disallow: /*/v2/
Disallow: /*/api/
Disallow: /*/oauth/
Disallow: /*/webhooks/
Disallow: /__health
Disallow: /mcp
Disallow: /sse
Disallow: /.well-known/
Disallow: /admin
Disallow: /*/admin
Disallow: /monitoring
Disallow: /*/monitoring
Disallow: /monitor
Disallow: /*/monitor
Disallow: /support
Disallow: /*/support
Disallow: /dashboard
Disallow: /*/dashboard
Disallow: /subscription
Disallow: /*/subscription
Disallow: /billing
Disallow: /*/billing
Disallow: /auth
Disallow: /*/auth

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
Allow: /

# Allow public documentation and machine-readable discovery
Allow: /mcp-domain-checker
Allow: /mcp/server-card
Allow: /.well-known/agents.md
Allow: /.well-known/api-catalog
Allow: /.well-known/ai-catalog.json
Allow: /.well-known/oauth-protected-resource
Allow: /.well-known/oauth-authorization-server

# Allow glossary pages that start with "auth" (before disallow rules)
Allow: /glossary/auth
Allow: /*/glossary/auth

# Disallow API endpoints and private interfaces from indexing
Disallow: /v1/
Disallow: /v2/
Disallow: /api/
Disallow: /oauth/
Disallow: /webhooks/
Disallow: /*/v1/
Disallow: /*/v2/
Disallow: /*/api/
Disallow: /*/oauth/
Disallow: /*/webhooks/
Disallow: /__health
Disallow: /mcp
Disallow: /sse
Disallow: /.well-known/
Disallow: /admin
Disallow: /*/admin
Disallow: /monitoring
Disallow: /*/monitoring
Disallow: /monitor
Disallow: /*/monitor
Disallow: /support
Disallow: /*/support
Disallow: /dashboard
Disallow: /*/dashboard
Disallow: /subscription
Disallow: /*/subscription
Disallow: /billing
Disallow: /*/billing
Disallow: /auth
Disallow: /*/auth

# Sitemap location
Sitemap: https://domscan.net/sitemap.xml