Robots.txt API API reference

Operations

GET /v1/robots View API reference

GET `/v1/robots`

Operation: getRobotsTxt Credits and authentication: 1 Authentication required

GET /v1/robots Credits: 1

Query parameters

Parameter	Type	required
url	string	optional
domain	string	optional
user_agent	string Default `*`	optional
path	string	optional
skip_cache	string Allowed values `1`	optional

Response fields

Field	Type
`cache_age_s`	integer
`classification`	object
`classification.status`	string
`classification.reason`	string
`classification.evidence`	object
`classification.evidence.source`	string
`classification.evidence.method`	string
`classification.evidence.observed_at`	string
`classification.evidence.detail`	string
`classification.checked_at`	string
`domain`	string
`url`	string
`robots_url`	string
`status`	string
`checked_at`	string
`fetch`	object
`fetch.checked_url`	string
`fetch.http_status`	integer \| null
`fetch.final_url`	string \| null
`fetch.via_proxy`	boolean \| null
`fetch.redirect_count`	integer \| null
`fetch.content_type`	string \| null
`fetch.bytes`	integer \| null
`fetch.truncated`	boolean
`fetch.error`	string \| null
`sitemaps[]`	string[]
`groups[]`	object[]
`groups[]`	object
`groups[].user_agents[]`	string[]
`groups[].rules[]`	object[]
`groups[].rules[]`	object
`groups[].rules[].type`	string
`groups[].rules[].path`	string
`groups[].rules[].line`	integer
`groups[].crawl_delay`	number \| null
`other_directives[]`	object[]
`other_directives[]`	object
`other_directives[].name`	string
`other_directives[].count`	integer
`other_directives[].standard`	boolean
`test`	object
`test.user_agent`	string
`test.path`	string
`test.allowed`	boolean \| null
`test.matched_rule`	object
`test.matched_rule.type`	string
`test.matched_rule.path`	string
`test.matched_rule.line`	integer
`test.matched_group`	string \| null
`test.matched_user_agents[]`	string[]
`test.reason`	string
`ai_crawlers[]`	object[]
`ai_crawlers[]`	object
`ai_crawlers[].user_agent`	string
`ai_crawlers[].owner`	string
`ai_crawlers[].purpose`	string
`ai_crawlers[].root_access`	string
`ai_crawlers[].group`	string \| null
`issues[]`	object[]
`issues[]`	object
`issues[].code`	string
`issues[].count`	integer
`issues[].lines[]`	integer[]
`caveats[]`	string[]
`summary`	object
`summary.groups`	integer
`summary.rules`	integer
`summary.allow_rules`	integer
`summary.disallow_rules`	integer
`summary.sitemaps`	integer
`summary.issues`	integer
`summary.lines`	integer
`summary.ai_crawlers_allowed`	integer
`summary.ai_crawlers_disallowed`	integer
`summary.ai_crawlers_partial`	integer
`summary.ai_crawlers_unknown`	integer
`summary.ai_crawlers_with_specific_group`	integer
`meta`	object
`meta.cached`	boolean
`meta.duration_ms`	integer
`meta.source_version`	string

Example request

curl -H "X-API-Key: $DOMSCAN_API_KEY" "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1"

const response = await fetch("https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1", {
  headers: { "X-API-Key": process.env.DOMSCAN_API_KEY },
});
const data = await response.json();
console.log(data);

import os
import requests

response = requests.get(
    "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1",
    headers={"X-API-Key": os.environ["DOMSCAN_API_KEY"]},
)
print(response.json())

package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
)

func main() {
	req, err := http.NewRequest(http.MethodGet, "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1", nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set("X-API-Key", os.Getenv("DOMSCAN_API_KEY"))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	fmt.Println(resp.Status, string(data))
}

require "json"
require "net/http"

uri = URI("https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1")
request = Net::HTTP::Get.new(uri)
request["X-API-Key"] = ENV.fetch("DOMSCAN_API_KEY")

response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
  http.request(request)
end
puts JSON.pretty_generate(JSON.parse(response.body))

Example response

{
  "domain": "example.com",
  "url": "https://example.com/",
  "robots_url": "https://example.com/robots.txt",
  "status": "found",
  "checked_at": "2026-10-09T10:15:30.000Z",
  "fetch": {
    "checked_url": "https://example.com/robots.txt",
    "http_status": 200,
    "final_url": "https://example.com/robots.txt",
    "via_proxy": true,
    "redirect_count": 0,
    "content_type": "text/plain; charset=utf-8",
    "bytes": 180,
    "truncated": false,
    "error": null
  },
  "sitemaps": [
    "https://example.com/sitemap.xml"
  ],
  "groups": [
    {
      "user_agents": [
        "GPTBot",
        "CCBot"
      ],
      "rules": [
        {
          "type": "disallow",
          "path": "/",
          "line": 3
        }
      ],
      "crawl_delay": null
    },
    {
      "user_agents": [
        "*"
      ],
      "rules": [
        {
          "type": "disallow",
          "path": "/admin/",
          "line": 6
        },
        {
          "type": "allow",
          "path": "/admin/public$",
          "line": 7
        }
      ],
      "crawl_delay": 10
    }
  ],
  "other_directives": [
    {
      "name": "host",
      "count": 1,
      "standard": false
    }
  ],
  "test": {
    "user_agent": "GPTBot",
    "path": "/admin/settings",
    "allowed": false,
    "matched_rule": {
      "type": "disallow",
      "path": "/",
      "line": 3
    },
    "matched_group": "specific",
    "matched_user_agents": [
      "GPTBot"
    ],
    "reason": "matched_disallow"
  },
  "ai_crawlers": [
    {
      "user_agent": "GPTBot",
      "owner": "openai",
      "purpose": "training",
      "root_access": "disallowed",
      "group": "specific"
    },
    {
      "user_agent": "OAI-SearchBot",
      "owner": "openai",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "ChatGPT-User",
      "owner": "openai",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "ClaudeBot",
      "owner": "anthropic",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Claude-SearchBot",
      "owner": "anthropic",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Claude-User",
      "owner": "anthropic",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Google-Extended",
      "owner": "google",
      "purpose": "training_control",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "PerplexityBot",
      "owner": "perplexity",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Perplexity-User",
      "owner": "perplexity",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "CCBot",
      "owner": "common_crawl",
      "purpose": "training",
      "root_access": "disallowed",
      "group": "specific"
    },
    {
      "user_agent": "Bytespider",
      "owner": "bytedance",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Applebot-Extended",
      "owner": "apple",
      "purpose": "training_control",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "meta-externalagent",
      "owner": "meta",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Amazonbot",
      "owner": "amazon",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    }
  ],
  "issues": [
    {
      "code": "crawl_delay_nonstandard",
      "count": 1,
      "lines": [
        8
      ]
    },
    {
      "code": "nonstandard_directive",
      "count": 1,
      "lines": [
        10
      ]
    }
  ],
  "caveats": [],
  "summary": {
    "groups": 2,
    "rules": 3,
    "allow_rules": 1,
    "disallow_rules": 2,
    "sitemaps": 1,
    "issues": 2,
    "lines": 12,
    "ai_crawlers_allowed": 0,
    "ai_crawlers_disallowed": 2,
    "ai_crawlers_partial": 12,
    "ai_crawlers_unknown": 0,
    "ai_crawlers_with_specific_group": 2
  },
  "meta": {
    "cached": false,
    "duration_ms": 184,
    "source_version": "2026-10-09"
  }
}

HTTP status codes

200 OK
400 Bad Request
401 Unauthorized
402 Payment Required
429 Too Many Requests
502 Bad Gateway

Reference resources

OpenAPI 3.0 Postman MCP

Support

Robots.txt API API

Operations

GET `/v1/robots`

Query parameters

Response fields

Example request

Example response

HTTP status codes

Reference resources

Ask AI about the DomScan API