API API robots.txt

Пример запроса

curl -H "X-API-Key: $DOMSCAN_API_KEY" "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1"

const response = await fetch("https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1", {
  headers: { "X-API-Key": process.env.DOMSCAN_API_KEY },
});
const data = await response.json();
console.log(data);

import os
import requests

response = requests.get(
    "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1",
    headers={"X-API-Key": os.environ["DOMSCAN_API_KEY"]},
)
print(response.json())

package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
)

func main() {
	req, err := http.NewRequest(http.MethodGet, "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1", nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set("X-API-Key", os.Getenv("DOMSCAN_API_KEY"))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	fmt.Println(resp.Status, string(data))
}

require "json"
require "net/http"

uri = URI("https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1")
request = Net::HTTP::Get.new(uri)
request["X-API-Key"] = ENV.fetch("DOMSCAN_API_KEY")

response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
  http.request(request)
end
puts JSON.pretty_generate(JSON.parse(response.body))

Пример ответа

{
  "domain": "example.com",
  "url": "https://example.com/",
  "robots_url": "https://example.com/robots.txt",
  "status": "found",
  "checked_at": "2026-10-09T10:15:30.000Z",
  "fetch": {
    "checked_url": "https://example.com/robots.txt",
    "http_status": 200,
    "final_url": "https://example.com/robots.txt",
    "via_proxy": true,
    "redirect_count": 0,
    "content_type": "text/plain; charset=utf-8",
    "bytes": 180,
    "truncated": false,
    "error": null
  },
  "sitemaps": [
    "https://example.com/sitemap.xml"
  ],
  "groups": [
    {
      "user_agents": [
        "GPTBot",
        "CCBot"
      ],
      "rules": [
        {
          "type": "disallow",
          "path": "/",
          "line": 3
        }
      ],
      "crawl_delay": null
    },
    {
      "user_agents": [
        "*"
      ],
      "rules": [
        {
          "type": "disallow",
          "path": "/admin/",
          "line": 6
        },
        {
          "type": "allow",
          "path": "/admin/public$",
          "line": 7
        }
      ],
      "crawl_delay": 10
    }
  ],
  "other_directives": [
    {
      "name": "host",
      "count": 1,
      "standard": false
    }
  ],
  "test": {
    "user_agent": "GPTBot",
    "path": "/admin/settings",
    "allowed": false,
    "matched_rule": {
      "type": "disallow",
      "path": "/",
      "line": 3
    },
    "matched_group": "specific",
    "matched_user_agents": [
      "GPTBot"
    ],
    "reason": "matched_disallow"
  },
  "ai_crawlers": [
    {
      "user_agent": "GPTBot",
      "owner": "openai",
      "purpose": "training",
      "root_access": "disallowed",
      "group": "specific"
    },
    {
      "user_agent": "OAI-SearchBot",
      "owner": "openai",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "ChatGPT-User",
      "owner": "openai",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "ClaudeBot",
      "owner": "anthropic",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Claude-SearchBot",
      "owner": "anthropic",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Claude-User",
      "owner": "anthropic",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Google-Extended",
      "owner": "google",
      "purpose": "training_control",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "PerplexityBot",
      "owner": "perplexity",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Perplexity-User",
      "owner": "perplexity",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "CCBot",
      "owner": "common_crawl",
      "purpose": "training",
      "root_access": "disallowed",
      "group": "specific"
    },
    {
      "user_agent": "Bytespider",
      "owner": "bytedance",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Applebot-Extended",
      "owner": "apple",
      "purpose": "training_control",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "meta-externalagent",
      "owner": "meta",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Amazonbot",
      "owner": "amazon",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    }
  ],
  "issues": [
    {
      "code": "crawl_delay_nonstandard",
      "count": 1,
      "lines": [
        8
      ]
    },
    {
      "code": "nonstandard_directive",
      "count": 1,
      "lines": [
        10
      ]
    }
  ],
  "caveats": [],
  "summary": {
    "groups": 2,
    "rules": 3,
    "allow_rules": 1,
    "disallow_rules": 2,
    "sitemaps": 1,
    "issues": 2,
    "lines": 12,
    "ai_crawlers_allowed": 0,
    "ai_crawlers_disallowed": 2,
    "ai_crawlers_partial": 12,
    "ai_crawlers_unknown": 0,
    "ai_crawlers_with_specific_group": 2
  },
  "meta": {
    "cached": false,
    "duration_ms": 184,
    "source_version": "2026-10-09"
  }
}

Коды состояния HTTP

• 200 OK
• 400 Неверный запрос
• 401 Не авторизован
• 402 Требуется оплата
• 429 Слишком много запросов
• 502 Ошибка шлюза