Référence API de API robots.txt

Opérations

GET /v1/robots Voir la référence API

GET `/v1/robots`

Opération: getRobotsTxt Crédits et authentification: 1 Authentification requise

GET /v1/robots Crédits: 1

Paramètres de requête

Paramètre	Type	requis
url	string	optionnel
domain	string	optionnel
user_agent	string Défaut `*`	optionnel
path	string	optionnel
skip_cache	string Valeurs autorisées `1`	optionnel

Champs de réponse

Champ	Type
`cache_age_s`	integer
`classification`	object
`classification.status`	string
`classification.reason`	string
`classification.evidence`	object
`classification.evidence.source`	string
`classification.evidence.method`	string
`classification.evidence.observed_at`	string
`classification.evidence.detail`	string
`classification.checked_at`	string
`domain`	string
`url`	string
`robots_url`	string
`status`	string
`checked_at`	string
`fetch`	object
`fetch.checked_url`	string
`fetch.http_status`	integer \| null
`fetch.final_url`	string \| null
`fetch.via_proxy`	boolean \| null
`fetch.redirect_count`	integer \| null
`fetch.content_type`	string \| null
`fetch.bytes`	integer \| null
`fetch.truncated`	boolean
`fetch.error`	string \| null
`sitemaps[]`	string[]
`groups[]`	object[]
`groups[]`	object
`groups[].user_agents[]`	string[]
`groups[].rules[]`	object[]
`groups[].rules[]`	object
`groups[].rules[].type`	string
`groups[].rules[].path`	string
`groups[].rules[].line`	integer
`groups[].crawl_delay`	number \| null
`other_directives[]`	object[]
`other_directives[]`	object
`other_directives[].name`	string
`other_directives[].count`	integer
`other_directives[].standard`	boolean
`test`	object
`test.user_agent`	string
`test.path`	string
`test.allowed`	boolean \| null
`test.matched_rule`	object
`test.matched_rule.type`	string
`test.matched_rule.path`	string
`test.matched_rule.line`	integer
`test.matched_group`	string \| null
`test.matched_user_agents[]`	string[]
`test.reason`	string
`ai_crawlers[]`	object[]
`ai_crawlers[]`	object
`ai_crawlers[].user_agent`	string
`ai_crawlers[].owner`	string
`ai_crawlers[].purpose`	string
`ai_crawlers[].root_access`	string
`ai_crawlers[].group`	string \| null
`issues[]`	object[]
`issues[]`	object
`issues[].code`	string
`issues[].count`	integer
`issues[].lines[]`	integer[]
`caveats[]`	string[]
`summary`	object
`summary.groups`	integer
`summary.rules`	integer
`summary.allow_rules`	integer
`summary.disallow_rules`	integer
`summary.sitemaps`	integer
`summary.issues`	integer
`summary.lines`	integer
`summary.ai_crawlers_allowed`	integer
`summary.ai_crawlers_disallowed`	integer
`summary.ai_crawlers_partial`	integer
`summary.ai_crawlers_unknown`	integer
`summary.ai_crawlers_with_specific_group`	integer
`meta`	object
`meta.cached`	boolean
`meta.duration_ms`	integer
`meta.source_version`	string

Exemple de requête

curl -H "X-API-Key: $DOMSCAN_API_KEY" "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1"

const response = await fetch("https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1", {
  headers: { "X-API-Key": process.env.DOMSCAN_API_KEY },
});
const data = await response.json();
console.log(data);

import os
import requests

response = requests.get(
    "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1",
    headers={"X-API-Key": os.environ["DOMSCAN_API_KEY"]},
)
print(response.json())

package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
)

func main() {
	req, err := http.NewRequest(http.MethodGet, "https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1", nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set("X-API-Key", os.Getenv("DOMSCAN_API_KEY"))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	fmt.Println(resp.Status, string(data))
}

require "json"
require "net/http"

uri = URI("https://domscan.net/v1/robots?url=https%3A%2F%2Fexample.com&domain=example.com&user_agent=GPTBot&path=%2Fadmin%2Fsettings&skip_cache=1")
request = Net::HTTP::Get.new(uri)
request["X-API-Key"] = ENV.fetch("DOMSCAN_API_KEY")

response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
  http.request(request)
end
puts JSON.pretty_generate(JSON.parse(response.body))

Exemple de réponse

{
  "domain": "example.com",
  "url": "https://example.com/",
  "robots_url": "https://example.com/robots.txt",
  "status": "found",
  "checked_at": "2026-10-09T10:15:30.000Z",
  "fetch": {
    "checked_url": "https://example.com/robots.txt",
    "http_status": 200,
    "final_url": "https://example.com/robots.txt",
    "via_proxy": true,
    "redirect_count": 0,
    "content_type": "text/plain; charset=utf-8",
    "bytes": 180,
    "truncated": false,
    "error": null
  },
  "sitemaps": [
    "https://example.com/sitemap.xml"
  ],
  "groups": [
    {
      "user_agents": [
        "GPTBot",
        "CCBot"
      ],
      "rules": [
        {
          "type": "disallow",
          "path": "/",
          "line": 3
        }
      ],
      "crawl_delay": null
    },
    {
      "user_agents": [
        "*"
      ],
      "rules": [
        {
          "type": "disallow",
          "path": "/admin/",
          "line": 6
        },
        {
          "type": "allow",
          "path": "/admin/public$",
          "line": 7
        }
      ],
      "crawl_delay": 10
    }
  ],
  "other_directives": [
    {
      "name": "host",
      "count": 1,
      "standard": false
    }
  ],
  "test": {
    "user_agent": "GPTBot",
    "path": "/admin/settings",
    "allowed": false,
    "matched_rule": {
      "type": "disallow",
      "path": "/",
      "line": 3
    },
    "matched_group": "specific",
    "matched_user_agents": [
      "GPTBot"
    ],
    "reason": "matched_disallow"
  },
  "ai_crawlers": [
    {
      "user_agent": "GPTBot",
      "owner": "openai",
      "purpose": "training",
      "root_access": "disallowed",
      "group": "specific"
    },
    {
      "user_agent": "OAI-SearchBot",
      "owner": "openai",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "ChatGPT-User",
      "owner": "openai",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "ClaudeBot",
      "owner": "anthropic",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Claude-SearchBot",
      "owner": "anthropic",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Claude-User",
      "owner": "anthropic",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Google-Extended",
      "owner": "google",
      "purpose": "training_control",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "PerplexityBot",
      "owner": "perplexity",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Perplexity-User",
      "owner": "perplexity",
      "purpose": "user_fetch",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "CCBot",
      "owner": "common_crawl",
      "purpose": "training",
      "root_access": "disallowed",
      "group": "specific"
    },
    {
      "user_agent": "Bytespider",
      "owner": "bytedance",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Applebot-Extended",
      "owner": "apple",
      "purpose": "training_control",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "meta-externalagent",
      "owner": "meta",
      "purpose": "training",
      "root_access": "partial",
      "group": "wildcard"
    },
    {
      "user_agent": "Amazonbot",
      "owner": "amazon",
      "purpose": "search",
      "root_access": "partial",
      "group": "wildcard"
    }
  ],
  "issues": [
    {
      "code": "crawl_delay_nonstandard",
      "count": 1,
      "lines": [
        8
      ]
    },
    {
      "code": "nonstandard_directive",
      "count": 1,
      "lines": [
        10
      ]
    }
  ],
  "caveats": [],
  "summary": {
    "groups": 2,
    "rules": 3,
    "allow_rules": 1,
    "disallow_rules": 2,
    "sitemaps": 1,
    "issues": 2,
    "lines": 12,
    "ai_crawlers_allowed": 0,
    "ai_crawlers_disallowed": 2,
    "ai_crawlers_partial": 12,
    "ai_crawlers_unknown": 0,
    "ai_crawlers_with_specific_group": 2
  },
  "meta": {
    "cached": false,
    "duration_ms": 184,
    "source_version": "2026-10-09"
  }
}

Codes de statut HTTP

200 OK
400 Requête incorrecte
401 Non autorisé
402 Paiement requis
429 Trop de requêtes
502 Mauvaise passerelle

Ressources de référence

OpenAPI 3.0 Postman MCP

Assistance

API API robots.txt

Opérations

GET `/v1/robots`

Paramètres de requête

Champs de réponse

Exemple de requête

Exemple de réponse

Codes de statut HTTP

Ressources de référence

Demander à l'IA à propos de l'API DomScan