مرجع API

API الخاص بـ واجهة API لمختبر Robots.txt

اختبر عددًا كبيرًا من الروابط وبرامج الزحف مقابل ملف robots.txt ملصق أو منشور. استخدم هذا المرجع لاختيار عملية ودمج عقد الاستجابة الموثق.

العمليات

POST /v1/robots/test عرض مرجع API

POST /v1/robots/test

العملية: testRobotsTxt الوحدات والمصادقة: 1 المصادقة مطلوبة
POST /v1/robots/test الرصيد: 1

معاملات متن الطلب

المعامل النوع مطلوب
robots_txt string اختياري
url string اختياري
urls string[] مطلوب
user_agents string[] اختياري

حقول الرد

الحقل النوع
source string
url string | null
robots_url string | null
status string
checked_at string
summary object
summary.tests integer
summary.allowed integer
summary.disallowed integer
summary.unknown integer
summary.groups integer
summary.rules integer
summary.issues integer
user_agents[] object[]
user_agents[] object
user_agents[].user_agent string
user_agents[].allowed integer
user_agents[].disallowed integer
user_agents[].unknown integer
results[] object[]
results[] object
results[].url string
results[].path string
results[].user_agent string
results[].token string
results[].allowed boolean | null
results[].reason string
results[].matched_rule object
results[].matched_rule.type string
results[].matched_rule.path string
results[].matched_rule.line integer
results[].matched_group string | null
results[].matched_user_agents[] string[]
results[].vendor object
results[].vendor.allowed boolean | null
results[].vendor.reason string
results[].vendor.matched_rule object
results[].vendor.matched_rule.type string
results[].vendor.matched_rule.path string
results[].vendor.matched_rule.line integer
results[].vendor.matched_group string
sitemaps[] string[]
issues[] object[]
issues[] object
issues[].code string
issues[].count integer
issues[].lines[] integer[]
caveats[] string[]
fetch object
fetch.checked_url string
fetch.http_status integer | null
fetch.final_url string | null
fetch.via_proxy boolean | null
fetch.redirect_count integer | null
fetch.content_type string | null
fetch.bytes integer | null
fetch.truncated boolean
fetch.error string | null
meta object
meta.cached boolean
meta.duration_ms integer

طلب مثال

curl -X POST "https://domscan.net/v1/robots/test" \
  -H "Content-Type: application/json" \
  -H "X-API-Key: $DOMSCAN_API_KEY" \
  -d '{
  "robots_txt": "User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nDisallow: /admin/\nAllow: /admin/public",
  "urls": [
    "/blog/post-1",
    "/admin/public"
  ],
  "user_agents": [
    "Googlebot",
    "GPTBot"
  ]
}'
const response = await fetch("https://domscan.net/v1/robots/test", {
  method: "POST",
  headers: { "X-API-Key": process.env.DOMSCAN_API_KEY, "Content-Type": "application/json" },
  body: JSON.stringify({
    "robots_txt": "User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nDisallow: /admin/\nAllow: /admin/public",
    "urls": [
      "/blog/post-1",
      "/admin/public"
    ],
    "user_agents": [
      "Googlebot",
      "GPTBot"
    ]
  }),
});
const data = await response.json();
console.log(data);
import os
import requests

response = requests.post(
    "https://domscan.net/v1/robots/test",
    headers={"X-API-Key": os.environ["DOMSCAN_API_KEY"]},
    json={
        "robots_txt": "User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nDisallow: /admin/\nAllow: /admin/public",
        "urls": [
            "/blog/post-1",
            "/admin/public",
        ],
        "user_agents": [
            "Googlebot",
            "GPTBot",
        ],
    },
)
print(response.json())
package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
)

func main() {
	body := strings.NewReader(`{
	  "robots_txt": "User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nDisallow: /admin/\nAllow: /admin/public",
	  "urls": [
	    "/blog/post-1",
	    "/admin/public"
	  ],
	  "user_agents": [
	    "Googlebot",
	    "GPTBot"
	  ]
	}`)
	req, err := http.NewRequest(http.MethodPost, "https://domscan.net/v1/robots/test", body)
	if err != nil {
		panic(err)
	}
	req.Header.Set("X-API-Key", os.Getenv("DOMSCAN_API_KEY"))
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	fmt.Println(resp.Status, string(data))
}
require "json"
require "net/http"

uri = URI("https://domscan.net/v1/robots/test")
request = Net::HTTP::Post.new(uri)
request["X-API-Key"] = ENV.fetch("DOMSCAN_API_KEY")
request["Content-Type"] = "application/json"
request.body = JSON.generate({
  "robots_txt" => "User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nDisallow: /admin/\nAllow: /admin/public",
  "urls" => [
    "/blog/post-1",
    "/admin/public"
  ],
  "user_agents" => [
    "Googlebot",
    "GPTBot"
  ]
})

response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
  http.request(request)
end
puts JSON.pretty_generate(JSON.parse(response.body))

مثال الرد

{
  "source": "provided",
  "url": null,
  "robots_url": null,
  "status": "found",
  "checked_at": "2026-10-13T10:00:00.000Z",
  "summary": {
    "tests": 4,
    "allowed": 2,
    "disallowed": 2,
    "unknown": 0,
    "groups": 2,
    "rules": 3,
    "issues": 0
  },
  "user_agents": [
    {
      "user_agent": "Googlebot",
      "allowed": 2,
      "disallowed": 0,
      "unknown": 0
    },
    {
      "user_agent": "GPTBot",
      "allowed": 0,
      "disallowed": 2,
      "unknown": 0
    }
  ],
  "results": [
    {
      "url": "/blog/post-1",
      "path": "/blog/post-1",
      "user_agent": "Googlebot",
      "token": "Googlebot",
      "allowed": true,
      "reason": "no_matching_rule",
      "matched_rule": null,
      "matched_group": "wildcard",
      "matched_user_agents": [
        "*"
      ],
      "vendor": null
    },
    {
      "url": "/admin/public",
      "path": "/admin/public",
      "user_agent": "Googlebot",
      "token": "Googlebot",
      "allowed": true,
      "reason": "matched_allow",
      "matched_rule": {
        "type": "allow",
        "path": "/admin/public",
        "line": 6
      },
      "matched_group": "wildcard",
      "matched_user_agents": [
        "*"
      ],
      "vendor": null
    },
    {
      "url": "/blog/post-1",
      "path": "/blog/post-1",
      "user_agent": "GPTBot",
      "token": "GPTBot",
      "allowed": false,
      "reason": "matched_disallow",
      "matched_rule": {
        "type": "disallow",
        "path": "/",
        "line": 2
      },
      "matched_group": "specific",
      "matched_user_agents": [
        "GPTBot"
      ],
      "vendor": null
    },
    {
      "url": "/admin/public",
      "path": "/admin/public",
      "user_agent": "GPTBot",
      "token": "GPTBot",
      "allowed": false,
      "reason": "matched_disallow",
      "matched_rule": {
        "type": "disallow",
        "path": "/",
        "line": 2
      },
      "matched_group": "specific",
      "matched_user_agents": [
        "GPTBot"
      ],
      "vendor": null
    }
  ],
  "sitemaps": [],
  "issues": [],
  "caveats": [],
  "fetch": null,
  "meta": {
    "cached": false,
    "duration_ms": 2
  }
}

رموز حالة HTTP

  • 200 OK
  • 400 طلب غير صالح
  • 401 غير مصرح
  • 402 الدفع مطلوب
  • 409 HTTP 409
  • 422 HTTP 422
  • 429 طلبات كثيرة جداً

موارد المرجع