{"openapi":"3.1.0","info":{"title":"robots.txt API","version":"1.0.0","description":"Fetch and evaluate any website's robots.txt. Pass a URL and a user-agent and the check endpoint tells you whether that URL is crawlable — selecting the most-specific user-agent group and applying the RFC 9309 longest-match Allow/Disallow rules (with * and $ wildcards, where Allow wins ties), and returning the matched rule, the group's crawl-delay and the sitemaps the site declares. The parse endpoint returns the whole file structured into per-user-agent groups (their allow and disallow lists and crawl-delay) plus the list of sitemaps. A missing robots.txt (404/403) means everything is allowed, exactly as the spec requires. The request is made server-side and private or internal targets are refused (SSRF-guarded). Built for SEO audits, crawler and scraper compliance, sitemap discovery and pre-flight \"am I allowed to fetch this?\" checks. A robots.txt evaluator — distinct from the on-page SEO audit (seo), the XML toolkit (xml) and link unfurling/preview (url). No upstream key, no cache.","contact":{"name":"PremiumApi","url":"https://www.oanor.com/by/premiumapi"}},"servers":[{"url":"https://api.oanor.com/robots-api","description":"oanor gateway"}],"tags":[{"name":"robots.txt"},{"name":"Meta"}],"components":{"securitySchemes":{"oanorKey":{"type":"apiKey","in":"header","name":"x-oanor-key","description":"Get your key at https://www.oanor.com/developer/keys"}}},"security":[{"oanorKey":[]}],"paths":{"/v1/check":{"get":{"operationId":"get_v1_check","tags":["robots.txt"],"summary":"Is a URL crawlable?","description":"","parameters":[{"name":"url","in":"query","required":true,"description":"The URL to check","schema":{"type":"string"},"example":"https://www.google.com/"},{"name":"user_agent","in":"query","required":false,"description":"User-agent (default *)","schema":{"type":"string"},"example":"Googlebot"}],"security":[{"oanorKey":[]}],"responses":{"200":{"description":"OK"},"401":{"description":"Missing or invalid x-oanor-key header"},"402":{"description":"Active subscription required"},"429":{"description":"Rate-limit or monthly quota reached"},"502":{"description":"Upstream did not respond"}}}},"/v1/parse":{"get":{"operationId":"get_v1_parse","tags":["robots.txt"],"summary":"Parse a site's robots.txt","description":"","parameters":[{"name":"url","in":"query","required":true,"description":"Site URL or domain","schema":{"type":"string"},"example":"https://www.google.com"}],"security":[{"oanorKey":[]}],"responses":{"200":{"description":"OK"},"401":{"description":"Missing or invalid x-oanor-key header"},"402":{"description":"Active subscription required"},"429":{"description":"Rate-limit or monthly quota reached"},"502":{"description":"Upstream did not respond"}}}},"/v1/meta":{"get":{"operationId":"get_v1_meta","tags":["Meta"],"summary":"Spec","description":"","parameters":[],"security":[{"oanorKey":[]}],"responses":{"200":{"description":"OK"},"401":{"description":"Missing or invalid x-oanor-key header"},"402":{"description":"Active subscription required"},"429":{"description":"Rate-limit or monthly quota reached"},"502":{"description":"Upstream did not respond"}}}}},"x-oanor-pricing":[{"slug":"free","name":"Free","price_cents_month":0,"monthly_call_quota":2000,"rps_limit":2,"hard_limit":true},{"slug":"starter","name":"Starter","price_cents_month":590,"monthly_call_quota":37500,"rps_limit":8,"hard_limit":true},{"slug":"pro","name":"Pro","price_cents_month":1970,"monthly_call_quota":202000,"rps_limit":20,"hard_limit":true},{"slug":"mega","name":"Mega","price_cents_month":5160,"monthly_call_quota":785000,"rps_limit":50,"hard_limit":true}],"x-oanor-marketplace-url":"https://www.oanor.com/api/robots-api"}