{
  "x402Version": 2,
  "id": 72533,
  "slug": "saylorinnovations-api-web-robots-check",
  "resource": "https://saylorinnovations.com/api/web/robots-check",
  "description": "Check whether a crawler may fetch a path under a robots.txt per RFC 9309 - matched group and rule, crawl delay and sitemaps, for Googlebot, GPTBot or any agent",
  "type": "http",
  "serviceName": "Saylor Innovations Data Desk",
  "mimeType": "application/json",
  "resourceType": "API",
  "accepts": [
    {
      "scheme": "exact",
      "network": "eip155:8453",
      "payTo": "0xA2A8dF0f7CaaBEa4e120a2568118f141569fc8aa",
      "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "eip155:137",
      "payTo": "0xA2A8dF0f7CaaBEa4e120a2568118f141569fc8aa",
      "asset": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "eip155:42161",
      "payTo": "0xA2A8dF0f7CaaBEa4e120a2568118f141569fc8aa",
      "asset": "0xaf88d065e77c8cC2239327C5EDb3A432268e5831",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "7LSjfrJf8fNsB8VA9u7N3WEn25smQLpob3SvyUvXacy7",
      "asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "7LSjfrJf8fNsB8VA9u7N3WEn25smQLpob3SvyUvXacy7",
      "asset": "Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "7LSjfrJf8fNsB8VA9u7N3WEn25smQLpob3SvyUvXacy7",
      "asset": "So11111111111111111111111111111111111111112",
      "amount": "95830",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    }
  ],
  "outputSchema": {
    "type": "object",
    "properties": {
      "user_agent": {
        "type": "string",
        "description": "User-agent the rules were evaluated for"
      },
      "path": {
        "type": "string",
        "description": "Path that was tested"
      },
      "allowed": {
        "type": "boolean",
        "description": "True when the crawler may fetch the path"
      },
      "decision": {
        "type": "string",
        "description": "allowed or blocked"
      },
      "matched_group": {
        "type": [
          "array",
          "null"
        ],
        "description": "Matched group"
      },
      "group_selection": {
        "type": "string",
        "description": "Group selection"
      },
      "matched_rule": {
        "type": [
          "object",
          "null"
        ],
        "description": "The rule that decided the outcome, or null when no rule matched",
        "properties": {
          "directive": {
            "description": "Allow or Disallow",
            "type": [
              "string",
              "null"
            ]
          },
          "pattern": {
            "description": "The path pattern of the deciding rule",
            "type": [
              "string",
              "null"
            ]
          }
        }
      },
      "crawl_delay_seconds": {
        "type": [
          "number",
          "null"
        ],
        "description": "Crawl delay, in seconds"
      },
      "sitemaps": {
        "type": "array",
        "description": "Sitemap URLs declared in the file"
      },
      "groups_found": {
        "type": "number",
        "description": "Groups found"
      },
      "rules_in_group": {
        "type": "number",
        "description": "Rules in group"
      },
      "note": {
        "type": "string",
        "description": "Caveat or context for reading the result"
      },
      "source": {
        "type": "string",
        "description": "Data source the values come from"
      }
    },
    "required": [
      "allowed",
      "decision"
    ]
  },
  "extensions": {
    "bazaar": {
      "info": {
        "input": {
          "type": "http",
          "method": "GET",
          "queryParams": {
            "path": "/private/public/a.html",
            "robots": "User-agent: *\nDisallow: /private/\nAllow: /private/public\n\nUser-agent: GPTBot\nDisallow: /\nSitemap: https://example.com/sitemap.xml",
            "user_agent": "Googlebot"
          }
        },
        "output": {
          "type": "json",
          "example": {
            "user_agent": "Googlebot",
            "path": "/private/public/a.html",
            "allowed": true,
            "decision": "allowed",
            "matched_group": [
              "*"
            ],
            "group_selection": "no group names this agent, so the * group applies",
            "matched_rule": {
              "directive": "Allow",
              "pattern": "/private/public"
            },
            "crawl_delay_seconds": null,
            "sitemaps": [
              "https://example.com/sitemap.xml"
            ],
            "groups_found": 2,
            "rules_in_group": 2,
            "note": "Implements RFC 9309: the longest matching rule wins, Allow wins ties, and * and $ are supported in patterns.",
            "source": "robots.txt parsing per RFC 9309 on the supplied text"
          }
        }
      },
      "schema": {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "type": "object",
        "properties": {
          "input": {
            "type": "object",
            "properties": {
              "type": {
                "type": "string",
                "const": "http"
              },
              "method": {
                "type": "string",
                "enum": [
                  "GET"
                ]
              },
              "queryParams": {
                "type": "object",
                "properties": {
                  "path": {
                    "type": "string",
                    "description": "optional path to test, default /"
                  },
                  "robots": {
                    "type": "string",
                    "description": "robots.txt contents, up to 12000 characters"
                  },
                  "user_agent": {
                    "type": "string",
                    "description": "optional crawler token such as Googlebot or GPTBot, default *"
                  }
                },
                "required": [
                  "robots"
                ]
              }
            },
            "required": [
              "type",
              "method"
            ],
            "additionalProperties": false
          },
          "output": {
            "type": "object",
            "properties": {
              "type": {
                "type": "string",
                "const": "json"
              },
              "example": {
                "type": "object",
                "properties": {
                  "user_agent": {
                    "type": "string",
                    "description": "User-agent the rules were evaluated for"
                  },
                  "path": {
                    "type": "string",
                    "description": "Path that was tested"
                  },
                  "allowed": {
                    "type": "boolean",
                    "description": "True when the crawler may fetch the path"
                  },
                  "decision": {
                    "type": "string",
                    "description": "allowed or blocked"
                  },
                  "matched_group": {
                    "type": [
                      "array",
                      "null"
                    ],
                    "description": "Matched group"
                  },
                  "group_selection": {
                    "type": "string",
                    "description": "Group selection"
                  },
                  "matched_rule": {
                    "type": [
                      "object",
                      "null"
                    ],
                    "description": "The rule that decided the outcome, or null when no rule matched",
                    "properties": {
                      "directive": {
                        "description": "Allow or Disallow",
                        "type": [
                          "string",
                          "null"
                        ]
                      },
                      "pattern": {
                        "description": "The path pattern of the deciding rule",
                        "type": [
                          "string",
                          "null"
                        ]
                      }
                    }
                  },
                  "crawl_delay_seconds": {
                    "type": [
                      "number",
                      "null"
                    ],
                    "description": "Crawl delay, in seconds"
                  },
                  "sitemaps": {
                    "type": "array",
                    "description": "Sitemap URLs declared in the file"
                  },
                  "groups_found": {
                    "type": "number",
                    "description": "Groups found"
                  },
                  "rules_in_group": {
                    "type": "number",
                    "description": "Rules in group"
                  },
                  "note": {
                    "type": "string",
                    "description": "Caveat or context for reading the result"
                  },
                  "source": {
                    "type": "string",
                    "description": "Data source the values come from"
                  }
                },
                "required": [
                  "allowed",
                  "decision"
                ]
              }
            },
            "required": [
              "type"
            ]
          }
        },
        "required": [
          "input"
        ]
      }
    }
  },
  "tags": [
    "can this bot crawl a url",
    "robots.txt allow disallow check",
    "is gptbot blocked by robots txt",
    "longest match robots rule",
    "extract sitemap and crawl delay"
  ],
  "sourceHost": "saylorinnovations.com",
  "lastUpdated": "2026-10-26T22:14:01.958Z",
  "quality": {},
  "metadata": {
    "serviceName": "Saylor Innovations Data Desk",
    "method": "GET",
    "mimeType": "application/json",
    "inputSchema": {
      "type": "object",
      "properties": {
        "path": {
          "type": "string",
          "description": "optional path to test, default /"
        },
        "robots": {
          "type": "string",
          "description": "robots.txt contents, up to 12000 characters"
        },
        "user_agent": {
          "type": "string",
          "description": "optional crawler token such as Googlebot or GPTBot, default *"
        }
      },
      "required": [
        "robots"
      ]
    },
    "exampleResponse": {
      "user_agent": "Googlebot",
      "path": "/private/public/a.html",
      "allowed": true,
      "decision": "allowed",
      "matched_group": [
        "*"
      ],
      "group_selection": "no group names this agent, so the * group applies",
      "matched_rule": {
        "directive": "Allow",
        "pattern": "/private/public"
      },
      "crawl_delay_seconds": null,
      "sitemaps": [
        "https://example.com/sitemap.xml"
      ],
      "groups_found": 2,
      "rules_in_group": 2,
      "note": "Implements RFC 9309: the longest matching rule wins, Allow wins ties, and * and $ are supported in patterns.",
      "source": "robots.txt parsing per RFC 9309 on the supplied text"
    },
    "parameterLocation": "query",
    "extensions": {
      "bazaar": {
        "info": {
          "input": {
            "type": "http",
            "method": "GET",
            "queryParams": {
              "path": "/private/public/a.html",
              "robots": "User-agent: *\nDisallow: /private/\nAllow: /private/public\n\nUser-agent: GPTBot\nDisallow: /\nSitemap: https://example.com/sitemap.xml",
              "user_agent": "Googlebot"
            }
          },
          "output": {
            "type": "json",
            "example": {
              "user_agent": "Googlebot",
              "path": "/private/public/a.html",
              "allowed": true,
              "decision": "allowed",
              "matched_group": [
                "*"
              ],
              "group_selection": "no group names this agent, so the * group applies",
              "matched_rule": {
                "directive": "Allow",
                "pattern": "/private/public"
              },
              "crawl_delay_seconds": null,
              "sitemaps": [
                "https://example.com/sitemap.xml"
              ],
              "groups_found": 2,
              "rules_in_group": 2,
              "note": "Implements RFC 9309: the longest matching rule wins, Allow wins ties, and * and $ are supported in patterns.",
              "source": "robots.txt parsing per RFC 9309 on the supplied text"
            }
          }
        },
        "schema": {
          "$schema": "https://json-schema.org/draft/2020-12/schema",
          "type": "object",
          "properties": {
            "input": {
              "type": "object",
              "properties": {
                "type": {
                  "type": "string",
                  "const": "http"
                },
                "method": {
                  "type": "string",
                  "enum": [
                    "GET"
                  ]
                },
                "queryParams": {
                  "type": "object",
                  "properties": {
                    "path": {
                      "type": "string",
                      "description": "optional path to test, default /"
                    },
                    "robots": {
                      "type": "string",
                      "description": "robots.txt contents, up to 12000 characters"
                    },
                    "user_agent": {
                      "type": "string",
                      "description": "optional crawler token such as Googlebot or GPTBot, default *"
                    }
                  },
                  "required": [
                    "robots"
                  ]
                }
              },
              "required": [
                "type",
                "method"
              ],
              "additionalProperties": false
            },
            "output": {
              "type": "object",
              "properties": {
                "type": {
                  "type": "string",
                  "const": "json"
                },
                "example": {
                  "type": "object",
                  "properties": {
                    "user_agent": {
                      "type": "string",
                      "description": "User-agent the rules were evaluated for"
                    },
                    "path": {
                      "type": "string",
                      "description": "Path that was tested"
                    },
                    "allowed": {
                      "type": "boolean",
                      "description": "True when the crawler may fetch the path"
                    },
                    "decision": {
                      "type": "string",
                      "description": "allowed or blocked"
                    },
                    "matched_group": {
                      "type": [
                        "array",
                        "null"
                      ],
                      "description": "Matched group"
                    },
                    "group_selection": {
                      "type": "string",
                      "description": "Group selection"
                    },
                    "matched_rule": {
                      "type": [
                        "object",
                        "null"
                      ],
                      "description": "The rule that decided the outcome, or null when no rule matched",
                      "properties": {
                        "directive": {
                          "description": "Allow or Disallow",
                          "type": [
                            "string",
                            "null"
                          ]
                        },
                        "pattern": {
                          "description": "The path pattern of the deciding rule",
                          "type": [
                            "string",
                            "null"
                          ]
                        }
                      }
                    },
                    "crawl_delay_seconds": {
                      "type": [
                        "number",
                        "null"
                      ],
                      "description": "Crawl delay, in seconds"
                    },
                    "sitemaps": {
                      "type": "array",
                      "description": "Sitemap URLs declared in the file"
                    },
                    "groups_found": {
                      "type": "number",
                      "description": "Groups found"
                    },
                    "rules_in_group": {
                      "type": "number",
                      "description": "Rules in group"
                    },
                    "note": {
                      "type": "string",
                      "description": "Caveat or context for reading the result"
                    },
                    "source": {
                      "type": "string",
                      "description": "Data source the values come from"
                    }
                  },
                  "required": [
                    "allowed",
                    "decision"
                  ]
                }
              },
              "required": [
                "type"
              ]
            }
          },
          "required": [
            "input"
          ]
        }
      }
    },
    "examples": [
      {
        "label": "Unpaid discovery call",
        "method": "GET",
        "url": "https://saylorinnovations.com/api/web/robots-check?robots=User-agent%3A%20*%0ADisallow%3A%20%2Fprivate%2F%0AAllow%3A%20%2Fprivate%2Fpublic%0A%0AUser-agent%3A%20GPTBot%0ADisallow%3A%20%2F%0ASitemap%3A%20https%3A%2F%2Fexample.com%2Fsitemap.xml&user_agent=Googlebot&path=%2Fprivate%2Fpublic%2Fa.html",
        "expectedStatus": 402
      }
    ],
    "cacheTtlSeconds": 0,
    "capabilities": [],
    "noAccountRequired": true,
    "noApiKeyRequired": true,
    "sideEffects": "Paid calls settle the advertised payment; see the provider contract for execution behavior.",
    "payment": {
      "protocol": "x402",
      "settlement": "direct-to-provider"
    },
    "provider": {
      "name": "Saylor Innovations",
      "host": "saylorinnovations.com",
      "manifestUrl": "https://saylorinnovations.com/.well-known/x402.json"
    }
  },
  "liveness": {
    "reliability": null
  },
  "verified": true,
  "featured": false,
  "provider": {
    "host": "saylorinnovations.com",
    "manifest_name": "Saylor Innovations Knowledge Base",
    "source": "submitted",
    "source_manifest_url": "https://saylorinnovations.com/.well-known/x402.json",
    "submitted_at": "2026-10-26T05:52:02.113Z"
  },
  "exampleResponse": {
    "user_agent": "string",
    "path": "string",
    "allowed": true,
    "decision": "string",
    "matched_group": [
      "value"
    ],
    "group_selection": "string",
    "matched_rule": {
      "directive": "string",
      "pattern": "string"
    },
    "crawl_delay_seconds": 0,
    "sitemaps": [
      "value"
    ],
    "groups_found": 0,
    "rules_in_group": 0,
    "note": "string",
    "source": "string"
  }
}