{
  "x402Version": 2,
  "id": 71972,
  "slug": "saylorinnovations-api-agent-robots",
  "resource": "https://saylorinnovations.com/api/agent/robots",
  "description": "robots.txt rules for a user-agent - allow and disallow rules, crawl delay and sitemap URLs",
  "type": "http",
  "serviceName": "Saylor Agent Tools",
  "mimeType": "application/json",
  "resourceType": "API",
  "accepts": [
    {
      "scheme": "exact",
      "network": "eip155:8453",
      "payTo": "0xA2A8dF0f7CaaBEa4e120a2568118f141569fc8aa",
      "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "eip155:137",
      "payTo": "0xA2A8dF0f7CaaBEa4e120a2568118f141569fc8aa",
      "asset": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "eip155:42161",
      "payTo": "0xA2A8dF0f7CaaBEa4e120a2568118f141569fc8aa",
      "asset": "0xaf88d065e77c8cC2239327C5EDb3A432268e5831",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "7LSjfrJf8fNsB8VA9u7N3WEn25smQLpob3SvyUvXacy7",
      "asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "7LSjfrJf8fNsB8VA9u7N3WEn25smQLpob3SvyUvXacy7",
      "asset": "Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "7LSjfrJf8fNsB8VA9u7N3WEn25smQLpob3SvyUvXacy7",
      "asset": "So11111111111111111111111111111111111111112",
      "amount": "95877",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 120
    }
  ],
  "outputSchema": {
    "type": "object",
    "properties": {
      "found": {
        "type": "boolean",
        "description": "True when the lookup matched a record"
      },
      "sitemaps": {
        "type": "array",
        "items": {
          "type": "string"
        }
      },
      "matching_group": {
        "type": [
          "object",
          "null"
        ],
        "description": "Matching group"
      },
      "url": {
        "type": [
          "string",
          "null"
        ],
        "description": "Canonical URL"
      },
      "final_url": {
        "type": [
          "string",
          "null"
        ],
        "description": "Final URL"
      },
      "http_status": {
        "type": [
          "number",
          "null"
        ],
        "description": "HTTP status code returned"
      },
      "user_agent": {
        "type": [
          "string",
          "null"
        ],
        "description": "User-agent the rules were evaluated for"
      },
      "groups_count": {
        "type": [
          "number",
          "null"
        ],
        "description": "Number of groups"
      },
      "truncated": {
        "type": [
          "boolean",
          "null"
        ],
        "description": "True when the list was cut at the limit"
      },
      "summary": {
        "description": "Short summary"
      },
      "note": {
        "type": [
          "string",
          "null"
        ],
        "description": "Caveat or context for reading the result"
      }
    },
    "required": [
      "found"
    ]
  },
  "extensions": {
    "bazaar": {
      "info": {
        "input": {
          "type": "http",
          "method": "GET",
          "queryParams": {
            "url": "saylorinnovations.com"
          }
        },
        "output": {
          "type": "json",
          "example": {
            "found": true,
            "url": "https://saylorinnovations.com/robots.txt",
            "final_url": "https://saylorinnovations.com/robots.txt",
            "http_status": 200,
            "user_agent": "*",
            "groups_count": 16,
            "matching_group": {
              "userAgents": [
                "*"
              ],
              "allow": [
                "/"
              ],
              "disallow": [],
              "crawlDelay": null
            },
            "sitemaps": [
              "https://saylorinnovations.com/sitemap.xml"
            ],
            "truncated": false,
            "summary": {
              "allow_count": 1,
              "disallow_count": 0,
              "crawl_delay": null
            },
            "note": "Robots rules are advisory and path-pattern semantics vary by crawler."
          }
        }
      },
      "schema": {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "type": "object",
        "properties": {
          "input": {
            "type": "object",
            "properties": {
              "type": {
                "type": "string",
                "const": "http"
              },
              "method": {
                "type": "string",
                "enum": [
                  "GET"
                ]
              },
              "queryParams": {
                "type": "object",
                "properties": {
                  "url": {
                    "type": "string",
                    "description": "site URL or domain (query)"
                  }
                },
                "required": [
                  "url"
                ]
              }
            },
            "required": [
              "type",
              "method"
            ],
            "additionalProperties": false
          },
          "output": {
            "type": "object",
            "properties": {
              "type": {
                "type": "string",
                "const": "json"
              },
              "example": {
                "type": "object",
                "properties": {
                  "found": {
                    "type": "boolean",
                    "description": "True when the lookup matched a record"
                  },
                  "sitemaps": {
                    "type": "array",
                    "items": {
                      "type": "string"
                    }
                  },
                  "matching_group": {
                    "type": [
                      "object",
                      "null"
                    ],
                    "description": "Matching group"
                  },
                  "url": {
                    "type": [
                      "string",
                      "null"
                    ],
                    "description": "Canonical URL"
                  },
                  "final_url": {
                    "type": [
                      "string",
                      "null"
                    ],
                    "description": "Final URL"
                  },
                  "http_status": {
                    "type": [
                      "number",
                      "null"
                    ],
                    "description": "HTTP status code returned"
                  },
                  "user_agent": {
                    "type": [
                      "string",
                      "null"
                    ],
                    "description": "User-agent the rules were evaluated for"
                  },
                  "groups_count": {
                    "type": [
                      "number",
                      "null"
                    ],
                    "description": "Number of groups"
                  },
                  "truncated": {
                    "type": [
                      "boolean",
                      "null"
                    ],
                    "description": "True when the list was cut at the limit"
                  },
                  "summary": {
                    "description": "Short summary"
                  },
                  "note": {
                    "type": [
                      "string",
                      "null"
                    ],
                    "description": "Caveat or context for reading the result"
                  }
                },
                "required": [
                  "found"
                ]
              }
            },
            "required": [
              "type"
            ]
          }
        },
        "required": [
          "input"
        ]
      }
    }
  },
  "tags": [
    "agents",
    "web",
    "robots",
    "discovery"
  ],
  "sourceHost": "saylorinnovations.com",
  "lastUpdated": "2026-10-26T22:14:01.835Z",
  "quality": {},
  "metadata": {
    "serviceName": "Saylor Agent Tools",
    "method": "GET",
    "mimeType": "application/json",
    "inputSchema": {
      "type": "object",
      "properties": {
        "url": {
          "type": "string",
          "description": "site URL or domain (query)"
        }
      },
      "required": [
        "url"
      ]
    },
    "exampleResponse": {
      "found": true,
      "url": "https://saylorinnovations.com/robots.txt",
      "final_url": "https://saylorinnovations.com/robots.txt",
      "http_status": 200,
      "user_agent": "*",
      "groups_count": 16,
      "matching_group": {
        "userAgents": [
          "*"
        ],
        "allow": [
          "/"
        ],
        "disallow": [],
        "crawlDelay": null
      },
      "sitemaps": [
        "https://saylorinnovations.com/sitemap.xml"
      ],
      "truncated": false,
      "summary": {
        "allow_count": 1,
        "disallow_count": 0,
        "crawl_delay": null
      },
      "note": "Robots rules are advisory and path-pattern semantics vary by crawler."
    },
    "parameterLocation": "query",
    "extensions": {
      "bazaar": {
        "info": {
          "input": {
            "type": "http",
            "method": "GET",
            "queryParams": {
              "url": "saylorinnovations.com"
            }
          },
          "output": {
            "type": "json",
            "example": {
              "found": true,
              "url": "https://saylorinnovations.com/robots.txt",
              "final_url": "https://saylorinnovations.com/robots.txt",
              "http_status": 200,
              "user_agent": "*",
              "groups_count": 16,
              "matching_group": {
                "userAgents": [
                  "*"
                ],
                "allow": [
                  "/"
                ],
                "disallow": [],
                "crawlDelay": null
              },
              "sitemaps": [
                "https://saylorinnovations.com/sitemap.xml"
              ],
              "truncated": false,
              "summary": {
                "allow_count": 1,
                "disallow_count": 0,
                "crawl_delay": null
              },
              "note": "Robots rules are advisory and path-pattern semantics vary by crawler."
            }
          }
        },
        "schema": {
          "$schema": "https://json-schema.org/draft/2020-12/schema",
          "type": "object",
          "properties": {
            "input": {
              "type": "object",
              "properties": {
                "type": {
                  "type": "string",
                  "const": "http"
                },
                "method": {
                  "type": "string",
                  "enum": [
                    "GET"
                  ]
                },
                "queryParams": {
                  "type": "object",
                  "properties": {
                    "url": {
                      "type": "string",
                      "description": "site URL or domain (query)"
                    }
                  },
                  "required": [
                    "url"
                  ]
                }
              },
              "required": [
                "type",
                "method"
              ],
              "additionalProperties": false
            },
            "output": {
              "type": "object",
              "properties": {
                "type": {
                  "type": "string",
                  "const": "json"
                },
                "example": {
                  "type": "object",
                  "properties": {
                    "found": {
                      "type": "boolean",
                      "description": "True when the lookup matched a record"
                    },
                    "sitemaps": {
                      "type": "array",
                      "items": {
                        "type": "string"
                      }
                    },
                    "matching_group": {
                      "type": [
                        "object",
                        "null"
                      ],
                      "description": "Matching group"
                    },
                    "url": {
                      "type": [
                        "string",
                        "null"
                      ],
                      "description": "Canonical URL"
                    },
                    "final_url": {
                      "type": [
                        "string",
                        "null"
                      ],
                      "description": "Final URL"
                    },
                    "http_status": {
                      "type": [
                        "number",
                        "null"
                      ],
                      "description": "HTTP status code returned"
                    },
                    "user_agent": {
                      "type": [
                        "string",
                        "null"
                      ],
                      "description": "User-agent the rules were evaluated for"
                    },
                    "groups_count": {
                      "type": [
                        "number",
                        "null"
                      ],
                      "description": "Number of groups"
                    },
                    "truncated": {
                      "type": [
                        "boolean",
                        "null"
                      ],
                      "description": "True when the list was cut at the limit"
                    },
                    "summary": {
                      "description": "Short summary"
                    },
                    "note": {
                      "type": [
                        "string",
                        "null"
                      ],
                      "description": "Caveat or context for reading the result"
                    }
                  },
                  "required": [
                    "found"
                  ]
                }
              },
              "required": [
                "type"
              ]
            }
          },
          "required": [
            "input"
          ]
        }
      }
    },
    "examples": [
      {
        "label": "Unpaid discovery call",
        "method": "GET",
        "url": "https://saylorinnovations.com/api/agent/robots?url=saylorinnovations.com",
        "expectedStatus": 402
      }
    ],
    "cacheTtlSeconds": 0,
    "capabilities": [],
    "noAccountRequired": true,
    "noApiKeyRequired": true,
    "sideEffects": "Paid calls settle the advertised payment; see the provider contract for execution behavior.",
    "payment": {
      "protocol": "x402",
      "settlement": "direct-to-provider"
    },
    "provider": {
      "name": "Saylor Innovations",
      "host": "saylorinnovations.com",
      "manifestUrl": "https://saylorinnovations.com/.well-known/x402.json"
    }
  },
  "liveness": {
    "reliability": null
  },
  "verified": true,
  "featured": false,
  "provider": {
    "host": "saylorinnovations.com",
    "manifest_name": "Saylor Innovations Knowledge Base",
    "source": "submitted",
    "source_manifest_url": "https://saylorinnovations.com/.well-known/x402.json",
    "submitted_at": "2026-10-26T05:52:02.113Z"
  },
  "exampleResponse": {
    "found": true,
    "sitemaps": [
      "string"
    ],
    "matching_group": {},
    "url": "string",
    "final_url": "string",
    "http_status": 0,
    "user_agent": "string",
    "groups_count": 0,
    "truncated": true,
    "summary": null,
    "note": "string"
  }
}