{"command":{"effect":"write","timeoutSeconds":30,"inputSchema":{"type":"object","description":"Starts a crawl job for the provided URL and its children. Check available options like `gotoOptions` and `waitFor*` to control page load behaviour.","properties":{"path":{"type":"object","properties":{"account_id":{"description":"Account ID.","type":"string"}},"required":["account_id"],"additionalProperties":false},"query":{"type":"object","properties":{"cacheTTL":{"description":"Cache TTL default is 5s. Set to 0 to disable.","type":"number","default":5,"maximum":86400,"minimum":0}},"required":[],"additionalProperties":false},"body":{"anyOf":[{"additionalProperties":false,"properties":{"actionTimeout":{"description":"The maximum duration allowed for the browser action to complete after the page has loaded (such as taking screenshots, extracting content, or generating PDFs). If this time limit is exceeded, the action stops and returns a timeout error.","type":"number","maximum":120000},"addScriptTag":{"description":"Adds a script element into the page with the desired URL or content.","type":"array","items":{"additionalProperties":false,"properties":{"content":{"type":"string"},"id":{"type":"string"},"type":{"type":"string"},"url":{"type":"string","format":"uri"}},"type":"object"}},"addStyleTag":{"description":"Adds a `<link rel=\"stylesheet\">` tag into the page with the desired URL or a `<style type=\"text/css\">` tag with the content.","type":"array","items":{"properties":{"content":{"type":"string"},"url":{"type":"string","format":"uri"}},"type":"object"}},"allowRequestPattern":{"description":"Only allow requests that match the provided regex patterns, eg. '/^.*\\.(css)'. Reject rules are applied first.","type":"array","items":{"type":"string"}},"allowResourceTypes":{"description":"Only allow requests that match the provided resource types, eg. 'image' or 'script'. Reject rules are applied first.","type":"array","items":{"anyOf":[{"enum":["document"],"type":"string"},{"enum":["stylesheet"],"type":"string"},{"enum":["image"],"type":"string"},{"enum":["media"],"type":"string"},{"enum":["font"],"type":"string"},{"enum":["script"],"type":"string"},{"enum":["texttrack"],"type":"string"},{"enum":["xhr"],"type":"string"},{"enum":["fetch"],"type":"string"},{"enum":["prefetch"],"type":"string"},{"enum":["eventsource"],"type":"string"},{"enum":["websocket"],"type":"string"},{"enum":["manifest"],"type":"string"},{"enum":["signedexchange"],"type":"string"},{"enum":["ping"],"type":"string"},{"enum":["cspviolationreport"],"type":"string"},{"enum":["preflight"],"type":"string"},{"enum":["other"],"type":"string"}]}},"authenticate":{"description":"Provide credentials for HTTP authentication.","type":"object","properties":{"password":{"type":"string","minLength":1},"username":{"type":"string","minLength":1}},"required":["username","password"]},"bestAttempt":{"description":"Attempt to proceed when 'awaited' events fail or timeout.","type":"boolean"},"browser":{"description":"Rendering backend for this crawl. Set to `kitesurf` to render pages with Kitesurf (beta). Only valid when `render` is `true`.","type":"string","enum":["kitesurf"]},"contentUse":{"description":"Intended content use level to respect the `use` Content-Signal directive in robots.txt. Levels (least to most permissive): 'reference', 'full'. A URL is disallowed when the publisher's declared `use` level is lower than this intent. Learn more: https://contentsignals.org/. Default: 'full'.","default":"full","anyOf":[{"enum":["reference"],"type":"string"},{"enum":["full"],"type":"string"}]},"cookies":{"description":"Check [options](https://pptr.dev/api/puppeteer.page.setcookie).","type":"array","items":{"properties":{"domain":{"type":"string"},"expires":{"type":"number"},"httpOnly":{"type":"boolean"},"name":{"description":"Cookie name.","type":"string"},"partitionKey":{"type":"string"},"path":{"type":"string"},"priority":{"anyOf":[{"enum":["Low"],"type":"string"},{"enum":["Medium"],"type":"string"},{"enum":["High"],"type":"string"}]},"sameParty":{"type":"boolean"},"sameSite":{"anyOf":[{"enum":["Strict"],"type":"string"},{"enum":["Lax"],"type":"string"},{"enum":["None"],"type":"string"}]},"secure":{"type":"boolean"},"sourcePort":{"type":"number"},"sourceScheme":{"anyOf":[{"enum":["Unset"],"type":"string"},{"enum":["NonSecure"],"type":"string"},{"enum":["Secure"],"type":"string"}]},"url":{"type":"string"},"value":{"type":"string"}},"required":["name","value"],"type":"object"}},"crawlPurposes":{"description":"List of crawl purposes to respect Content-Signal directives in robots.txt. Allowed values: 'search', 'ai-input', 'ai-train'. Learn more: https://contentsignals.org/. Default: ['search', 'ai-input', 'ai-train'].","type":"array","items":{"anyOf":[{"enum":["search"],"type":"string"},{"enum":["ai-input"],"type":"string"},{"enum":["ai-train"],"type":"string"}]},"default":["search","ai-input","ai-train"],"maxItems":3,"minItems":1},"depth":{"description":"Maximum number of levels deep the crawler will traverse from the starting URL.","type":"number","default":100000,"maximum":100000,"minimum":1},"emulateMediaType":{"type":"string"},"formats":{"description":"Formats to return. Default is `html`.","type":"array","items":{"anyOf":[{"enum":["html"],"type":"string"},{"enum":["markdown"],"type":"string"},{"enum":["json"],"type":"string"}]},"default":["html"]},"gotoOptions":{"description":"Check [options](https://pptr.dev/api/puppeteer.gotooptions).","type":"object","default":{},"properties":{"referer":{"type":"string"},"referrerPolicy":{"type":"string"},"timeout":{"type":"number","default":30000,"maximum":60000},"waitUntil":{"default":"domcontentloaded","anyOf":[{"enum":["load"],"type":"string"},{"enum":["domcontentloaded"],"type":"string"},{"enum":["networkidle0"],"type":"string"},{"enum":["networkidle2"],"type":"string"},{"items":{"anyOf":[{"enum":["load"],"type":"string"},{"enum":["domcontentloaded"],"type":"string"},{"enum":["networkidle0"],"type":"string"},{"enum":["networkidle2"],"type":"string"}]},"type":"array"}]}}},"jsonOptions":{"description":"Options for JSON extraction.","type":"object","additionalProperties":false,"properties":{"custom_ai":{"description":"Optional list of custom AI models to use for the request. The models will be tried in the order provided, and in case a model returns an error, the next one will be used as fallback.","type":"array","items":{"properties":{"authorization":{"description":"Authorization token for the AI model: `Bearer <token>`. Not needed for workers-ai models.","type":"string"},"model":{"description":"AI model to use for the request. Must be formed as `<provider>/<model_name>`, e.g. `workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast`.","type":"string"}},"required":["model"],"type":"object"}},"prompt":{"type":"string"},"response_format":{"type":"object","properties":{"json_schema":{"description":"Schema for the response format. More information here: https://developers.cloudflare.com/workers-ai/json-mode/","type":["object","null"],"additionalProperties":true},"type":{"type":"string"}},"required":["type"]}}},"limit":{"description":"Maximum number of URLs to crawl.","type":"number","default":10,"maximum":100000,"minimum":1},"maxAge":{"description":"Maximum age of a resource that can be returned from cache in seconds. Default is 1 day.","type":"number","default":86400,"maximum":604800,"minimum":0},"modifiedSince":{"description":"Unix timestamp (seconds since epoch) indicating to only crawl pages that were modified since this time. For sitemap URLs with a lastmod field, this is compared directly. For other URLs, the crawler will use If-Modified-Since header when fetching. URLs without modification information (no lastmod in sitemap and no Last-Modified header support) will be crawled. Note: This works in conjunction with maxAge - both filters must pass for a cached resource to be used. Must be within the last year and not in the future.","type":"integer","exclusiveMinimum":0},"options":{"description":"Additional options for the crawler.","type":"object","default":{"includeExternalLinks":false,"includeSubdomains":false},"properties":{"excludePatterns":{"description":"Exclude links matching the provided wildcard patterns in the crawl job. Example: 'https://example.com/privacy/**'.","type":"array","items":{"maxLength":500,"minLength":1,"type":"string"},"maxItems":100},"includeExternalLinks":{"description":"Include external links in the crawl job. If set to true, includeSubdomains is ignored.","type":"boolean","default":false},"includePatterns":{"description":"Include only links matching the provided wildcard patterns in the crawl job. Include patterns are evaluated before exclude patterns. URLs that match any of the specified include patterns will be included in the crawl job. Example: 'https://example.com/blog/**'.","type":"array","items":{"maxLength":500,"minLength":1,"type":"string"},"maxItems":100},"includeSubdomains":{"description":"Include links to subdomains in the crawl job. This option is ignored if includeExternalLinks is true.","type":"boolean","default":false}}},"rejectRequestPattern":{"description":"Block undesired requests that match the provided regex patterns, eg. '/^.*\\.(css)'.","type":"array","items":{"type":"string"}},"rejectResourceTypes":{"description":"Block undesired requests that match the provided resource types, eg. 'image' or 'script'.","type":"array","items":{"anyOf":[{"enum":["document"],"type":"string"},{"enum":["stylesheet"],"type":"string"},{"enum":["image"],"type":"string"},{"enum":["media"],"type":"string"},{"enum":["font"],"type":"string"},{"enum":["script"],"type":"string"},{"enum":["texttrack"],"type":"string"},{"enum":["xhr"],"type":"string"},{"enum":["fetch"],"type":"string"},{"enum":["prefetch"],"type":"string"},{"enum":["eventsource"],"type":"string"},{"enum":["websocket"],"type":"string"},{"enum":["manifest"],"type":"string"},{"enum":["signedexchange"],"type":"string"},{"enum":["ping"],"type":"string"},{"enum":["cspviolationreport"],"type":"string"},{"enum":["preflight"],"type":"string"},{"enum":["other"],"type":"string"}]}},"render":{"description":"Whether to render the page or fetch static content. True by default.","type":"boolean","default":true,"enum":[true]},"setExtraHTTPHeaders":{"type":"object","additionalProperties":{"type":"string"}},"setJavaScriptEnabled":{"type":"boolean"},"source":{"description":"Source of links to crawl. 'sitemaps' - only crawl URLs from sitemaps, 'links' - only crawl URLs scraped from pages, 'all' - crawl both sitemap and scraped links (default).","default":"all","anyOf":[{"enum":["sitemaps"],"type":"string"},{"enum":["links"],"type":"string"},{"enum":["all"],"type":"string"}]},"url":{"description":"URL to navigate to, eg. `https://example.com`.","type":"string","format":"uri"},"viewport":{"description":"Check [options](https://pptr.dev/api/puppeteer.page.setviewport).","type":"object","default":{"height":1080,"width":1920},"properties":{"deviceScaleFactor":{"type":"number"},"hasTouch":{"type":"boolean"},"height":{"type":"number"},"isLandscape":{"type":"boolean"},"isMobile":{"type":"boolean"},"width":{"type":"number"}},"required":["width","height"]},"waitForSelector":{"description":"Wait for the selector to appear in page. Check [options](https://pptr.dev/api/puppeteer.page.waitforselector).","type":"object","properties":{"hidden":{"type":"boolean","enum":[true]},"selector":{"type":"string"},"timeout":{"type":"number","maximum":120000},"visible":{"type":"boolean","enum":[true]}},"required":["selector"]},"waitForTimeout":{"description":"Waits for a specified timeout before continuing.","type":"number","maximum":120000}},"required":["url"],"type":"object"},{"additionalProperties":false,"properties":{"contentUse":{"description":"Intended content use level to respect the `use` Content-Signal directive in robots.txt. Levels (least to most permissive): 'reference', 'full'. A URL is disallowed when the publisher's declared `use` level is lower than this intent. Learn more: https://contentsignals.org/. Default: 'full'.","default":"full","anyOf":[{"enum":["reference"],"type":"string"},{"enum":["full"],"type":"string"}]},"crawlPurposes":{"description":"List of crawl purposes to respect Content-Signal directives in robots.txt. Allowed values: 'search', 'ai-input', 'ai-train'. Learn more: https://contentsignals.org/. Default: ['search', 'ai-input', 'ai-train'].","type":"array","items":{"anyOf":[{"enum":["search"],"type":"string"},{"enum":["ai-input"],"type":"string"},{"enum":["ai-train"],"type":"string"}]},"default":["search","ai-input","ai-train"],"maxItems":3,"minItems":1},"depth":{"description":"Maximum number of levels deep the crawler will traverse from the starting URL.","type":"number","default":100000,"maximum":100000,"minimum":1},"formats":{"description":"Formats to return. Default is `html`.","type":"array","items":{"anyOf":[{"enum":["html"],"type":"string"},{"enum":["markdown"],"type":"string"},{"enum":["json"],"type":"string"}]},"default":["html"]},"jsonOptions":{"description":"Options for JSON extraction.","type":"object","additionalProperties":false,"properties":{"custom_ai":{"description":"Optional list of custom AI models to use for the request. The models will be tried in the order provided, and in case a model returns an error, the next one will be used as fallback.","type":"array","items":{"properties":{"authorization":{"description":"Authorization token for the AI model: `Bearer <token>`. Not needed for workers-ai models.","type":"string"},"model":{"description":"AI model to use for the request. Must be formed as `<provider>/<model_name>`, e.g. `workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast`.","type":"string"}},"required":["model"],"type":"object"}},"prompt":{"type":"string"},"response_format":{"type":"object","properties":{"json_schema":{"description":"Schema for the response format. More information here: https://developers.cloudflare.com/workers-ai/json-mode/","type":["object","null"],"additionalProperties":true},"type":{"type":"string"}},"required":["type"]}}},"limit":{"description":"Maximum number of URLs to crawl.","type":"number","default":10,"maximum":100000,"minimum":1},"maxAge":{"description":"Maximum age of a resource that can be returned from cache in seconds. Default is 1 day.","type":"number","default":86400,"maximum":604800,"minimum":0},"modifiedSince":{"description":"Unix timestamp (seconds since epoch) indicating to only crawl pages that were modified since this time. For sitemap URLs with a lastmod field, this is compared directly. For other URLs, the crawler will use If-Modified-Since header when fetching. URLs without modification information (no lastmod in sitemap and no Last-Modified header support) will be crawled. Note: This works in conjunction with maxAge - both filters must pass for a cached resource to be used. Must be within the last year and not in the future.","type":"integer","exclusiveMinimum":0},"options":{"description":"Additional options for the crawler.","type":"object","default":{"includeExternalLinks":false,"includeSubdomains":false},"properties":{"excludePatterns":{"description":"Exclude links matching the provided wildcard patterns in the crawl job. Example: 'https://example.com/privacy/**'.","type":"array","items":{"maxLength":500,"minLength":1,"type":"string"},"maxItems":100},"includeExternalLinks":{"description":"Include external links in the crawl job. If set to true, includeSubdomains is ignored.","type":"boolean","default":false},"includePatterns":{"description":"Include only links matching the provided wildcard patterns in the crawl job. Include patterns are evaluated before exclude patterns. URLs that match any of the specified include patterns will be included in the crawl job. Example: 'https://example.com/blog/**'.","type":"array","items":{"maxLength":500,"minLength":1,"type":"string"},"maxItems":100},"includeSubdomains":{"description":"Include links to subdomains in the crawl job. This option is ignored if includeExternalLinks is true.","type":"boolean","default":false}}},"render":{"description":"Whether to render the page or fetch static content. True by default.","type":"boolean","enum":[false]},"source":{"description":"Source of links to crawl. 'sitemaps' - only crawl URLs from sitemaps, 'links' - only crawl URLs scraped from pages, 'all' - crawl both sitemap and scraped links (default).","default":"all","anyOf":[{"enum":["sitemaps"],"type":"string"},{"enum":["links"],"type":"string"},{"enum":["all"],"type":"string"}]},"url":{"description":"URL to navigate to, eg. `https://example.com`.","type":"string","format":"uri"}},"required":["url","render"],"type":"object"}],"examples":[{"url":"https://www.example.com/"}]}},"required":["path","body"],"additionalProperties":false},"name":"Crawl websites.","id":"cloudflare.brapi-post_crawl","example":{"path":{"account_id":"example"},"body":{"url":"https://www.example.com/"}},"maxRequests":1,"summary":false}}