{ "openapi": "3.1.0", "info": { "title": "decrawl API", "version": "1.0.0", "description": "Scrape a page, crawl a website, or map its URLs. Full documentation: https://decrawl.sixteenlabs.space/api/docs" }, "servers": [ { "url": "https://api.decrawl.sixteenlabs.space", "description": "Production API" } ], "tags": [ { "name": "Crawling" }, { "name": "Scraping" } ], "paths": { "/v1/scrape": { "post": { "operationId": "scrape_page", "summary": "scrape page", "tags": [ "Scraping" ], "description": "Scrape one URL with a 120-second default deadline. Successful complete responses (including model output) and source pages are cached separately for up to two days. Identical requests reuse the response; changed prompts reuse source content for new extraction. Failed results are never cached. Custom headers and explicit actions bypass caching.", "security": [ { "ApiKey": [] } ], "responses": { "200": { "description": "Success", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeResponse" } } }, "headers": { "X-Cache": { "description": "hit or miss for eligible cached requests; absent on cache bypass.", "schema": { "type": "string", "enum": [ "hit", "miss" ] } } } }, "default": { "description": "Error; see API_REFERENCE.md for status/code details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing, invalid, expired, or revoked credential.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The request cannot be accepted. Read the returned error for details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "LLM_NOT_CONFIGURED or BROWSER_NOT_CONFIGURED: the requested feature needs server configuration. HTTP-only requests work without a browser.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "BROWSER_CONNECTION_FAILED, BROWSER_AUTH_FAILED, BROWSER_RATE_LIMITED, TARGET_BLOCKED, TARGET_AUTH_REQUIRED, TARGET_RATE_LIMITED, TARGET_HTTP_ERROR, TARGET_NAVIGATION_FAILED, BROWSER_OPERATION_FAILED, SCRAPE_FAILED or EXTRACTION_FAILED; see the error code and server request_id log. Recognized challenges and upstream HTTP errors are not returned as successful documents.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeRequest" }, "example": { "url": "https://thedeconstruct.in/", "formats": [ "markdown", "links" ], "maxAge": 0 } } } } } }, "/v1/crawl": { "post": { "operationId": "start_crawl", "summary": "start crawl", "tags": [ "Crawling" ], "description": "Start an asynchronous website crawl. Poll the returned URL for status and page results.", "security": [ { "ApiKey": [] } ], "responses": { "200": { "description": "Success", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CrawlStarted" } } } }, "default": { "description": "Error; see API_REFERENCE.md for status/code details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing, invalid, expired, or revoked credential.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The request cannot be accepted. Read the returned error for details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CrawlRequest" } } } } } }, "/v1/crawl/{id}": { "get": { "operationId": "crawl_status", "summary": "crawl status", "tags": [ "Crawling" ], "description": "", "security": [ { "ApiKey": [] } ], "responses": { "200": { "description": "Success", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CrawlStatus" } } } }, "default": { "description": "Error; see API_REFERENCE.md for status/code details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing, invalid, expired, or revoked credential.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string", "pattern": "^[0-9A-Za-z-]{8,64}$" }, "description": "Crawl ID returned by POST /v1/crawl." }, { "name": "skip", "in": "query", "schema": { "type": "integer", "minimum": 0, "default": 0 }, "description": "Page offset. Invalid/negative values behave as zero." } ] }, "delete": { "operationId": "cancel_crawl", "summary": "cancel crawl", "tags": [ "Crawling" ], "description": "Signal cancellation. Partial results remain readable.", "security": [ { "ApiKey": [] } ], "responses": { "200": { "description": "Success", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Cancelled" } } } }, "default": { "description": "Error; see API_REFERENCE.md for status/code details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing, invalid, expired, or revoked credential.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "No requests left on this account.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string", "pattern": "^[0-9A-Za-z-]{8,64}$" }, "description": "Crawl ID returned by POST /v1/crawl." } ] } }, "/v1/crawl/{id}/errors": { "get": { "operationId": "crawl_errors", "summary": "crawl errors", "tags": [ "Crawling" ], "description": "", "security": [ { "ApiKey": [] } ], "responses": { "200": { "description": "Success", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CrawlErrors" } } } }, "default": { "description": "Error; see API_REFERENCE.md for status/code details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing, invalid, expired, or revoked credential.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "No requests left on this account.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string", "pattern": "^[0-9A-Za-z-]{8,64}$" }, "description": "Crawl ID returned by POST /v1/crawl." } ] } }, "/v1/map": { "post": { "operationId": "map_urls", "summary": "map urls", "tags": [ "Scraping" ], "description": "Discover URLs from page links and sitemaps.", "security": [ { "ApiKey": [] } ], "responses": { "200": { "description": "Success", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MapResponse" } } } }, "default": { "description": "Error; see API_REFERENCE.md for status/code details.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing, invalid, expired, or revoked credential.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "No requests remaining; reads and cancellation remain available.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MapRequest" } } } } } } }, "components": { "securitySchemes": { "ApiKey": { "type": "http", "scheme": "bearer", "description": "Customer API key created in the dashboard." } }, "schemas": { "Error": { "type": "object", "properties": { "success": { "type": "boolean", "const": false }, "error": { "type": "string" }, "code": { "type": "string", "description": "Optional machine-readable code." } }, "required": [ "success", "error" ] }, "Format": { "oneOf": [ { "type": "string", "enum": [ "markdown", "html", "rawHtml", "links", "images", "records", "summary", "json" ] }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "markdown", "html", "rawHtml", "links", "images", "records", "summary", "json" ] }, "prompt": { "type": "string", "description": "JSON extraction prompt." }, "schema": { "type": "object", "properties": {}, "additionalProperties": true, "description": "JSON Schema passed to the model; constraints are not separately validated server-side." } }, "required": [ "type" ] } ] }, "Agent": { "type": "object", "properties": { "prompt": { "type": "string", "description": "Task; falls back to top-level prompt. One task must be nonempty." }, "maxSteps": { "type": "integer", "description": "Action budget. Zero selects the default.", "minimum": 0, "maximum": 25, "default": 10 } } }, "Action": { "oneOf": [ { "type": "object", "properties": { "type": { "type": "string", "const": "wait" }, "milliseconds": { "type": "integer", "minimum": 1, "maximum": 60000 }, "selector": { "type": "string", "minLength": 1 } }, "required": [ "type" ], "oneOf": [ { "required": [ "milliseconds" ], "not": { "required": [ "selector" ] } }, { "required": [ "selector" ], "not": { "required": [ "milliseconds" ] } } ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "click" }, "selector": { "type": "string", "minLength": 1 }, "all": { "type": "boolean", "default": false } }, "required": [ "type", "selector" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "write" }, "text": { "type": "string", "minLength": 1 } }, "required": [ "type", "text" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "press" }, "key": { "type": "string", "minLength": 1 } }, "required": [ "type", "key" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "scroll" }, "direction": { "type": "string", "enum": [ "up", "down" ], "default": "down" }, "selector": { "type": "string" } }, "required": [ "type" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "scrape" } }, "required": [ "type" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "executeJavascript" }, "script": { "type": "string", "minLength": 1 } }, "required": [ "type", "script" ] } ] }, "ScrapeOptions": { "type": "object", "properties": { "formats": { "type": "array", "items": { "$ref": "#/components/schemas/Format" }, "default": [ "markdown" ] }, "prompt": { "type": "string", "description": "Adds JSON extraction and provides the default agent task." }, "agent": { "$ref": "#/components/schemas/Agent" }, "onlyMainContent": { "type": "boolean", "default": true }, "includeTags": { "type": "array", "items": { "type": "string", "description": "CSS selector" } }, "excludeTags": { "type": "array", "items": { "type": "string", "description": "CSS selector" } }, "actions": { "type": "array", "items": { "$ref": "#/components/schemas/Action" } }, "waitFor": { "type": "integer", "description": "Milliseconds; positive values force browser rendering.", "minimum": 0, "maximum": 60000, "default": 0 }, "timeout": { "description": "Whole page deadline in milliseconds, including extraction/agent work. Omitted or zero selects 120000.", "default": 120000, "anyOf": [ { "const": 0 }, { "type": "integer", "minimum": 1000, "maximum": 300000 } ] }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Headers sent to the target page. Standalone scrapes with headers bypass shared caching." } } }, "ScrapeRequest": { "type": "object", "properties": { "url": { "type": "string", "description": "HTTP(S) target. Missing scheme defaults to HTTPS.", "minLength": 1 }, "formats": { "type": "array", "items": { "$ref": "#/components/schemas/Format" }, "default": [ "markdown" ] }, "prompt": { "type": "string", "description": "Adds JSON extraction and provides the default agent task." }, "agent": { "$ref": "#/components/schemas/Agent" }, "onlyMainContent": { "type": "boolean", "default": true }, "includeTags": { "type": "array", "items": { "type": "string", "description": "CSS selector" } }, "excludeTags": { "type": "array", "items": { "type": "string", "description": "CSS selector" } }, "actions": { "type": "array", "items": { "$ref": "#/components/schemas/Action" } }, "waitFor": { "type": "integer", "description": "Milliseconds; positive values force browser rendering.", "minimum": 0, "maximum": 60000, "default": 0 }, "timeout": { "description": "Whole page deadline in milliseconds, including extraction/agent work. Omitted or zero selects 120000.", "default": 120000, "anyOf": [ { "const": 0 }, { "type": "integer", "minimum": 1000, "maximum": 300000 } ] }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Headers sent to the target page. Standalone scrapes with headers bypass shared caching." }, "maxAge": { "type": "number", "default": 172800000, "description": "Acceptable cache age in milliseconds; <=0 skips reads/writes. Standalone scrape only." }, "storeInCache": { "type": "boolean", "default": true, "description": "Prevents writes when false; can still read cached results. Standalone scrape only." } }, "required": [ "url" ] }, "CrawlRequest": { "type": "object", "properties": { "url": { "type": "string", "minLength": 1 }, "includePaths": { "type": "array", "items": { "type": "string", "description": "Regular expression" } }, "excludePaths": { "type": "array", "items": { "type": "string", "description": "Regular expression" } }, "regexOnFullURL": { "type": "boolean", "default": false }, "maxDiscoveryDepth": { "type": "integer", "description": "Link depth, start=0. Sitemap entries are admitted separately; omit for unlimited.", "minimum": 0 }, "sitemap": { "type": "string", "description": "Start URL is still attempted with only.", "enum": [ "include", "skip", "only" ], "default": "include" }, "ignoreQueryParameters": { "type": "boolean", "default": false }, "limit": { "type": "integer", "description": "Maximum admitted page attempts. Zero selects the default.", "minimum": 0, "default": 10000 }, "crawlEntireDomain": { "type": "boolean", "default": false }, "allowExternalLinks": { "type": "boolean", "default": false }, "allowSubdomains": { "type": "boolean", "default": false }, "ignoreRobotsTxt": { "type": "boolean", "default": false }, "robotsUserAgent": { "type": "string", "default": "*" }, "delay": { "type": "number", "minimum": 0, "maximum": 60, "default": 0, "description": "Seconds before each scrape; positive delay forces concurrency to one." }, "maxConcurrency": { "type": "integer", "description": "Zero selects default; positive delay overrides to one.", "minimum": 0, "maximum": 50, "default": 10 }, "scrapeOptions": { "$ref": "#/components/schemas/ScrapeOptions", "description": "Options per page. url is ignored; Standalone scrape cache options do not apply." } }, "required": [ "url" ] }, "Metadata": { "type": "object", "properties": { "title": { "type": "string" }, "description": { "type": "string" }, "language": { "type": "string" }, "sourceURL": { "type": "string" }, "url": { "type": "string" }, "statusCode": { "type": "integer" }, "contentType": { "type": "string" }, "engine": { "type": "string", "enum": [ "http", "browser" ] }, "elapsedMs": { "type": "integer" }, "numPages": { "type": "integer" }, "cacheState": { "type": "string", "enum": [ "hit" ] }, "cachedAt": { "type": "string", "format": "date-time" } }, "required": [ "sourceURL", "url", "statusCode", "engine", "elapsedMs" ] }, "Document": { "type": "object", "properties": { "markdown": { "type": "string", "description": "Clean Markdown including image references, also used as input to model extraction." }, "html": { "type": "string" }, "rawHtml": { "type": "string" }, "links": { "type": "array", "items": { "type": "string" } }, "images": { "type": "array", "items": { "type": "string" }, "description": "All deduplicated image URLs discovered in markup, responsive/lazy sources, gallery attributes, product JSON-LD and embedded gallery data. Includes page-wide images, not only the primary product photo." }, "records": { "type": "array", "items": { "type": "object", "properties": { "type": { "type": "string" }, "columns": { "type": "array", "items": { "type": "string" } }, "rows": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } } }, "required": [ "type", "columns", "rows" ] } }, "summary": { "type": "string" }, "json": {}, "warning": { "type": "string" }, "metadata": { "$ref": "#/components/schemas/Metadata" }, "actions": { "type": "object", "properties": { "scrapes": { "type": "array", "items": { "type": "object", "properties": { "url": { "type": "string" }, "html": { "type": "string" } }, "required": [ "url", "html" ] } }, "javascriptReturns": { "type": "array", "items": { "type": "object", "properties": { "type": { "type": "string" }, "value": {} }, "required": [ "type", "value" ] } } }, "required": [ "scrapes", "javascriptReturns" ] }, "agent": { "type": "object", "properties": { "steps": { "type": "array", "items": { "type": "object", "properties": { "action": { "type": "string" }, "input": {}, "error": { "type": "string" } }, "required": [ "action", "input" ] } }, "finished": { "type": "boolean" } }, "required": [ "steps", "finished" ] } }, "required": [ "metadata" ] }, "ScrapeResponse": { "type": "object", "properties": { "success": { "type": "boolean" }, "data": { "$ref": "#/components/schemas/Document" } }, "required": [ "success", "data" ] }, "CrawlStarted": { "type": "object", "properties": { "success": { "type": "boolean" }, "id": { "type": "string" }, "url": { "type": "string" } }, "required": [ "success", "id", "url" ] }, "CrawlStatus": { "type": "object", "properties": { "success": { "type": "boolean" }, "status": { "type": "string", "enum": [ "scraping", "completed", "failed", "cancelled" ] }, "total": { "type": "integer" }, "completed": { "type": "integer" }, "creditsUsed": { "type": "integer", "description": "Successful page count, not the platform request charge." }, "createdAt": { "type": "string", "format": "date-time" }, "completedAt": { "type": "string", "format": "date-time" }, "expiresAt": { "type": "string", "format": "date-time" }, "duration": { "type": "number", "description": "Seconds" }, "next": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "data": { "type": "array", "items": { "$ref": "#/components/schemas/Document" } } }, "required": [ "success", "status", "total", "completed", "creditsUsed", "createdAt", "expiresAt", "duration", "next", "data" ] }, "Cancelled": { "type": "object", "properties": { "status": { "type": "string", "const": "cancelled" } }, "required": [ "status" ] }, "MapResponse": { "type": "object", "properties": { "success": { "type": "boolean" }, "links": { "type": "array", "items": { "type": "object", "properties": { "url": { "type": "string" } }, "required": [ "url" ] } } }, "required": [ "success", "links" ] }, "MapRequest": { "type": "object", "properties": { "url": { "type": "string", "minLength": 1 }, "search": { "type": "string", "description": "Ranks matching URL words; not a strict filter." }, "sitemap": { "type": "string", "enum": [ "include", "skip", "only" ], "default": "include" }, "includeSubdomains": { "type": "boolean", "default": true }, "ignoreQueryParameters": { "type": "boolean", "default": true }, "limit": { "type": "integer", "description": "Zero selects the default.", "minimum": 0, "maximum": 100000, "default": 5000 }, "timeout": { "type": "integer", "description": "Discovery deadline in milliseconds. Nonpositive values select the default.", "default": 120000 } }, "required": [ "url" ] }, "CrawlErrors": { "type": "object", "properties": { "errors": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string" }, "timestamp": { "type": "string", "format": "date-time" }, "url": { "type": "string" }, "error": { "type": "string" } }, "required": [ "id", "timestamp", "url", "error" ] } }, "robotsBlocked": { "type": "array", "items": { "type": "string" } } }, "required": [ "errors", "robotsBlocked" ] } } } }