diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 3cddc06..08e24b5 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,6 +1,6 @@ { "name": "spicrawl-plugins", - "description": "Spicrawl's plugin for coding agents: scrape web pages, run batch jobs, keep login sessions.", + "description": "Web scraping for AI coding agents: the Spicrawl plugin fetches web pages as Markdown, runs batch jobs and keeps login sessions.", "owner": { "name": "Spicrawl", "email": "dev@spicrawl.com" @@ -9,10 +9,10 @@ { "name": "spicrawl", "source": "./plugins/spicrawl", - "description": "Scrape web pages to Markdown, HTML, text or JSON, run batch jobs, keep login sessions, and read request history and usage, through the hosted Spicrawl MCP server.", + "description": "Web scraping for AI coding agents through the hosted Spicrawl MCP server: fetch a URL as clean Markdown, HTML, text or JSON, render JavaScript pages, extract fields with CSS selectors, run batch jobs and keep login sessions.", "homepage": "https://docs.spicrawl.com/agents/mcp", "license": "Apache-2.0", - "keywords": ["scraping", "web", "markdown", "batch", "sessions", "mcp"] + "keywords": ["scraping", "web-scraping", "web-scraper", "markdown", "html-to-markdown", "mcp", "mcp-server", "ai-agents", "llm", "rag", "batch", "sessions", "data-extraction", "claude-code"] } ] } diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index 5c20a72..639b077 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -5,13 +5,13 @@ "email": "dev@spicrawl.com" }, "metadata": { - "description": "Spicrawl's plugin for coding agents: scrape web pages, run batch jobs, keep login sessions." + "description": "Web scraping for AI coding agents: the Spicrawl plugin fetches web pages as Markdown, runs batch jobs and keeps login sessions." }, "plugins": [ { "name": "spicrawl", "source": "plugins/spicrawl", - "description": "Scrape web pages to Markdown, HTML, text or JSON, run batch jobs, keep login sessions, and read request history and usage, through the hosted Spicrawl MCP server." + "description": "Web scraping for AI coding agents through the hosted Spicrawl MCP server: fetch a URL as clean Markdown, HTML, text or JSON, render JavaScript pages, extract fields with CSS selectors, run batch jobs and keep login sessions." } ] } diff --git a/.factory-plugin/marketplace.json b/.factory-plugin/marketplace.json index b9f9d0a..f72ee19 100644 --- a/.factory-plugin/marketplace.json +++ b/.factory-plugin/marketplace.json @@ -1,13 +1,13 @@ { "name": "spicrawl-plugins", - "description": "Spicrawl's plugin for coding agents: scrape web pages, run batch jobs, keep login sessions.", + "description": "Web scraping for AI coding agents: the Spicrawl plugin fetches web pages as Markdown, runs batch jobs and keeps login sessions.", "owner": { "name": "Spicrawl" }, "plugins": [ { "name": "spicrawl", - "description": "Scrape web pages to Markdown, HTML, text or JSON, run batch jobs, keep login sessions, and read request history and usage, through the hosted Spicrawl MCP server.", + "description": "Web scraping for AI coding agents through the hosted Spicrawl MCP server: fetch a URL as clean Markdown, HTML, text or JSON, render JavaScript pages, extract fields with CSS selectors, run batch jobs and keep login sessions.", "source": "./plugins/spicrawl", "homepage": "https://docs.spicrawl.com/agents/mcp" } diff --git a/README.md b/README.md index 5192fc2..a98f2f9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,17 @@ -# Spicrawl agent plugins +# Spicrawl agent plugins: web scraping for Claude Code, Cursor, Codex and Gemini CLI [![smithery badge](https://smithery.ai/badge/spicrawl/web-scraper)](https://smithery.ai/servers/spicrawl/web-scraper) -Also available as an MCP server: [Smithery](https://smithery.ai/servers/spicrawl/web-scraper) ยท [MCP Registry](https://registry.modelcontextprotocol.io/v0/servers?search=io.github.OfficialSpicrawl/mcp) +**How do I give Claude Code, Cursor, Codex or Gemini CLI web scraping?** Connect the agent to +Spicrawl's hosted [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server at +`https://mcp.spicrawl.com/mcp`. This repository packages that connection as a plugin or extension +for each tool, so setup is one install command plus an API key from . +Through the full server the agent gets 24 `spicrawl_*` tools: fetch a web page as clean Markdown +(or HTML, text or JSON), render pages that need JavaScript, extract fields with CSS selectors, run +batch jobs over lists of URLs, keep login sessions, and read its own request history and usage. For +Claude Code, run `claude plugin marketplace add OfficialSpicrawl/agent-plugins && claude plugin install spicrawl@spicrawl-plugins`. +For Gemini CLI, run `gemini extensions install https://github.com/OfficialSpicrawl/agent-plugins`. +Cursor and Codex are in the table below. Spicrawl's plugins for coding agents and workflow tools, in one repository. For coding agents it connects the agent to the hosted Spicrawl MCP server and, in the tools that support it, gives the @@ -17,23 +26,37 @@ Spicrawl is a web scraping API for AI agents. Through the MCP server an agent ca - read its request history and its usage; - search Spicrawl's docs. -This repository holds the packaging for these tools: +## Supported tools -| Tool | What is installed | -|---|---| -| Claude Code | plugin (MCP server + skill) | -| Codex | plugin (skill + MCP server entry), in OpenAI's portable format; the key goes in your Codex config | -| ChatGPT | the same plugin, connecting through OAuth at `/chatgpt/mcp` once that endpoint is live; not in the directory yet (see [ChatGPT and Codex](#chatgpt-and-codex)) | -| Cursor | plugin (MCP server + skill) | -| OpenCode | npm plugin `opencode-spicrawl` (adds the MCP server) | -| Factory Droid | plugin (MCP server + skill) | -| Devin | plugin (MCP server + skill), installed from this repository | -| Gemini CLI | extension (MCP server + context file) | -| GitHub Copilot CLI | plugin (skill), plus one command for the MCP server | -| Dify | plugin `.difypkg` (Scrape, Extract and Batch scrape tools), from a GitHub release of this repository | -| n8n | community node `n8n-nodes-spicrawl` (Scrape URL and Batch Scrape), on npm; source in [OfficialSpicrawl/n8n-nodes-spicrawl](https://github.com/OfficialSpicrawl/n8n-nodes-spicrawl) | -| Any tool that reads Agent Skills | the skill only (`npx skills add`) | -| Any MCP client | the MCP server, by hand | +Each tool needs a Spicrawl API key; see [Get an API key](#get-an-api-key) and the section for your +tool under [Install](#install). + +| Tool | Install | What you get | +|---|---|---| +| Claude Code | `claude plugin marketplace add OfficialSpicrawl/agent-plugins && claude plugin install spicrawl@spicrawl-plugins` | plugin (MCP server + skill); Claude Code asks for the key when you enable it | +| Cursor | `git clone https://github.com/OfficialSpicrawl/agent-plugins.git && mkdir -p ~/.cursor/plugins/local && cp -r agent-plugins/plugins/spicrawl ~/.cursor/plugins/local/spicrawl` | plugin (MCP server + skill); restart Cursor, then set `SPICRAWL_API_KEY` under Plugins, Configure | +| Codex | `codex plugin marketplace add OfficialSpicrawl/agent-plugins` | plugin (skill + MCP server entry), in OpenAI's portable format; then install `spicrawl` from `/plugins`. The key goes in your Codex config (see [ChatGPT and Codex](#chatgpt-and-codex)) | +| ChatGPT | not in the directory yet | the same plugin, connecting through OAuth at `/chatgpt/mcp` once that endpoint is live (see [ChatGPT and Codex](#chatgpt-and-codex)) | +| Gemini CLI | `gemini extensions install https://github.com/OfficialSpicrawl/agent-plugins` | extension (MCP server + context file); asks for the key as a setting | +| OpenCode | `"plugin": ["opencode-spicrawl"]` in `opencode.json` | npm plugin `opencode-spicrawl` (adds the MCP server); reads `SPICRAWL_API_KEY` | +| Factory Droid | `droid plugin marketplace add OfficialSpicrawl/agent-plugins && droid plugin install spicrawl@spicrawl-plugins --scope user` | plugin (MCP server + skill); reads `SPICRAWL_AUTH` | +| GitHub Copilot CLI | `copilot plugin marketplace add OfficialSpicrawl/agent-plugins && copilot plugin install spicrawl@spicrawl-plugins` | plugin (skill), plus one command for the MCP server (see [GitHub Copilot CLI](#github-copilot-cli)) | +| Devin | `devin plugins install --local ./plugins/spicrawl` | plugin (MCP server + skill), run from a clone of this repository | +| Dify | Plugins, Marketplace, search "Spicrawl" | plugin (Scrape, Extract and Batch scrape tools), from the [Dify Marketplace](https://marketplace.dify.ai/plugin/spicrawl/spicrawl) | +| n8n | `npm install n8n-nodes-spicrawl` in `~/.n8n/nodes`, or Settings, Community Nodes, Install | community node `n8n-nodes-spicrawl` (Scrape URL and Batch Scrape), on npm; source in [OfficialSpicrawl/n8n-nodes-spicrawl](https://github.com/OfficialSpicrawl/n8n-nodes-spicrawl) | +| Any tool that reads Agent Skills | `npx skills add OfficialSpicrawl/agent-plugins` | the skill only | +| Any MCP client | `https://mcp.spicrawl.com/mcp` (Streamable HTTP, header `Authorization: Bearer `) | the MCP server, by hand | + +## Where to find Spicrawl + +- Dify Marketplace: +- Smithery: +- Cursor Directory: +- npm: [`@spicrawl/mcp`](https://www.npmjs.com/package/@spicrawl/mcp) (the MCP server), + [`opencode-spicrawl`](https://www.npmjs.com/package/opencode-spicrawl) and + [`n8n-nodes-spicrawl`](https://www.npmjs.com/package/n8n-nodes-spicrawl) +- Official MCP Registry: [`io.github.OfficialSpicrawl/mcp`](https://registry.modelcontextprotocol.io/v0/servers?search=io.github.OfficialSpicrawl/mcp) +- Glama: ## Get an API key @@ -205,11 +228,15 @@ Extract and Batch scrape. It calls the Spicrawl API directly, not the MCP server API key (with the `scrape` scope, plus `batch` for Batch scrape) in the plugin's authorization settings. -1. Download `spicrawl-.difypkg` from the [`dify-v*` releases](https://github.com/OfficialSpicrawl/agent-plugins/releases) - of this repository. -2. In Dify open Plugins, Install plugin, and either pick GitHub (this repository, the release and - the file) or upload the downloaded file. -3. Choose To Authorize on the Spicrawl plugin and paste your API key. +1. In Dify open Plugins, Marketplace, search for Spicrawl and select Install, or open the + [Spicrawl listing](https://marketplace.dify.ai/plugin/spicrawl/spicrawl). It needs Dify 1.14.2 + or later. +2. Choose To Authorize on the Spicrawl plugin and paste your API key. + +To install a specific release, or on a Dify that cannot reach the Marketplace, download +`spicrawl-.difypkg` from the [`dify-v*` releases](https://github.com/OfficialSpicrawl/agent-plugins/releases) +of this repository and pick it under Plugins, Install plugin. The package on GitHub is unsigned, so +a self-hosted Dify with signature verification on rejects it; install from the Marketplace there. See [`dify/README.md`](dify/README.md) for the tools and their parameters. @@ -246,6 +273,81 @@ describes the Spicrawl tools, the CLI and the HTTP API. It does not connect the | Header | `Authorization: Bearer ` | | OAuth | not used; turn automatic OAuth off if the client tries it | +## FAQ + +### How do I add web scraping to Claude Code? + +Install the plugin: `claude plugin marketplace add OfficialSpicrawl/agent-plugins`, then +`claude plugin install spicrawl@spicrawl-plugins`. Or add only the server: + +```sh +claude mcp add --transport http spicrawl https://mcp.spicrawl.com/mcp \ + --header "Authorization: Bearer $SPICRAWL_API_KEY" +``` + +Then ask Claude Code to read a URL; it calls `spicrawl_scrape`. + +### How do I add web scraping to Cursor? + +Install the `spicrawl` plugin and set `SPICRAWL_API_KEY` under Plugins, Configure (see +[Cursor](#cursor)), or put the server in `~/.cursor/mcp.json`. The plugin is also listed on the +[Cursor Directory](https://cursor.directory/plugins/spicrawl). + +### How do I add web scraping to Codex or Gemini CLI? + +Codex: `codex plugin marketplace add OfficialSpicrawl/agent-plugins`, then add the server with your +key to `~/.codex/config.toml` (see [ChatGPT and Codex](#chatgpt-and-codex)). Gemini CLI: +`gemini extensions install https://github.com/OfficialSpicrawl/agent-plugins`. + +### What is MCP, and which server do these plugins use? + +MCP, the Model Context Protocol, is an open protocol that lets an AI app call external tools. The +plugins connect to Spicrawl's hosted server at `https://mcp.spicrawl.com/mcp` over Streamable HTTP, +with your API key as a bearer token. Spicrawl fetches the page and returns the content, so nothing +else runs on your machine. + +### What can the agent do with Spicrawl? + +Fetch one URL as Markdown, HTML, text or JSON; extract named fields with CSS selectors; take a +screenshot; run batch jobs over many URLs; keep a session's cookies and storage between requests; +read its request history and credit usage; and search Spicrawl's documentation. The Markdown is the +page's main content, which keeps token use down. + +### Does it work on pages that need JavaScript? + +Yes. If a plain fetch comes back empty, the agent can ask for JavaScript rendering. A plain fetch +costs 1 credit and JavaScript rendering costs 3. + +### Can it crawl a whole website? + +No. Spicrawl fetches the URLs you give it. To cover a site, collect the URLs (from a sitemap, for +example) and submit them as a batch job. + +### Can it read pages behind a login? + +A session keeps cookies and storage between requests, so the agent can sign in once and read +further pages with the same session. Use it only for sites and accounts you are allowed to access. + +### How much does it cost? + +Credits are charged only for successful requests, and a failed request costs 0. See + for the price of each kind of request. + +### Where does my API key go? + +Only to Spicrawl. See [What is sent where](#what-is-sent-where). No file in this repository contains +a key. + +### Which other tools does it support? + +ChatGPT, OpenCode, Factory Droid, GitHub Copilot CLI, Devin, Dify, n8n, and any tool that reads +Agent Skills or speaks MCP. See [Supported tools](#supported-tools). + +### What license is it under? + +The plugins in this repository are Apache-2.0. Spicrawl itself is a hosted service that needs an +account and an API key. + ## What is sent where The API key is sent only to Spicrawl: to `mcp.spicrawl.com` in the `Authorization` header of MCP diff --git a/gemini-extension.json b/gemini-extension.json index 1b21f24..730761e 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,7 +1,7 @@ { "name": "spicrawl", "version": "0.2.2", - "description": "Scrape web pages to Markdown, HTML, text or JSON, run batch jobs, keep login sessions, and read request history and usage, through the hosted Spicrawl MCP server.", + "description": "Web scraping for AI coding agents through the hosted Spicrawl MCP server: fetch a URL as clean Markdown, HTML, text or JSON, render JavaScript pages, extract fields with CSS selectors, run batch jobs and keep login sessions.", "contextFileName": "GEMINI.md", "mcpServers": { "spicrawl": { diff --git a/opencode/package.json b/opencode/package.json index 3d0dfa9..4869239 100644 --- a/opencode/package.json +++ b/opencode/package.json @@ -17,7 +17,17 @@ "opencode-plugin", "spicrawl", "mcp", - "web-scraping" + "mcp-server", + "model-context-protocol", + "web-scraping", + "web-scraper", + "scraper", + "markdown", + "html-to-markdown", + "ai-agents", + "llm", + "rag", + "data-extraction" ], "license": "Apache-2.0", "author": { diff --git a/plugins/spicrawl/.claude-plugin/plugin.json b/plugins/spicrawl/.claude-plugin/plugin.json index 87a99eb..6440c42 100644 --- a/plugins/spicrawl/.claude-plugin/plugin.json +++ b/plugins/spicrawl/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "spicrawl", "displayName": "Spicrawl", "version": "0.2.2", - "description": "Scrape web pages to Markdown, HTML, text or JSON, run batch jobs, keep login sessions, and read request history and usage, through the hosted Spicrawl MCP server.", + "description": "Web scraping for AI coding agents through the hosted Spicrawl MCP server: fetch a URL as clean Markdown, HTML, text or JSON, render JavaScript pages, extract fields with CSS selectors, run batch jobs and keep login sessions.", "author": { "name": "Spicrawl", "email": "dev@spicrawl.com", @@ -11,7 +11,7 @@ "homepage": "https://docs.spicrawl.com/agents/mcp", "repository": "https://github.com/OfficialSpicrawl/agent-plugins", "license": "Apache-2.0", - "keywords": ["scraping", "web", "markdown", "batch", "sessions", "mcp"], + "keywords": ["scraping", "web-scraping", "web-scraper", "markdown", "html-to-markdown", "mcp", "mcp-server", "ai-agents", "llm", "rag", "batch", "sessions", "data-extraction", "claude-code"], "mcpServers": "./mcp/claude.json", "userConfig": { "spicrawl_api_key": { diff --git a/plugins/spicrawl/.cursor-plugin/plugin.json b/plugins/spicrawl/.cursor-plugin/plugin.json index 53c28d1..584ded3 100644 --- a/plugins/spicrawl/.cursor-plugin/plugin.json +++ b/plugins/spicrawl/.cursor-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "spicrawl", "displayName": "Spicrawl", "version": "0.2.2", - "description": "Scrape web pages to Markdown, HTML, text or JSON, run batch jobs, keep login sessions, and read request history and usage, through the hosted Spicrawl MCP server.", + "description": "Web scraping for AI coding agents through the hosted Spicrawl MCP server: fetch a URL as clean Markdown, HTML, text or JSON, render JavaScript pages, extract fields with CSS selectors, run batch jobs and keep login sessions.", "author": { "name": "Spicrawl", "email": "dev@spicrawl.com" @@ -11,9 +11,9 @@ "repository": "https://github.com/OfficialSpicrawl/agent-plugins", "license": "Apache-2.0", "logo": "logo.png", - "keywords": ["scraping", "web", "markdown", "batch", "sessions", "mcp"], + "keywords": ["scraping", "web-scraping", "web-scraper", "markdown", "html-to-markdown", "mcp", "mcp-server", "ai-agents", "llm", "rag", "batch", "sessions", "data-extraction", "cursor"], "category": "integrations", - "tags": ["web-scraping", "markdown", "batch", "sessions", "mcp"], + "tags": ["web-scraping", "web-scraper", "markdown", "batch", "sessions", "mcp", "ai-agents", "data-extraction"], "skills": "skills", "mcpServers": "./mcp/cursor.json", "variables": { diff --git a/plugins/spicrawl/.factory-plugin/plugin.json b/plugins/spicrawl/.factory-plugin/plugin.json index 6cf36c8..acacc7d 100644 --- a/plugins/spicrawl/.factory-plugin/plugin.json +++ b/plugins/spicrawl/.factory-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "spicrawl", - "description": "Scrape web pages to Markdown, HTML, text or JSON, run batch jobs, keep login sessions, and read request history and usage, through the hosted Spicrawl MCP server.", + "description": "Web scraping for AI coding agents through the hosted Spicrawl MCP server: fetch a URL as clean Markdown, HTML, text or JSON, render JavaScript pages, extract fields with CSS selectors, run batch jobs and keep login sessions.", "version": "0.2.2", "author": { "name": "Spicrawl", @@ -10,5 +10,5 @@ "homepage": "https://docs.spicrawl.com/agents/mcp", "repository": "https://github.com/OfficialSpicrawl/agent-plugins", "license": "Apache-2.0", - "keywords": ["scraping", "web", "markdown", "batch", "sessions", "mcp"] + "keywords": ["scraping", "web-scraping", "web-scraper", "markdown", "html-to-markdown", "mcp", "mcp-server", "ai-agents", "llm", "rag", "batch", "sessions", "data-extraction", "factory", "droid"] } diff --git a/plugins/spicrawl/.github/plugin/plugin.json b/plugins/spicrawl/.github/plugin/plugin.json index 234645a..a2d768c 100644 --- a/plugins/spicrawl/.github/plugin/plugin.json +++ b/plugins/spicrawl/.github/plugin/plugin.json @@ -10,6 +10,6 @@ "homepage": "https://docs.spicrawl.com/agents/mcp", "repository": "https://github.com/OfficialSpicrawl/agent-plugins", "license": "Apache-2.0", - "keywords": ["scraping", "web", "markdown", "batch", "sessions"], + "keywords": ["scraping", "web-scraping", "web-scraper", "markdown", "html-to-markdown", "ai-agents", "llm", "rag", "batch", "sessions", "data-extraction", "copilot-cli"], "skills": "skills/" } diff --git a/plugins/spicrawl/README.md b/plugins/spicrawl/README.md index 758f506..caf7980 100644 --- a/plugins/spicrawl/README.md +++ b/plugins/spicrawl/README.md @@ -1,8 +1,16 @@ -# Spicrawl +# Spicrawl: web scraping for AI coding agents -Spicrawl is a web scraping API for AI agents. This plugin connects Cursor to the hosted Spicrawl -[Model Context Protocol](https://modelcontextprotocol.io/) server and adds the `spicrawl` skill, -which teaches the agent how to use it. +**How do I give Claude Code, Cursor, Codex or Gemini CLI web scraping?** Connect the agent to +Spicrawl's hosted [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server at +`https://mcp.spicrawl.com/mcp`. This plugin does that for Cursor, and the same folder also serves +Claude Code, Factory Droid, Devin and GitHub Copilot CLI; Codex and Gemini CLI have their own +packages in the same repository (see the table below). Through the full server the agent has 24 +`spicrawl_*` tools to fetch a web page as clean Markdown (or HTML, text or JSON), render pages that +need JavaScript, extract fields with CSS selectors, run batch jobs and keep login sessions. You +need an API key from . + +Spicrawl is a web scraping API for AI agents. This plugin connects Cursor to the hosted Spicrawl MCP +server and adds the `spicrawl` skill, which teaches the agent how to use it. With it, Cursor's agent can: @@ -60,6 +68,34 @@ The server offers 24 `spicrawl_*` tools: Each successful scrape is billed in credits and a failure costs nothing; see . +## Supported tools + +The same Spicrawl server works in more than Cursor. Each tool needs an API key; the +[repository README](https://github.com/OfficialSpicrawl/agent-plugins#readme) has the key setup for each. + +| Tool | Install | +|---|---| +| Cursor | the steps under [Install](#install) below | +| Claude Code | `claude plugin marketplace add OfficialSpicrawl/agent-plugins && claude plugin install spicrawl@spicrawl-plugins` | +| Codex | `codex plugin marketplace add OfficialSpicrawl/agent-plugins`, then install `spicrawl` from `/plugins` (the key goes in your Codex config; see the [repository README](https://github.com/OfficialSpicrawl/agent-plugins#chatgpt-and-codex)) | +| Gemini CLI | `gemini extensions install https://github.com/OfficialSpicrawl/agent-plugins` | +| OpenCode | `"plugin": ["opencode-spicrawl"]` in `opencode.json` | +| Factory Droid | `droid plugin marketplace add OfficialSpicrawl/agent-plugins && droid plugin install spicrawl@spicrawl-plugins --scope user` | +| GitHub Copilot CLI | `copilot plugin marketplace add OfficialSpicrawl/agent-plugins && copilot plugin install spicrawl@spicrawl-plugins` (skill only; add the MCP server as the [repository README](https://github.com/OfficialSpicrawl/agent-plugins#github-copilot-cli) shows) | +| Devin | `devin plugins install --local ./plugins/spicrawl`, from a clone of the repository | +| Dify | Plugins, Marketplace, search "Spicrawl" | +| n8n | `npm install n8n-nodes-spicrawl` in `~/.n8n/nodes`, or Settings, Community Nodes, Install | +| Any MCP client | `https://mcp.spicrawl.com/mcp` (Streamable HTTP, header `Authorization: Bearer `) | + +## Where to find Spicrawl + +- Cursor Directory: +- Smithery: +- Dify Marketplace: +- npm: [`@spicrawl/mcp`](https://www.npmjs.com/package/@spicrawl/mcp) (the MCP server), + [`opencode-spicrawl`](https://www.npmjs.com/package/opencode-spicrawl) and + [`n8n-nodes-spicrawl`](https://www.npmjs.com/package/n8n-nodes-spicrawl) + ## Data A tool call sends your API key and the arguments the agent chose (for example the URL to fetch) to @@ -67,6 +103,46 @@ A tool call sends your API key and the arguments the agent chose (for example th history, which the request tools read back. The plugin itself runs no local code. See the privacy policy below. +## FAQ + +### How do I add web scraping to Cursor? + +Copy this plugin into Cursor's local plugins folder (see [Install](#install)), then set +`SPICRAWL_API_KEY` under Plugins, Configure. Ask the agent to read a URL and it calls +`spicrawl_scrape`. + +### What is MCP, and which server does this plugin use? + +MCP, the Model Context Protocol, is an open protocol that lets an AI app call external tools. This +plugin connects to Spicrawl's hosted server at `https://mcp.spicrawl.com/mcp` over Streamable HTTP, +with your API key as a bearer token. Spicrawl fetches the page and returns the content, so the +plugin itself runs no code on your machine. + +### Does it work on pages that need JavaScript? + +Yes. If a plain fetch comes back empty, the agent can ask for JavaScript rendering. A plain fetch +costs 1 credit and JavaScript rendering costs 3. + +### Can it crawl a whole website? + +No. Spicrawl fetches the URLs you give it. To cover a site, collect the URLs (from a sitemap, for +example) and submit them as a batch job. + +### Can it read pages behind a login? + +A session keeps cookies and storage between requests, so the agent can sign in once and read +further pages with the same session. Use it only for sites and accounts you are allowed to access. + +### How much does it cost? + +Credits are charged only for successful requests, and a failed request costs 0. See + for the price of each kind of request. + +### Where does my API key go? + +Only to Spicrawl, in the `Authorization` header of requests to `mcp.spicrawl.com`. See +[Data](#data). No key is stored in this repository. + ## Links - Setup guide: