diff --git a/AGENTS.md b/AGENTS.md index ac320a4d..688aa825 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,6 +100,7 @@ Each example is self-contained and demonstrates specific patterns and capabiliti - `calculator/` - Mathematical computation tool - `datetime_tool/` - Timezone-aware now / parse / convert / shift / diff - `fia_signals/` - Crypto market intelligence (regime, signals, yields, gas, trending, wallet risk) +- `firecrawl_search/` - Web and news search via the Firecrawl API, with optional page content as Markdown - `hackernews_search/` - Hacker News search via the free Algolia API - `pdf_reader/` - PDF text + metadata extraction (local file or URL, page-range support) - `searxng_search/` - Privacy-focused search integration diff --git a/README.md b/README.md index 270f5588..eaa6a167 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,7 @@ Atomic Forge is a collection of tools that can be used with Atomic Agents to ext - Calculator - DateTime - Fía Signals +- Firecrawl Search - Hacker News Search - PDF Reader - SearXNG Search diff --git a/atomic-forge/README.md b/atomic-forge/README.md index d34c5a0f..19304267 100644 --- a/atomic-forge/README.md +++ b/atomic-forge/README.md @@ -18,6 +18,7 @@ The Atomic Forge project includes the following tools: - [Calculator](/atomic-forge/tools/calculator/README.md) - [DateTime](/atomic-forge/tools/datetime_tool/README.md) — timezone-aware now/parse/convert/shift/diff. - [Fía Signals](/atomic-forge/tools/fia_signals/README.md) +- [Firecrawl Search](/atomic-forge/tools/firecrawl_search/README.md) — web and news search via the Firecrawl API, with optional page content as Markdown. - [Hacker News Search](/atomic-forge/tools/hackernews_search/README.md) — search HN stories, comments, Show HN, Ask HN via the free Algolia API. - [PDF Reader](/atomic-forge/tools/pdf_reader/README.md) — extract text and metadata from a local or remote PDF, with page-range filtering. - [SearXNG Search](/atomic-forge/tools/searxng_search/README.md) diff --git a/atomic-forge/tools/firecrawl_search/.coveragerc b/atomic-forge/tools/firecrawl_search/.coveragerc new file mode 100644 index 00000000..c45136ef --- /dev/null +++ b/atomic-forge/tools/firecrawl_search/.coveragerc @@ -0,0 +1,8 @@ +[run] +source = tool +omit = */tests/* + +[report] +exclude_lines = + if __name__ == "__main__": +show_missing = True diff --git a/atomic-forge/tools/firecrawl_search/README.md b/atomic-forge/tools/firecrawl_search/README.md new file mode 100644 index 00000000..9910002d --- /dev/null +++ b/atomic-forge/tools/firecrawl_search/README.md @@ -0,0 +1,62 @@ +# Firecrawl Search Tool + +## Overview +Searches the web or recent news through the [Firecrawl Search](https://www.firecrawl.dev/search) API and returns each result's title, URL, snippet, and rank, plus the publish date for news. With `include_content=True` the main content of each result page also comes back as Markdown in the same call, so an agent does not need a separate fetch step to read the pages. Requires a Firecrawl API key. + +## Prerequisites and Dependencies +- Python 3.12 or later +- `atomic-agents` +- `pydantic` +- `aiohttp` +- A Firecrawl API key from [firecrawl.dev/app/api-keys](https://www.firecrawl.dev/app/api-keys), set as `FIRECRAWL_API_KEY` or passed in the config + +## Installation +1. Use the Atomic Assembler CLI: run `atomic` and pick `firecrawl_search`. +2. Or copy the `tool/` folder directly into your project. + +## Configuration +- `api_key` (`SecretStr`, a plain string is accepted): Firecrawl API key. Falls back to the `FIRECRAWL_API_KEY` environment variable when empty. It is only sent in the `Authorization` header. Raised errors, log messages and `failures` redact the configured key (when it is at least 8 characters) and anything in Firecrawl's `fc-` key format. In tracebacks that capture local variables, the key does not appear in the tool's own state or in anything derived from a response or a transport error; the inputs you pass in, such as a query, appear there as given, the same as in your own code's frames. +- `base_url` (str): API base URL (default `https://api.firecrawl.dev/v2`). +- `location` (str, optional): location to localise results, e.g. `Germany`. +- `time_range` (str, optional): `day`, `week`, `month`, or `year`. Only applies to web results. +- `max_content_chars` (int, optional): truncate each result's `content` to this many characters. No limit by default. +- `timeout` (float): HTTP timeout in seconds (default 75, since requests with `include_content` fetch each page). + +## Input & Output Structure + +### Input Schema +- `queries` (list[str], at least one): search queries to run. Duplicates are sent once. Operators such as `site:`, `"exact phrase"`, `-term`, and `filetype:pdf` are supported. +- `search_type` (str): `web` or `news`. Default `web`. +- `max_results_per_query` (int): 1-100 (default 5, kept low because each result can carry a full page with `include_content`). +- `include_content` (bool): also return each result page as Markdown (default `False`). + +### Output Schema +- `results`: a list of `FirecrawlSearchResultItem` items. Each has `query`, `title`, `url`, and optional `description` (a query-relevant excerpt or snippet), `position`, `published` (news), and `content` (Markdown, when `include_content` is set). Results without a well-formed http(s) URL (a valid host name, IPv4 or IPv6 address, a valid port, no whitespace or control characters; Unicode host names are checked in their IDNA 2003 encoded form, as Python's `idna` codec produces it) are skipped, and badly typed optional fields are left empty. +- `failures`: queries that failed while others returned results, each with `query` and `error` (the HTTP status and the API's error message). Keys are redacted from both as described for `api_key`. They are also logged as warnings. + +## Usage +If no query returns results and at least one failed (for example a wrong API key, no credits, or a malformed response), `run` raises the first error instead of returning an empty list. Redirects are not followed; they are reported as errors. + +```python +from tool.firecrawl_search import FirecrawlSearchTool, FirecrawlSearchToolConfig, FirecrawlSearchToolInputSchema + +tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key="your-firecrawl-api-key")) + +output = tool.run(FirecrawlSearchToolInputSchema( + queries=["retrieval augmented generation"], + max_results_per_query=3, + include_content=True, +)) + +for item in output.results: + print(item.title, "-", item.url) + print(item.content[:500] if item.content else item.description) +``` + +The request and response formats are documented at [docs.firecrawl.dev/features/search](https://docs.firecrawl.dev/features/search). + +## Contributing +PRs welcome. See the main repo `CONTRIBUTING.md`. + +## License +Same as the main Atomic Agents project. diff --git a/atomic-forge/tools/firecrawl_search/pyproject.toml b/atomic-forge/tools/firecrawl_search/pyproject.toml new file mode 100644 index 00000000..efdc59ab --- /dev/null +++ b/atomic-forge/tools/firecrawl_search/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["tool"] + +[project] +name = "firecrawl-search" +version = "1.0.0" +description = "Web and news search for Atomic Agents via the Firecrawl Search API, with optional page content" +readme = "README.md" +authors = [ + { name = "Juampi (Firecrawl)", email = "juan@sideguide.dev" } +] +requires-python = ">=3.12" +dependencies = [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "aiohttp>=3.9.0,<4.0.0", +] + +[dependency-groups] +dev = [ + "coverage>=7.6.1,<8.0.0", + "pytest>=8.3.3,<9.0.0", + "pytest-asyncio>=0.23.5,<1.0.0", + "pytest-cov>=5.0.0,<6.0.0", + "python-dotenv>=1.0.0,<2.0.0", + "rich>=13.7.0,<14.0.0", +] + +[tool.uv.sources] +atomic-agents = { workspace = true } diff --git a/atomic-forge/tools/firecrawl_search/requirements.txt b/atomic-forge/tools/firecrawl_search/requirements.txt new file mode 100644 index 00000000..469c7e98 --- /dev/null +++ b/atomic-forge/tools/firecrawl_search/requirements.txt @@ -0,0 +1,3 @@ +atomic-agents>=2.0.0,<3.0.0 +pydantic>=2.10.3,<3.0.0 +aiohttp>=3.9.0,<4.0.0 diff --git a/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py new file mode 100644 index 00000000..e16000e0 --- /dev/null +++ b/atomic-forge/tools/firecrawl_search/tests/test_firecrawl_search.py @@ -0,0 +1,800 @@ +import asyncio +import copy +import errno +import logging +import os +import socket +import sys +import traceback +from contextlib import asynccontextmanager +from typing import ( + AsyncIterator, + Awaitable, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + Sequence, + Tuple, + TypedDict, + Union, +) +from unittest.mock import AsyncMock, patch + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestServer +from pydantic import SecretStr, ValidationError + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from tool.firecrawl_search import ( # noqa: E402 + FirecrawlSearchFailure, + FirecrawlSearchResultItem, + FirecrawlSearchTool, + FirecrawlSearchToolConfig, + FirecrawlSearchToolInputSchema, + FirecrawlSearchToolOutputSchema, +) + +Body = Dict[str, object] +Responder = Callable[[Body], Union[web.Response, Awaitable[web.Response]]] + + +Address = Union[Tuple[object, ...], str, bytes] +AddrInfo = Tuple[ + socket.AddressFamily, + socket.SocketKind, + int, + str, + Union[Tuple[str, int], Tuple[str, int, int, int], Tuple[int, bytes]], +] + + +class Seen(TypedDict): + """One request as the fake server received it.""" + + headers: Dict[str, str] + body: Body + + +DUMMY_KEY = "fc-dummy0secret0key0123456789" # a configured key in a custom format +CANONICAL_KEY = "fc-0123456789abcdef0123456789abcdef" # a key in Firecrawl's own format +KEYS = (DUMMY_KEY, DUMMY_KEY.upper(), CANONICAL_KEY, CANONICAL_KEY.upper()) +TOOL_FILE = os.path.join("tool", "firecrawl_search.py") +LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"} +# Inputs the caller passes in. They are in the tool's frames as given (as they are in the caller's own frames), +# so the traceback-locals checks for a key that came in through the query skip these names. +CALLER_INPUT_LOCALS = {"query", "queries", "params"} + +WEB_HIT = { + "url": "https://www.python.org/", + "title": "Welcome to Python.org", + "description": "Python is a programming language that lets you work quickly.", + "position": 1, +} + +NEWS_HIT = { + "url": "https://example.com/python-3-14", + "title": "Python 3.14 released", + "snippet": "The Python core team shipped 3.14 today.", + "date": "2 days ago", + "imageUrl": "https://example.com/thumb.png", + "position": 2, +} + +CONTENT_HIT = { + "url": "https://example.com/guide", + "title": "Agent guide", + "description": "A guide to agents.", + "position": 3, + "markdown": "# Agent guide\n\nBuild agents step by step.", + "metadata": {"title": "Agent guide | Example", "statusCode": 200}, +} + + +@pytest.fixture(autouse=True) +def offline(monkeypatch: pytest.MonkeyPatch) -> Iterator[List[str]]: + """Keep the suite offline. Resolving a host other than loopback raises gaierror, and connecting to a + non-loopback address raises OSError; both are recorded, and the test fails unless it expected them.""" + attempts: List[str] = [] + real_getaddrinfo = socket.getaddrinfo + real_connect = socket.socket.connect + real_connect_ex = socket.socket.connect_ex + + def resolve_loopback_only( + host: Union[bytes, str, None], port: Union[bytes, str, int, None], *args: int, **kwargs: int + ) -> Sequence[AddrInfo]: + if host not in LOOPBACK_HOSTS: + attempts.append(str(host)) + raise socket.gaierror("network access is disabled in these tests") + return real_getaddrinfo(host, port, *args, **kwargs) + + def blocked(address: Address) -> bool: + if isinstance(address, tuple) and address[0] not in LOOPBACK_HOSTS: + attempts.append(str(address[0])) + return True + return False + + def connect_loopback_only(sock: socket.socket, address: Address) -> None: + if blocked(address): + raise OSError("network access is disabled in these tests") + real_connect(sock, address) + + def connect_ex_loopback_only(sock: socket.socket, address: Address) -> int: + return errno.ECONNREFUSED if blocked(address) else real_connect_ex(sock, address) + + monkeypatch.setattr(socket, "getaddrinfo", resolve_loopback_only) + monkeypatch.setattr(socket.socket, "connect", connect_loopback_only) + monkeypatch.setattr(socket.socket, "connect_ex", connect_ex_loopback_only) + yield attempts + assert attempts == [] + + +##################### +# Response builders # +##################### +def respond_json(payload: object, status: int = 200) -> Responder: + def respond(body: Body) -> web.Response: + return web.json_response(payload, status=status) + + return respond + + +def respond_hits(source: str, hits: Sequence[object]) -> Responder: + return respond_json({"success": True, "data": {source: list(hits)}}) + + +def respond_raw( + status: int, text: Optional[str] = None, content_type: Optional[str] = None, location: Optional[str] = None +) -> Responder: + def respond(body: Body) -> web.Response: + headers = {"Location": location} if location else None + return web.Response(status=status, text=text, content_type=content_type, headers=headers) + + return respond + + +def respond_per_query(bad_query: str, bad: Responder, good: Responder) -> Responder: + def respond(body: Body) -> Union[web.Response, Awaitable[web.Response]]: + return bad(body) if body["query"] == bad_query else good(body) + + return respond + + +################ +# Test harness # +################ +@asynccontextmanager +async def fake_firecrawl(respond: Responder) -> AsyncIterator[Tuple[str, List[Seen]]]: + """Serve POST /v2/search on a local aiohttp server; `respond(body)` builds each response.""" + seen: List[Seen] = [] + + async def handler(request: web.Request) -> web.Response: + body = await request.json() + seen.append(Seen(headers=dict(request.headers), body=body)) + response = respond(body) + if isinstance(response, web.Response): + return response + return await response + + app = web.Application() + app.router.add_post("/v2/search", handler) + server = TestServer(app, host="127.0.0.1") + await server.start_server() + try: + yield str(server.make_url("/v2")), seen + finally: + await server.close() + + +def http_tool(base_url: str, api_key: str = DUMMY_KEY) -> FirecrawlSearchTool: + return FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr(api_key), base_url=base_url)) + + +def search_input( + queries: Sequence[str], + search_type: Literal["web", "news"] = "web", + max_results_per_query: int = 5, + include_content: bool = False, +) -> FirecrawlSearchToolInputSchema: + return FirecrawlSearchToolInputSchema( + queries=list(queries), + search_type=search_type, + max_results_per_query=max_results_per_query, + include_content=include_content, + ) + + +async def search( + respond: Responder, + queries: Sequence[str] = ("q",), + api_key: str = DUMMY_KEY, + search_type: Literal["web", "news"] = "web", + max_results_per_query: int = 5, + include_content: bool = False, +) -> Tuple[FirecrawlSearchToolOutputSchema, List[Seen]]: + """Run one search against a fake server and return (output, requests seen).""" + params = search_input(queries, search_type, max_results_per_query, include_content) + async with fake_firecrawl(respond) as (base_url, seen): + out = await http_tool(base_url, api_key).run_async(params) + return out, seen + + +async def search_error(respond: Responder, queries: Sequence[str] = ("q",), api_key: str = DUMMY_KEY) -> BaseException: + with pytest.raises(Exception) as raised: + await search(respond, queries, api_key) + return raised.value + + +def kept_item(hit: object, query: str = "q", max_content_chars: Optional[int] = None) -> FirecrawlSearchResultItem: + """_to_item for a hit the test expects to be kept.""" + result = FirecrawlSearchTool._to_item(hit, query, max_content_chars) + assert result is not None + return result + + +def without_names(names: Dict[str, str], skip_names: Sequence[str]) -> Dict[str, str]: + return {name: value for name, value in names.items() if name not in skip_names} + + +def frame_locals(error: BaseException, keep: Callable[[str], bool], skip_names: Sequence[str] = ()) -> List[str]: + """The captured locals of every frame in the error's traceback whose file passes `keep`.""" + stack = traceback.TracebackException.from_exception(error, capture_locals=True).stack + return [repr(without_names(frame.locals or {}, skip_names)) for frame in stack if keep(frame.filename)] + + +def is_tool_frame(filename: str) -> bool: + return filename.endswith(TOOL_FILE) + + +def is_outside_this_test_file(filename: str) -> bool: + """Library and tool frames; this file's own frames hold the test's inputs, which may contain the key.""" + return filename != __file__ + + +def error_surfaces(error: BaseException) -> str: + """Every text an error surfaces: str, repr, its chain, and the locals of every tool frame in its traceback.""" + tool_frames = frame_locals(error, is_tool_frame) + assert tool_frames, "expected the traceback to pass through the tool" + return "\n".join([str(error), repr(error), repr(error.__context__), repr(error.__cause__), *tool_frames]) + + +def assert_no_key(*texts: str) -> None: + for key in KEYS: + assert not any(key in text for text in texts), key + + +################## +# Result parsing # +################## +def test_to_item_web() -> None: + item = kept_item(WEB_HIT, "python") + assert item == FirecrawlSearchResultItem( + query="python", + title="Welcome to Python.org", + url="https://www.python.org/", + description="Python is a programming language that lets you work quickly.", + position=1, + ) + + +def test_to_item_news_uses_snippet_and_date() -> None: + item = kept_item(NEWS_HIT, "python") + assert (item.description, item.published, item.position) == ("The Python core team shipped 3.14 today.", "2 days ago", 2) + + +def test_to_item_content_and_truncation() -> None: + assert kept_item(CONTENT_HIT, "q").content == CONTENT_HIT["markdown"] + assert kept_item(CONTENT_HIT, "q", max_content_chars=7).content == "# Agent" + + +@pytest.mark.parametrize( + "field, value, attribute, expected", + [ + ("title", {"nested": True}, "title", "https://example.com/a"), + ("title", 5, "title", "https://example.com/a"), + ("description", ["a", "b"], "description", None), + ("position", "N/A", "position", None), + ("position", 1.5, "position", None), + ("position", True, "position", None), + ("position", -3, "position", None), + ("position", 0, "position", None), + ("date", 20250101, "published", None), + ("markdown", 42, "content", None), + ("markdown", {}, "content", None), + ], +) +def test_to_item_drops_badly_typed_fields(field: str, value: object, attribute: str, expected: object) -> None: + """One badly typed field must not discard the result or raise a raw ValidationError.""" + kept = kept_item({"url": "https://example.com/a", field: value}) + assert getattr(kept, attribute) == expected + + +@pytest.mark.parametrize( + "url", + [ + "HTTPS://Example.com/a", + "http://example.com", + "https://example.com:8443/a?b=c#d", + "http://[::1]:8080/x", + "https://user@example.com/", + "http://192.0.2.1:8080/x", + "https://müller.de/straße", + "https://xn--mller-kva.de/", + "https://example.com/%C3%BC%20x.pdf", + "https://example.com/a\\b", + "https://example.com./", + "https://my_host.example.com/", + f"https://{'a' * 63}.com/", + f"https://{'a' * 63}.{'b' * 63}.{'c' * 63}.{'d' * 61}/", + "https://1.2.3.4/", + "https://stra\u00dfe.de/", + "https://\u05e9\u05dc\u05d5\u05dd.co.il/", + "https://\u4f8b\u5b50.\u6d4b\u8bd5/", + f"https://{'\u00e9' * 20}.com/", + "https://example\u3002com/", + "https://example.com\u3002/", + "https://1\uff0e2\uff0e3\uff0e4/", + "https://3com.example.123abc/", + ], +) +def test_to_item_keeps_valid_http_urls(url: str) -> None: + assert kept_item({"url": url}).url == url + + +@pytest.mark.parametrize( + "hit", + [ + 1, + "s", + None, + [], + {"title": "no url"}, + {"url": ""}, + {"url": 123}, + {"url": ["https://example.com"]}, + {"url": "javascript:alert(1)"}, + {"url": "ftp://example.com/file"}, + {"url": "httpx://example.com"}, + {"url": "https://"}, + {"url": "http:///path-without-host"}, + {"url": "https://@/a"}, + {"url": "https://:443/a"}, + {"url": "https://user@:443/a"}, + {"url": " https://example.com"}, + {"url": "https://example.com/a b"}, + {"url": "https://exa mple.com/a"}, + {"url": "https://example.com/a\n"}, + {"url": "https://example.com\t/a"}, + {"url": "https://./"}, + {"url": "https://-/"}, + {"url": "https://%/"}, + {"url": "https://_x.com/"}, + {"url": "https://[:]/"}, + {"url": "https://[zz]/"}, + {"url": "https://a\x00b.com/"}, + {"url": "https://example.com/a\x7f"}, + {"url": "https://exa\u200bmple.com/"}, + {"url": "https://example.com/\xa0"}, + {"url": "https://example.com:abc/"}, + {"url": "https://example.com:99999/"}, + {"url": "https://example.com:65536/"}, + {"url": "https://example.com:0/"}, + {"url": "https://example.com:\u0664\u0664\u0663/"}, + {"url": "https://a b@example.com/"}, + {"url": "https://[v1.x]/"}, + {"url": "https://[::1%25eth0]/"}, + {"url": "https://a.com\\@evil.com/"}, + {"url": "https://evil.com\\.example.com/"}, + {"url": "https://ev\\il.com/"}, + {"url": "https://example..com/a"}, + {"url": "https://a%.com/a"}, + {"url": "https://a%41b.com/a"}, + {"url": "https://.example.com/"}, + {"url": "https://-a.com/"}, + {"url": "https://a-.com/"}, + {"url": f"https://{'a' * 64}.com/"}, + {"url": "https://example.com../"}, + {"url": f"https://{('a' * 63 + '.') * 4}com/"}, + {"url": f"https://{'a' * 63}.{'b' * 63}.{'c' * 63}.{'d' * 62}/"}, # 254 characters + {"url": "https://a.com:1\\@b.com/"}, + {"url": "https://a.com:x@/"}, + {"url": "https://256.0.0.1/"}, + {"url": "https://1.2.3/"}, + {"url": "https://999999/"}, + {"url": "https://example.\u0661\u0662\u0663/"}, + {"url": "https://a\u05d0.com/a"}, # mixes left-to-right and right-to-left in one label + {"url": f"https://{'\u00e9' * 63}.com/"}, # 63 characters, but over 63 bytes once IDNA-encoded + {"url": f"https://{('\u00e9' * 20 + '.') * 10}com/"}, # under 253 characters, over 253 once encoded + {"url": "https://256.0.0\uff0e1/"}, # fullwidth dot: encodes to 256.0.0.1 + {"url": "https://1.2\u30023/"}, # ideographic dot: encodes to 1.2.3 + {"url": "https://a\uff61999999/"}, # halfwidth ideographic dot: numeric last label + {"url": "https://a\u2488./"}, # "\u2488" encodes to "1.", so this ends in an empty label + ], +) +def test_to_item_without_usable_url_is_none(hit: object) -> None: + assert FirecrawlSearchTool._to_item(hit, "q") is None + + +################ +# Request body # +################ +def test_build_body_defaults() -> None: + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr("k"))) + assert tool._build_body("python", FirecrawlSearchToolInputSchema(queries=["python"])) == { + "query": "python", + "limit": 5, + "sources": ["web"], + } + + +def test_build_body_with_options() -> None: + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr("k"), location="Germany", time_range="week")) + params = FirecrawlSearchToolInputSchema(queries=["python"], max_results_per_query=3, include_content=True) + assert tool._build_body("python", params) == { + "query": "python", + "limit": 3, + "sources": ["web"], + "location": "Germany", + "tbs": "qdr:w", + "scrapeOptions": {"formats": ["markdown"], "onlyMainContent": True}, + } + + +def test_build_body_skips_time_range_for_news() -> None: + """The API's tbs filter only applies to web results.""" + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr("k"), time_range="day")) + body = tool._build_body("python", FirecrawlSearchToolInputSchema(queries=["python"], search_type="news")) + assert body == {"query": "python", "limit": 5, "sources": ["news"]} + + +def test_empty_queries_rejected() -> None: + with pytest.raises(ValidationError): + FirecrawlSearchToolInputSchema(queries=[]) + + +################################### +# Round trips over real transport # +################################### +@pytest.mark.asyncio +async def test_web_round_trip() -> None: + out, seen = await search(respond_hits("web", [CONTENT_HIT]), queries=["agents"], include_content=True) + + assert out == FirecrawlSearchToolOutputSchema(results=[kept_item(CONTENT_HIT, "agents")]) + assert seen[0]["headers"]["Authorization"] == f"Bearer {DUMMY_KEY}" + assert seen[0]["body"] == { + "query": "agents", + "limit": 5, + "sources": ["web"], + "scrapeOptions": {"formats": ["markdown"], "onlyMainContent": True}, + } + + +@pytest.mark.asyncio +async def test_news_round_trip_reads_news_key_only() -> None: + out, _ = await search(respond_json({"success": True, "data": {"web": [WEB_HIT], "news": [NEWS_HIT]}}), search_type="news") + assert [r.url for r in out.results] == [NEWS_HIT["url"]] + + +@pytest.mark.asyncio +async def test_missing_source_key_is_an_empty_search() -> None: + """The API omits the source key when there are no results; that is a real empty result.""" + out, _ = await search(respond_json({"success": True, "data": {}})) + assert out == FirecrawlSearchToolOutputSchema(results=[], failures=[]) + + +@pytest.mark.asyncio +async def test_results_trimmed_to_max_results() -> None: + hits = [{"url": f"https://example.com/{i}"} for i in range(5)] + out, _ = await search(respond_hits("web", hits), max_results_per_query=2) + assert [r.url for r in out.results] == ["https://example.com/0", "https://example.com/1"] + + +@pytest.mark.asyncio +async def test_invalid_hits_skipped_valid_ones_kept() -> None: + hits = [1, "s", None, {"title": "x"}, {"url": "https://@/a"}, WEB_HIT, {"url": "https://example.com/b", "position": "N/A"}] + out, _ = await search(respond_hits("web", hits)) + assert [(r.url, r.position) for r in out.results] == [("https://www.python.org/", 1), ("https://example.com/b", None)] + + +@pytest.mark.asyncio +async def test_all_hits_invalid_raises() -> None: + error = await search_error(respond_hits("web", [1, None, {"title": "x"}, {"url": "https://:443/a"}])) + assert str(error) == "Firecrawl search failed for 'q': 200 none of the 4 'web' results had a usable url" + + +@pytest.mark.asyncio +async def test_duplicate_queries_sent_once() -> None: + out, seen = await search(respond_hits("web", [WEB_HIT]), queries=["q", "q"]) + assert len(seen) == 1 + assert [r.query for r in out.results] == ["q"] + + +####################### +# Response validation # +####################### +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload", + [ + {"success": True, "data": None}, + {"success": True}, + {"success": True, "data": {"web": {"url": "https://example.com/not-a-list"}}}, + {"success": True, "data": {"web": None}}, + {"success": True, "data": [WEB_HIT]}, + ], +) +async def test_malformed_success_response_raises(payload: Body) -> None: + error = await search_error(respond_json(payload)) + assert str(error) == "Firecrawl search failed for 'q': 200 unexpected response shape for 'web' results" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("payload", [{"success": "false"}, {"success": "true"}, {"success": 1}, {}]) +async def test_success_must_be_literally_true(payload: Body) -> None: + error = await search_error(respond_json({**payload, "data": {"web": [WEB_HIT]}})) + assert str(error) == "Firecrawl search failed for 'q': 200 response did not report success" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status, reason", [(201, "Created"), (500, "Internal Server Error")]) +async def test_non_200_with_success_true_raises(status: int, reason: str) -> None: + error = await search_error(respond_json({"success": True, "data": {"web": [WEB_HIT]}}, status=status)) + assert str(error) == f"Firecrawl search failed for 'q': {status} {reason}" + + +@pytest.mark.asyncio +async def test_provider_error_message_is_reported() -> None: + error = await search_error(respond_json({"success": False, "error": "Payment required"}, status=402)) + assert str(error) == "Firecrawl search failed for 'q': 402 Payment required" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status, text, content_type, expected", + [ + (502, "Bad Gateway", "text/html", "502 Bad Gateway"), + (200, "", None, "200 response did not report success"), + (204, None, None, "204 No Content"), + ], +) +async def test_non_json_bodies_raise_with_status( + status: int, text: Optional[str], content_type: Optional[str], expected: str +) -> None: + error = await search_error(respond_raw(status, text, content_type)) + assert str(error) == f"Firecrawl search failed for 'q': {expected}" + + +@pytest.mark.asyncio +async def test_deeply_nested_json_is_a_wrapped_failure() -> None: + """JSON too deep for the parser must fail like any other bad body, not escape with the response in scope.""" + error = await search_error(respond_raw(200, "[" * 200_000, "application/json")) + assert str(error) == "Firecrawl search failed for 'q': 200 response did not report success" + assert_no_key(error_surfaces(error)) + + +@pytest.mark.asyncio +async def test_redirect_is_not_followed() -> None: + """A redirect, even to another server, is reported as an error; the query is never re-sent elsewhere.""" + async with fake_firecrawl(respond_hits("web", [WEB_HIT])) as (other_url, other_seen): + error = await search_error(respond_raw(307, location=f"{other_url}/search")) + + assert str(error) == "Firecrawl search failed for 'q': 307 Temporary Redirect" + assert other_seen == [] + + +################### +# Batch semantics # +################### +@pytest.mark.asyncio +async def test_partial_failure_reports_failures(caplog: pytest.LogCaptureFixture) -> None: + rate_limited = respond_json({"success": False, "error": "Rate limit exceeded"}, status=429) + with caplog.at_level(logging.WARNING, logger="tool.firecrawl_search"): + out, _ = await search(respond_per_query("bad", rate_limited, respond_hits("web", [WEB_HIT])), queries=["bad", "good"]) + + reason = "Firecrawl search failed for 'bad': 429 Rate limit exceeded" + assert [r.query for r in out.results] == ["good"] + assert out.failures == [FirecrawlSearchFailure(query="bad", error=reason)] + assert caplog.messages == [reason] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad, expected", + [ + ({"success": True, "data": None}, "200 unexpected response shape for 'web' results"), + ({"success": False, "error": "Rate limit exceeded"}, "200 Rate limit exceeded"), + ], +) +async def test_empty_plus_failed_query_raises(bad: Body, expected: str) -> None: + """No results anywhere plus a failure must not look like an empty search.""" + respond = respond_per_query("bad", respond_json(bad), respond_json({"success": True, "data": {}})) + error = await search_error(respond, queries=["empty", "bad"]) + assert str(error) == f"Firecrawl search failed for 'bad': {expected}" + + +@pytest.mark.asyncio +async def test_every_query_failing_raises_first_error() -> None: + error = await search_error(respond_json({"success": False, "error": "Unauthorized"}, status=401), queries=["a", "b"]) + assert str(error) == "Firecrawl search failed for 'a': 401 Unauthorized" + + +#################### +# Transport errors # +#################### +@pytest.mark.asyncio +async def test_connection_error_is_wrapped_without_context() -> None: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + + with pytest.raises(Exception) as raised: + await http_tool(f"http://127.0.0.1:{port}/v2").run_async(FirecrawlSearchToolInputSchema(queries=["q"])) + + assert str(raised.value).startswith("Firecrawl search failed for 'q': request failed: ClientConnectorError") + assert raised.value.__context__ is None and raised.value.__cause__ is None + assert_no_key(error_surfaces(raised.value)) + + +@pytest.mark.asyncio +async def test_timeout_is_wrapped_with_query() -> None: + async def slow(body: Body) -> web.Response: + await asyncio.sleep(1) + return web.json_response({"success": True, "data": {"web": [WEB_HIT]}}) + + async with fake_firecrawl(slow) as (base_url, _): + tool = http_tool(base_url) + tool.timeout = 0.05 + with pytest.raises(Exception) as raised: + await tool.run_async(FirecrawlSearchToolInputSchema(queries=["q"])) + + assert str(raised.value) == "Firecrawl search failed for 'q': request failed: TimeoutError after 0.05s" + assert raised.value.__context__ is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "base_url, error_type", + [ + (f"{DUMMY_KEY}/v2", "InvalidUrlClientError"), # the error echoes the URL, so its text must be redacted + ("http://[::1/v2", "InvalidUrlClientError"), + ("http://127.0.0.1:99999/v2", "InvalidUrlClientError"), + ("http://user:pw@127.0.0.1:1/v2", "ValueError"), + ], +) +async def test_invalid_base_url_is_wrapped_and_no_frame_holds_the_key(base_url: str, error_type: str) -> None: + """aiohttp raises for such URLs from frames whose locals hold the request headers; none of that may surface.""" + with pytest.raises(Exception) as raised: + await http_tool(base_url).run_async(FirecrawlSearchToolInputSchema(queries=["q"])) + + assert str(raised.value).startswith(f"Firecrawl search failed for 'q': request failed: {error_type} ") + assert raised.value.__context__ is None + assert_no_key(str(raised.value), repr(raised.value), *frame_locals(raised.value, is_outside_this_test_file)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("base_url", ["http://localhost:9/v2", "http://local\nhost:9/v2"]) # yarl drops the newline +async def test_unresolvable_base_url_is_wrapped(monkeypatch: pytest.MonkeyPatch, base_url: str) -> None: + """A DNS failure is reported as a request failure. The lookup is for localhost and is answered by a stub that + fails it, so no external host name is involved.""" + lookups: List[str] = [] + + def unresolvable(host: Union[bytes, str, None], *args: object, **kwargs: object) -> Sequence[AddrInfo]: + lookups.append(str(host)) + raise socket.gaierror("name resolution failed") + + monkeypatch.setattr(socket, "getaddrinfo", unresolvable) + with pytest.raises(Exception) as raised: + await http_tool(base_url).run_async(search_input(["q"])) + + assert str(raised.value).startswith("Firecrawl search failed for 'q': request failed: ClientConnectorDNSError ") + assert_no_key(str(raised.value), repr(raised.value), *frame_locals(raised.value, is_outside_this_test_file)) + assert lookups == ["localhost"] + + +################ +# Key handling # +################ +@pytest.mark.asyncio +@pytest.mark.parametrize( + "echo", [DUMMY_KEY, f"Bearer {DUMMY_KEY}", DUMMY_KEY.upper(), {"token": DUMMY_KEY}, CANONICAL_KEY, CANONICAL_KEY.upper()] +) +async def test_echoed_key_kept_out_of_errors_and_logs(caplog: pytest.LogCaptureFixture, echo: object) -> None: + """Even when the provider echoes a key back, it never reaches raised errors, tracebacks, logs or failures.""" + respond = respond_per_query( + "bad", respond_json({"success": False, "error": echo}, status=401), respond_hits("web", [WEB_HIT]) + ) + with caplog.at_level(logging.DEBUG): + out, _ = await search(respond, queries=["bad", "good"]) + error = await search_error(respond, queries=["bad"]) + + assert out.failures == [FirecrawlSearchFailure(query="bad", error=str(error))] + assert "[redacted]" in str(error) + assert_no_key(caplog.text, error_surfaces(error), repr(out)) + + +def malformed_body_echoing(key: str) -> Body: + return {"success": True, "data": None, "error": {"token": key}} + + +def invalid_hits_echoing(key: str) -> Body: + return {"success": True, "data": {"web": [{"url": "https://@/a", "title": key}, {"description": key}]}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_key", [DUMMY_KEY, CANONICAL_KEY]) +@pytest.mark.parametrize("build_body", [malformed_body_echoing, invalid_hits_echoing]) +async def test_rejected_success_body_not_kept_in_traceback_locals(api_key: str, build_body: Callable[[str], Body]) -> None: + """A malformed or all-invalid success body is dropped before the raise, so captured locals cannot show a key.""" + error = await search_error(respond_json(build_body(api_key)), api_key=api_key) + assert str(error).startswith("Firecrawl search failed for 'q': 200 ") + assert_no_key(error_surfaces(error)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_key", [DUMMY_KEY, CANONICAL_KEY]) +async def test_key_in_query_text_is_redacted_in_diagnostics(caplog: pytest.LogCaptureFixture, api_key: str) -> None: + """A query that contains the key (for example pasted by mistake) must not leak it through errors, logs, + failures, or any tool-frame local other than the caller's own inputs.""" + query = f"why does {api_key} fail" + respond = respond_per_query( + query, respond_json({"success": False, "error": "Unauthorized"}, status=401), respond_hits("web", [WEB_HIT]) + ) + with caplog.at_level(logging.DEBUG): + out, _ = await search(respond, queries=[query, "good"], api_key=api_key) + error = await search_error(respond, queries=[query], api_key=api_key) + + assert out.failures == [FirecrawlSearchFailure(query="why does [redacted] fail", error=str(error))] + assert str(error) == "Firecrawl search failed for 'why does [redacted] fail': 401 Unauthorized" + assert_no_key(caplog.text, repr(out.failures), str(error), repr(error), repr(error.__context__)) + # Traceback locals: the caller's own inputs appear as given; nothing the tool derives from them may hold the key. + assert_no_key(*frame_locals(error, is_tool_frame, skip_names=sorted(CALLER_INPUT_LOCALS))) + + +def test_key_kept_out_of_tool_and_config_state() -> None: + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr(DUMMY_KEY))) + assert_no_key( + repr(tool), + repr(tool.config), + str(tool.config), + tool.config.model_dump_json(), + repr(vars(tool)), + repr(vars(copy.copy(tool))), + ) + assert tool._api_key.get_secret_value() == DUMMY_KEY + + +def test_redaction_does_not_mangle_ordinary_text() -> None: + """A one-letter key is not redacted (it would hit every 'a'), and fc- words that are not keys are left alone.""" + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr("a"))) + text = "Firecrawl search failed for 'fc-barcelona-2024' and 'fc-0123abcd'" + assert tool._redact(text) == text + + +@pytest.mark.asyncio +async def test_missing_key_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False) + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig()) + + with pytest.raises(ValueError, match="FIRECRAWL_API_KEY"): + await tool.run_async(FirecrawlSearchToolInputSchema(queries=["q"])) + + +def test_api_key_falls_back_to_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FIRECRAWL_API_KEY", "env-key") + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig()) + assert tool._api_key.get_secret_value() == "env-key" + + +def test_run_invokes_run_async() -> None: + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr("k"))) + sentinel = FirecrawlSearchToolOutputSchema(results=[]) + with patch.object(FirecrawlSearchTool, "run_async", AsyncMock(return_value=sentinel)): + assert tool.run(FirecrawlSearchToolInputSchema(queries=["x"])) is sentinel + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py new file mode 100644 index 00000000..f811b50f --- /dev/null +++ b/atomic-forge/tools/firecrawl_search/tool/firecrawl_search.py @@ -0,0 +1,395 @@ +import asyncio +import ipaddress +import logging +import os +import re +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, List, Literal, Optional, Tuple, Union +from urllib.parse import urlsplit + +import aiohttp +from pydantic import Field, SecretStr, field_validator + +from atomic_agents import BaseIOSchema, BaseTool, BaseToolConfig + + +logger = logging.getLogger(__name__) + + +################ +# INPUT SCHEMA # +################ +class FirecrawlSearchToolInputSchema(BaseIOSchema): + """ + Search the web or recent news through the Firecrawl Search API. + Use this to find current information on a topic. Set include_content to true to also get + each result page as Markdown in the same call. Requires a Firecrawl API key. + """ + + queries: List[str] = Field( + ..., + min_length=1, + description='Search queries to run. Supports operators such as site:, "exact phrase", -term, and filetype:pdf.', + ) + search_type: Literal["web", "news"] = Field( + default="web", + description="Which results to return. 'web' = web pages; 'news' = news articles.", + ) + max_results_per_query: int = Field(default=5, ge=1, le=100, description="Maximum results per query.") + include_content: bool = Field( + default=False, + description="Also return the main content of each result page as Markdown. Slower than a plain search.", + ) + + +#################### +# OUTPUT SCHEMA(S) # +#################### +class FirecrawlSearchResultItem(BaseIOSchema): + """A single Firecrawl search result (web page or news article).""" + + query: str = Field(..., description="The query that produced this result.") + title: str = Field(..., description="Title of the result.") + url: str = Field(..., description="URL of the result.") + description: Optional[str] = Field(default=None, description="Query-relevant excerpt or snippet for the result.") + position: Optional[int] = Field(default=None, description="Rank of the result.") + published: Optional[str] = Field(default=None, description="Publication date string (news only).") + content: Optional[str] = Field(default=None, description="Page content as Markdown (only when include_content is set).") + + @field_validator("position", mode="before") + @classmethod + def _positive_rank_or_none(cls, value: object) -> Optional[int]: + # Ranks come from the provider; anything other than a positive int (bool, float, "N/A") is dropped. + return value if type(value) is int and value > 0 else None + + +class FirecrawlSearchFailure(BaseIOSchema): + """A query that failed while other queries returned results.""" + + query: str = Field(..., description="The query that failed.") + error: str = Field(..., description="Why it failed, e.g. an HTTP status and the API's error message.") + + +class FirecrawlSearchToolOutputSchema(BaseIOSchema): + """Output of the Firecrawl search tool.""" + + results: List[FirecrawlSearchResultItem] = Field(..., description="Matching results across all queries.") + failures: List[FirecrawlSearchFailure] = Field( + default_factory=list, description="Queries that failed while others returned results, with the reason." + ) + + +################# +# CONFIGURATION # +################# +class FirecrawlSearchToolConfig(BaseToolConfig): + """Configuration for the FirecrawlSearchTool.""" + + api_key: SecretStr = Field( + default=SecretStr(""), + description="Firecrawl API key. Falls back to the FIRECRAWL_API_KEY environment variable. Hidden in repr.", + ) + base_url: str = Field(default="https://api.firecrawl.dev/v2", description="Firecrawl API base URL.") + location: Optional[str] = Field( + default=None, description="Location to localise results, e.g. 'Germany' or 'San Francisco,California,United States'." + ) + time_range: Optional[Literal["day", "week", "month", "year"]] = Field( + default=None, description="Only return web results from this time window. Does not apply to news." + ) + max_content_chars: Optional[int] = Field( + default=None, ge=1, description="Truncate each result's content to this many characters. No limit when unset." + ) + timeout: float = Field(default=75.0, ge=1.0, le=120.0, description="HTTP request timeout in seconds.") + + +##################### +# MAIN TOOL & LOGIC # +##################### +class FirecrawlSearchTool(BaseTool[FirecrawlSearchToolInputSchema, FirecrawlSearchToolOutputSchema]): + """Tool for searching the web and news via the Firecrawl Search API.""" + + TIME_RANGES: Dict[str, str] = {"day": "qdr:d", "week": "qdr:w", "month": "qdr:m", "year": "qdr:y"} + # Anything in the Firecrawl key format is redacted from error and log text, plus the configured key itself + # when it is long enough that replacing it cannot mangle ordinary words. + KEY_PATTERN = re.compile(r"\bfc-[0-9a-f]{32}\b", re.IGNORECASE) + MIN_REDACTED_KEY_LENGTH = 8 + # Shape of a usable result URL: http(s), optional userinfo, a host or IPv6 literal, an optional port, and no + # whitespace anywhere. No backslash or extra "@" before the path, since parsers disagree on what host + # "a.com\@b.com" or "a.com:1\@b.com" means. urlsplit then validates the port and the IPv6 literal, and + # _valid_host the host name. + RESULT_URL_PATTERN = re.compile( + r"https?://(?:[^\s/?#@\\]*@)?(?:[^\s/?#@:\[\]]+|\[[0-9a-f:.]+\])(?::[^\s/?#@\\]*)?(?:[/?#]\S*)?", re.IGNORECASE + ) + + # One label of a host name in its ASCII (IDNA) form: letters, digits, hyphens and underscores, starting and ending + # with a letter or digit. The IDNA codec enforces the 63-byte label limit. No DNS lookup happens; this only + # rejects host names that cannot be valid, such as "example..com", "a%.com" or "aא.com". + HOST_LABEL = r"[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?" + HOST_NAME_PATTERN = re.compile(rf"(?:{HOST_LABEL}\.)*{HOST_LABEL}", re.IGNORECASE) + MAX_HOST_NAME_LENGTH = 253 + + def __init__(self, config: FirecrawlSearchToolConfig = FirecrawlSearchToolConfig()): + super().__init__(config) + self._api_key = SecretStr(config.api_key.get_secret_value() or os.getenv("FIRECRAWL_API_KEY", "")) + self.base_url = config.base_url.rstrip("/") + self.location = config.location + self.time_range = config.time_range + self.max_content_chars = config.max_content_chars + self.timeout = config.timeout + + @staticmethod + def _first_text(*values: object) -> Optional[str]: + """Return the first non-empty string; result fields are provider data and may have any type.""" + return next((value for value in values if isinstance(value, str) and value), None) + + @classmethod + def _valid_host(cls, host: Optional[str]) -> bool: + """True for an IPv6 literal (urlsplit has already validated it), an IPv4 address, or a host name whose ASCII + (IDNA) form is at most 253 characters of labels that start and end with a letter or digit, with one optional + trailing dot. Unicode names are checked after encoding, which is what DNS sees.""" + name = host or "" + if ":" in name: + return True + ascii_name = cls._idna(name) + if ascii_name is None: + return False + # A numeric last label is never a TLD, so only a real IPv4 address is usable (not 256.0.0.1 or 1.2.3). Both + # forms are checked: encoding turns dots such as "。" into ".", and leaves digits such as "١٢٣" non-ASCII. + if cls._numeric_last_label(name.removesuffix("."), ascii_name): + return cls._is_ipv4(ascii_name) + return cls.HOST_NAME_PATTERN.fullmatch(ascii_name) is not None + + @staticmethod + def _numeric_last_label(*names: str) -> bool: + return any(name.rsplit(".", 1)[-1].isdigit() for name in names) + + @classmethod + def _idna(cls, name: str) -> Optional[str]: + """The ASCII (IDNA 2003, as Python's codec implements it) form of a host name without its trailing dot, or + None when it is longer than 253 characters or cannot be encoded: the codec rejects labels that are empty or + over 63 bytes once encoded, and Unicode labels that break the IDNA rules, such as mixing left-to-right and + right-to-left scripts. Encoding also turns dots such as "。" into ".".""" + try: + ascii_name = name.encode("idna").decode("ascii").removesuffix(".") + except UnicodeError: + return None + return ascii_name if len(ascii_name) <= cls.MAX_HOST_NAME_LENGTH else None + + @staticmethod + def _is_ipv4(name: str) -> bool: + try: + ipaddress.IPv4Address(name) + except ValueError: + return False + return True + + @classmethod + def _usable_url(cls, url: str) -> bool: + """True for a printable http(s) URL with a valid host and an in-range, non-zero port.""" + if not (url.isprintable() and cls.RESULT_URL_PATTERN.fullmatch(url)): + return False + try: + # urlsplit raises ValueError for an out-of-range port or an invalid IPv6 literal. + parts = urlsplit(url) + return parts.port != 0 and cls._valid_host(parts.hostname) + except ValueError: + return False + + @classmethod + def _to_item(cls, hit: object, query: str, max_content_chars: Optional[int] = None) -> Optional[FirecrawlSearchResultItem]: + """Normalise one result, or return None when it has no usable http(s) URL.""" + fields: Dict[str, object] = hit if isinstance(hit, dict) else {} + url = str(fields.get("url")) + if not cls._usable_url(url): + return None + content = cls._first_text(fields.get("markdown")) + return FirecrawlSearchResultItem( + query=query, + title=cls._first_text(fields.get("title")) or url, + url=url, + description=cls._first_text(fields.get("description"), fields.get("snippet")), + position=FirecrawlSearchResultItem._positive_rank_or_none(fields.get("position")), + published=cls._first_text(fields.get("date")), + content=content[:max_content_chars] if content else None, + ) + + def _build_body(self, query: str, params: FirecrawlSearchToolInputSchema) -> Dict[str, object]: + body: Dict[str, object] = {"query": query, "limit": params.max_results_per_query, "sources": [params.search_type]} + if self.location: + body["location"] = self.location + if self.time_range and params.search_type == "web": + body["tbs"] = self.TIME_RANGES[self.time_range] + if params.include_content: + body["scrapeOptions"] = {"formats": ["markdown"], "onlyMainContent": True} + return body + + def _redact(self, text: str) -> str: + key = self._api_key.get_secret_value() + if len(key) >= self.MIN_REDACTED_KEY_LENGTH: + text = re.sub(re.escape(key), "[redacted]", text, flags=re.IGNORECASE) + return self.KEY_PATTERN.sub("[redacted]", text) + + def _error(self, query: str, detail: str) -> Exception: + # Redacted after the query is interpolated, since the query text can contain a key too. + return Exception(self._redact(f"Firecrawl search failed for '{query}': {detail}")) + + @staticmethod + async def _read_json(resp: aiohttp.ClientResponse) -> object: + try: + return await resp.json(content_type=None) + except (ValueError, RecursionError): + # Invalid JSON, or JSON nested too deeply for the parser. + return None + + def _check_response(self, status: int, reason: Optional[str], data: object) -> Union[Dict[str, object], str]: + """Return the body of a successful search (a 200 whose body says success is true), or why the response + is not one. Provider text is redacted here, where it enters the tool.""" + payload: Dict[str, object] = data if isinstance(data, dict) else {} + if status == 200 and payload.get("success") is True: + return payload + return self._redact(f"{status} {payload.get('error') or reason}") + + async def _exchange( + self, session: aiohttp.ClientSession, query: str, params: FirecrawlSearchToolInputSchema + ) -> Union[Dict[str, object], str]: + """Send one search request and return the body of a successful response, or why it failed. + + Transport errors, timeouts and invalid URLs are turned into a failure reason here instead of propagating, + because aiohttp's own frames and errors hold the request headers, Authorization included.""" + try: + async with session.post( + f"{self.base_url}/search", json=self._build_body(query, params), allow_redirects=False + ) as resp: + payload = await self._read_json(resp) + reason = resp.reason if resp.status != 200 else "response did not report success" + return self._check_response(resp.status, reason, payload) + except (aiohttp.ClientError, asyncio.TimeoutError, ValueError) as error: + return self._redact(f"request failed: {type(error).__name__} {str(error) or f'after {self.timeout:g}s'}") + + async def _post( + self, session: aiohttp.ClientSession, query: str, params: FirecrawlSearchToolInputSchema + ) -> Dict[str, object]: + """Return the body of a successful search response, or raise. + + The raise happens in this frame, which only ever holds the redacted failure reason: no response, no aiohttp + error and no failed body, so a traceback that captures locals cannot show a key echoed back. The caller's + own inputs (the query and params) are in these frames as given, as they are in the caller's frames.""" + outcome = await self._exchange(session, query, params) + if isinstance(outcome, str): + raise self._error(query, outcome) + return outcome + + @staticmethod + def _extract_hits(payload: Dict[str, object], search_type: str) -> Optional[List[object]]: + """Return the result list for the requested source, or None when the response has an unexpected shape. + The API leaves the source key out when there are no results.""" + groups = payload.get("data") + hits = groups.get(search_type, []) if isinstance(groups, dict) else None + return hits if isinstance(hits, list) else None + + def _parse( + self, payload: Dict[str, object], query: str, search_type: str + ) -> Tuple[List[FirecrawlSearchResultItem], Optional[str]]: + """Turn a successful response body into result items, or say why it cannot be used. It never raises, so + the rejected body is not left in the locals of a raising frame.""" + hits = self._extract_hits(payload, search_type) + if hits is None: + return [], f"200 unexpected response shape for '{search_type}' results" + items = list(filter(None, (self._to_item(hit, query, self.max_content_chars) for hit in hits))) + if hits and not items: + return [], f"200 none of the {len(hits)} '{search_type}' results had a usable url" + return items, None + + async def _fetch( + self, + session: aiohttp.ClientSession, + query: str, + params: FirecrawlSearchToolInputSchema, + ) -> List[FirecrawlSearchResultItem]: + items, problem = self._parse(await self._post(session, query, params), query, params.search_type) + if problem: + raise self._error(query, problem) + return items[: params.max_results_per_query] + + def _collect( + self, queries: List[str], grouped: List[Union[List[FirecrawlSearchResultItem], BaseException]] + ) -> FirecrawlSearchToolOutputSchema: + """Merge per-query results. Raise when no query produced results and at least one failed, so a bad key, + exhausted credits or a malformed response never looks like an empty search.""" + results: List[FirecrawlSearchResultItem] = [] + failures: List[FirecrawlSearchFailure] = [] + errors: List[BaseException] = [] + for query, group in zip(queries, grouped): + if isinstance(group, BaseException): + errors.append(group) + failures.append(FirecrawlSearchFailure(query=self._redact(query), error=self._redact(str(group)))) + logger.warning("%s", failures[-1].error) + continue + results.extend(group) + + if errors and not results: + raise errors[0] + return FirecrawlSearchToolOutputSchema(results=results, failures=failures) + + async def run_async(self, params: FirecrawlSearchToolInputSchema) -> FirecrawlSearchToolOutputSchema: + if not self._api_key.get_secret_value(): + raise ValueError( + "Firecrawl API key is missing. Set FirecrawlSearchToolConfig.api_key or the FIRECRAWL_API_KEY " + "environment variable." + ) + + # Duplicate queries would be billed twice and return the same results. + queries = list(dict.fromkeys(params.queries)) + # The Authorization header is built inline so no local variable holds the key. + async with aiohttp.ClientSession( + headers={"Authorization": f"Bearer {self._api_key.get_secret_value()}", "Content-Type": "application/json"}, + timeout=aiohttp.ClientTimeout(total=self.timeout), + ) as session: + grouped: List[Union[List[FirecrawlSearchResultItem], BaseException]] = await asyncio.gather( + *(self._fetch(session, q, params) for q in queries), return_exceptions=True + ) + + return self._collect(queries, grouped) + + def run(self, params: FirecrawlSearchToolInputSchema) -> FirecrawlSearchToolOutputSchema: + with ThreadPoolExecutor() as executor: + return executor.submit(asyncio.run, self.run_async(params)).result() + + +################# +# EXAMPLE USAGE # +################# +if __name__ == "__main__": # pragma: no cover + from dotenv import load_dotenv + from rich.console import Console + + load_dotenv() + console = Console() + tool = FirecrawlSearchTool(config=FirecrawlSearchToolConfig(api_key=SecretStr(os.getenv("FIRECRAWL_API_KEY", "")))) + + examples: List[Tuple[Literal["web", "news"], bool, str]] = [ + ("web", False, "atomic agents framework"), + ("news", False, "AI agents"), + ("web", True, "atomic agents framework"), + ] + for search_type, include_content, query in examples: + output = tool.run( + FirecrawlSearchToolInputSchema( + queries=[query], + search_type=search_type, + max_results_per_query=3, + include_content=include_content, + ) + ) + console.rule(f"[bold cyan]{search_type}{' + content' if include_content else ''}") + for item in output.results: + console.print(f"[bold]{item.title}[/bold]") + console.print(item.url) + if item.description: + console.print(item.description[:200]) + if item.published: + console.print(f"[bold]Published:[/bold] {item.published}") + if item.content: + console.print(f"[bold]Content:[/bold] {len(item.content)} chars") + console.print(item.content[:300]) + console.print() diff --git a/docs/guides/tools.md b/docs/guides/tools.md index 5d290363..7d9e2a23 100644 --- a/docs/guides/tools.md +++ b/docs/guides/tools.md @@ -150,6 +150,7 @@ The Atomic Forge ships with several pre-built tools: - **Calculator**: Perform mathematical calculations - **DateTime**: Timezone-aware now / parse / convert / shift / diff (no key required) - **Fía Signals**: Crypto market intelligence — market regime, trading signals, DeFi yields, gas prices, Solana trending tokens, and wallet risk scoring +- **Firecrawl Search**: Web and news search via the Firecrawl API, with optional page content as Markdown - **Hacker News Search**: Search HN stories, comments, Show HN, Ask HN, polls (free Algolia API) - **PDF Reader**: Extract text and metadata from local or remote PDFs, with page-range filtering - **SearXNG Search**: Search the web using SearXNG diff --git a/uv.lock b/uv.lock index 6c39d504..a357fa2f 100644 --- a/uv.lock +++ b/uv.lock @@ -21,6 +21,7 @@ members = [ "example-mcp-server", "fastapi-memory", "fia-signals", + "firecrawl-search", "hackernews-search", "hooks-example", "nested-multimodal", @@ -1329,6 +1330,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/91/7216b27286936c16f5b4d0c530087e4a54eead683e6b0b73dd0c64844af6/filelock-3.20.0-py3-none-any.whl", hash = "sha256:339b4732ffda5cd79b13f4e2711a31b0365ce445d95d243bb996273d072546a2", size = 16054, upload-time = "2025-10-08T18:03:48.35Z" }, ] +[[package]] +name = "firecrawl-search" +version = "1.0.0" +source = { editable = "atomic-forge/tools/firecrawl_search" } +dependencies = [ + { name = "aiohttp" }, + { name = "atomic-agents" }, + { name = "pydantic" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "python-dotenv" }, + { name = "rich" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", specifier = ">=3.9.0,<4.0.0" }, + { name = "atomic-agents", editable = "." }, + { name = "pydantic", specifier = ">=2.10.3,<3.0.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", specifier = ">=7.6.1,<8.0.0" }, + { name = "pytest", specifier = ">=8.3.3,<9.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.23.5,<1.0.0" }, + { name = "pytest-cov", specifier = ">=5.0.0,<6.0.0" }, + { name = "python-dotenv", specifier = ">=1.0.0,<2.0.0" }, + { name = "rich", specifier = ">=13.7.0,<14.0.0" }, +] + [[package]] name = "flake8" version = "7.3.0"