diff --git a/.env.example b/.env.example index f0ac87e..8f5375b 100644 --- a/.env.example +++ b/.env.example @@ -13,5 +13,9 @@ NUEXTRACT_MODEL=nuextract3 #NUEXTRACT_INSTRUCTIONS= #NUEXTRACT_DPI= +# URL Download Proxy (for SSRF mitigation) +# All URL downloads will be routed through this proxy. +#BIBRA_URL_PROXY=http://proxy.example.com:8080 + # System Settings (optional) # LOG_LEVEL=INFO diff --git a/bibra/api/v0/routes.py b/bibra/api/v0/routes.py index 864b5ef..e5618a9 100644 --- a/bibra/api/v0/routes.py +++ b/bibra/api/v0/routes.py @@ -5,10 +5,17 @@ import tempfile from typing import Annotated -from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile +import httpx +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile +from pydantic import HttpUrl from bibra import __version__ -from bibra.config import ConfigError, ProjectNotFoundError, ProjectRegistry +from bibra.config import ( + ConfigError, + ProjectNotFoundError, + ProjectRegistry, + get_url_proxy, +) from bibra.types import PublicationMetadata logger = logging.getLogger(__name__) @@ -94,3 +101,67 @@ async def extract( logger.debug( "Failed to remove temporary file: %s", tmp_path, exc_info=True ) + + +@router.post( + "/projects/{project_id}/extract-url", + responses={400: {"description": "Bad Request - malformed data"}}, +) +async def extract_url( + project_id: str, + registry: Annotated[ProjectRegistry, Depends(get_registry)], + url: HttpUrl = Form(...), # noqa: B008 +) -> PublicationMetadata: + """ + Extract publication metadata from a PDF or image file at a given URL for a + specific project. + + Args: + project_id: The ID of the project to extract metadata for + url: URL pointing to a file to process + + Returns: + PublicationMetadata: Extracted metadata as JSON + """ + try: + backend = registry.get_backend(project_id) + except ProjectNotFoundError as e: + raise HTTPException(status_code=404, detail=str(e)) + except ConfigError as e: + logger.exception("Configuration error") + raise HTTPException(status_code=500, detail=str(e)) + + url_str = str(url) + + try: + proxy = get_url_proxy() + async with httpx.AsyncClient(proxy=proxy) as client: + response = await client.get(url_str) + + content_type = response.headers.get("content-type", "") + if content_type != "application/pdf": + expected = "application/pdf" + detail = ( + f"'{url}' does not point to a PDF file. " + f"Expected '{expected}', got '{content_type}'." + ) + raise HTTPException(status_code=400, detail=detail) + + status_code = response.status_code + if status_code >= 400: + raise HTTPException( + status_code=status_code, + detail=str(response.reason_phrase), + ) + + with tempfile.NamedTemporaryFile(suffix=".pdf") as tmp: + async for chunk in response.aiter_bytes(chunk_size=1024 * 1024): + tmp.write(chunk) + tmp.flush() + return await backend.extract([tmp.name]) + except httpx.HTTPError as e: + logger.exception("HTTP Error downloading %s", url_str) + raise HTTPException(status_code=500, detail=str(e)) + except Exception as e: + logger.exception("Unexpected error during download from %s", url) + raise HTTPException(status_code=500, detail=str(e)) diff --git a/bibra/cli.py b/bibra/cli.py index 8faf46a..8e681da 100644 --- a/bibra/cli.py +++ b/bibra/cli.py @@ -1,14 +1,17 @@ """CLI interface for BIBRA.""" import asyncio +import tempfile import click +import httpx from dotenv import load_dotenv from bibra.config import ( ConfigError, ProjectNotFoundError, ProjectRegistry, + get_url_proxy, ) @@ -103,3 +106,53 @@ def extract(project_id: str, file_path: str, config: str | None, output: str | N click.echo(f"Output written to {output}") else: click.echo(json_output) + + +@cli.command("extract-url") +@click.argument("project_id") +@click.argument("url") +@click.option( + "--config", + "-c", + default=None, + help="Path to the project configuration file (overrides BIBRA_CONFIG).", +) +@click.option( + "--output", + "-o", + type=click.Path(dir_okay=False, writable=True, resolve_path=True), + default=None, + help="Write JSON output to file instead of stdout", +) +def extract_url(project_id: str, url: str, config: str | None, output: str | None): + """Extract publication metadata from a PDF or image file at a URL.""" + registry = ProjectRegistry(config) + + try: + backend = registry.get_backend(project_id) + except ProjectNotFoundError as e: + raise click.UsageError(str(e)) from None + except ConfigError as e: + raise click.ClickException(str(e)) from None + + try: + with tempfile.NamedTemporaryFile(suffix=".pdf") as tmp: + proxy = get_url_proxy() + with httpx.stream("GET", url, proxy=proxy) as response: + if response.status_code >= 400: + raise httpx.HTTPError(f"HTTP {response.status_code} for {url}") + for chunk in response.iter_bytes(chunk_size=1024 * 1024): + tmp.write(chunk) + tmp.flush() + result = asyncio.run(backend.extract([tmp.name])) + except Exception as e: + raise click.ClickException(f"Extraction failed: {e}") from e + + json_output = result.model_dump_json(indent=2) + + if output: + with open(output, "w", encoding="utf-8") as f: + f.write(json_output + "\n") + click.echo(f"Output written to {output}") + else: + click.echo(json_output) diff --git a/bibra/config.py b/bibra/config.py index 308080b..8417edb 100644 --- a/bibra/config.py +++ b/bibra/config.py @@ -277,3 +277,12 @@ def list_projects(self) -> list[dict[str, Any]]: } for project in self._projects.values() ] + + +def get_url_proxy() -> str | None: + """Return the BIBRA_URL_PROXY environment variable value. + + Returns: + The proxy URL string, or None if not set. + """ + return os.environ.get("BIBRA_URL_PROXY") diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 5c9fa0b..7ae85ed 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -1,13 +1,15 @@ """Tests for API routes.""" -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException, Request from fastapi.routing import APIRoute +from httpx import Headers from bibra.api.v0.routes import ( extract, + extract_url, get_registry, list_projects, router, @@ -80,6 +82,66 @@ async def test_extract_returns_example_metadata(self): assert result.p_isbn == [] assert result.e_issn is None + def test_extract_url_route_exists(self): + """The router should have a project-specific extract-url route.""" + routes = [str(r.path) for r in router.routes] + assert "/projects/{project_id}/extract-url" in routes + + def test_extract_url_route_is_post_method(self): + """The extract-url route should use POST method.""" + extract_url_routes = [ + r + for r in router.routes + if str(r.path) == "/projects/{project_id}/extract-url" + ] + assert len(extract_url_routes) >= 1 + # Check that the route uses POST method + route = extract_url_routes[0] + assert isinstance(route, APIRoute) + + async def test_extract_url_returns_example_metadata(self): + """The /projects/{project_id}/extract-url endpoint should return example + publication metadata.""" + from pydantic import HttpUrl + + registry = ProjectRegistry() + + async def mock_aiter_bytes(*args, **kwargs): + yield b"%PDF-1.4 mock content" + + mock_response = MagicMock() + mock_response.headers = Headers({"content-type": "application/pdf"}) + mock_response.status_code = 200 + mock_response.aiter_bytes.return_value = mock_aiter_bytes() + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_cls.return_value = mock_client + result = await extract_url( + project_id="dummy", + registry=registry, + url=HttpUrl("https://example.com/paper.pdf"), + ) + + assert isinstance(result, PublicationMetadata) + assert result.language == "en" + assert ( + result.title == "Machine Learning Approaches for Software Defect Prediction" + ) + assert result.creator == ["Smith, John", "Johnson, Emily"] + assert result.year == "2023" + assert result.publisher == ["Springer", "ACM"] + assert result.doi == "10.1234/example.doi.12345" + assert result.e_isbn == ["978-0-123456-78-9"] + assert result.type_coar == "article" + # Verify fields that don't have values are None or empty lists + assert result.alt_title is None + assert result.p_isbn == [] + assert result.e_issn is None + def test_get_registry_returns_existing_registry(self): """get_registry should return the registry already on app.state.""" mock_state = MagicMock() @@ -155,3 +217,65 @@ async def test_extract_handles_project_not_found_error(self): assert exc_info.value.status_code == 404 assert exc_info.value.detail == "Project 'unknown' not found" + + async def test_extract_url_passes_proxy_when_set(self, monkeypatch): + """Test that extract-url passes the proxy to httpx.AsyncClient when set.""" + from pydantic import HttpUrl + + monkeypatch.setenv("BIBRA_URL_PROXY", "http://proxy.example.com:8080") + + registry = ProjectRegistry() + + async def mock_aiter_bytes(*args, **kwargs): + yield b"%PDF-1.4 mock content" + + mock_response = MagicMock() + mock_response.headers = Headers({"content-type": "application/pdf"}) + mock_response.status_code = 200 + mock_response.aiter_bytes.return_value = mock_aiter_bytes() + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_cls.return_value = mock_client + result = await extract_url( + project_id="dummy", + registry=registry, + url=HttpUrl("https://example.com/paper.pdf"), + ) + + mock_client_cls.assert_called_once_with(proxy="http://proxy.example.com:8080") + assert isinstance(result, PublicationMetadata) + + async def test_extract_url_no_proxy_when_not_set(self, monkeypatch): + """Test that extract-url passes proxy=None when env var is not set.""" + from pydantic import HttpUrl + + monkeypatch.delenv("BIBRA_URL_PROXY", raising=False) + + registry = ProjectRegistry() + + async def mock_aiter_bytes(*args, **kwargs): + yield b"%PDF-1.4 mock content" + + mock_response = MagicMock() + mock_response.headers = Headers({"content-type": "application/pdf"}) + mock_response.status_code = 200 + mock_response.aiter_bytes.return_value = mock_aiter_bytes() + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_cls.return_value = mock_client + result = await extract_url( + project_id="dummy", + registry=registry, + url=HttpUrl("https://example.com/paper.pdf"), + ) + + mock_client_cls.assert_called_once_with(proxy=None) + assert isinstance(result, PublicationMetadata) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5575e60..fd2786b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,8 +7,8 @@ import pytest from click.testing import CliRunner -from bibra.cli import _make_list_template, cli, extract, list_projects -from bibra.config import ConfigError +from bibra.cli import _make_list_template, cli, extract, extract_url, list_projects +from bibra.config import ConfigError, ProjectNotFoundError class TestCli: @@ -247,6 +247,264 @@ def test_extract_config_error_converted_to_click_exception(self, tmp_path): assert "Invalid config syntax" in result.output +"""Tests for the extract-url command.""" + + +def _make_httpx_stream_mock(chunks=(b"%PDF-1.4 dummy content",)): + """Build a mock for httpx stream response that yields the given chunks.""" + mock_response = MagicMock() + mock_response.headers.get.return_value = "application/pdf" + mock_response.status_code = 200 + mock_response.iter_bytes.return_value = chunks + mock_response.__enter__.return_value = mock_response + mock_response.__exit__.return_value = False + return mock_response + + +def _make_backend(json_payload=None): + """Build a mock backend whose .extract() returns an object with a + model_dump_json method, matching what asyncio.run(backend.extract(...)) + is expected to produce.""" + if json_payload is None: + json_payload = {"title": "Some Paper", "authors": ["A. Author"]} + + mock_result = MagicMock() + mock_result.model_dump_json.return_value = json.dumps(json_payload, indent=2) + + mock_backend = MagicMock() + + async def _extract(*args, **kwargs): + return mock_result + + mock_backend.extract.side_effect = _extract + return mock_backend + + +class TestExtractUrl: + """Tests for the extract-url command.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + def test_extract_url_help(self): + """Test extract-url help output.""" + result = self.runner.invoke(extract_url, ["--help"]) + assert result.exit_code == 0 + assert not result.exception + assert "Extract publication metadata" in result.output + assert "PROJECT_ID" in result.output + assert "URL" in result.output + assert "--output" in result.output + assert "-o" in result.output + + def test_extract_url_missing_url(self): + """Test extract-url command with only a project id (missing URL).""" + result = self.runner.invoke(extract_url, ["test-project"]) + assert result.exit_code != 0 + assert result.exception + + def test_extract_url_with_valid_url(self): + """Test extract-url command with a valid URL and successful extraction.""" + with ( + patch("bibra.cli.ProjectRegistry") as mock_registry_cls, + patch("bibra.cli.httpx.stream") as mock_stream, + ): + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.return_value = _make_backend() + mock_stream.return_value = _make_httpx_stream_mock() + + result = self.runner.invoke( + extract_url, ["dummy", "https://example.com/paper.pdf"] + ) + + assert result.exit_code == 0 + assert not result.exception + + output = result.output.strip() + data = json.loads(output) + assert "title" in data or "authors" in data + + def test_extract_url_with_output_option(self, tmp_path): + """Test extract-url command with --output option to write JSON to file.""" + output_file = tmp_path / "output.json" + + with ( + patch("bibra.cli.ProjectRegistry") as mock_registry_cls, + patch("bibra.cli.httpx.stream") as mock_stream, + ): + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.return_value = _make_backend() + mock_stream.return_value = _make_httpx_stream_mock() + + result = self.runner.invoke( + extract_url, + [ + "dummy", + "https://example.com/paper.pdf", + "--output", + str(output_file), + ], + ) + + assert result.exit_code == 0 + assert not result.exception + assert "Output written to" in result.output + + assert output_file.exists() + content = output_file.read_text(encoding="utf-8") + data = json.loads(content.strip()) + assert data is not None + + def test_extract_url_with_short_output_option(self, tmp_path): + """Test extract-url command with -o short option.""" + output_file = tmp_path / "output.json" + + with ( + patch("bibra.cli.ProjectRegistry") as mock_registry_cls, + patch("bibra.cli.httpx.stream") as mock_stream, + ): + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.return_value = _make_backend() + mock_stream.return_value = _make_httpx_stream_mock() + + result = self.runner.invoke( + extract_url, + ["dummy", "https://example.com/paper.pdf", "-o", str(output_file)], + ) + + assert result.exit_code == 0 + assert not result.exception + assert "Output written to" in result.output + + def test_extract_url_with_nonexistent_project(self): + """Test extract-url command with a project that isn't found.""" + with patch("bibra.cli.ProjectRegistry") as mock_registry_cls: + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.side_effect = ProjectNotFoundError( + "Project 'nonexistent-project' not found" + ) + + result = self.runner.invoke( + extract_url, + ["nonexistent-project", "https://example.com/paper.pdf"], + ) + + assert result.exit_code != 0 + assert result.exception + assert "not found" in result.output + + def test_extract_url_config_error_converted_to_click_exception(self): + """Test ConfigError during backend resolution becomes ClickException.""" + with patch("bibra.cli.ProjectRegistry") as mock_registry_cls: + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.side_effect = ConfigError("Invalid config syntax") + + result = self.runner.invoke( + extract_url, ["dummy", "https://example.com/paper.pdf"] + ) + + assert result.exit_code != 0 + assert "Invalid config syntax" in result.output + + def test_extract_url_download_failure_converted_to_click_exception(self): + """Test download failure is wrapped as 'Extraction failed:'.""" + with ( + patch("bibra.cli.ProjectRegistry") as mock_registry_cls, + patch("bibra.cli.httpx.stream") as mock_stream, + ): + import httpx as _httpx + + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.return_value = _make_backend() + mock_stream.side_effect = _httpx.HTTPError("Name or service not known") + + result = self.runner.invoke( + extract_url, ["dummy", "https://bad.example.invalid/paper.pdf"] + ) + + assert result.exit_code != 0 + assert "Extraction failed:" in result.output + + def test_extract_url_generic_exception_converted_to_click_exception(self): + """Test that a generic Exception during extraction is wrapped in + ClickException with the 'Extraction failed:' prefix.""" + with ( + patch("bibra.cli.ProjectRegistry") as mock_registry_cls, + patch("bibra.cli.httpx.stream") as mock_stream, + ): + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_backend = MagicMock() + + async def _extract(*args, **kwargs): + raise RuntimeError("PDF corrupted") + + mock_backend.extract.side_effect = _extract + mock_registry.get_backend.return_value = mock_backend + mock_stream.return_value = _make_httpx_stream_mock() + + result = self.runner.invoke( + extract_url, ["dummy", "https://example.com/paper.pdf"] + ) + + assert result.exit_code != 0 + assert "Extraction failed: PDF corrupted" in result.output + + def test_extract_url_passes_proxy_when_set(self): + """Test that extract-url passes the proxy to httpx.stream when set.""" + with ( + patch("bibra.cli.ProjectRegistry") as mock_registry_cls, + patch("bibra.cli.httpx.stream") as mock_stream, + ): + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.return_value = _make_backend() + mock_stream.return_value = _make_httpx_stream_mock() + + result = self.runner.invoke( + extract_url, + ["dummy", "https://example.com/paper.pdf"], + env={"BIBRA_URL_PROXY": "http://proxy.example.com:8080"}, + ) + + assert result.exit_code == 0 + mock_stream.assert_called_once_with( + "GET", + "https://example.com/paper.pdf", + proxy="http://proxy.example.com:8080", + ) + + def test_extract_url_no_proxy_when_not_set(self): + """Test that extract-url passes proxy=None when env var is not set.""" + with ( + patch("bibra.cli.ProjectRegistry") as mock_registry_cls, + patch("bibra.cli.httpx.stream") as mock_stream, + ): + mock_registry = MagicMock() + mock_registry_cls.return_value = mock_registry + mock_registry.get_backend.return_value = _make_backend() + mock_stream.return_value = _make_httpx_stream_mock() + + result = self.runner.invoke( + extract_url, + ["dummy", "https://example.com/paper.pdf"], + ) + + assert result.exit_code == 0 + mock_stream.assert_called_once_with( + "GET", + "https://example.com/paper.pdf", + proxy=None, + ) + + class TestMakeListTemplate: """Tests for the _make_list_template helper function.""" diff --git a/tests/test_config.py b/tests/test_config.py index d241e28..7534730 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -607,3 +607,25 @@ def test_unclosed_placeholder_preserved(self): def test_unclosed_placeholder_with_trailing_text(self): """Test that an unclosed ${ followed by text is left as-is.""" assert _interpolate_env_vars("prefix${BAR suffix") == "prefix${BAR suffix" + + +class TestGetUrlProxy: + """Tests for get_url_proxy helper function.""" + + def test_returns_proxy_when_set(self, monkeypatch): + """Test that get_url_proxy returns the proxy URL when set.""" + monkeypatch.setenv("BIBRA_URL_PROXY", "http://proxy.example.com:8080") + from bibra.config import get_url_proxy + + assert get_url_proxy() == "http://proxy.example.com:8080" + + def test_returns_none_when_not_set(self, monkeypatch): + """Test that get_url_proxy returns None when the env var is not set.""" + monkeypatch.delenv("BIBRA_URL_PROXY", raising=False) + # Re-import to get fresh import + import importlib + + import bibra.config + + importlib.reload(bibra.config) + assert bibra.config.get_url_proxy() is None diff --git a/tests/test_schemathesis.py b/tests/test_schemathesis.py index 2083c35..ccb544f 100644 --- a/tests/test_schemathesis.py +++ b/tests/test_schemathesis.py @@ -1,3 +1,5 @@ +from unittest.mock import MagicMock, patch + import schemathesis from bibra.main import app @@ -6,6 +8,22 @@ schema = schemathesis.openapi.from_asgi("/openapi.json", app) +async def _mock_aiter_bytes(*args, **kwargs): + """Async generator that yields mock PDF content.""" + yield b"%PDF-1.4 mock content" + + +def _mock_httpx_response(*args, **kwargs): + """Build a mock httpx stream response for PDF content.""" + mock_response = MagicMock() + mock_response.headers.get.return_value = "application/pdf" + mock_response.status_code = 200 + mock_response.aiter_bytes.return_value = _mock_aiter_bytes() + mock_response.__aenter__.return_value = mock_response + mock_response.__aexit__.return_value = False + return mock_response + + @schema.parametrize() def test_api(case): # Skip extract cases missing the required `files` field. @@ -23,4 +41,38 @@ def test_api(case): if hasattr(case, "path") and case.path == "/v0/projects/{project_id}/extract": # Modify the path to use dummy project case.path = "/v0/projects/dummy/extract" + # Skip extract-url cases missing the required `url` field. + is_extract_url = ( + case.path == "/v0/projects/{project_id}/extract-url" + and case.method.upper() == "POST" + ) + if is_extract_url: + body = case.body + # Skip if `url` is absent or empty + has_url = isinstance(body, dict) and bool(body.get("url")) + if not has_url: + return + # Use dummy backend for testing to avoid real network downloads/API calls + if ( + hasattr(case, "path") + and case.path == "/v0/projects/{project_id}/extract-url" + ): + # Modify the path to use dummy project + case.path = "/v0/projects/dummy/extract-url" + + # Mock httpx.AsyncClient stream response + mock_response = _mock_httpx_response() + + async def async_get(*args, **kwargs): + return mock_response + + with patch("httpx.AsyncClient") as mock_client: + mock_client.__aenter__.return_value = mock_client + mock_client.__aexit__.return_value = False + mock_client.stream = MagicMock(return_value=mock_response) + mock_client.get = async_get + with patch("httpx.AsyncClient", return_value=mock_client): + case.call_and_validate() + return + case.call_and_validate() diff --git a/uv.lock b/uv.lock index 576d841..445ca84 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,10 @@ version = 1 revision = 3 requires-python = ">=3.11" +[options] +exclude-newer = "2026-08-06T09:44:44.7887759Z" +exclude-newer-span = "P1W" + [[package]] name = "annotated-doc" version = "0.0.4"