Loading PDF files from URL in REST API and command line

.env.example

@@ -13,5 +13,9 @@ NUEXTRACT_MODEL=nuextract3
 #NUEXTRACT_INSTRUCTIONS=
 #NUEXTRACT_DPI=
 
+# URL Download Proxy (for SSRF mitigation)
+# All URL downloads will be routed through this proxy.
+#BIBRA_URL_PROXY=http://proxy.example.com:8080
+
 # System Settings (optional)
 # LOG_LEVEL=INFO

bibra/api/v0/routes.py

@@ -5,10 +5,17 @@
 import tempfile
 from typing import Annotated
 
-from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
+import httpx
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
+from pydantic import HttpUrl
 
 from bibra import __version__
-from bibra.config import ConfigError, ProjectNotFoundError, ProjectRegistry
+from bibra.config import (
+    ConfigError,
+    ProjectNotFoundError,
+    ProjectRegistry,
+    get_url_proxy,
+)
 from bibra.types import PublicationMetadata
 
 logger = logging.getLogger(__name__)
@@ -94,3 +101,67 @@
                 logger.debug(
                     "Failed to remove temporary file: %s", tmp_path, exc_info=True
                 )
+
+
+@router.post(
+    "/projects/{project_id}/extract-url",
+    responses={400: {"description": "Bad Request - malformed data"}},
+)
+async def extract_url(
+    project_id: str,
+    registry: Annotated[ProjectRegistry, Depends(get_registry)],
+    url: HttpUrl = Form(...),  # noqa: B008
+) -> PublicationMetadata:
+    """
+    Extract publication metadata from a PDF or image file at a given URL for a
+    specific project.
+
+    Args:
+        project_id: The ID of the project to extract metadata for
+        url: URL pointing to a file to process
+
+    Returns:
+        PublicationMetadata: Extracted metadata as JSON
+    """
+    try:
+        backend = registry.get_backend(project_id)
+    except ProjectNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except ConfigError as e:
+        logger.exception("Configuration error")
+        raise HTTPException(status_code=500, detail=str(e))
+
+    url_str = str(url)
+
+    try:
+        proxy = get_url_proxy()
+        async with httpx.AsyncClient(proxy=proxy) as client:
+            response = await client.get(url_str)
+
+            content_type = response.headers.get("content-type", "")
+            if content_type != "application/pdf":
+                expected = "application/pdf"
+                detail = (
+                    f"'{url}' does not point to a PDF file. "
+                    f"Expected '{expected}', got '{content_type}'."
+                )
+                raise HTTPException(status_code=400, detail=detail)
+
+            status_code = response.status_code
+            if status_code >= 400:
+                raise HTTPException(
+                    status_code=status_code,
+                    detail=str(response.reason_phrase),
+                )
+
+            with tempfile.NamedTemporaryFile(suffix=".pdf") as tmp:
+                async for chunk in response.aiter_bytes(chunk_size=1024 * 1024):
+                    tmp.write(chunk)
+                tmp.flush()
+                return await backend.extract([tmp.name])
+    except httpx.HTTPError as e:
+        logger.exception("HTTP Error downloading %s", url_str)
+        raise HTTPException(status_code=500, detail=str(e))
+    except Exception as e:
+        logger.exception("Unexpected error during download from %s", url)
+        raise HTTPException(status_code=500, detail=str(e))

bibra/cli.py

@@ -1,14 +1,17 @@
 """CLI interface for BIBRA."""
 
 import asyncio
+import tempfile
 
 import click
+import httpx
 from dotenv import load_dotenv
 
 from bibra.config import (
     ConfigError,
     ProjectNotFoundError,
     ProjectRegistry,
+    get_url_proxy,
 )
 
 
@@ -103,3 +106,53 @@ def extract(project_id: str, file_path: str, config: str | None, output: str | N
         click.echo(f"Output written to {output}")
     else:
         click.echo(json_output)
+
+
+@cli.command("extract-url")
+@click.argument("project_id")
+@click.argument("url")
+@click.option(
+    "--config",
+    "-c",
+    default=None,
+    help="Path to the project configuration file (overrides BIBRA_CONFIG).",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(dir_okay=False, writable=True, resolve_path=True),
+    default=None,
+    help="Write JSON output to file instead of stdout",
+)
+def extract_url(project_id: str, url: str, config: str | None, output: str | None):
+    """Extract publication metadata from a PDF or image file at a URL."""
+    registry = ProjectRegistry(config)
+
+    try:
+        backend = registry.get_backend(project_id)
+    except ProjectNotFoundError as e:
+        raise click.UsageError(str(e)) from None
+    except ConfigError as e:
+        raise click.ClickException(str(e)) from None
+
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".pdf") as tmp:
+            proxy = get_url_proxy()
+            with httpx.stream("GET", url, proxy=proxy) as response:
+                if response.status_code >= 400:
+                    raise httpx.HTTPError(f"HTTP {response.status_code} for {url}")
+                for chunk in response.iter_bytes(chunk_size=1024 * 1024):
+                    tmp.write(chunk)
+                tmp.flush()
+            result = asyncio.run(backend.extract([tmp.name]))
+    except Exception as e:
+        raise click.ClickException(f"Extraction failed: {e}") from e
+
+    json_output = result.model_dump_json(indent=2)
+
+    if output:
+        with open(output, "w", encoding="utf-8") as f:
+            f.write(json_output + "\n")
+        click.echo(f"Output written to {output}")
+    else:
+        click.echo(json_output)

bibra/config.py

@@ -277,3 +277,12 @@ def list_projects(self) -> list[dict[str, Any]]:
             }
             for project in self._projects.values()
         ]
+
+
+def get_url_proxy() -> str | None:
+    """Return the BIBRA_URL_PROXY environment variable value.
+
+    Returns:
+        The proxy URL string, or None if not set.
+    """
+    return os.environ.get("BIBRA_URL_PROXY")

tests/test_api_routes.py

@@ -1,13 +1,15 @@
 """Tests for API routes."""
 
-from unittest.mock import MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from fastapi import HTTPException, Request
 from fastapi.routing import APIRoute
+from httpx import Headers
 
 from bibra.api.v0.routes import (
     extract,
+    extract_url,
     get_registry,
     list_projects,
     router,
@@ -80,6 +82,66 @@ async def test_extract_returns_example_metadata(self):
         assert result.p_isbn == []
         assert result.e_issn is None
 
+    def test_extract_url_route_exists(self):
+        """The router should have a project-specific extract-url route."""
+        routes = [str(r.path) for r in router.routes]
+        assert "/projects/{project_id}/extract-url" in routes
+
+    def test_extract_url_route_is_post_method(self):
+        """The extract-url route should use POST method."""
+        extract_url_routes = [
+            r
+            for r in router.routes
+            if str(r.path) == "/projects/{project_id}/extract-url"
+        ]
+        assert len(extract_url_routes) >= 1
+        # Check that the route uses POST method
+        route = extract_url_routes[0]
+        assert isinstance(route, APIRoute)
+
+    async def test_extract_url_returns_example_metadata(self):
+        """The /projects/{project_id}/extract-url endpoint should return example
+        publication metadata."""
+        from pydantic import HttpUrl
+
+        registry = ProjectRegistry()
+
+        async def mock_aiter_bytes(*args, **kwargs):
+            yield b"%PDF-1.4 mock content"
+
+        mock_response = MagicMock()
+        mock_response.headers = Headers({"content-type": "application/pdf"})
+        mock_response.status_code = 200
+        mock_response.aiter_bytes.return_value = mock_aiter_bytes()
+
+        with patch("httpx.AsyncClient") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+            mock_client.__aexit__ = AsyncMock(return_value=False)
+            mock_client.get = AsyncMock(return_value=mock_response)
+            mock_client_cls.return_value = mock_client
+            result = await extract_url(
+                project_id="dummy",
+                registry=registry,
+                url=HttpUrl("https://example.com/paper.pdf"),
+            )
+
+        assert isinstance(result, PublicationMetadata)
+        assert result.language == "en"
+        assert (
+            result.title == "Machine Learning Approaches for Software Defect Prediction"
+        )
+        assert result.creator == ["Smith, John", "Johnson, Emily"]
+        assert result.year == "2023"
+        assert result.publisher == ["Springer", "ACM"]
+        assert result.doi == "10.1234/example.doi.12345"
+        assert result.e_isbn == ["978-0-123456-78-9"]
+        assert result.type_coar == "article"
+        # Verify fields that don't have values are None or empty lists
+        assert result.alt_title is None
+        assert result.p_isbn == []
+        assert result.e_issn is None
+
     def test_get_registry_returns_existing_registry(self):
         """get_registry should return the registry already on app.state."""
         mock_state = MagicMock()
@@ -155,3 +217,65 @@ async def test_extract_handles_project_not_found_error(self):
 
         assert exc_info.value.status_code == 404
         assert exc_info.value.detail == "Project 'unknown' not found"
+
+    async def test_extract_url_passes_proxy_when_set(self, monkeypatch):
+        """Test that extract-url passes the proxy to httpx.AsyncClient when set."""
+        from pydantic import HttpUrl
+
+        monkeypatch.setenv("BIBRA_URL_PROXY", "http://proxy.example.com:8080")
+
+        registry = ProjectRegistry()
+
+        async def mock_aiter_bytes(*args, **kwargs):
+            yield b"%PDF-1.4 mock content"
+
+        mock_response = MagicMock()
+        mock_response.headers = Headers({"content-type": "application/pdf"})
+        mock_response.status_code = 200
+        mock_response.aiter_bytes.return_value = mock_aiter_bytes()
+
+        with patch("httpx.AsyncClient") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+            mock_client.__aexit__ = AsyncMock(return_value=False)
+            mock_client.get = AsyncMock(return_value=mock_response)
+            mock_client_cls.return_value = mock_client
+            result = await extract_url(
+                project_id="dummy",
+                registry=registry,
+                url=HttpUrl("https://example.com/paper.pdf"),
+            )
+
+        mock_client_cls.assert_called_once_with(proxy="http://proxy.example.com:8080")
+        assert isinstance(result, PublicationMetadata)
+
+    async def test_extract_url_no_proxy_when_not_set(self, monkeypatch):
+        """Test that extract-url passes proxy=None when env var is not set."""
+        from pydantic import HttpUrl
+
+        monkeypatch.delenv("BIBRA_URL_PROXY", raising=False)
+
+        registry = ProjectRegistry()
+
+        async def mock_aiter_bytes(*args, **kwargs):
+            yield b"%PDF-1.4 mock content"
+
+        mock_response = MagicMock()
+        mock_response.headers = Headers({"content-type": "application/pdf"})
+        mock_response.status_code = 200
+        mock_response.aiter_bytes.return_value = mock_aiter_bytes()
+
+        with patch("httpx.AsyncClient") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+            mock_client.__aexit__ = AsyncMock(return_value=False)
+            mock_client.get = AsyncMock(return_value=mock_response)
+            mock_client_cls.return_value = mock_client
+            result = await extract_url(
+                project_id="dummy",
+                registry=registry,
+                url=HttpUrl("https://example.com/paper.pdf"),
+            )
+
+        mock_client_cls.assert_called_once_with(proxy=None)
+        assert isinstance(result, PublicationMetadata)