feat: enforce max-redirects and response-size limit on crawler (#1816)

backend/secuscan/crawler.py

@@ -78,19 +78,41 @@ async def crawl_target(
     timeout: int = 10,
     cookies: Dict[str, str] | None = None,
     extra_headers: Dict[str, Any] | None = None,
+    max_redirects: int = 10,
+    max_size: int = 5 * 1024 * 1024,
 ) -> Dict[str, Any]:
     """Fetch a target and normalize discovered links/forms/scripts/API hints."""
     headers = _build_headers(extra_headers)
     async with httpx.AsyncClient(
         follow_redirects=True,
+        max_redirects=max_redirects,
         timeout=timeout,
         headers=headers,
         cookies=cookies or {},
         verify=False,
     ) as client:
-        response = await client.get(url)
+        async with client.stream("GET", url) as response:
+            # Check Content-Length header if present
+            content_length = response.headers.get("content-length")
+            if content_length:
+                try:
+                    cl_val = int(content_length)
+                except ValueError:
+                    cl_val = 0
+                if cl_val > max_size:
+                    raise ValueError(f"Response size exceeds limit of {max_size} bytes")
 
-    body = response.text
+            # Read response in chunks to enforce size limit
+            body_chunks = []
+            bytes_read = 0
+            async for chunk in response.aiter_bytes():
+                bytes_read += len(chunk)
+                if bytes_read > max_size:
+                    raise ValueError(f"Response size exceeds limit of {max_size} bytes")
+                body_chunks.append(chunk)
+
+            body_bytes = b"".join(body_chunks)
+            body = body_bytes.decode("utf-8", errors="replace")
     parser = _SurfaceParser()
     parser.feed(body)
 

testing/backend/unit/test_crawler_limits.py

@@ -0,0 +1,82 @@
+"""Unit tests for verifying crawl_target max-redirects and max-size constraints."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+import httpx
+import pytest
+
+from backend.secuscan.crawler import crawl_target
+
+
+@pytest.mark.asyncio
+async def test_crawl_target_max_size_via_content_length():
+    """Verify that crawl_target raises ValueError if the Content-Length header exceeds max_size."""
+    mock_response = MagicMock()
+    mock_response.headers = {"content-length": "10000000"}  # 10MB
+    mock_response.status_code = 200
+    mock_response.url = "http://example.com"
+    mock_response.history = []
+
+    mock_client = MagicMock()
+    mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+    mock_client.__aexit__ = AsyncMock(return_value=None)
+
+    mock_stream_ctx = MagicMock()
+    mock_stream_ctx.__aenter__ = AsyncMock(return_value=mock_response)
+    mock_stream_ctx.__aexit__ = AsyncMock(return_value=None)
+    mock_client.stream.return_value = mock_stream_ctx
+
+    with patch("httpx.AsyncClient", return_value=mock_client):
+        with pytest.raises(ValueError, match="Response size exceeds limit"):
+            await crawl_target("http://example.com", max_size=1000)
+
+
+@pytest.mark.asyncio
+async def test_crawl_target_max_size_via_streaming():
+    """Verify that crawl_target raises ValueError if the streamed chunks exceed max_size."""
+    mock_response = MagicMock()
+    mock_response.headers = {}
+    mock_response.status_code = 200
+    mock_response.url = "http://example.com"
+    mock_response.history = []
+
+    async def mock_aiter_bytes():
+        yield b"hello "
+        yield b"world of pentesting"
+
+    mock_response.aiter_bytes = mock_aiter_bytes
+
+    mock_client = MagicMock()
+    mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+    mock_client.__aexit__ = AsyncMock(return_value=None)
+
+    mock_stream_ctx = MagicMock()
+    mock_stream_ctx.__aenter__ = AsyncMock(return_value=mock_response)
+    mock_stream_ctx.__aexit__ = AsyncMock(return_value=None)
+    mock_client.stream.return_value = mock_stream_ctx
+
+    with patch("httpx.AsyncClient", return_value=mock_client):
+        # Setting max_size to 10 bytes:
+        # First chunk (b"hello ") is 6 bytes (ok).
+        # Second chunk adds 19 bytes, total 25 bytes (exceeds limit).
+        with pytest.raises(ValueError, match="Response size exceeds limit"):
+            await crawl_target("http://example.com", max_size=10)
+
+
+@pytest.mark.asyncio
+async def test_crawl_target_max_redirects_exceeded():
+    """Verify that crawl_target raises httpx.TooManyRedirects when the redirect limit is hit."""
+    mock_client = MagicMock()
+    mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+    mock_client.__aexit__ = AsyncMock(return_value=None)
+    mock_request = httpx.Request("GET", "http://example.com")
+
+    mock_stream_ctx = MagicMock()
+    mock_stream_ctx.__aenter__ = AsyncMock(side_effect=httpx.TooManyRedirects("Too many redirects", request=mock_request))
+    mock_stream_ctx.__aexit__ = AsyncMock(return_value=None)
+    mock_client.stream.return_value = mock_stream_ctx
+
+    with patch("httpx.AsyncClient", return_value=mock_client):
+        with pytest.raises(httpx.TooManyRedirects):
+            await crawl_target("http://example.com", max_redirects=2)