Detect unknown MIME types without copying uploads

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## 0.1.11
 
+### Improvements
+
+- Detect unknown upload MIME types from the existing stream without a full-file copy; preserve filename hints and rewind on success or failure.
+
 ### Fixes
 
 - **Restore `MAX_LIFETIME_SECONDS` support in the Docker image**: `scripts/app-start.sh` invokes GNU `timeout` with `--preserve-status` and `--foreground`, flags the Wolfi base's BusyBox `timeout` does not support. Setting `MAX_LIFETIME_SECONDS` therefore caused the server to fail to start and the container to restart-loop. Added `coreutils` to the image so GNU `timeout` is available. This regressed when the base image moved from RockyLinux (which shipped GNU coreutils) to Wolfi.

prepline_general/api/filetypes.py

@@ -1,13 +1,31 @@
-import os
-from typing import Optional
-from io import BytesIO
+from typing import IO, Any, Optional, cast
 
 from fastapi import HTTPException, UploadFile
 
 from unstructured.file_utils.filetype import detect_filetype
 from unstructured.file_utils.model import FileType
 
 
+class _SpooledFileProxy:
+    """Expose a spooled upload as a normal file without copying its complete body.
+
+    `unstructured.detect_filetype()` copies `SpooledTemporaryFile` inputs into a `BytesIO`, even
+    though its detector only seeks and reads small portions. Hiding the concrete type preserves the
+    normal file interface while avoiding that document-sized allocation.
+
+    `name` is set explicitly rather than forwarded: the detector reads `.name` to derive the
+    filename-extension, and a spooled file's own `.name` is the temporary file rather than the
+    uploaded filename.
+    """
+
+    def __init__(self, file: IO[bytes], name: str | None):
+        self._file = file
+        self.name = name
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._file, name)
+
+
 def _remove_optional_info_from_mime_type(content_type: str | None) -> str | None:
     """removes charset information from mime types, e.g.,
     "application/json; charset=utf-8" -> "application/json"
@@ -37,14 +55,14 @@ def get_validated_mimetype(file: UploadFile, content_type_hint: str | None = Non
     filetype = FileType.from_mime_type(content_type)
 
     # If content_type was not specified, use the library to identify the file
-    # We inspect the bytes to do this, so we need to buffer the file
+    # The detector seeks and reads bounded portions of the upload. Proxy the spooled file so the
+    # dependency does not make a complete in-memory `BytesIO` copy first.
     if not filetype or filetype == FileType.UNK:
-        file_buffer = BytesIO(file.file.read())
-        file.file.seek(0)
-
-        file_buffer.name = file.filename
-
-        filetype = detect_filetype(file=file_buffer)
+        file_proxy = cast(IO[bytes], _SpooledFileProxy(file.file, file.filename))
+        try:
+            filetype = detect_filetype(file=file_proxy)
+        finally:
+            file.file.seek(0)
 
     if not filetype.is_partitionable:
         raise HTTPException(

test_general/api/test_filetypes.py

@@ -0,0 +1,88 @@
+from tempfile import SpooledTemporaryFile
+
+import pytest
+from fastapi import UploadFile
+
+from prepline_general.api import filetypes
+from unstructured.file_utils.model import FileType
+
+
+def test_unknown_mimetype_is_detected_from_existing_upload_stream(monkeypatch):
+    upload_stream = SpooledTemporaryFile()
+    upload_stream.write(b"sample text")
+    upload_stream.seek(0)
+    upload = UploadFile(file=upload_stream, filename="sample.txt")
+
+    def fake_detect_filetype(*, file):
+        assert file._file is upload_stream
+        assert file.name == "sample.txt"
+        file.seek(4)
+        return FileType.TXT
+
+    monkeypatch.setattr(filetypes, "detect_filetype", fake_detect_filetype)
+
+    assert filetypes.get_validated_mimetype(upload) == "text/plain"
+    assert upload_stream.tell() == 0
+
+
+def test_unknown_mimetype_rewinds_upload_stream_when_detection_fails(monkeypatch):
+    upload_stream = SpooledTemporaryFile()
+    upload_stream.write(b"sample text")
+    upload_stream.seek(0)
+    upload = UploadFile(file=upload_stream, filename="sample.txt")
+
+    def fake_detect_filetype(*, file):
+        file.seek(4)
+        raise RuntimeError("detection failed")
+
+    monkeypatch.setattr(filetypes, "detect_filetype", fake_detect_filetype)
+
+    with pytest.raises(RuntimeError, match="detection failed"):
+        filetypes.get_validated_mimetype(upload)
+
+    assert upload_stream.tell() == 0
+
+
+@pytest.mark.parametrize("max_size", [1, 1024 * 1024])
+@pytest.mark.parametrize(
+    "filename,payload",
+    [
+        ("sample.txt", b"A sample paragraph of ordinary text."),
+        ("sample.html", b"<!doctype html><html><body><p>Hello</p></body></html>"),
+        ("sample.csv", b"name,value\nAlice,1\nBob,2\n"),
+        ("sample.json", b'{"name": "Alice", "value": 1}'),
+        (None, b"A sample paragraph of ordinary text."),
+    ],
+)
+def test_real_detector_matches_copied_upload(filename, payload, max_size):
+    from io import BytesIO
+
+    copied = BytesIO(payload)
+    copied.name = filename
+    expected = filetypes.detect_filetype(file=copied)
+    with SpooledTemporaryFile(max_size=max_size) as stream:
+        stream.write(payload)
+        stream.seek(0)
+        upload = UploadFile(file=stream, filename=filename)
+        assert filetypes.get_validated_mimetype(upload) == expected.mime_type
+        assert stream.tell() == 0
+        assert not stream.closed
+
+
+@pytest.mark.parametrize("filename", ["layout-parser-paper.pdf", "notes.pptx", "stanley-cups.xlsx"])
+@pytest.mark.parametrize("max_size", [1, 10 * 1024 * 1024])
+def test_real_detector_matches_copied_binary_upload(filename, max_size):
+    from io import BytesIO
+    from pathlib import Path
+
+    payload = (Path("sample-docs") / filename).read_bytes()
+    copied = BytesIO(payload)
+    copied.name = filename
+    expected = filetypes.detect_filetype(file=copied)
+    with SpooledTemporaryFile(max_size=max_size) as stream:
+        stream.write(payload)
+        stream.seek(0)
+        upload = UploadFile(file=stream, filename=filename)
+        assert filetypes.get_validated_mimetype(upload) == expected.mime_type
+        assert stream.tell() == 0
+        assert not stream.closed