Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ chunks = client.documents.list_chunks(
document_id,
page=1,
page_size=50,
chunk_type="text",
chunk_type="image",
include_asset_urls=True,
)
print(chunks.pagination.total)
if chunks.chunks:
Expand All @@ -106,16 +107,22 @@ if chunks.chunks:
include_asset_urls=True,
)
print(chunk.chunk.content)
print(chunk.chunk.asset_url) # Requested 7-day URL for image/table chunks.

client.documents.archive(document_id)
```

You can also list documents in a namespace:

```python
documents = client.documents.list(namespace="support-center")
documents = client.documents.list(
namespace="support-center",
page=1,
page_size=50,
)
for document in documents.documents:
print(document.document_id, document.status)
print(documents.pagination.total_pages)
```

Retrieval supports exclusions when clients want follow-up results that avoid
Expand Down
10 changes: 9 additions & 1 deletion docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -537,9 +537,14 @@ response = client.retrieval.query(
### List, get, and archive documents

```python
document_list = client.documents.list(namespace="support-center")
document_list = client.documents.list(
namespace="support-center",
page=1,
page_size=50,
)
for document in document_list.documents:
print(document.document_id, document.status, document.source_file_name)
print(document_list.pagination.total_pages)

document = client.documents.get("doc_123")
print(document.current_job_result_id)
Expand All @@ -553,6 +558,9 @@ chunks = client.documents.list_chunks(
for chunk in chunks.chunks:
print(chunk.id, chunk.content)

# Set include_asset_urls=True to request 7-day asset_url values for
# image and table chunks when available.

image_chunk = client.documents.get_chunk(
"doc_123",
"dchk_123",
Expand Down
2 changes: 2 additions & 0 deletions src/knowhere/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
DocumentChunkPagination,
DocumentChunkResponse,
DocumentChunkType,
DocumentListPagination,
DocumentListResponse,
)
from knowhere.types.job import Job, JobError, JobProgress, JobResult
Expand Down Expand Up @@ -112,6 +113,7 @@
"DocumentChunkPagination",
"DocumentChunkResponse",
"DocumentChunkType",
"DocumentListPagination",
"DocumentListResponse",
# Retrieval types
"RetrievalChannel",
Expand Down
68 changes: 50 additions & 18 deletions src/knowhere/resources/documents.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,19 @@
class Documents(SyncAPIResource):
"""Synchronous interface for ``/v1/documents`` endpoints."""

def list(self, *, namespace: Optional[str] = None) -> DocumentListResponse:
def list(
self,
*,
namespace: Optional[str] = None,
page: int = 1,
page_size: int = 50,
) -> DocumentListResponse:
"""List canonical documents in a namespace."""
params: Dict[str, Any] = {}
if namespace is not None:
params["namespace"] = namespace
params: Dict[str, Any] = _build_document_list_params(
namespace=namespace,
page=page,
page_size=page_size,
)

return self._request(
"GET",
Expand All @@ -45,7 +53,7 @@ def list_chunks(
page: int = 1,
page_size: int = 50,
chunk_type: Optional[DocumentChunkType] = None,
include_asset_urls: bool = False,
include_asset_urls: Optional[bool] = None,
) -> DocumentChunkListResponse:
"""List current-revision chunks for one canonical document."""
params: Dict[str, Any] = _build_chunk_list_params(
Expand All @@ -67,7 +75,7 @@ def get_chunk(
document_id: str,
document_chunk_id: str,
*,
include_asset_urls: bool = False,
include_asset_urls: Optional[bool] = None,
) -> DocumentChunkResponse:
"""Get one current-revision chunk for one canonical document."""
params: Dict[str, Any] = _build_chunk_get_params(
Expand All @@ -93,11 +101,19 @@ def archive(self, document_id: str) -> Document:
class AsyncDocuments(AsyncAPIResource):
"""Asynchronous interface for ``/v1/documents`` endpoints."""

async def list(self, *, namespace: Optional[str] = None) -> DocumentListResponse:
async def list(
self,
*,
namespace: Optional[str] = None,
page: int = 1,
page_size: int = 50,
) -> DocumentListResponse:
"""List canonical documents in a namespace."""
params: Dict[str, Any] = {}
if namespace is not None:
params["namespace"] = namespace
params: Dict[str, Any] = _build_document_list_params(
namespace=namespace,
page=page,
page_size=page_size,
)

return await self._request(
"GET",
Expand All @@ -121,7 +137,7 @@ async def list_chunks(
page: int = 1,
page_size: int = 50,
chunk_type: Optional[DocumentChunkType] = None,
include_asset_urls: bool = False,
include_asset_urls: Optional[bool] = None,
) -> DocumentChunkListResponse:
"""List current-revision chunks for one canonical document."""
params: Dict[str, Any] = _build_chunk_list_params(
Expand All @@ -143,7 +159,7 @@ async def get_chunk(
document_id: str,
document_chunk_id: str,
*,
include_asset_urls: bool = False,
include_asset_urls: Optional[bool] = None,
) -> DocumentChunkResponse:
"""Get one current-revision chunk for one canonical document."""
params: Dict[str, Any] = _build_chunk_get_params(
Expand All @@ -166,12 +182,28 @@ async def archive(self, document_id: str) -> Document:
)


def _build_document_list_params(
*,
namespace: Optional[str],
page: int,
page_size: int,
) -> Dict[str, Any]:
params: Dict[str, Any] = {}
if namespace is not None:
params["namespace"] = namespace
if page != 1:
params["page"] = page
if page_size != 50:
params["page_size"] = page_size
return params


def _build_chunk_list_params(
*,
page: int,
page_size: int,
chunk_type: Optional[DocumentChunkType],
include_asset_urls: bool,
include_asset_urls: Optional[bool],
) -> Dict[str, Any]:
params: Dict[str, Any] = {}
if page != 1:
Expand All @@ -180,12 +212,12 @@ def _build_chunk_list_params(
params["page_size"] = page_size
if chunk_type is not None:
params["chunk_type"] = chunk_type
if include_asset_urls:
params["include_asset_urls"] = True
if include_asset_urls is not None:
params["include_asset_urls"] = include_asset_urls
return params


def _build_chunk_get_params(*, include_asset_urls: bool) -> Dict[str, Any]:
if not include_asset_urls:
def _build_chunk_get_params(*, include_asset_urls: Optional[bool]) -> Dict[str, Any]:
if include_asset_urls is None:
return {}
return {"include_asset_urls": True}
return {"include_asset_urls": include_asset_urls}
2 changes: 2 additions & 0 deletions src/knowhere/types/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
DocumentChunkPagination,
DocumentChunkResponse,
DocumentChunkType,
DocumentListPagination,
DocumentListResponse,
)
from knowhere.types.job import Job, JobError, JobResult
Expand Down Expand Up @@ -53,6 +54,7 @@
"DocumentChunkPagination",
"DocumentChunkResponse",
"DocumentChunkType",
"DocumentListPagination",
"DocumentListResponse",
# retrieval
"RetrievalChannel",
Expand Down
31 changes: 29 additions & 2 deletions src/knowhere/types/document.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from datetime import datetime
from typing import Any, Dict, Literal, Optional

from pydantic import BaseModel
from pydantic import BaseModel, model_validator


class Document(BaseModel):
Expand All @@ -21,11 +21,38 @@ class Document(BaseModel):
archived_at: Optional[datetime] = None


class DocumentListPagination(BaseModel):
"""Pagination metadata returned by document list endpoints."""

page: int
page_size: int
total: int
total_pages: int


class DocumentListResponse(BaseModel):
"""Response from ``GET /v1/documents``."""

namespace: str
documents: list[Document]
pagination: DocumentListPagination

@model_validator(mode="before")
@classmethod
def populate_legacy_pagination(cls, value: Any) -> Any:
if not isinstance(value, dict) or "pagination" in value:
return value

payload: Dict[str, Any] = dict(value)
documents = payload.get("documents")
document_count = len(documents) if isinstance(documents, list) else 0
payload["pagination"] = {
"page": 1,
"page_size": document_count,
"total": document_count,
"total_pages": 1 if document_count else 0,
}
return payload


DocumentChunkType = Literal["text", "image", "table"]
Expand Down Expand Up @@ -53,7 +80,7 @@ class DocumentChunk(BaseModel):
file_path: Optional[str] = None
sort_order: int
metadata: Dict[str, Any]
asset_url: Optional[str] = None
asset_url: Optional[str] = None # 7-day media asset URL when available.
created_at: Optional[datetime] = None


Expand Down
77 changes: 74 additions & 3 deletions tests/test_documents.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,23 +55,45 @@ def test_list_documents_sends_namespace_query(self, sync_client: Any) -> None:
json={
"namespace": "support-center",
"documents": [_make_document()],
"pagination": {
"page": 2,
"page_size": 25,
"total": 26,
"total_pages": 2,
},
},
)
)

response = sync_client.documents.list(namespace="support-center")
response = sync_client.documents.list(
namespace="support-center",
page=2,
page_size=25,
)

assert route.called
assert route.calls[0].request.url.params["namespace"] == "support-center"
assert route.calls[0].request.url.params["page"] == "2"
assert route.calls[0].request.url.params["page_size"] == "25"
assert response.namespace == "support-center"
assert response.documents[0].document_id == "doc_123"
assert response.pagination.total_pages == 2

@respx.mock
def test_list_documents_omits_namespace_when_defaulted(self, sync_client: Any) -> None:
route = respx.get(DOCUMENTS_URL).mock(
return_value=httpx.Response(
200,
json={"namespace": "default", "documents": []},
json={
"namespace": "default",
"documents": [],
"pagination": {
"page": 1,
"page_size": 50,
"total": 0,
"total_pages": 0,
},
},
)
)

Expand All @@ -81,6 +103,27 @@ def test_list_documents_omits_namespace_when_defaulted(self, sync_client: Any) -
assert dict(route.calls[0].request.url.params) == {}
assert response.namespace == "default"
assert response.documents == []
assert response.pagination.total == 0

@respx.mock
def test_list_documents_accepts_legacy_unpaginated_response(self, sync_client: Any) -> None:
route = respx.get(DOCUMENTS_URL).mock(
return_value=httpx.Response(
200,
json={
"namespace": "default",
"documents": [_make_document()],
},
)
)

response = sync_client.documents.list()

assert route.called
assert response.pagination.page == 1
assert response.pagination.page_size == 1
assert response.pagination.total == 1
assert response.pagination.total_pages == 1

@respx.mock
def test_get_document_returns_document_state(self, sync_client: Any) -> None:
Expand Down Expand Up @@ -160,9 +203,37 @@ def test_list_chunks_omits_default_query_params(self, sync_client: Any) -> None:
assert response.chunks == []
assert response.pagination.total == 0

@respx.mock
def test_list_chunks_accepts_explicit_asset_url_control(
self, sync_client: Any
) -> None:
route = respx.get(f"{DOCUMENTS_URL}/doc_123/chunks").mock(
return_value=httpx.Response(
200,
json={
"document_id": "doc_123",
"namespace": "support-center",
"job_result_id": "result_123",
"job_id": "job_123",
"chunks": [],
"pagination": {
"page": 1,
"page_size": 50,
"total": 0,
"total_pages": 0,
},
},
)
)

sync_client.documents.list_chunks("doc_123", include_asset_urls=False)

assert route.called
assert route.calls[0].request.url.params["include_asset_urls"] == "false"

@respx.mock
@pytest.mark.asyncio
async def test_async_get_chunk_requests_asset_urls_only_when_needed(
async def test_async_get_chunk_accepts_explicit_asset_url_control(
self,
async_client: Any,
) -> None:
Expand Down
Loading