From f26574e3aeb508fec57231ff301588ec02b19aae Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 30 Jun 2026 06:05:52 +0000 Subject: [PATCH 1/4] feat: support document list pagination --- README.md | 7 ++++- docs/usage.md | 7 ++++- src/knowhere/__init__.py | 2 ++ src/knowhere/resources/documents.py | 48 ++++++++++++++++++++++++----- src/knowhere/types/__init__.py | 2 ++ src/knowhere/types/document.py | 10 ++++++ tests/test_documents.py | 27 ++++++++++++++-- 7 files changed, 91 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9c7416e..8449da7 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,14 @@ client.documents.archive(document_id) You can also list documents in a namespace: ```python -documents = client.documents.list(namespace="support-center") +documents = client.documents.list( + namespace="support-center", + page=1, + page_size=50, +) for document in documents.documents: print(document.document_id, document.status) +print(documents.pagination.total_pages) ``` Retrieval supports exclusions when clients want follow-up results that avoid diff --git a/docs/usage.md b/docs/usage.md index 6735cd1..790364f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -537,9 +537,14 @@ response = client.retrieval.query( ### List, get, and archive documents ```python -document_list = client.documents.list(namespace="support-center") +document_list = client.documents.list( + namespace="support-center", + page=1, + page_size=50, +) for document in document_list.documents: print(document.document_id, document.status, document.source_file_name) +print(document_list.pagination.total_pages) document = client.documents.get("doc_123") print(document.current_job_result_id) diff --git a/src/knowhere/__init__.py b/src/knowhere/__init__.py index bf36dfe..2b963d1 100644 --- a/src/knowhere/__init__.py +++ b/src/knowhere/__init__.py @@ -42,6 +42,7 @@ DocumentChunkPagination, DocumentChunkResponse, DocumentChunkType, + DocumentListPagination, DocumentListResponse, ) from knowhere.types.job import Job, JobError, JobProgress, JobResult @@ -112,6 +113,7 @@ "DocumentChunkPagination", "DocumentChunkResponse", "DocumentChunkType", + "DocumentListPagination", "DocumentListResponse", # Retrieval types "RetrievalChannel", diff --git a/src/knowhere/resources/documents.py b/src/knowhere/resources/documents.py index c58aebb..e7db7fc 100644 --- a/src/knowhere/resources/documents.py +++ b/src/knowhere/resources/documents.py @@ -17,11 +17,19 @@ class Documents(SyncAPIResource): """Synchronous interface for ``/v1/documents`` endpoints.""" - def list(self, *, namespace: Optional[str] = None) -> DocumentListResponse: + def list( + self, + *, + namespace: Optional[str] = None, + page: int = 1, + page_size: int = 50, + ) -> DocumentListResponse: """List canonical documents in a namespace.""" - params: Dict[str, Any] = {} - if namespace is not None: - params["namespace"] = namespace + params: Dict[str, Any] = _build_document_list_params( + namespace=namespace, + page=page, + page_size=page_size, + ) return self._request( "GET", @@ -93,11 +101,19 @@ def archive(self, document_id: str) -> Document: class AsyncDocuments(AsyncAPIResource): """Asynchronous interface for ``/v1/documents`` endpoints.""" - async def list(self, *, namespace: Optional[str] = None) -> DocumentListResponse: + async def list( + self, + *, + namespace: Optional[str] = None, + page: int = 1, + page_size: int = 50, + ) -> DocumentListResponse: """List canonical documents in a namespace.""" - params: Dict[str, Any] = {} - if namespace is not None: - params["namespace"] = namespace + params: Dict[str, Any] = _build_document_list_params( + namespace=namespace, + page=page, + page_size=page_size, + ) return await self._request( "GET", @@ -166,6 +182,22 @@ async def archive(self, document_id: str) -> Document: ) +def _build_document_list_params( + *, + namespace: Optional[str], + page: int, + page_size: int, +) -> Dict[str, Any]: + params: Dict[str, Any] = {} + if namespace is not None: + params["namespace"] = namespace + if page != 1: + params["page"] = page + if page_size != 50: + params["page_size"] = page_size + return params + + def _build_chunk_list_params( *, page: int, diff --git a/src/knowhere/types/__init__.py b/src/knowhere/types/__init__.py index 17f2cbc..14274c2 100644 --- a/src/knowhere/types/__init__.py +++ b/src/knowhere/types/__init__.py @@ -9,6 +9,7 @@ DocumentChunkPagination, DocumentChunkResponse, DocumentChunkType, + DocumentListPagination, DocumentListResponse, ) from knowhere.types.job import Job, JobError, JobResult @@ -53,6 +54,7 @@ "DocumentChunkPagination", "DocumentChunkResponse", "DocumentChunkType", + "DocumentListPagination", "DocumentListResponse", # retrieval "RetrievalChannel", diff --git a/src/knowhere/types/document.py b/src/knowhere/types/document.py index fcd1b80..693a368 100644 --- a/src/knowhere/types/document.py +++ b/src/knowhere/types/document.py @@ -21,11 +21,21 @@ class Document(BaseModel): archived_at: Optional[datetime] = None +class DocumentListPagination(BaseModel): + """Pagination metadata returned by document list endpoints.""" + + page: int + page_size: int + total: int + total_pages: int + + class DocumentListResponse(BaseModel): """Response from ``GET /v1/documents``.""" namespace: str documents: list[Document] + pagination: DocumentListPagination DocumentChunkType = Literal["text", "image", "table"] diff --git a/tests/test_documents.py b/tests/test_documents.py index 205d57e..7740ec2 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -55,23 +55,45 @@ def test_list_documents_sends_namespace_query(self, sync_client: Any) -> None: json={ "namespace": "support-center", "documents": [_make_document()], + "pagination": { + "page": 2, + "page_size": 25, + "total": 26, + "total_pages": 2, + }, }, ) ) - response = sync_client.documents.list(namespace="support-center") + response = sync_client.documents.list( + namespace="support-center", + page=2, + page_size=25, + ) assert route.called assert route.calls[0].request.url.params["namespace"] == "support-center" + assert route.calls[0].request.url.params["page"] == "2" + assert route.calls[0].request.url.params["page_size"] == "25" assert response.namespace == "support-center" assert response.documents[0].document_id == "doc_123" + assert response.pagination.total_pages == 2 @respx.mock def test_list_documents_omits_namespace_when_defaulted(self, sync_client: Any) -> None: route = respx.get(DOCUMENTS_URL).mock( return_value=httpx.Response( 200, - json={"namespace": "default", "documents": []}, + json={ + "namespace": "default", + "documents": [], + "pagination": { + "page": 1, + "page_size": 50, + "total": 0, + "total_pages": 0, + }, + }, ) ) @@ -81,6 +103,7 @@ def test_list_documents_omits_namespace_when_defaulted(self, sync_client: Any) - assert dict(route.calls[0].request.url.params) == {} assert response.namespace == "default" assert response.documents == [] + assert response.pagination.total == 0 @respx.mock def test_get_document_returns_document_state(self, sync_client: Any) -> None: From 3176fa8ee4964fb854b148a75aa56ca560ece67e Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 30 Jun 2026 06:31:11 +0000 Subject: [PATCH 2/4] Expose document chunk asset URL controls --- README.md | 2 +- docs/usage.md | 4 +++- src/knowhere/resources/documents.py | 20 ++++++++++---------- src/knowhere/types/document.py | 2 +- tests/test_documents.py | 28 +++++++++++++++++++++++++++- 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8449da7..2263a4d 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ if chunks.chunks: chunk = client.documents.get_chunk( document_id, chunks.chunks[0].id, - include_asset_urls=True, ) print(chunk.chunk.content) + print(chunk.chunk.asset_url) # 7-day URL for image/table chunks when available. client.documents.archive(document_id) ``` diff --git a/docs/usage.md b/docs/usage.md index 790364f..0899db8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -558,10 +558,12 @@ chunks = client.documents.list_chunks( for chunk in chunks.chunks: print(chunk.id, chunk.content) +# Image and table chunks include 7-day asset_url values when available. +# Pass include_asset_urls=False to opt out for bulk scans. + image_chunk = client.documents.get_chunk( "doc_123", "dchk_123", - include_asset_urls=True, ) print(image_chunk.chunk.asset_url) diff --git a/src/knowhere/resources/documents.py b/src/knowhere/resources/documents.py index e7db7fc..7551dc7 100644 --- a/src/knowhere/resources/documents.py +++ b/src/knowhere/resources/documents.py @@ -53,7 +53,7 @@ def list_chunks( page: int = 1, page_size: int = 50, chunk_type: Optional[DocumentChunkType] = None, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkListResponse: """List current-revision chunks for one canonical document.""" params: Dict[str, Any] = _build_chunk_list_params( @@ -75,7 +75,7 @@ def get_chunk( document_id: str, document_chunk_id: str, *, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkResponse: """Get one current-revision chunk for one canonical document.""" params: Dict[str, Any] = _build_chunk_get_params( @@ -137,7 +137,7 @@ async def list_chunks( page: int = 1, page_size: int = 50, chunk_type: Optional[DocumentChunkType] = None, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkListResponse: """List current-revision chunks for one canonical document.""" params: Dict[str, Any] = _build_chunk_list_params( @@ -159,7 +159,7 @@ async def get_chunk( document_id: str, document_chunk_id: str, *, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkResponse: """Get one current-revision chunk for one canonical document.""" params: Dict[str, Any] = _build_chunk_get_params( @@ -203,7 +203,7 @@ def _build_chunk_list_params( page: int, page_size: int, chunk_type: Optional[DocumentChunkType], - include_asset_urls: bool, + include_asset_urls: Optional[bool], ) -> Dict[str, Any]: params: Dict[str, Any] = {} if page != 1: @@ -212,12 +212,12 @@ def _build_chunk_list_params( params["page_size"] = page_size if chunk_type is not None: params["chunk_type"] = chunk_type - if include_asset_urls: - params["include_asset_urls"] = True + if include_asset_urls is not None: + params["include_asset_urls"] = include_asset_urls return params -def _build_chunk_get_params(*, include_asset_urls: bool) -> Dict[str, Any]: - if not include_asset_urls: +def _build_chunk_get_params(*, include_asset_urls: Optional[bool]) -> Dict[str, Any]: + if include_asset_urls is None: return {} - return {"include_asset_urls": True} + return {"include_asset_urls": include_asset_urls} diff --git a/src/knowhere/types/document.py b/src/knowhere/types/document.py index 693a368..23b1eb5 100644 --- a/src/knowhere/types/document.py +++ b/src/knowhere/types/document.py @@ -63,7 +63,7 @@ class DocumentChunk(BaseModel): file_path: Optional[str] = None sort_order: int metadata: Dict[str, Any] - asset_url: Optional[str] = None + asset_url: Optional[str] = None # 7-day media asset URL when available. created_at: Optional[datetime] = None diff --git a/tests/test_documents.py b/tests/test_documents.py index 7740ec2..c5d8ac0 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -183,9 +183,35 @@ def test_list_chunks_omits_default_query_params(self, sync_client: Any) -> None: assert response.chunks == [] assert response.pagination.total == 0 + @respx.mock + def test_list_chunks_can_opt_out_of_asset_urls(self, sync_client: Any) -> None: + route = respx.get(f"{DOCUMENTS_URL}/doc_123/chunks").mock( + return_value=httpx.Response( + 200, + json={ + "document_id": "doc_123", + "namespace": "support-center", + "job_result_id": "result_123", + "job_id": "job_123", + "chunks": [], + "pagination": { + "page": 1, + "page_size": 50, + "total": 0, + "total_pages": 0, + }, + }, + ) + ) + + sync_client.documents.list_chunks("doc_123", include_asset_urls=False) + + assert route.called + assert route.calls[0].request.url.params["include_asset_urls"] == "false" + @respx.mock @pytest.mark.asyncio - async def test_async_get_chunk_requests_asset_urls_only_when_needed( + async def test_async_get_chunk_accepts_explicit_asset_url_control( self, async_client: Any, ) -> None: From 5d3855d91073b19383a7a693680026f096f7b717 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 30 Jun 2026 07:15:42 +0000 Subject: [PATCH 3/4] Handle legacy document list responses --- src/knowhere/types/document.py | 19 ++++++++++++++++++- tests/test_documents.py | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/knowhere/types/document.py b/src/knowhere/types/document.py index 23b1eb5..e6e813c 100644 --- a/src/knowhere/types/document.py +++ b/src/knowhere/types/document.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Any, Dict, Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, model_validator class Document(BaseModel): @@ -37,6 +37,23 @@ class DocumentListResponse(BaseModel): documents: list[Document] pagination: DocumentListPagination + @model_validator(mode="before") + @classmethod + def populate_legacy_pagination(cls, value: Any) -> Any: + if not isinstance(value, dict) or "pagination" in value: + return value + + payload: Dict[str, Any] = dict(value) + documents = payload.get("documents") + document_count = len(documents) if isinstance(documents, list) else 0 + payload["pagination"] = { + "page": 1, + "page_size": document_count, + "total": document_count, + "total_pages": 1 if document_count else 0, + } + return payload + DocumentChunkType = Literal["text", "image", "table"] diff --git a/tests/test_documents.py b/tests/test_documents.py index c5d8ac0..71180b8 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -105,6 +105,26 @@ def test_list_documents_omits_namespace_when_defaulted(self, sync_client: Any) - assert response.documents == [] assert response.pagination.total == 0 + @respx.mock + def test_list_documents_accepts_legacy_unpaginated_response(self, sync_client: Any) -> None: + route = respx.get(DOCUMENTS_URL).mock( + return_value=httpx.Response( + 200, + json={ + "namespace": "default", + "documents": [_make_document()], + }, + ) + ) + + response = sync_client.documents.list() + + assert route.called + assert response.pagination.page == 1 + assert response.pagination.page_size == 1 + assert response.pagination.total == 1 + assert response.pagination.total_pages == 1 + @respx.mock def test_get_document_returns_document_state(self, sync_client: Any) -> None: route = respx.get(f"{DOCUMENTS_URL}/doc_123").mock( From 9754eaa552bedba7caf98181624724cd03436357 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 30 Jun 2026 07:57:34 +0000 Subject: [PATCH 4/4] Request document asset URLs explicitly --- README.md | 6 ++++-- docs/usage.md | 5 +++-- tests/test_documents.py | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2263a4d..782cc02 100644 --- a/README.md +++ b/README.md @@ -96,16 +96,18 @@ chunks = client.documents.list_chunks( document_id, page=1, page_size=50, - chunk_type="text", + chunk_type="image", + include_asset_urls=True, ) print(chunks.pagination.total) if chunks.chunks: chunk = client.documents.get_chunk( document_id, chunks.chunks[0].id, + include_asset_urls=True, ) print(chunk.chunk.content) - print(chunk.chunk.asset_url) # 7-day URL for image/table chunks when available. + print(chunk.chunk.asset_url) # Requested 7-day URL for image/table chunks. client.documents.archive(document_id) ``` diff --git a/docs/usage.md b/docs/usage.md index 0899db8..b311032 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -558,12 +558,13 @@ chunks = client.documents.list_chunks( for chunk in chunks.chunks: print(chunk.id, chunk.content) -# Image and table chunks include 7-day asset_url values when available. -# Pass include_asset_urls=False to opt out for bulk scans. +# Set include_asset_urls=True to request 7-day asset_url values for +# image and table chunks when available. image_chunk = client.documents.get_chunk( "doc_123", "dchk_123", + include_asset_urls=True, ) print(image_chunk.chunk.asset_url) diff --git a/tests/test_documents.py b/tests/test_documents.py index 71180b8..eb6c7c8 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -204,7 +204,9 @@ def test_list_chunks_omits_default_query_params(self, sync_client: Any) -> None: assert response.pagination.total == 0 @respx.mock - def test_list_chunks_can_opt_out_of_asset_urls(self, sync_client: Any) -> None: + def test_list_chunks_accepts_explicit_asset_url_control( + self, sync_client: Any + ) -> None: route = respx.get(f"{DOCUMENTS_URL}/doc_123/chunks").mock( return_value=httpx.Response( 200,