diff --git a/README.md b/README.md index 9c7416e..782cc02 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,8 @@ chunks = client.documents.list_chunks( document_id, page=1, page_size=50, - chunk_type="text", + chunk_type="image", + include_asset_urls=True, ) print(chunks.pagination.total) if chunks.chunks: @@ -106,6 +107,7 @@ if chunks.chunks: include_asset_urls=True, ) print(chunk.chunk.content) + print(chunk.chunk.asset_url) # Requested 7-day URL for image/table chunks. client.documents.archive(document_id) ``` @@ -113,9 +115,14 @@ client.documents.archive(document_id) You can also list documents in a namespace: ```python -documents = client.documents.list(namespace="support-center") +documents = client.documents.list( + namespace="support-center", + page=1, + page_size=50, +) for document in documents.documents: print(document.document_id, document.status) +print(documents.pagination.total_pages) ``` Retrieval supports exclusions when clients want follow-up results that avoid diff --git a/docs/usage.md b/docs/usage.md index 6735cd1..b311032 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -537,9 +537,14 @@ response = client.retrieval.query( ### List, get, and archive documents ```python -document_list = client.documents.list(namespace="support-center") +document_list = client.documents.list( + namespace="support-center", + page=1, + page_size=50, +) for document in document_list.documents: print(document.document_id, document.status, document.source_file_name) +print(document_list.pagination.total_pages) document = client.documents.get("doc_123") print(document.current_job_result_id) @@ -553,6 +558,9 @@ chunks = client.documents.list_chunks( for chunk in chunks.chunks: print(chunk.id, chunk.content) +# Set include_asset_urls=True to request 7-day asset_url values for +# image and table chunks when available. + image_chunk = client.documents.get_chunk( "doc_123", "dchk_123", diff --git a/src/knowhere/__init__.py b/src/knowhere/__init__.py index bf36dfe..2b963d1 100644 --- a/src/knowhere/__init__.py +++ b/src/knowhere/__init__.py @@ -42,6 +42,7 @@ DocumentChunkPagination, DocumentChunkResponse, DocumentChunkType, + DocumentListPagination, DocumentListResponse, ) from knowhere.types.job import Job, JobError, JobProgress, JobResult @@ -112,6 +113,7 @@ "DocumentChunkPagination", "DocumentChunkResponse", "DocumentChunkType", + "DocumentListPagination", "DocumentListResponse", # Retrieval types "RetrievalChannel", diff --git a/src/knowhere/resources/documents.py b/src/knowhere/resources/documents.py index c58aebb..7551dc7 100644 --- a/src/knowhere/resources/documents.py +++ b/src/knowhere/resources/documents.py @@ -17,11 +17,19 @@ class Documents(SyncAPIResource): """Synchronous interface for ``/v1/documents`` endpoints.""" - def list(self, *, namespace: Optional[str] = None) -> DocumentListResponse: + def list( + self, + *, + namespace: Optional[str] = None, + page: int = 1, + page_size: int = 50, + ) -> DocumentListResponse: """List canonical documents in a namespace.""" - params: Dict[str, Any] = {} - if namespace is not None: - params["namespace"] = namespace + params: Dict[str, Any] = _build_document_list_params( + namespace=namespace, + page=page, + page_size=page_size, + ) return self._request( "GET", @@ -45,7 +53,7 @@ def list_chunks( page: int = 1, page_size: int = 50, chunk_type: Optional[DocumentChunkType] = None, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkListResponse: """List current-revision chunks for one canonical document.""" params: Dict[str, Any] = _build_chunk_list_params( @@ -67,7 +75,7 @@ def get_chunk( document_id: str, document_chunk_id: str, *, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkResponse: """Get one current-revision chunk for one canonical document.""" params: Dict[str, Any] = _build_chunk_get_params( @@ -93,11 +101,19 @@ def archive(self, document_id: str) -> Document: class AsyncDocuments(AsyncAPIResource): """Asynchronous interface for ``/v1/documents`` endpoints.""" - async def list(self, *, namespace: Optional[str] = None) -> DocumentListResponse: + async def list( + self, + *, + namespace: Optional[str] = None, + page: int = 1, + page_size: int = 50, + ) -> DocumentListResponse: """List canonical documents in a namespace.""" - params: Dict[str, Any] = {} - if namespace is not None: - params["namespace"] = namespace + params: Dict[str, Any] = _build_document_list_params( + namespace=namespace, + page=page, + page_size=page_size, + ) return await self._request( "GET", @@ -121,7 +137,7 @@ async def list_chunks( page: int = 1, page_size: int = 50, chunk_type: Optional[DocumentChunkType] = None, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkListResponse: """List current-revision chunks for one canonical document.""" params: Dict[str, Any] = _build_chunk_list_params( @@ -143,7 +159,7 @@ async def get_chunk( document_id: str, document_chunk_id: str, *, - include_asset_urls: bool = False, + include_asset_urls: Optional[bool] = None, ) -> DocumentChunkResponse: """Get one current-revision chunk for one canonical document.""" params: Dict[str, Any] = _build_chunk_get_params( @@ -166,12 +182,28 @@ async def archive(self, document_id: str) -> Document: ) +def _build_document_list_params( + *, + namespace: Optional[str], + page: int, + page_size: int, +) -> Dict[str, Any]: + params: Dict[str, Any] = {} + if namespace is not None: + params["namespace"] = namespace + if page != 1: + params["page"] = page + if page_size != 50: + params["page_size"] = page_size + return params + + def _build_chunk_list_params( *, page: int, page_size: int, chunk_type: Optional[DocumentChunkType], - include_asset_urls: bool, + include_asset_urls: Optional[bool], ) -> Dict[str, Any]: params: Dict[str, Any] = {} if page != 1: @@ -180,12 +212,12 @@ def _build_chunk_list_params( params["page_size"] = page_size if chunk_type is not None: params["chunk_type"] = chunk_type - if include_asset_urls: - params["include_asset_urls"] = True + if include_asset_urls is not None: + params["include_asset_urls"] = include_asset_urls return params -def _build_chunk_get_params(*, include_asset_urls: bool) -> Dict[str, Any]: - if not include_asset_urls: +def _build_chunk_get_params(*, include_asset_urls: Optional[bool]) -> Dict[str, Any]: + if include_asset_urls is None: return {} - return {"include_asset_urls": True} + return {"include_asset_urls": include_asset_urls} diff --git a/src/knowhere/types/__init__.py b/src/knowhere/types/__init__.py index 17f2cbc..14274c2 100644 --- a/src/knowhere/types/__init__.py +++ b/src/knowhere/types/__init__.py @@ -9,6 +9,7 @@ DocumentChunkPagination, DocumentChunkResponse, DocumentChunkType, + DocumentListPagination, DocumentListResponse, ) from knowhere.types.job import Job, JobError, JobResult @@ -53,6 +54,7 @@ "DocumentChunkPagination", "DocumentChunkResponse", "DocumentChunkType", + "DocumentListPagination", "DocumentListResponse", # retrieval "RetrievalChannel", diff --git a/src/knowhere/types/document.py b/src/knowhere/types/document.py index fcd1b80..e6e813c 100644 --- a/src/knowhere/types/document.py +++ b/src/knowhere/types/document.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Any, Dict, Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, model_validator class Document(BaseModel): @@ -21,11 +21,38 @@ class Document(BaseModel): archived_at: Optional[datetime] = None +class DocumentListPagination(BaseModel): + """Pagination metadata returned by document list endpoints.""" + + page: int + page_size: int + total: int + total_pages: int + + class DocumentListResponse(BaseModel): """Response from ``GET /v1/documents``.""" namespace: str documents: list[Document] + pagination: DocumentListPagination + + @model_validator(mode="before") + @classmethod + def populate_legacy_pagination(cls, value: Any) -> Any: + if not isinstance(value, dict) or "pagination" in value: + return value + + payload: Dict[str, Any] = dict(value) + documents = payload.get("documents") + document_count = len(documents) if isinstance(documents, list) else 0 + payload["pagination"] = { + "page": 1, + "page_size": document_count, + "total": document_count, + "total_pages": 1 if document_count else 0, + } + return payload DocumentChunkType = Literal["text", "image", "table"] @@ -53,7 +80,7 @@ class DocumentChunk(BaseModel): file_path: Optional[str] = None sort_order: int metadata: Dict[str, Any] - asset_url: Optional[str] = None + asset_url: Optional[str] = None # 7-day media asset URL when available. created_at: Optional[datetime] = None diff --git a/tests/test_documents.py b/tests/test_documents.py index 205d57e..eb6c7c8 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -55,23 +55,45 @@ def test_list_documents_sends_namespace_query(self, sync_client: Any) -> None: json={ "namespace": "support-center", "documents": [_make_document()], + "pagination": { + "page": 2, + "page_size": 25, + "total": 26, + "total_pages": 2, + }, }, ) ) - response = sync_client.documents.list(namespace="support-center") + response = sync_client.documents.list( + namespace="support-center", + page=2, + page_size=25, + ) assert route.called assert route.calls[0].request.url.params["namespace"] == "support-center" + assert route.calls[0].request.url.params["page"] == "2" + assert route.calls[0].request.url.params["page_size"] == "25" assert response.namespace == "support-center" assert response.documents[0].document_id == "doc_123" + assert response.pagination.total_pages == 2 @respx.mock def test_list_documents_omits_namespace_when_defaulted(self, sync_client: Any) -> None: route = respx.get(DOCUMENTS_URL).mock( return_value=httpx.Response( 200, - json={"namespace": "default", "documents": []}, + json={ + "namespace": "default", + "documents": [], + "pagination": { + "page": 1, + "page_size": 50, + "total": 0, + "total_pages": 0, + }, + }, ) ) @@ -81,6 +103,27 @@ def test_list_documents_omits_namespace_when_defaulted(self, sync_client: Any) - assert dict(route.calls[0].request.url.params) == {} assert response.namespace == "default" assert response.documents == [] + assert response.pagination.total == 0 + + @respx.mock + def test_list_documents_accepts_legacy_unpaginated_response(self, sync_client: Any) -> None: + route = respx.get(DOCUMENTS_URL).mock( + return_value=httpx.Response( + 200, + json={ + "namespace": "default", + "documents": [_make_document()], + }, + ) + ) + + response = sync_client.documents.list() + + assert route.called + assert response.pagination.page == 1 + assert response.pagination.page_size == 1 + assert response.pagination.total == 1 + assert response.pagination.total_pages == 1 @respx.mock def test_get_document_returns_document_state(self, sync_client: Any) -> None: @@ -160,9 +203,37 @@ def test_list_chunks_omits_default_query_params(self, sync_client: Any) -> None: assert response.chunks == [] assert response.pagination.total == 0 + @respx.mock + def test_list_chunks_accepts_explicit_asset_url_control( + self, sync_client: Any + ) -> None: + route = respx.get(f"{DOCUMENTS_URL}/doc_123/chunks").mock( + return_value=httpx.Response( + 200, + json={ + "document_id": "doc_123", + "namespace": "support-center", + "job_result_id": "result_123", + "job_id": "job_123", + "chunks": [], + "pagination": { + "page": 1, + "page_size": 50, + "total": 0, + "total_pages": 0, + }, + }, + ) + ) + + sync_client.documents.list_chunks("doc_123", include_asset_urls=False) + + assert route.called + assert route.calls[0].request.url.params["include_asset_urls"] == "false" + @respx.mock @pytest.mark.asyncio - async def test_async_get_chunk_requests_asset_urls_only_when_needed( + async def test_async_get_chunk_accepts_explicit_asset_url_control( self, async_client: Any, ) -> None: