From ebd038950d454826bec2658cf7e908db43ea3755 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 28 Jan 2026 11:48:52 -0600 Subject: [PATCH 1/6] speed up update entities wip Signed-off-by: phernandez --- src/basic_memory/api/app.py | 25 +- src/basic_memory/api/routers/__init__.py | 11 - .../api/routers/directory_router.py | 84 - .../api/routers/importer_router.py | 152 -- .../api/routers/knowledge_router.py | 364 ----- .../api/routers/management_router.py | 80 - src/basic_memory/api/routers/memory_router.py | 90 -- .../api/routers/project_router.py | 472 ------ src/basic_memory/api/routers/prompt_router.py | 260 --- .../api/routers/resource_router.py | 252 --- src/basic_memory/api/routers/search_router.py | 36 - .../api/v2/routers/knowledge_router.py | 96 +- .../api/v2/routers/memory_router.py | 2 +- .../api/v2/routers/project_router.py | 193 ++- .../api/v2/routers/prompt_router.py | 2 +- .../api/v2/routers/search_router.py | 2 +- src/basic_memory/api/{routers => v2}/utils.py | 0 .../cli/commands/cloud/cloud_utils.py | 4 +- .../cli/commands/command_utils.py | 4 +- src/basic_memory/cli/commands/project.py | 42 +- src/basic_memory/cli/commands/status.py | 2 +- src/basic_memory/mcp/clients/knowledge.py | 26 +- src/basic_memory/mcp/clients/project.py | 4 +- src/basic_memory/mcp/project_context.py | 23 +- .../mcp/prompts/continue_conversation.py | 10 +- src/basic_memory/mcp/prompts/search.py | 12 +- .../mcp/resources/project_info.py | 3 +- src/basic_memory/mcp/tools/edit_note.py | 2 +- src/basic_memory/mcp/tools/recent_activity.py | 2 +- src/basic_memory/mcp/tools/write_note.py | 6 +- src/basic_memory/schemas/cloud.py | 2 +- src/basic_memory/services/entity_service.py | 189 +++ tests/api/conftest.py | 40 - tests/api/test_api_container.py | 62 - tests/api/test_async_client.py | 53 - .../test_continue_conversation_template.py | 145 -- tests/api/test_directory_router.py | 212 --- tests/api/test_importer_router.py | 465 ------ tests/api/test_knowledge_router.py | 1406 ----------------- tests/api/test_management_router.py | 121 -- tests/api/test_memory_router.py | 146 -- tests/api/test_project_router.py | 843 ---------- tests/api/test_project_router_operations.py | 55 - tests/api/test_prompt_router.py | 155 -- .../test_relation_background_resolution.py | 52 - tests/api/test_resource_router.py | 454 ------ tests/api/test_search_router.py | 179 --- tests/api/test_search_template.py | 158 -- tests/api/test_template_loader.py | 219 --- tests/api/test_template_loader_helpers.py | 203 --- tests/api/v2/conftest.py | 23 + tests/api/v2/test_knowledge_router.py | 74 +- .../cloud/test_cloud_api_client_and_utils.py | 4 +- tests/mcp/clients/test_clients.py | 2 +- tests/services/test_entity_service.py | 50 + 55 files changed, 700 insertions(+), 6873 deletions(-) delete mode 100644 src/basic_memory/api/routers/__init__.py delete mode 100644 src/basic_memory/api/routers/directory_router.py delete mode 100644 src/basic_memory/api/routers/importer_router.py delete mode 100644 src/basic_memory/api/routers/knowledge_router.py delete mode 100644 src/basic_memory/api/routers/management_router.py delete mode 100644 src/basic_memory/api/routers/memory_router.py delete mode 100644 src/basic_memory/api/routers/project_router.py delete mode 100644 src/basic_memory/api/routers/prompt_router.py delete mode 100644 src/basic_memory/api/routers/resource_router.py delete mode 100644 src/basic_memory/api/routers/search_router.py rename src/basic_memory/api/{routers => v2}/utils.py (100%) delete mode 100644 tests/api/conftest.py delete mode 100644 tests/api/test_api_container.py delete mode 100644 tests/api/test_async_client.py delete mode 100644 tests/api/test_continue_conversation_template.py delete mode 100644 tests/api/test_directory_router.py delete mode 100644 tests/api/test_importer_router.py delete mode 100644 tests/api/test_knowledge_router.py delete mode 100644 tests/api/test_management_router.py delete mode 100644 tests/api/test_memory_router.py delete mode 100644 tests/api/test_project_router.py delete mode 100644 tests/api/test_project_router_operations.py delete mode 100644 tests/api/test_prompt_router.py delete mode 100644 tests/api/test_relation_background_resolution.py delete mode 100644 tests/api/test_resource_router.py delete mode 100644 tests/api/test_search_router.py delete mode 100644 tests/api/test_search_template.py delete mode 100644 tests/api/test_template_loader.py delete mode 100644 tests/api/test_template_loader_helpers.py diff --git a/src/basic_memory/api/app.py b/src/basic_memory/api/app.py index 455d9a5a..9dcf3f37 100644 --- a/src/basic_memory/api/app.py +++ b/src/basic_memory/api/app.py @@ -8,17 +8,6 @@ from basic_memory import __version__ as version from basic_memory.api.container import ApiContainer, set_container -from basic_memory.api.routers import ( - directory_router, - importer_router, - knowledge, - management, - memory, - project, - resource, - search, - prompt_router, -) from basic_memory.api.v2.routers import ( knowledge_router as v2_knowledge, project_router as v2_project, @@ -90,19 +79,7 @@ async def lifespan(app: FastAPI): # pragma: no cover app.include_router(v2_importer, prefix="/v2/projects/{project_id}") app.include_router(v2_project, prefix="/v2") -# Include v1 routers (/{project} is a catch-all, must come after specific prefixes) -app.include_router(knowledge.router, prefix="/{project}") -app.include_router(memory.router, prefix="/{project}") -app.include_router(resource.router, prefix="/{project}") -app.include_router(search.router, prefix="/{project}") -app.include_router(project.project_router, prefix="/{project}") -app.include_router(directory_router.router, prefix="/{project}") -app.include_router(prompt_router.router, prefix="/{project}") -app.include_router(importer_router.router, prefix="/{project}") - -# Project resource router works across projects -app.include_router(project.project_resource_router) -app.include_router(management.router) +# V2 routers are the only public API surface @app.exception_handler(Exception) diff --git a/src/basic_memory/api/routers/__init__.py b/src/basic_memory/api/routers/__init__.py deleted file mode 100644 index e08b61c0..00000000 --- a/src/basic_memory/api/routers/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""API routers.""" - -from . import knowledge_router as knowledge -from . import management_router as management -from . import memory_router as memory -from . import project_router as project -from . import resource_router as resource -from . import search_router as search -from . import prompt_router as prompt - -__all__ = ["knowledge", "management", "memory", "project", "resource", "search", "prompt"] diff --git a/src/basic_memory/api/routers/directory_router.py b/src/basic_memory/api/routers/directory_router.py deleted file mode 100644 index bb2150ab..00000000 --- a/src/basic_memory/api/routers/directory_router.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Router for directory tree operations.""" - -from typing import List, Optional - -from fastapi import APIRouter, Query - -from basic_memory.deps import DirectoryServiceDep, ProjectIdDep -from basic_memory.schemas.directory import DirectoryNode - -router = APIRouter(prefix="/directory", tags=["directory"]) - - -@router.get("/tree", response_model=DirectoryNode, response_model_exclude_none=True) -async def get_directory_tree( - directory_service: DirectoryServiceDep, - project_id: ProjectIdDep, -): - """Get hierarchical directory structure from the knowledge base. - - Args: - directory_service: Service for directory operations - project_id: ID of the current project - - Returns: - DirectoryNode representing the root of the hierarchical tree structure - """ - # Get a hierarchical directory tree for the specific project - tree = await directory_service.get_directory_tree() - - # Return the hierarchical tree - return tree - - -@router.get("/structure", response_model=DirectoryNode, response_model_exclude_none=True) -async def get_directory_structure( - directory_service: DirectoryServiceDep, - project_id: ProjectIdDep, -): - """Get folder structure for navigation (no files). - - Optimized endpoint for folder tree navigation. Returns only directory nodes - without file metadata. For full tree with files, use /directory/tree. - - Args: - directory_service: Service for directory operations - project_id: ID of the current project - - Returns: - DirectoryNode tree containing only folders (type="directory") - """ - structure = await directory_service.get_directory_structure() - return structure - - -@router.get("/list", response_model=List[DirectoryNode], response_model_exclude_none=True) -async def list_directory( - directory_service: DirectoryServiceDep, - project_id: ProjectIdDep, - dir_name: str = Query("/", description="Directory path to list"), - depth: int = Query(1, ge=1, le=10, description="Recursion depth (1-10)"), - file_name_glob: Optional[str] = Query( - None, description="Glob pattern for filtering file names" - ), -): - """List directory contents with filtering and depth control. - - Args: - directory_service: Service for directory operations - project_id: ID of the current project - dir_name: Directory path to list (default: root "/") - depth: Recursion depth (1-10, default: 1 for immediate children only) - file_name_glob: Optional glob pattern for filtering file names (e.g., "*.md", "*meeting*") - - Returns: - List of DirectoryNode objects matching the criteria - """ - # Get directory listing with filtering - nodes = await directory_service.list_directory( - dir_name=dir_name, - depth=depth, - file_name_glob=file_name_glob, - ) - - return nodes diff --git a/src/basic_memory/api/routers/importer_router.py b/src/basic_memory/api/routers/importer_router.py deleted file mode 100644 index 9fd716ba..00000000 --- a/src/basic_memory/api/routers/importer_router.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Import router for Basic Memory API.""" - -import json -import logging - -from fastapi import APIRouter, Form, HTTPException, UploadFile, status - -from basic_memory.deps import ( - ChatGPTImporterDep, - ClaudeConversationsImporterDep, - ClaudeProjectsImporterDep, - MemoryJsonImporterDep, -) -from basic_memory.importers import Importer -from basic_memory.schemas.importer import ( - ChatImportResult, - EntityImportResult, - ProjectImportResult, -) - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/import", tags=["import"]) - - -@router.post("/chatgpt", response_model=ChatImportResult) -async def import_chatgpt( - importer: ChatGPTImporterDep, - file: UploadFile, - directory: str = Form("conversations"), -) -> ChatImportResult: - """Import conversations from ChatGPT JSON export. - - Args: - file: The ChatGPT conversations.json file. - directory: The directory to place the files in. - markdown_processor: MarkdownProcessor instance. - - Returns: - ChatImportResult with import statistics. - - Raises: - HTTPException: If import fails. - """ - return await import_file(importer, file, directory) - - -@router.post("/claude/conversations", response_model=ChatImportResult) -async def import_claude_conversations( - importer: ClaudeConversationsImporterDep, - file: UploadFile, - directory: str = Form("conversations"), -) -> ChatImportResult: - """Import conversations from Claude conversations.json export. - - Args: - file: The Claude conversations.json file. - directory: The directory to place the files in. - markdown_processor: MarkdownProcessor instance. - - Returns: - ChatImportResult with import statistics. - - Raises: - HTTPException: If import fails. - """ - return await import_file(importer, file, directory) - - -@router.post("/claude/projects", response_model=ProjectImportResult) -async def import_claude_projects( - importer: ClaudeProjectsImporterDep, - file: UploadFile, - directory: str = Form("projects"), -) -> ProjectImportResult: - """Import projects from Claude projects.json export. - - Args: - file: The Claude projects.json file. - directory: The directory to place the files in. - markdown_processor: MarkdownProcessor instance. - - Returns: - ProjectImportResult with import statistics. - - Raises: - HTTPException: If import fails. - """ - return await import_file(importer, file, directory) - - -@router.post("/memory-json", response_model=EntityImportResult) -async def import_memory_json( - importer: MemoryJsonImporterDep, - file: UploadFile, - directory: str = Form("conversations"), -) -> EntityImportResult: - """Import entities and relations from a memory.json file. - - Args: - file: The memory.json file. - directory: Optional destination directory within the project. - markdown_processor: MarkdownProcessor instance. - - Returns: - EntityImportResult with import statistics. - - Raises: - HTTPException: If import fails. - """ - try: - file_data = [] - file_bytes = await file.read() - file_str = file_bytes.decode("utf-8") - for line in file_str.splitlines(): - json_data = json.loads(line) - file_data.append(json_data) - - result = await importer.import_data(file_data, directory) - if not result.success: # pragma: no cover - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message or "Import failed", - ) - except Exception as e: - logger.exception("Import failed") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Import failed: {str(e)}", - ) - return result - - -async def import_file(importer: Importer, file: UploadFile, destination_folder: str): - try: - # Process file - json_data = json.load(file.file) - result = await importer.import_data(json_data, destination_folder) - if not result.success: # pragma: no cover - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message or "Import failed", - ) - - return result - - except Exception as e: - logger.exception("Import failed") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Import failed: {str(e)}", - ) diff --git a/src/basic_memory/api/routers/knowledge_router.py b/src/basic_memory/api/routers/knowledge_router.py deleted file mode 100644 index 7c501392..00000000 --- a/src/basic_memory/api/routers/knowledge_router.py +++ /dev/null @@ -1,364 +0,0 @@ -"""Router for knowledge graph operations. - -⚠️ DEPRECATED: This v1 API is deprecated and will be removed on June 30, 2026. -Please migrate to /v2/{project}/knowledge endpoints which use entity IDs instead -of path-based identifiers for improved performance and stability. - -Migration guide: See docs/migration/v1-to-v2.md -""" - -from typing import Annotated - -from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Query, Response -from loguru import logger - -from basic_memory.deps import ( - EntityServiceDep, - get_search_service, - SearchServiceDep, - LinkResolverDep, - ProjectPathDep, - FileServiceDep, - ProjectConfigDep, - AppConfigDep, - SyncServiceDep, -) -from basic_memory.schemas import ( - EntityListResponse, - EntityResponse, - DeleteEntitiesResponse, - DeleteEntitiesRequest, -) -from basic_memory.schemas.request import EditEntityRequest, MoveEntityRequest, MoveDirectoryRequest -from basic_memory.schemas.response import DirectoryMoveResult -from basic_memory.schemas.base import Permalink, Entity - -router = APIRouter( - prefix="/knowledge", - tags=["knowledge"], - deprecated=True, # Marks entire router as deprecated in OpenAPI docs -) - - -async def resolve_relations_background(sync_service, entity_id: int, entity_permalink: str) -> None: - """Background task to resolve relations for a specific entity. - - This runs asynchronously after the API response is sent, preventing - long delays when creating entities with many relations. - """ - try: - # Only resolve relations for the newly created entity - await sync_service.resolve_relations(entity_id=entity_id) - logger.debug( - f"Background: Resolved relations for entity {entity_permalink} (id={entity_id})" - ) - except Exception as e: # pragma: no cover - # Log but don't fail - this is a background task. - # Avoid forcing synthetic failures just for coverage. - logger.warning( # pragma: no cover - f"Background: Failed to resolve relations for entity {entity_permalink}: {e}" - ) - - -## Create endpoints - - -@router.post("/entities", response_model=EntityResponse) -async def create_entity( - data: Entity, - background_tasks: BackgroundTasks, - entity_service: EntityServiceDep, - search_service: SearchServiceDep, -) -> EntityResponse: - """Create an entity.""" - logger.info( - "API request", endpoint="create_entity", entity_type=data.entity_type, title=data.title - ) - - entity = await entity_service.create_entity(data) - - # reindex - await search_service.index_entity(entity, background_tasks=background_tasks) - result = EntityResponse.model_validate(entity) - - logger.info( - f"API response: endpoint='create_entity' title={result.title}, permalink={result.permalink}, status_code=201" - ) - return result - - -@router.put("/entities/{permalink:path}", response_model=EntityResponse) -async def create_or_update_entity( - project: ProjectPathDep, - permalink: Permalink, - data: Entity, - response: Response, - background_tasks: BackgroundTasks, - entity_service: EntityServiceDep, - search_service: SearchServiceDep, - file_service: FileServiceDep, - sync_service: SyncServiceDep, -) -> EntityResponse: - """Create or update an entity. If entity exists, it will be updated, otherwise created.""" - logger.info( - f"API request: create_or_update_entity for {project=}, {permalink=}, {data.entity_type=}, {data.title=}" - ) - - # Validate permalink matches - if data.permalink != permalink: - logger.warning( - f"API validation error: creating/updating entity with permalink mismatch - url={permalink}, data={data.permalink}", - ) - raise HTTPException( - status_code=400, - detail=f"Entity permalink {data.permalink} must match URL path: '{permalink}'", - ) - - # Try create_or_update operation - entity, created = await entity_service.create_or_update_entity(data) - response.status_code = 201 if created else 200 - - # reindex - await search_service.index_entity(entity, background_tasks=background_tasks) - - # Schedule relation resolution as a background task for new entities - # This prevents blocking the API response while resolving potentially many relations - if created: - background_tasks.add_task( - resolve_relations_background, sync_service, entity.id, entity.permalink or "" - ) - - result = EntityResponse.model_validate(entity) - - logger.info( - f"API response: {result.title=}, {result.permalink=}, {created=}, status_code={response.status_code}" - ) - return result - - -@router.patch("/entities/{identifier:path}", response_model=EntityResponse) -async def edit_entity( - identifier: str, - data: EditEntityRequest, - background_tasks: BackgroundTasks, - entity_service: EntityServiceDep, - search_service: SearchServiceDep, -) -> EntityResponse: - """Edit an existing entity using various operations like append, prepend, find_replace, or replace_section. - - This endpoint allows for targeted edits without requiring the full entity content. - """ - logger.info( - f"API request: endpoint='edit_entity', identifier='{identifier}', operation='{data.operation}'" - ) - - try: - # Edit the entity using the service - entity = await entity_service.edit_entity( - identifier=identifier, - operation=data.operation, - content=data.content, - section=data.section, - find_text=data.find_text, - expected_replacements=data.expected_replacements, - ) - - # Reindex the updated entity - await search_service.index_entity(entity, background_tasks=background_tasks) - - # Return the updated entity response - result = EntityResponse.model_validate(entity) - - logger.info( - "API response", - endpoint="edit_entity", - identifier=identifier, - operation=data.operation, - permalink=result.permalink, - status_code=200, - ) - - return result - - except Exception as e: - logger.error(f"Error editing entity: {e}") - raise HTTPException(status_code=400, detail=str(e)) - - -@router.post("/move") -async def move_entity( - data: MoveEntityRequest, - background_tasks: BackgroundTasks, - entity_service: EntityServiceDep, - project_config: ProjectConfigDep, - app_config: AppConfigDep, - search_service: SearchServiceDep, -) -> EntityResponse: - """Move an entity to a new file location with project consistency. - - This endpoint moves a note to a different path while maintaining project - consistency and optionally updating permalinks based on configuration. - """ - logger.info( - f"API request: endpoint='move_entity', identifier='{data.identifier}', destination='{data.destination_path}'" - ) - - try: - # Move the entity using the service - moved_entity = await entity_service.move_entity( - identifier=data.identifier, - destination_path=data.destination_path, - project_config=project_config, - app_config=app_config, - ) - - # Get the moved entity to reindex it - entity = await entity_service.link_resolver.resolve_link(data.destination_path) - if entity: - await search_service.index_entity(entity, background_tasks=background_tasks) - - logger.info( - "API response", - endpoint="move_entity", - identifier=data.identifier, - destination=data.destination_path, - status_code=200, - ) - result = EntityResponse.model_validate(moved_entity) - return result - - except Exception as e: - logger.error(f"Error moving entity: {e}") - raise HTTPException(status_code=400, detail=str(e)) - - -@router.post("/move-directory") -async def move_directory( - data: MoveDirectoryRequest, - background_tasks: BackgroundTasks, - entity_service: EntityServiceDep, - project_config: ProjectConfigDep, - app_config: AppConfigDep, - search_service: SearchServiceDep, -) -> DirectoryMoveResult: - """Move all entities in a directory to a new location. - - This endpoint moves all files within a source directory to a destination - directory, updating database records and optionally updating permalinks. - """ - logger.info( - f"API request: endpoint='move_directory', source='{data.source_directory}', destination='{data.destination_directory}'" - ) - - try: - # Move the directory using the service - result = await entity_service.move_directory( - source_directory=data.source_directory, - destination_directory=data.destination_directory, - project_config=project_config, - app_config=app_config, - ) - - # Reindex moved entities - for file_path in result.moved_files: - entity = await entity_service.link_resolver.resolve_link(file_path) - if entity: - await search_service.index_entity(entity, background_tasks=background_tasks) - - logger.info( - f"API response: endpoint='move_directory', " - f"total={result.total_files}, success={result.successful_moves}, failed={result.failed_moves}" - ) - return result - - except Exception as e: - logger.error(f"Error moving directory: {e}") - raise HTTPException(status_code=400, detail=str(e)) - - -## Read endpoints - - -@router.get("/entities/{identifier:path}", response_model=EntityResponse) -async def get_entity( - entity_service: EntityServiceDep, - link_resolver: LinkResolverDep, - identifier: str, -) -> EntityResponse: - """Get a specific entity by file path or permalink.. - - Args: - identifier: Entity file path or permalink - :param entity_service: EntityService - :param link_resolver: LinkResolver - """ - logger.info(f"request: get_entity with identifier={identifier}") - entity = await link_resolver.resolve_link(identifier) - if not entity: - raise HTTPException(status_code=404, detail=f"Entity {identifier} not found") - - result = EntityResponse.model_validate(entity) - return result - - -@router.get("/entities", response_model=EntityListResponse) -async def get_entities( - entity_service: EntityServiceDep, - permalink: Annotated[list[str] | None, Query()] = None, -) -> EntityListResponse: - """Open specific entities""" - logger.info(f"request: get_entities with permalinks={permalink}") - - entities = await entity_service.get_entities_by_permalinks(permalink) if permalink else [] - result = EntityListResponse( - entities=[EntityResponse.model_validate(entity) for entity in entities] - ) - return result - - -## Delete endpoints - - -@router.delete("/entities/{identifier:path}", response_model=DeleteEntitiesResponse) -async def delete_entity( - identifier: str, - background_tasks: BackgroundTasks, - entity_service: EntityServiceDep, - link_resolver: LinkResolverDep, - search_service=Depends(get_search_service), -) -> DeleteEntitiesResponse: - """Delete a single entity and remove from search index.""" - logger.info(f"request: delete_entity with identifier={identifier}") - - entity = await link_resolver.resolve_link(identifier) - if entity is None: - return DeleteEntitiesResponse(deleted=False) - - # Delete the entity - deleted = await entity_service.delete_entity(entity.permalink or entity.id) - - # Remove from search index (entity, observations, and relations) - background_tasks.add_task(search_service.handle_delete, entity) - - result = DeleteEntitiesResponse(deleted=deleted) - return result - - -@router.post("/entities/delete", response_model=DeleteEntitiesResponse) -async def delete_entities( - data: DeleteEntitiesRequest, - background_tasks: BackgroundTasks, - entity_service: EntityServiceDep, - search_service=Depends(get_search_service), -) -> DeleteEntitiesResponse: - """Delete entities and remove from search index.""" - logger.info(f"request: delete_entities with data={data}") - deleted = False - - # Remove each deleted entity from search index - for permalink in data.permalinks: - deleted = await entity_service.delete_entity(permalink) - background_tasks.add_task(search_service.delete_by_permalink, permalink) - - result = DeleteEntitiesResponse(deleted=deleted) - return result diff --git a/src/basic_memory/api/routers/management_router.py b/src/basic_memory/api/routers/management_router.py deleted file mode 100644 index 5be51723..00000000 --- a/src/basic_memory/api/routers/management_router.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Management router for basic-memory API.""" - -import asyncio - -from fastapi import APIRouter, Request -from loguru import logger -from pydantic import BaseModel - -from basic_memory.config import ConfigManager -from basic_memory.deps import SyncServiceDep, ProjectRepositoryDep - -router = APIRouter(prefix="/management", tags=["management"]) - - -class WatchStatusResponse(BaseModel): - """Response model for watch status.""" - - running: bool - """Whether the watch service is currently running.""" - - -@router.get("/watch/status", response_model=WatchStatusResponse) -async def get_watch_status(request: Request) -> WatchStatusResponse: - """Get the current status of the watch service.""" - return WatchStatusResponse( - running=request.app.state.watch_task is not None and not request.app.state.watch_task.done() - ) - - -@router.post("/watch/start", response_model=WatchStatusResponse) -async def start_watch_service( - request: Request, project_repository: ProjectRepositoryDep, sync_service: SyncServiceDep -) -> WatchStatusResponse: - """Start the watch service if it's not already running.""" - - # needed because of circular imports from sync -> app - from basic_memory.sync import WatchService - from basic_memory.sync.background_sync import create_background_sync_task - - if request.app.state.watch_task is not None and not request.app.state.watch_task.done(): - # Watch service is already running - return WatchStatusResponse(running=True) - - app_config = ConfigManager().config - - # Create and start a new watch service - logger.info("Starting watch service via management API") - - # Get services needed for the watch task - watch_service = WatchService( - app_config=app_config, - project_repository=project_repository, - ) - - # Create and store the task - watch_task = create_background_sync_task(sync_service, watch_service) - request.app.state.watch_task = watch_task - - return WatchStatusResponse(running=True) - - -@router.post("/watch/stop", response_model=WatchStatusResponse) -async def stop_watch_service(request: Request) -> WatchStatusResponse: # pragma: no cover - """Stop the watch service if it's running.""" - if request.app.state.watch_task is None or request.app.state.watch_task.done(): - # Watch service is not running - return WatchStatusResponse(running=False) - - # Cancel the running task - logger.info("Stopping watch service via management API") - request.app.state.watch_task.cancel() - - # Wait for it to be properly cancelled - try: - await request.app.state.watch_task - except asyncio.CancelledError: - pass - - request.app.state.watch_task = None - return WatchStatusResponse(running=False) diff --git a/src/basic_memory/api/routers/memory_router.py b/src/basic_memory/api/routers/memory_router.py deleted file mode 100644 index e71260c5..00000000 --- a/src/basic_memory/api/routers/memory_router.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Routes for memory:// URI operations.""" - -from typing import Annotated, Optional - -from fastapi import APIRouter, Query -from loguru import logger - -from basic_memory.deps import ContextServiceDep, EntityRepositoryDep -from basic_memory.schemas.base import TimeFrame, parse_timeframe -from basic_memory.schemas.memory import ( - GraphContext, - normalize_memory_url, -) -from basic_memory.schemas.search import SearchItemType -from basic_memory.api.routers.utils import to_graph_context - -router = APIRouter(prefix="/memory", tags=["memory"]) - - -@router.get("/recent", response_model=GraphContext) -async def recent( - context_service: ContextServiceDep, - entity_repository: EntityRepositoryDep, - type: Annotated[list[SearchItemType] | None, Query()] = None, - depth: int = 1, - timeframe: TimeFrame = "7d", - page: int = 1, - page_size: int = 10, - max_related: int = 10, -) -> GraphContext: - # return all types by default - types = ( - [SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION] - if not type - else type - ) - - logger.debug( - f"Getting recent context: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`" - ) - # Parse timeframe - since = parse_timeframe(timeframe) - limit = page_size - offset = (page - 1) * page_size - - # Build context - context = await context_service.build_context( - types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related - ) - recent_context = await to_graph_context( - context, entity_repository=entity_repository, page=page, page_size=page_size - ) - logger.debug(f"Recent context: {recent_context.model_dump_json()}") - return recent_context - - -# get_memory_context needs to be declared last so other paths can match - - -@router.get("/{uri:path}", response_model=GraphContext) -async def get_memory_context( - context_service: ContextServiceDep, - entity_repository: EntityRepositoryDep, - uri: str, - depth: int = 1, - timeframe: Optional[TimeFrame] = None, - page: int = 1, - page_size: int = 10, - max_related: int = 10, -) -> GraphContext: - """Get rich context from memory:// URI.""" - # add the project name from the config to the url as the "host - # Parse URI - logger.debug( - f"Getting context for URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`" - ) - memory_url = normalize_memory_url(uri) - - # Parse timeframe - since = parse_timeframe(timeframe) if timeframe else None - limit = page_size - offset = (page - 1) * page_size - - # Build context - context = await context_service.build_context( - memory_url, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related - ) - return await to_graph_context( - context, entity_repository=entity_repository, page=page, page_size=page_size - ) diff --git a/src/basic_memory/api/routers/project_router.py b/src/basic_memory/api/routers/project_router.py deleted file mode 100644 index b951dc42..00000000 --- a/src/basic_memory/api/routers/project_router.py +++ /dev/null @@ -1,472 +0,0 @@ -"""Router for project management.""" - -import os -from fastapi import APIRouter, HTTPException, Path, Body, BackgroundTasks, Response, Query -from typing import Optional -from loguru import logger - -from basic_memory.deps import ( - ProjectConfigDep, - ProjectServiceDep, - ProjectPathDep, - SyncServiceDep, -) -from basic_memory.schemas import ProjectInfoResponse, SyncReportResponse -from basic_memory.schemas.project_info import ( - ProjectList, - ProjectItem, - ProjectInfoRequest, - ProjectStatusResponse, -) -from basic_memory.utils import normalize_project_path - -# Router for resources in a specific project -# The ProjectPathDep is used in the path as a prefix, so the request path is like /{project}/project/info -project_router = APIRouter(prefix="/project", tags=["project"]) - -# Router for managing project resources -project_resource_router = APIRouter(prefix="/projects", tags=["project_management"]) - - -@project_router.get("/info", response_model=ProjectInfoResponse) -async def get_project_info( - project_service: ProjectServiceDep, - project: ProjectPathDep, -) -> ProjectInfoResponse: - """Get comprehensive information about the specified Basic Memory project.""" - return await project_service.get_project_info(project) - - -@project_router.get("/item", response_model=ProjectItem) -async def get_project( - project_service: ProjectServiceDep, - project: ProjectPathDep, -) -> ProjectItem: - """Get bassic info about the specified Basic Memory project.""" - found_project = await project_service.get_project(project) - if not found_project: - raise HTTPException( - status_code=404, detail=f"Project: '{project}' does not exist" - ) # pragma: no cover - - return ProjectItem( - id=found_project.id, - external_id=found_project.external_id, - name=found_project.name, - path=normalize_project_path(found_project.path), - is_default=found_project.is_default or False, - ) - - -# Update a project -@project_router.patch("/{name}", response_model=ProjectStatusResponse) -async def update_project( - project_service: ProjectServiceDep, - name: str = Path(..., description="Name of the project to update"), - path: Optional[str] = Body(None, description="New absolute path for the project"), - is_active: Optional[bool] = Body(None, description="Status of the project (active/inactive)"), -) -> ProjectStatusResponse: - """Update a project's information in configuration and database. - - Args: - name: The name of the project to update - path: Optional new absolute path for the project - is_active: Optional status update for the project - - Returns: - Response confirming the project was updated - """ - try: - # Validate that path is absolute if provided - if path and not os.path.isabs(path): - raise HTTPException(status_code=400, detail="Path must be absolute") - - # Get original project info for the response - old_project = await project_service.get_project(name) - if not old_project: - raise HTTPException( - status_code=400, detail=f"Project '{name}' not found in configuration" - ) - - old_project_info = ProjectItem( - id=old_project.id, - external_id=old_project.external_id, - name=old_project.name, - path=old_project.path, - is_default=old_project.is_default or False, - ) - - if path: - await project_service.move_project(name, path) - elif is_active is not None: - await project_service.update_project(name, is_active=is_active) - - # Get updated project info - updated_project = await project_service.get_project(name) - if not updated_project: - raise HTTPException( # pragma: no cover - status_code=404, detail=f"Project '{name}' not found after update" - ) - - return ProjectStatusResponse( - message=f"Project '{name}' updated successfully", - status="success", - default=(name == project_service.default_project), - old_project=old_project_info, - new_project=ProjectItem( - id=updated_project.id, - external_id=updated_project.external_id, - name=updated_project.name, - path=updated_project.path, - is_default=updated_project.is_default or False, - ), - ) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) # pragma: no cover - - -# Sync project filesystem -@project_router.post("/sync") -async def sync_project( - background_tasks: BackgroundTasks, - sync_service: SyncServiceDep, - project_config: ProjectConfigDep, - force_full: bool = Query( - False, description="Force full scan, bypassing watermark optimization" - ), - run_in_background: bool = Query(True, description="Run in background"), -): - """Force project filesystem sync to database. - - Scans the project directory and updates the database with any new or modified files. - - Args: - background_tasks: FastAPI background tasks - sync_service: Sync service for this project - project_config: Project configuration - force_full: If True, force a full scan even if watermark exists - run_in_background: If True, run sync in background and return immediately - - Returns: - Response confirming sync was initiated (background) or SyncReportResponse (foreground) - """ - if run_in_background: - background_tasks.add_task( - sync_service.sync, project_config.home, project_config.name, force_full=force_full - ) - logger.info( - f"Filesystem sync initiated for project: {project_config.name} (force_full={force_full})" - ) - - return { - "status": "sync_started", - "message": f"Filesystem sync initiated for project '{project_config.name}'", - } - else: - report = await sync_service.sync( - project_config.home, project_config.name, force_full=force_full - ) - logger.info( - f"Filesystem sync completed for project: {project_config.name} (force_full={force_full})" - ) - return SyncReportResponse.from_sync_report(report) - - -@project_router.post("/status", response_model=SyncReportResponse) -async def project_sync_status( - sync_service: SyncServiceDep, - project_config: ProjectConfigDep, -) -> SyncReportResponse: - """Scan directory for changes compared to database state. - - Args: - sync_service: Sync service for this project - project_config: Project configuration - - Returns: - Scan report with details on files that need syncing - """ - logger.info(f"Scanning filesystem for project: {project_config.name}") # pragma: no cover - sync_report = await sync_service.scan(project_config.home) # pragma: no cover - - return SyncReportResponse.from_sync_report(sync_report) # pragma: no cover - - -# List all available projects -@project_resource_router.get("/projects", response_model=ProjectList) -async def list_projects( - project_service: ProjectServiceDep, -) -> ProjectList: - """List all configured projects. - - Returns: - A list of all projects with metadata - """ - projects = await project_service.list_projects() - default_project = project_service.default_project - - project_items = [ - ProjectItem( - id=project.id, - external_id=project.external_id, - name=project.name, - path=normalize_project_path(project.path), - is_default=project.is_default or False, - ) - for project in projects - ] - - return ProjectList( - projects=project_items, - default_project=default_project, - ) - - -# Add a new project -@project_resource_router.post("/projects", response_model=ProjectStatusResponse, status_code=201) -async def add_project( - response: Response, - project_data: ProjectInfoRequest, - project_service: ProjectServiceDep, -) -> ProjectStatusResponse: - """Add a new project to configuration and database. - - Args: - project_data: The project name and path, with option to set as default - - Returns: - Response confirming the project was added - """ - # Check if project already exists before attempting to add - existing_project = await project_service.get_project(project_data.name) - if existing_project: - # Project exists - check if paths match for true idempotency - # Normalize paths for comparison (resolve symlinks, etc.) - from pathlib import Path - - requested_path = Path(project_data.path).resolve() - existing_path = Path(existing_project.path).resolve() - - if requested_path == existing_path: - # Same name, same path - return 200 OK (idempotent) - response.status_code = 200 - return ProjectStatusResponse( # pyright: ignore [reportCallIssue] - message=f"Project '{project_data.name}' already exists", - status="success", - default=existing_project.is_default or False, - new_project=ProjectItem( - id=existing_project.id, - external_id=existing_project.external_id, - name=existing_project.name, - path=existing_project.path, - is_default=existing_project.is_default or False, - ), - ) - else: - # Same name, different path - this is an error - raise HTTPException( - status_code=400, - detail=f"Project '{project_data.name}' already exists with different path. Existing: {existing_project.path}, Requested: {project_data.path}", - ) - - try: # pragma: no cover - # The service layer now handles cloud mode validation and path sanitization - await project_service.add_project( - project_data.name, project_data.path, set_default=project_data.set_default - ) - - # Fetch the newly created project to get its ID - new_project = await project_service.get_project(project_data.name) - if not new_project: - raise HTTPException(status_code=500, detail="Failed to retrieve newly created project") - - return ProjectStatusResponse( # pyright: ignore [reportCallIssue] - message=f"Project '{new_project.name}' added successfully", - status="success", - default=project_data.set_default, - new_project=ProjectItem( - id=new_project.id, - external_id=new_project.external_id, - name=new_project.name, - path=new_project.path, - is_default=new_project.is_default or False, - ), - ) - except ValueError as e: # pragma: no cover - raise HTTPException(status_code=400, detail=str(e)) - - -# Remove a project -@project_resource_router.delete("/{name}", response_model=ProjectStatusResponse) -async def remove_project( - project_service: ProjectServiceDep, - name: str = Path(..., description="Name of the project to remove"), - delete_notes: bool = Query( - False, description="If True, delete project directory from filesystem" - ), -) -> ProjectStatusResponse: - """Remove a project from configuration and database. - - Args: - name: The name of the project to remove - delete_notes: If True, delete the project directory from the filesystem - - Returns: - Response confirming the project was removed - """ - try: - old_project = await project_service.get_project(name) - if not old_project: # pragma: no cover - raise HTTPException( - status_code=404, detail=f"Project: '{name}' does not exist" - ) # pragma: no cover - - # Check if trying to delete the default project - # In cloud mode, database is source of truth; in local mode, check config - config_default = project_service.default_project - db_default = await project_service.repository.get_default_project() - - # Use database default if available, otherwise fall back to config default - default_project_name = db_default.name if db_default else config_default - - if name == default_project_name: - available_projects = await project_service.list_projects() - other_projects = [p.name for p in available_projects if p.name != name] - detail = f"Cannot delete default project '{name}'. " - if other_projects: - detail += ( - f"Set another project as default first. Available: {', '.join(other_projects)}" - ) - else: - detail += "This is the only project in your configuration." - raise HTTPException(status_code=400, detail=detail) - - await project_service.remove_project(name, delete_notes=delete_notes) - - return ProjectStatusResponse( - message=f"Project '{old_project.name}' removed successfully", - status="success", - default=False, - old_project=ProjectItem( - id=old_project.id, - external_id=old_project.external_id, - name=old_project.name, - path=old_project.path, - is_default=old_project.is_default or False, - ), - new_project=None, - ) - except ValueError as e: # pragma: no cover - raise HTTPException(status_code=400, detail=str(e)) - - -# Set a project as default -@project_resource_router.put("/{name}/default", response_model=ProjectStatusResponse) -async def set_default_project( - project_service: ProjectServiceDep, - name: str = Path(..., description="Name of the project to set as default"), -) -> ProjectStatusResponse: - """Set a project as the default project. - - Args: - name: The name of the project to set as default - - Returns: - Response confirming the project was set as default - """ - try: - # Get the old default project - default_name = project_service.default_project - default_project = await project_service.get_project(default_name) - if not default_project: # pragma: no cover - raise HTTPException( # pragma: no cover - status_code=404, detail=f"Default Project: '{default_name}' does not exist" - ) - - # get the new project - new_default_project = await project_service.get_project(name) - if not new_default_project: # pragma: no cover - raise HTTPException( - status_code=404, detail=f"Project: '{name}' does not exist" - ) # pragma: no cover - - await project_service.set_default_project(name) - - return ProjectStatusResponse( - message=f"Project '{name}' set as default successfully", - status="success", - default=True, - old_project=ProjectItem( - id=default_project.id, - external_id=default_project.external_id, - name=default_name, - path=default_project.path, - is_default=False, - ), - new_project=ProjectItem( - id=new_default_project.id, - external_id=new_default_project.external_id, - name=name, - path=new_default_project.path, - is_default=True, - ), - ) - except ValueError as e: # pragma: no cover - raise HTTPException(status_code=400, detail=str(e)) - - -# Get the default project -@project_resource_router.get("/default", response_model=ProjectItem) -async def get_default_project( - project_service: ProjectServiceDep, -) -> ProjectItem: - """Get the default project. - - Returns: - Response with project default information - """ - # Get the default project - # In cloud mode, database is source of truth; in local mode, check config - config_default = project_service.default_project - db_default = await project_service.repository.get_default_project() - - # Use database default if available, otherwise fall back to config default - default_name = db_default.name if db_default else config_default - default_project = await project_service.get_project(default_name) - if not default_project: # pragma: no cover - raise HTTPException( # pragma: no cover - status_code=404, detail=f"Default Project: '{default_name}' does not exist" - ) - - return ProjectItem( - id=default_project.id, - external_id=default_project.external_id, - name=default_project.name, - path=default_project.path, - is_default=True, - ) - - -# Synchronize projects between config and database -@project_resource_router.post("/config/sync", response_model=ProjectStatusResponse) -async def synchronize_projects( - project_service: ProjectServiceDep, -) -> ProjectStatusResponse: - """Synchronize projects between configuration file and database. - - Ensures that all projects in the configuration file exist in the database - and vice versa. - - Returns: - Response confirming synchronization was completed - """ - try: # pragma: no cover - await project_service.synchronize_projects() - - return ProjectStatusResponse( # pyright: ignore [reportCallIssue] - message="Projects synchronized successfully between configuration and database", - status="success", - default=False, - ) - except ValueError as e: # pragma: no cover - raise HTTPException(status_code=400, detail=str(e)) diff --git a/src/basic_memory/api/routers/prompt_router.py b/src/basic_memory/api/routers/prompt_router.py deleted file mode 100644 index f391bb9d..00000000 --- a/src/basic_memory/api/routers/prompt_router.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Router for prompt-related operations. - -This router is responsible for rendering various prompts using Handlebars templates. -It centralizes all prompt formatting logic that was previously in the MCP prompts. -""" - -from datetime import datetime, timezone -from fastapi import APIRouter, HTTPException, status -from loguru import logger - -from basic_memory.api.routers.utils import to_graph_context, to_search_results -from basic_memory.api.template_loader import template_loader -from basic_memory.schemas.base import parse_timeframe -from basic_memory.deps import ( - ContextServiceDep, - EntityRepositoryDep, - SearchServiceDep, - EntityServiceDep, -) -from basic_memory.schemas.prompt import ( - ContinueConversationRequest, - SearchPromptRequest, - PromptResponse, - PromptMetadata, -) -from basic_memory.schemas.search import SearchItemType, SearchQuery - -router = APIRouter(prefix="/prompt", tags=["prompt"]) - - -@router.post("/continue-conversation", response_model=PromptResponse) -async def continue_conversation( - search_service: SearchServiceDep, - entity_service: EntityServiceDep, - context_service: ContextServiceDep, - entity_repository: EntityRepositoryDep, - request: ContinueConversationRequest, -) -> PromptResponse: - """Generate a prompt for continuing a conversation. - - This endpoint takes a topic and/or timeframe and generates a prompt with - relevant context from the knowledge base. - - Args: - request: The request parameters - - Returns: - Formatted continuation prompt with context - """ - logger.info( - f"Generating continue conversation prompt, topic: {request.topic}, timeframe: {request.timeframe}" - ) - - since = parse_timeframe(request.timeframe) if request.timeframe else None - - # Initialize search results - search_results = [] - - # Get data needed for template - if request.topic: - query = SearchQuery(text=request.topic, after_date=request.timeframe) - results = await search_service.search(query, limit=request.search_items_limit) - search_results = await to_search_results(entity_service, results) - - # Build context from results - all_hierarchical_results = [] - for result in search_results: - if hasattr(result, "permalink") and result.permalink: - # Get hierarchical context using the new dataclass-based approach - context_result = await context_service.build_context( - result.permalink, - depth=request.depth, - since=since, - max_related=request.related_items_limit, - include_observations=True, # Include observations for entities - ) - - # Process results into the schema format - graph_context = await to_graph_context( - context_result, entity_repository=entity_repository - ) - - # Add results to our collection (limit to top results for each permalink) - if graph_context.results: - all_hierarchical_results.extend(graph_context.results[:3]) - - # Limit to a reasonable number of total results - all_hierarchical_results = all_hierarchical_results[:10] - - template_context = { - "topic": request.topic, - "timeframe": request.timeframe, - "hierarchical_results": all_hierarchical_results, - "has_results": len(all_hierarchical_results) > 0, - } - else: - # If no topic, get recent activity - context_result = await context_service.build_context( - types=[SearchItemType.ENTITY], - depth=request.depth, - since=since, - max_related=request.related_items_limit, - include_observations=True, - ) - recent_context = await to_graph_context(context_result, entity_repository=entity_repository) - - hierarchical_results = recent_context.results[:5] # Limit to top 5 recent items - - template_context = { - "topic": f"Recent Activity from ({request.timeframe})", - "timeframe": request.timeframe, - "hierarchical_results": hierarchical_results, - "has_results": len(hierarchical_results) > 0, - } - - try: - # Render template - rendered_prompt = await template_loader.render( - "prompts/continue_conversation.hbs", template_context - ) - - # Calculate metadata - # Count items of different types - observation_count = 0 - relation_count = 0 - entity_count = 0 - - # Get the hierarchical results from the template context - hierarchical_results_for_count = template_context.get("hierarchical_results", []) - - # For topic-based search - if request.topic: - for item in hierarchical_results_for_count: - if hasattr(item, "observations"): - observation_count += len(item.observations) if item.observations else 0 - - if hasattr(item, "related_results"): - for related in item.related_results or []: - if hasattr(related, "type"): - if related.type == "relation": - relation_count += 1 - elif related.type == "entity": # pragma: no cover - entity_count += 1 # pragma: no cover - # For recent activity - else: - for item in hierarchical_results_for_count: - if hasattr(item, "observations"): - observation_count += len(item.observations) if item.observations else 0 - - if hasattr(item, "related_results"): - for related in item.related_results or []: - if hasattr(related, "type"): - if related.type == "relation": - relation_count += 1 - elif related.type == "entity": # pragma: no cover - entity_count += 1 # pragma: no cover - - # Build metadata - metadata = { - "query": request.topic, - "timeframe": request.timeframe, - "search_count": len(search_results) - if request.topic - else 0, # Original search results count - "context_count": len(hierarchical_results_for_count), - "observation_count": observation_count, - "relation_count": relation_count, - "total_items": ( - len(hierarchical_results_for_count) - + observation_count - + relation_count - + entity_count - ), - "search_limit": request.search_items_limit, - "context_depth": request.depth, - "related_limit": request.related_items_limit, - "generated_at": datetime.now(timezone.utc).isoformat(), - } - - prompt_metadata = PromptMetadata(**metadata) - - return PromptResponse( - prompt=rendered_prompt, context=template_context, metadata=prompt_metadata - ) - except Exception as e: - logger.error(f"Error rendering continue conversation template: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error rendering prompt template: {str(e)}", - ) - - -@router.post("/search", response_model=PromptResponse) -async def search_prompt( - search_service: SearchServiceDep, - entity_service: EntityServiceDep, - request: SearchPromptRequest, - page: int = 1, - page_size: int = 10, -) -> PromptResponse: - """Generate a prompt for search results. - - This endpoint takes a search query and formats the results into a helpful - prompt with context and suggestions. - - Args: - request: The search parameters - page: The page number for pagination - page_size: The number of results per page, defaults to 10 - - Returns: - Formatted search results prompt with context - """ - logger.info(f"Generating search prompt, query: {request.query}, timeframe: {request.timeframe}") - - limit = page_size - offset = (page - 1) * page_size - - query = SearchQuery(text=request.query, after_date=request.timeframe) - results = await search_service.search(query, limit=limit, offset=offset) - search_results = await to_search_results(entity_service, results) - - template_context = { - "query": request.query, - "timeframe": request.timeframe, - "results": search_results, - "has_results": len(search_results) > 0, - "result_count": len(search_results), - } - - try: - # Render template - rendered_prompt = await template_loader.render("prompts/search.hbs", template_context) - - # Build metadata - metadata = { - "query": request.query, - "timeframe": request.timeframe, - "search_count": len(search_results), - "context_count": len(search_results), - "observation_count": 0, # Search results don't include observations - "relation_count": 0, # Search results don't include relations - "total_items": len(search_results), - "search_limit": limit, - "context_depth": 0, # No context depth for basic search - "related_limit": 0, # No related items for basic search - "generated_at": datetime.now(timezone.utc).isoformat(), - } - - prompt_metadata = PromptMetadata(**metadata) - - return PromptResponse( - prompt=rendered_prompt, context=template_context, metadata=prompt_metadata - ) - except Exception as e: - logger.error(f"Error rendering search template: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error rendering prompt template: {str(e)}", - ) diff --git a/src/basic_memory/api/routers/resource_router.py b/src/basic_memory/api/routers/resource_router.py deleted file mode 100644 index ad852c4f..00000000 --- a/src/basic_memory/api/routers/resource_router.py +++ /dev/null @@ -1,252 +0,0 @@ -"""Routes for getting entity content.""" - -import tempfile -import uuid -from pathlib import Path -from typing import Annotated, Union - -from fastapi import APIRouter, HTTPException, BackgroundTasks, Body, Response -from fastapi.responses import FileResponse, JSONResponse -from loguru import logger - -from basic_memory.deps import ( - ProjectConfigDep, - LinkResolverDep, - SearchServiceDep, - EntityServiceDep, - FileServiceDep, - EntityRepositoryDep, -) -from basic_memory.repository.search_repository import SearchIndexRow -from basic_memory.schemas.memory import normalize_memory_url -from basic_memory.schemas.search import SearchQuery, SearchItemType -from basic_memory.models.knowledge import Entity as EntityModel -from datetime import datetime - -router = APIRouter(prefix="/resource", tags=["resources"]) - - -def _mtime_to_datetime(entity: EntityModel) -> datetime: - """Convert entity mtime (file modification time) to datetime. - - Returns the file's actual modification time, falling back to updated_at - if mtime is not available. - """ - if entity.mtime: # pragma: no cover - return datetime.fromtimestamp(entity.mtime).astimezone() # pragma: no cover - return entity.updated_at - - -def get_entity_ids(item: SearchIndexRow) -> set[int]: - match item.type: - case SearchItemType.ENTITY: - return {item.id} - case SearchItemType.OBSERVATION: - return {item.entity_id} # pyright: ignore [reportReturnType] - case SearchItemType.RELATION: - from_entity = item.from_id - to_entity = item.to_id # pyright: ignore [reportReturnType] - return {from_entity, to_entity} if to_entity else {from_entity} # pyright: ignore [reportReturnType] - case _: # pragma: no cover - raise ValueError(f"Unexpected type: {item.type}") - - -@router.get("/{identifier:path}", response_model=None) -async def get_resource_content( - config: ProjectConfigDep, - link_resolver: LinkResolverDep, - search_service: SearchServiceDep, - entity_service: EntityServiceDep, - file_service: FileServiceDep, - background_tasks: BackgroundTasks, - identifier: str, - page: int = 1, - page_size: int = 10, -) -> Union[Response, FileResponse]: - """Get resource content by identifier: name or permalink.""" - logger.debug(f"Getting content for: {identifier}") - - # Find single entity by permalink - entity = await link_resolver.resolve_link(identifier) - results = [entity] if entity else [] - - # pagination for multiple results - limit = page_size - offset = (page - 1) * page_size - - # search using the identifier as a permalink - if not results: - # if the identifier contains a wildcard, use GLOB search - query = ( - SearchQuery(permalink_match=identifier) - if "*" in identifier - else SearchQuery(permalink=identifier) - ) - search_results = await search_service.search(query, limit, offset) - if not search_results: - raise HTTPException(status_code=404, detail=f"Resource not found: {identifier}") - - # get the deduplicated entities related to the search results - entity_ids = {id for result in search_results for id in get_entity_ids(result)} - results = await entity_service.get_entities_by_id(list(entity_ids)) - - # return single response - if len(results) == 1: - entity = results[0] - # Check file exists via file_service (for cloud compatibility) - if not await file_service.exists(entity.file_path): - raise HTTPException( - status_code=404, - detail=f"File not found: {entity.file_path}", - ) - # Read content via file_service as bytes (works with both local and S3) - content = await file_service.read_file_bytes(entity.file_path) - content_type = file_service.content_type(entity.file_path) - return Response(content=content, media_type=content_type) - - # for multiple files, initialize a temporary file for writing the results - with tempfile.NamedTemporaryFile(delete=False, mode="w", suffix=".md") as tmp_file: - temp_file_path = tmp_file.name - - for result in results: - # Read content for each entity - content = await file_service.read_entity_content(result) - memory_url = normalize_memory_url(result.permalink) - modified_date = _mtime_to_datetime(result).isoformat() - checksum = result.checksum[:8] if result.checksum else "" - - # Prepare the delimited content - response_content = f"--- {memory_url} {modified_date} {checksum}\n" - response_content += f"\n{content}\n" - response_content += "\n" - - # Write content directly to the temporary file in append mode - tmp_file.write(response_content) - - # Ensure all content is written to disk - tmp_file.flush() - - # Schedule the temporary file to be deleted after the response - background_tasks.add_task(cleanup_temp_file, temp_file_path) - - # Return the file response - return FileResponse(path=temp_file_path) - - -def cleanup_temp_file(file_path: str): - """Delete the temporary file.""" - try: - Path(file_path).unlink() # Deletes the file - logger.debug(f"Temporary file deleted: {file_path}") - except Exception as e: # pragma: no cover - logger.error(f"Error deleting temporary file {file_path}: {e}") - - -@router.put("/{file_path:path}") -async def write_resource( - config: ProjectConfigDep, - file_service: FileServiceDep, - entity_repository: EntityRepositoryDep, - search_service: SearchServiceDep, - file_path: str, - content: Annotated[str, Body()], -) -> JSONResponse: - """Write content to a file in the project. - - This endpoint allows writing content directly to a file in the project. - Also creates an entity record and indexes the file for search. - - Args: - file_path: Path to write to, relative to project root - request: Contains the content to write - - Returns: - JSON response with file information - """ - try: - # Get content from request body - - # Defensive type checking: ensure content is a string - # FastAPI should validate this, but if a dict somehow gets through - # (e.g., via JSON body parsing), we need to catch it here - if isinstance(content, dict): - logger.error( # pragma: no cover - f"Error writing resource {file_path}: " - f"content is a dict, expected string. Keys: {list(content.keys())}" - ) - raise HTTPException( # pragma: no cover - status_code=400, - detail="content must be a string, not a dict. " - "Ensure request body is sent as raw string content, not JSON object.", - ) - - # Ensure it's UTF-8 string content - if isinstance(content, bytes): # pragma: no cover - content_str = content.decode("utf-8") - else: - content_str = str(content) - - # Cloud compatibility: do not assume a local filesystem path structure. - # Delegate directory creation + writes to the configured FileService (local or S3). - await file_service.ensure_directory(Path(file_path).parent) - checksum = await file_service.write_file(file_path, content_str) - - # Get file info - file_metadata = await file_service.get_file_metadata(file_path) - - # Determine file details - file_name = Path(file_path).name - content_type = file_service.content_type(file_path) - - entity_type = "canvas" if file_path.endswith(".canvas") else "file" - - # Check if entity already exists - existing_entity = await entity_repository.get_by_file_path(file_path) - - if existing_entity: - # Update existing entity - entity = await entity_repository.update( - existing_entity.id, - { - "title": file_name, - "entity_type": entity_type, - "content_type": content_type, - "file_path": file_path, - "checksum": checksum, - "updated_at": file_metadata.modified_at, - }, - ) - status_code = 200 - else: - # Create a new entity model - # Explicitly set external_id to ensure NOT NULL constraint is satisfied (fixes #512) - entity = EntityModel( - external_id=str(uuid.uuid4()), - title=file_name, - entity_type=entity_type, - content_type=content_type, - file_path=file_path, - checksum=checksum, - created_at=file_metadata.created_at, - updated_at=file_metadata.modified_at, - ) - entity = await entity_repository.add(entity) - status_code = 201 - - # Index the file for search - await search_service.index_entity(entity) # pyright: ignore - - # Return success response - return JSONResponse( - status_code=status_code, - content={ - "file_path": file_path, - "checksum": checksum, - "size": file_metadata.size, - "created_at": file_metadata.created_at.timestamp(), - "modified_at": file_metadata.modified_at.timestamp(), - }, - ) - except Exception as e: # pragma: no cover - logger.error(f"Error writing resource {file_path}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to write resource: {str(e)}") diff --git a/src/basic_memory/api/routers/search_router.py b/src/basic_memory/api/routers/search_router.py deleted file mode 100644 index 67c259ad..00000000 --- a/src/basic_memory/api/routers/search_router.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Router for search operations.""" - -from fastapi import APIRouter, BackgroundTasks - -from basic_memory.api.routers.utils import to_search_results -from basic_memory.schemas.search import SearchQuery, SearchResponse -from basic_memory.deps import SearchServiceDep, EntityServiceDep - -router = APIRouter(prefix="/search", tags=["search"]) - - -@router.post("/", response_model=SearchResponse) -async def search( - query: SearchQuery, - search_service: SearchServiceDep, - entity_service: EntityServiceDep, - page: int = 1, - page_size: int = 10, -): - """Search across all knowledge and documents.""" - limit = page_size - offset = (page - 1) * page_size - results = await search_service.search(query, limit=limit, offset=offset) - search_results = await to_search_results(entity_service, results) - return SearchResponse( - results=search_results, - current_page=page, - page_size=page_size, - ) - - -@router.post("/reindex") -async def reindex(background_tasks: BackgroundTasks, search_service: SearchServiceDep): - """Recreate and populate the search index.""" - await search_service.reindex_all(background_tasks=background_tasks) - return {"status": "ok", "message": "Reindex initiated"} diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 3ed221d3..35a5ade2 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -10,7 +10,7 @@ - Simplified caching strategies """ -from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response, Path +from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response, Path, Query from loguru import logger from basic_memory.deps import ( @@ -182,11 +182,15 @@ async def create_entity( background_tasks: BackgroundTasks, entity_service: EntityServiceV2ExternalDep, search_service: SearchServiceV2ExternalDep, + fast: bool = Query( + True, description="If true, write quickly and defer indexing to background tasks." + ), ) -> EntityResponseV2: """Create a new entity. Args: data: Entity data to create + fast: If True, defer indexing to background tasks Returns: Created entity with generated external_id (UUID) @@ -195,11 +199,16 @@ async def create_entity( "API v2 request", endpoint="create_entity", entity_type=data.entity_type, title=data.title ) - entity = await entity_service.create_entity(data) + if fast: + entity = await entity_service.fast_write_entity(data) + background_tasks.add_task(entity_service.reindex_entity, entity.id) + else: + entity = await entity_service.create_entity(data) + await search_service.index_entity(entity, background_tasks=background_tasks) - # reindex - await search_service.index_entity(entity, background_tasks=background_tasks) result = EntityResponseV2.model_validate(entity) + if fast: + result = result.model_copy(update={"observations": [], "relations": []}) logger.info( f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201" @@ -221,6 +230,9 @@ async def update_entity_by_id( sync_service: SyncServiceV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), + fast: bool = Query( + True, description="If true, write quickly and defer indexing to background tasks." + ), ) -> EntityResponseV2: """Update an entity by external ID. @@ -229,22 +241,42 @@ async def update_entity_by_id( Args: entity_id: External ID (UUID string) data: Updated entity data + fast: If True, defer indexing to background tasks Returns: Updated entity """ logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}") - # Check if entity exists + # Check if entity exists (external_id is the source of truth for v2) existing = await entity_repository.get_by_external_id(entity_id) created = existing is None - # Perform update or create - entity, _ = await entity_service.create_or_update_entity(data) - response.status_code = 201 if created else 200 - - # reindex - await search_service.index_entity(entity, background_tasks=background_tasks) + if fast: + entity = await entity_service.fast_write_entity(data, external_id=entity_id) + response.status_code = 200 if existing else 201 + background_tasks.add_task(entity_service.reindex_entity, entity.id) + else: + if existing: + # Update the existing entity in-place to avoid path-based duplication + entity = await entity_service.update_entity(existing, data) + response.status_code = 200 + else: + # Create new entity, then bind external_id to the requested UUID + entity = await entity_service.create_entity(data) + if entity.external_id != entity_id: + entity = await entity_repository.update( + entity.id, + {"external_id": entity_id}, + ) + if not entity: + raise HTTPException( + status_code=404, + detail=f"Entity with external_id '{entity_id}' not found", + ) + response.status_code = 201 + + await search_service.index_entity(entity, background_tasks=background_tasks) # Schedule relation resolution for new entities if created: @@ -253,6 +285,8 @@ async def update_entity_by_id( ) result = EntityResponseV2.model_validate(entity) + if fast: + result = result.model_copy(update={"observations": [], "relations": []}) logger.info( f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}" @@ -269,12 +303,16 @@ async def edit_entity_by_id( search_service: SearchServiceV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), + fast: bool = Query( + True, description="If true, write quickly and defer indexing to background tasks." + ), ) -> EntityResponseV2: """Edit an existing entity by external ID using operations like append, prepend, etc. Args: entity_id: External ID (UUID string) data: Edit operation details + fast: If True, defer indexing to background tasks Returns: Updated entity @@ -294,21 +332,33 @@ async def edit_entity_by_id( ) try: - # Edit using the entity's permalink or path - identifier = entity.permalink or entity.file_path - updated_entity = await entity_service.edit_entity( - identifier=identifier, - operation=data.operation, - content=data.content, - section=data.section, - find_text=data.find_text, - expected_replacements=data.expected_replacements, - ) + if fast: + updated_entity = await entity_service.fast_edit_entity( + entity=entity, + operation=data.operation, + content=data.content, + section=data.section, + find_text=data.find_text, + expected_replacements=data.expected_replacements, + ) + background_tasks.add_task(entity_service.reindex_entity, updated_entity.id) + else: + # Edit using the entity's permalink or path + identifier = entity.permalink or entity.file_path + updated_entity = await entity_service.edit_entity( + identifier=identifier, + operation=data.operation, + content=data.content, + section=data.section, + find_text=data.find_text, + expected_replacements=data.expected_replacements, + ) - # Reindex - await search_service.index_entity(updated_entity, background_tasks=background_tasks) + await search_service.index_entity(updated_entity, background_tasks=background_tasks) result = EntityResponseV2.model_validate(updated_entity) + if fast: + result = result.model_copy(update={"observations": [], "relations": []}) logger.info( f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200" diff --git a/src/basic_memory/api/v2/routers/memory_router.py b/src/basic_memory/api/v2/routers/memory_router.py index 42aacec5..91de4a08 100644 --- a/src/basic_memory/api/v2/routers/memory_router.py +++ b/src/basic_memory/api/v2/routers/memory_router.py @@ -16,7 +16,7 @@ normalize_memory_url, ) from basic_memory.schemas.search import SearchItemType -from basic_memory.api.routers.utils import to_graph_context +from basic_memory.api.v2.utils import to_graph_context # Note: No prefix here - it's added during registration as /v2/{project_id}/memory router = APIRouter(tags=["memory"]) diff --git a/src/basic_memory/api/v2/routers/project_router.py b/src/basic_memory/api/v2/routers/project_router.py index 4ca8e945..5cf885fd 100644 --- a/src/basic_memory/api/v2/routers/project_router.py +++ b/src/basic_memory/api/v2/routers/project_router.py @@ -13,15 +13,21 @@ import os from typing import Optional -from fastapi import APIRouter, HTTPException, Body, Query, Path +from fastapi import APIRouter, HTTPException, Body, Query, Path, BackgroundTasks from loguru import logger from basic_memory.deps import ( ProjectServiceDep, ProjectRepositoryDep, + ProjectConfigV2ExternalDep, + SyncServiceV2ExternalDep, ) +from basic_memory.schemas import SyncReportResponse from basic_memory.schemas.project_info import ( ProjectItem, + ProjectList, + ProjectInfoRequest, + ProjectInfoResponse, ProjectStatusResponse, ) from basic_memory.schemas.v2 import ProjectResolveRequest, ProjectResolveResponse @@ -30,6 +36,175 @@ router = APIRouter(prefix="/projects", tags=["project_management-v2"]) +@router.get("/", response_model=ProjectList) +async def list_projects( + project_service: ProjectServiceDep, +) -> ProjectList: + """List all configured projects. + + Returns: + A list of all projects with metadata + """ + projects = await project_service.list_projects() + default_project = project_service.default_project + + project_items = [ + ProjectItem( + id=project.id, + external_id=project.external_id, + name=project.name, + path=normalize_project_path(project.path), + is_default=project.is_default or False, + ) + for project in projects + ] + + return ProjectList( + projects=project_items, + default_project=default_project, + ) + + +@router.post("/", response_model=ProjectStatusResponse, status_code=201) +async def add_project( + project_data: ProjectInfoRequest, + project_service: ProjectServiceDep, +) -> ProjectStatusResponse: + """Add a new project to configuration and database. + + Args: + project_data: The project name and path, with option to set as default + + Returns: + Response confirming the project was added + """ + # Check if project already exists before attempting to add + existing_project = await project_service.get_project(project_data.name) + if existing_project: + # Project exists - check if paths match for true idempotency + # Normalize paths for comparison (resolve symlinks, etc.) + from pathlib import Path + + requested_path = Path(project_data.path).resolve() + existing_path = Path(existing_project.path).resolve() + + if requested_path == existing_path: + # Same name, same path - return 200 OK (idempotent) + return ProjectStatusResponse( # pyright: ignore [reportCallIssue] + message=f"Project '{project_data.name}' already exists", + status="success", + default=existing_project.is_default or False, + new_project=ProjectItem( + id=existing_project.id, + external_id=existing_project.external_id, + name=existing_project.name, + path=existing_project.path, + is_default=existing_project.is_default or False, + ), + ) + else: + # Same name, different path - this is an error + raise HTTPException( + status_code=400, + detail=( + f"Project '{project_data.name}' already exists with different path. " + f"Existing: {existing_project.path}, Requested: {project_data.path}" + ), + ) + + try: # pragma: no cover + # The service layer handles cloud mode validation and path sanitization + await project_service.add_project( + project_data.name, project_data.path, set_default=project_data.set_default + ) + + # Fetch the newly created project to get its ID + new_project = await project_service.get_project(project_data.name) + if not new_project: + raise HTTPException(status_code=500, detail="Failed to retrieve newly created project") + + return ProjectStatusResponse( # pyright: ignore [reportCallIssue] + message=f"Project '{new_project.name}' added successfully", + status="success", + default=project_data.set_default, + new_project=ProjectItem( + id=new_project.id, + external_id=new_project.external_id, + name=new_project.name, + path=new_project.path, + is_default=new_project.is_default or False, + ), + ) + except ValueError as e: # pragma: no cover + raise HTTPException(status_code=400, detail=str(e)) + + +@router.post("/config/sync", response_model=ProjectStatusResponse) +async def synchronize_projects( + project_service: ProjectServiceDep, +) -> ProjectStatusResponse: + """Synchronize projects between configuration file and database.""" + try: # pragma: no cover + await project_service.synchronize_projects() + + return ProjectStatusResponse( # pyright: ignore [reportCallIssue] + message="Projects synchronized successfully between configuration and database", + status="success", + default=False, + ) + except ValueError as e: # pragma: no cover + raise HTTPException(status_code=400, detail=str(e)) + + +@router.post("/{project_id}/sync") +async def sync_project( + background_tasks: BackgroundTasks, + sync_service: SyncServiceV2ExternalDep, + project_config: ProjectConfigV2ExternalDep, + project_id: str = Path(..., description="Project external ID (UUID)"), + force_full: bool = Query( + False, description="Force full scan, bypassing watermark optimization" + ), + run_in_background: bool = Query(True, description="Run in background"), +): + """Force project filesystem sync to database.""" + if run_in_background: + background_tasks.add_task( + sync_service.sync, project_config.home, project_config.name, force_full=force_full + ) + logger.info( + f"Filesystem sync initiated for project: {project_config.name} (force_full={force_full})" + ) + + return { + "status": "sync_started", + "message": f"Filesystem sync initiated for project '{project_config.name}'", + } + + report = await sync_service.sync( + project_config.home, project_config.name, force_full=force_full + ) + logger.info( + f"Filesystem sync completed for project: {project_config.name} (force_full={force_full})" + ) + return SyncReportResponse.from_sync_report(report) + + +@router.post("/{project_id}/status", response_model=SyncReportResponse) +async def get_project_status( + sync_service: SyncServiceV2ExternalDep, + project_config: ProjectConfigV2ExternalDep, + project_id: str = Path(..., description="Project external ID (UUID)"), + force_full: bool = Query( + False, description="Force full scan, bypassing watermark optimization" + ), +) -> SyncReportResponse: + """Get sync status of files vs database for a project.""" + logger.info(f"API v2 request: get_project_status for project_id={project_id}") + report = await sync_service.scan(project_config.home, force_full=force_full) + return SyncReportResponse.from_sync_report(report) + + @router.post("/resolve", response_model=ProjectResolveResponse) async def resolve_project_identifier( data: ProjectResolveRequest, @@ -147,6 +322,22 @@ async def get_project_by_id( ) +@router.get("/{project_id}/info", response_model=ProjectInfoResponse) +async def get_project_info_by_id( + project_service: ProjectServiceDep, + project_repository: ProjectRepositoryDep, + project_id: str = Path(..., description="Project external ID (UUID)"), +) -> ProjectInfoResponse: + """Get detailed project information by external ID.""" + logger.info(f"API v2 request: get_project_info_by_id for project_id={project_id}") + project = await project_repository.get_by_external_id(project_id) + if not project: + raise HTTPException( + status_code=404, detail=f"Project with external_id '{project_id}' not found" + ) + return await project_service.get_project_info(project.name) + + @router.patch("/{project_id}", response_model=ProjectStatusResponse) async def update_project_by_id( project_service: ProjectServiceDep, diff --git a/src/basic_memory/api/v2/routers/prompt_router.py b/src/basic_memory/api/v2/routers/prompt_router.py index d27d161e..c6e58ec0 100644 --- a/src/basic_memory/api/v2/routers/prompt_router.py +++ b/src/basic_memory/api/v2/routers/prompt_router.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, HTTPException, status, Path from loguru import logger -from basic_memory.api.routers.utils import to_graph_context, to_search_results +from basic_memory.api.v2.utils import to_graph_context, to_search_results from basic_memory.api.template_loader import template_loader from basic_memory.schemas.base import parse_timeframe from basic_memory.deps import ( diff --git a/src/basic_memory/api/v2/routers/search_router.py b/src/basic_memory/api/v2/routers/search_router.py index b8b62e78..eae643ab 100644 --- a/src/basic_memory/api/v2/routers/search_router.py +++ b/src/basic_memory/api/v2/routers/search_router.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, BackgroundTasks, Path -from basic_memory.api.routers.utils import to_search_results +from basic_memory.api.v2.utils import to_search_results from basic_memory.schemas.search import SearchQuery, SearchResponse from basic_memory.deps import SearchServiceV2ExternalDep, EntityServiceV2ExternalDep diff --git a/src/basic_memory/api/routers/utils.py b/src/basic_memory/api/v2/utils.py similarity index 100% rename from src/basic_memory/api/routers/utils.py rename to src/basic_memory/api/v2/utils.py diff --git a/src/basic_memory/cli/commands/cloud/cloud_utils.py b/src/basic_memory/cli/commands/cloud/cloud_utils.py index 28790b77..b4c1158a 100644 --- a/src/basic_memory/cli/commands/cloud/cloud_utils.py +++ b/src/basic_memory/cli/commands/cloud/cloud_utils.py @@ -30,7 +30,7 @@ async def fetch_cloud_projects( config = config_manager.config host_url = config.cloud_host.rstrip("/") - response = await api_request(method="GET", url=f"{host_url}/proxy/projects/projects") + response = await api_request(method="GET", url=f"{host_url}/proxy/v2/projects/") return CloudProjectList.model_validate(response.json()) except Exception as e: @@ -66,7 +66,7 @@ async def create_cloud_project( response = await api_request( method="POST", - url=f"{host_url}/proxy/projects/projects", + url=f"{host_url}/proxy/v2/projects/", headers={"Content-Type": "application/json"}, json_data=project_data.model_dump(), ) diff --git a/src/basic_memory/cli/commands/command_utils.py b/src/basic_memory/cli/commands/command_utils.py index f5b1214c..0e26ecc6 100644 --- a/src/basic_memory/cli/commands/command_utils.py +++ b/src/basic_memory/cli/commands/command_utils.py @@ -58,7 +58,7 @@ async def run_sync( try: async with get_client() as client: project_item = await get_active_project(client, project, None) - url = f"{project_item.project_url}/project/sync" + url = f"/v2/projects/{project_item.external_id}/sync" params = [] if force_full: params.append("force_full=true") @@ -92,7 +92,7 @@ async def get_project_info(project: str): try: async with get_client() as client: project_item = await get_active_project(client, project, None) - response = await call_get(client, f"{project_item.project_url}/project/info") + response = await call_get(client, f"/v2/projects/{project_item.external_id}/info") return ProjectInfoResponse.model_validate(response.json()) except (ToolError, ValueError) as e: console.print(f"[red]Sync failed: {e}[/red]") diff --git a/src/basic_memory/cli/commands/project.py b/src/basic_memory/cli/commands/project.py index 8744c57a..4c2c2a26 100644 --- a/src/basic_memory/cli/commands/project.py +++ b/src/basic_memory/cli/commands/project.py @@ -17,6 +17,7 @@ from basic_memory.mcp.async_client import get_client from basic_memory.mcp.tools.utils import call_delete, call_get, call_patch, call_post, call_put from basic_memory.schemas.project_info import ProjectList, ProjectStatusResponse +from basic_memory.schemas.v2 import ProjectResolveResponse from basic_memory.utils import generate_permalink, normalize_project_path # Import rclone commands for project sync @@ -65,7 +66,7 @@ def list_projects( async def _list_projects(): async with get_client() as client: - response = await call_get(client, "/projects/projects") + response = await call_get(client, "/v2/projects/") return ProjectList.model_validate(response.json()) try: @@ -167,7 +168,7 @@ async def _add_project(): "local_sync_path": local_sync_path, "set_default": set_default, } - response = await call_post(client, "/projects/projects", json=data) + response = await call_post(client, "/v2/projects/", json=data) return ProjectStatusResponse.model_validate(response.json()) else: # Local mode: path is required @@ -181,7 +182,7 @@ async def _add_project(): async def _add_project(): async with get_client() as client: data = {"name": name, "path": resolved_path, "set_default": set_default} - response = await call_post(client, "/projects/projects", json=data) + response = await call_post(client, "/v2/projects/", json=data) return ProjectStatusResponse.model_validate(response.json()) try: @@ -234,7 +235,7 @@ def setup_project_sync( async def _verify_project_exists(): """Verify the project exists on cloud by listing all projects.""" async with get_client() as client: - response = await call_get(client, "/projects/projects") + response = await call_get(client, "/v2/projects/") project_list = response.json() project_names = [p["name"] for p in project_list["projects"]] if name not in project_names: @@ -433,7 +434,7 @@ def synchronize_projects( async def _sync_config(): async with get_client() as client: - response = await call_post(client, "/projects/config/sync") + response = await call_post(client, "/v2/projects/config/sync") return ProjectStatusResponse.model_validate(response.json()) try: @@ -475,10 +476,15 @@ def move_project( async def _move_project(): async with get_client() as client: data = {"path": resolved_path} - project_permalink = generate_permalink(name) - - # TODO fix route to use ProjectPathDep - response = await call_patch(client, f"/{name}/project/{project_permalink}", json=data) + resolve_response = await call_post( + client, + "/v2/projects/resolve", + json={"identifier": name}, + ) + project_info = ProjectResolveResponse.model_validate(resolve_response.json()) + response = await call_patch( + client, f"/v2/projects/{project_info.external_id}", json=data + ) return ProjectStatusResponse.model_validate(response.json()) try: @@ -530,7 +536,7 @@ def sync_project_command( # Get project info async def _get_project(): async with get_client() as client: - response = await call_get(client, "/projects/projects") + response = await call_get(client, "/v2/projects/") projects_list = ProjectList.model_validate(response.json()) for proj in projects_list.projects: if generate_permalink(proj.name) == generate_permalink(name): @@ -571,9 +577,10 @@ async def _get_project(): async def _trigger_db_sync(): async with get_client() as client: - permalink = generate_permalink(name) response = await call_post( - client, f"/{permalink}/project/sync?force_full=true", json={} + client, + f"/v2/projects/{project_data.external_id}/sync?force_full=true", + json={}, ) return response.json() @@ -621,7 +628,7 @@ def bisync_project_command( # Get project info async def _get_project(): async with get_client() as client: - response = await call_get(client, "/projects/projects") + response = await call_get(client, "/v2/projects/") projects_list = ProjectList.model_validate(response.json()) for proj in projects_list.projects: if generate_permalink(proj.name) == generate_permalink(name): @@ -669,9 +676,10 @@ async def _get_project(): async def _trigger_db_sync(): async with get_client() as client: - permalink = generate_permalink(name) response = await call_post( - client, f"/{permalink}/project/sync?force_full=true", json={} + client, + f"/v2/projects/{project_data.external_id}/sync?force_full=true", + json={}, ) return response.json() @@ -715,7 +723,7 @@ def check_project_command( # Get project info async def _get_project(): async with get_client() as client: - response = await call_get(client, "/projects/projects") + response = await call_get(client, "/v2/projects/") projects_list = ProjectList.model_validate(response.json()) for proj in projects_list.projects: if generate_permalink(proj.name) == generate_permalink(name): @@ -816,7 +824,7 @@ def ls_project_command( # Get project info async def _get_project(): async with get_client() as client: - response = await call_get(client, "/projects/projects") + response = await call_get(client, "/v2/projects/") projects_list = ProjectList.model_validate(response.json()) for proj in projects_list.projects: if generate_permalink(proj.name) == generate_permalink(name): diff --git a/src/basic_memory/cli/commands/status.py b/src/basic_memory/cli/commands/status.py index d8ef3620..6d5ca6b0 100644 --- a/src/basic_memory/cli/commands/status.py +++ b/src/basic_memory/cli/commands/status.py @@ -146,7 +146,7 @@ async def run_status(project: Optional[str] = None, verbose: bool = False): # p try: async with get_client() as client: project_item = await get_active_project(client, project, None) - response = await call_post(client, f"{project_item.project_url}/project/status") + response = await call_post(client, f"/v2/projects/{project_item.external_id}/status") sync_report = SyncReportResponse.model_validate(response.json()) display_changes(project_item.name, "Status", sync_report, verbose) diff --git a/src/basic_memory/mcp/clients/knowledge.py b/src/basic_memory/mcp/clients/knowledge.py index cf4ebbcc..fc657b3e 100644 --- a/src/basic_memory/mcp/clients/knowledge.py +++ b/src/basic_memory/mcp/clients/knowledge.py @@ -43,7 +43,9 @@ def __init__(self, http_client: AsyncClient, project_id: str): # --- Entity CRUD Operations --- - async def create_entity(self, entity_data: dict[str, Any]) -> EntityResponse: + async def create_entity( + self, entity_data: dict[str, Any], *, fast: bool | None = None + ) -> EntityResponse: """Create a new entity. Args: @@ -55,14 +57,22 @@ async def create_entity(self, entity_data: dict[str, Any]) -> EntityResponse: Raises: ToolError: If the request fails """ + params = {"fast": fast} if fast is not None else None response = await call_post( self.http_client, f"{self._base_path}/entities", json=entity_data, + params=params, ) return EntityResponse.model_validate(response.json()) - async def update_entity(self, entity_id: str, entity_data: dict[str, Any]) -> EntityResponse: + async def update_entity( + self, + entity_id: str, + entity_data: dict[str, Any], + *, + fast: bool | None = None, + ) -> EntityResponse: """Update an existing entity (full replacement). Args: @@ -75,10 +85,12 @@ async def update_entity(self, entity_id: str, entity_data: dict[str, Any]) -> En Raises: ToolError: If the request fails """ + params = {"fast": fast} if fast is not None else None response = await call_put( self.http_client, f"{self._base_path}/entities/{entity_id}", json=entity_data, + params=params, ) return EntityResponse.model_validate(response.json()) @@ -100,7 +112,13 @@ async def get_entity(self, entity_id: str) -> EntityResponse: ) return EntityResponse.model_validate(response.json()) - async def patch_entity(self, entity_id: str, patch_data: dict[str, Any]) -> EntityResponse: + async def patch_entity( + self, + entity_id: str, + patch_data: dict[str, Any], + *, + fast: bool | None = None, + ) -> EntityResponse: """Partially update an entity. Args: @@ -113,10 +131,12 @@ async def patch_entity(self, entity_id: str, patch_data: dict[str, Any]) -> Enti Raises: ToolError: If the request fails """ + params = {"fast": fast} if fast is not None else None response = await call_patch( self.http_client, f"{self._base_path}/entities/{entity_id}", json=patch_data, + params=params, ) return EntityResponse.model_validate(response.json()) diff --git a/src/basic_memory/mcp/clients/project.py b/src/basic_memory/mcp/clients/project.py index 89500490..50e06401 100644 --- a/src/basic_memory/mcp/clients/project.py +++ b/src/basic_memory/mcp/clients/project.py @@ -47,7 +47,7 @@ async def list_projects(self) -> ProjectList: """ response = await call_get( self.http_client, - "/projects/projects", + "/v2/projects/", ) return ProjectList.model_validate(response.json()) @@ -65,7 +65,7 @@ async def create_project(self, project_data: dict[str, Any]) -> ProjectStatusRes """ response = await call_post( self.http_client, - "/projects/projects", + "/v2/projects/", json=project_data, ) return ProjectStatusResponse.model_validate(response.json()) diff --git a/src/basic_memory/mcp/project_context.py b/src/basic_memory/mcp/project_context.py index 786ff699..69a9822d 100644 --- a/src/basic_memory/mcp/project_context.py +++ b/src/basic_memory/mcp/project_context.py @@ -19,7 +19,7 @@ from basic_memory.config import ConfigManager from basic_memory.project_resolver import ProjectResolver from basic_memory.schemas.project_info import ProjectItem, ProjectList -from basic_memory.utils import generate_permalink +from basic_memory.schemas.v2 import ProjectResolveResponse async def resolve_project_parameter( @@ -78,7 +78,7 @@ async def get_project_names(client: AsyncClient, headers: HeaderTypes | None = N # Deferred import to avoid circular dependency with tools from basic_memory.mcp.tools.utils import call_get - response = await call_get(client, "/projects/projects", headers=headers) + response = await call_get(client, "/v2/projects/", headers=headers) project_list = ProjectList.model_validate(response.json()) return [project.name for project in project_list.projects] @@ -104,7 +104,7 @@ async def get_active_project( HTTPError: If project doesn't exist or is inaccessible """ # Deferred import to avoid circular dependency with tools - from basic_memory.mcp.tools.utils import call_get + from basic_memory.mcp.tools.utils import call_post resolved_project = await resolve_project_parameter(project) if not resolved_project: @@ -126,9 +126,20 @@ async def get_active_project( # Validate project exists by calling API logger.debug(f"Validating project: {project}") - permalink = generate_permalink(project) - response = await call_get(client, f"/{permalink}/project/item", headers=headers) - active_project = ProjectItem.model_validate(response.json()) + response = await call_post( + client, + "/v2/projects/resolve", + json={"identifier": project}, + headers=headers, + ) + resolved = ProjectResolveResponse.model_validate(response.json()) + active_project = ProjectItem( + id=resolved.project_id, + external_id=resolved.external_id, + name=resolved.name, + path=resolved.path, + is_default=resolved.is_default, + ) # Cache in context if available if context: diff --git a/src/basic_memory/mcp/prompts/continue_conversation.py b/src/basic_memory/mcp/prompts/continue_conversation.py index 8abfd5dd..a2f5de56 100644 --- a/src/basic_memory/mcp/prompts/continue_conversation.py +++ b/src/basic_memory/mcp/prompts/continue_conversation.py @@ -9,8 +9,9 @@ from loguru import logger from pydantic import Field -from basic_memory.config import get_project_config +from basic_memory.config import ConfigManager from basic_memory.mcp.async_client import get_client +from basic_memory.mcp.project_context import get_active_project from basic_memory.mcp.server import mcp from basic_memory.mcp.tools.utils import call_post from basic_memory.schemas.prompt import ContinueConversationRequest @@ -42,17 +43,18 @@ async def continue_conversation( logger.info(f"Continuing session, topic: {topic}, timeframe: {timeframe}") async with get_client() as client: + config = ConfigManager().config + active_project = await get_active_project(client, project=config.default_project) + # Create request model request = ContinueConversationRequest( # pyright: ignore [reportCallIssue] topic=topic, timeframe=timeframe ) - project_url = get_project_config().project_url - # Call the prompt API endpoint response = await call_post( client, - f"{project_url}/prompt/continue-conversation", + f"/v2/projects/{active_project.external_id}/prompt/continue-conversation", json=request.model_dump(exclude_none=True), ) diff --git a/src/basic_memory/mcp/prompts/search.py b/src/basic_memory/mcp/prompts/search.py index c9a13f8c..1aa12b3a 100644 --- a/src/basic_memory/mcp/prompts/search.py +++ b/src/basic_memory/mcp/prompts/search.py @@ -8,8 +8,9 @@ from loguru import logger from pydantic import Field -from basic_memory.config import get_project_config +from basic_memory.config import ConfigManager from basic_memory.mcp.async_client import get_client +from basic_memory.mcp.project_context import get_active_project from basic_memory.mcp.server import mcp from basic_memory.mcp.tools.utils import call_post from basic_memory.schemas.prompt import SearchPromptRequest @@ -41,14 +42,17 @@ async def search_prompt( logger.info(f"Searching knowledge base, query: {query}, timeframe: {timeframe}") async with get_client() as client: + config = ConfigManager().config + active_project = await get_active_project(client, project=config.default_project) + # Create request model request = SearchPromptRequest(query=query, timeframe=timeframe) - project_url = get_project_config().project_url - # Call the prompt API endpoint response = await call_post( - client, f"{project_url}/prompt/search", json=request.model_dump(exclude_none=True) + client, + f"/v2/projects/{active_project.external_id}/prompt/search", + json=request.model_dump(exclude_none=True), ) # Extract the rendered prompt from the response diff --git a/src/basic_memory/mcp/resources/project_info.py b/src/basic_memory/mcp/resources/project_info.py index 0dc159df..6762ea80 100644 --- a/src/basic_memory/mcp/resources/project_info.py +++ b/src/basic_memory/mcp/resources/project_info.py @@ -62,10 +62,9 @@ async def project_info( async with get_client() as client: project_config = await get_active_project(client, project, context) - project_url = project_config.permalink # Call the API endpoint - response = await call_get(client, f"{project_url}/project/info") + response = await call_get(client, f"/v2/projects/{project_config.external_id}/info") # Convert response to ProjectInfoResponse return ProjectInfoResponse.model_validate(response.json()) diff --git a/src/basic_memory/mcp/tools/edit_note.py b/src/basic_memory/mcp/tools/edit_note.py index 7edaa2dc..c706825b 100644 --- a/src/basic_memory/mcp/tools/edit_note.py +++ b/src/basic_memory/mcp/tools/edit_note.py @@ -256,7 +256,7 @@ async def edit_note( edit_data["expected_replacements"] = str(expected_replacements) # Call the PATCH endpoint - result = await knowledge_client.patch_entity(entity_id, edit_data) + result = await knowledge_client.patch_entity(entity_id, edit_data, fast=False) # Format summary summary = [ diff --git a/src/basic_memory/mcp/tools/recent_activity.py b/src/basic_memory/mcp/tools/recent_activity.py index 3083921f..84aa4cd1 100644 --- a/src/basic_memory/mcp/tools/recent_activity.py +++ b/src/basic_memory/mcp/tools/recent_activity.py @@ -144,7 +144,7 @@ async def recent_activity( ) # Get list of all projects - response = await call_get(client, "/projects/projects") + response = await call_get(client, "/v2/projects/") project_list = ProjectList.model_validate(response.json()) projects_activity = {} diff --git a/src/basic_memory/mcp/tools/write_note.py b/src/basic_memory/mcp/tools/write_note.py index ca30c701..a9a79c89 100644 --- a/src/basic_memory/mcp/tools/write_note.py +++ b/src/basic_memory/mcp/tools/write_note.py @@ -159,7 +159,7 @@ async def write_note( logger.debug(f"Attempting to create entity permalink={entity.permalink}") action = "Created" # Default to created try: - result = await knowledge_client.create_entity(entity.model_dump()) + result = await knowledge_client.create_entity(entity.model_dump(), fast=False) action = "Created" except Exception as e: # If creation failed due to conflict (already exists), try to update @@ -175,7 +175,9 @@ async def write_note( "Entity permalink is required for updates" ) # pragma: no cover entity_id = await knowledge_client.resolve_entity(entity.permalink) - result = await knowledge_client.update_entity(entity_id, entity.model_dump()) + result = await knowledge_client.update_entity( + entity_id, entity.model_dump(), fast=False + ) action = "Updated" except Exception as update_error: # pragma: no cover # Re-raise the original error if update also fails diff --git a/src/basic_memory/schemas/cloud.py b/src/basic_memory/schemas/cloud.py index 4e52edd9..e3a61df1 100644 --- a/src/basic_memory/schemas/cloud.py +++ b/src/basic_memory/schemas/cloud.py @@ -25,7 +25,7 @@ class CloudProject(BaseModel): class CloudProjectList(BaseModel): - """Response from /proxy/projects/projects endpoint.""" + """Response from /proxy/v2/projects endpoint.""" projects: list[CloudProject] = Field(default_factory=list, description="List of cloud projects") diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 4dd7dcc4..0e72d7a2 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1,5 +1,6 @@ """Service for managing entities in the database.""" +from datetime import datetime from pathlib import Path from typing import List, Optional, Sequence, Tuple, Union @@ -344,6 +345,194 @@ async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> Enti return entity + async def fast_write_entity( + self, + schema: EntitySchema, + external_id: Optional[str] = None, + ) -> EntityModel: + """Write file and upsert a minimal entity row for fast responses.""" + logger.debug( + "Fast-writing entity", + title=schema.title, + external_id=external_id, + content_type=schema.content_type, + ) + + # --- Identity & File Path --- + existing = await self.repository.get_by_external_id(external_id) if external_id else None + + # Trigger: external_id already exists + # Why: avoid duplicate entities when title-derived paths change + # Outcome: update in-place and keep the existing file path + file_path = Path(existing.file_path) if existing else Path(schema.file_path) + + if not existing and await self.file_service.exists(file_path): + raise EntityCreationError( + f"file for entity {schema.directory}/{schema.title} already exists: {file_path}" + ) + + # --- Frontmatter Overrides --- + content_markdown = None + if schema.content and has_frontmatter(schema.content): + content_frontmatter = parse_frontmatter(schema.content) + + if "type" in content_frontmatter: + schema.entity_type = content_frontmatter["type"] + + if "permalink" in content_frontmatter: + from basic_memory.markdown.schemas import EntityFrontmatter + + frontmatter_metadata = { + "title": schema.title, + "type": schema.entity_type, + "permalink": content_frontmatter["permalink"], + } + frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) + content_markdown = EntityMarkdown( + frontmatter=frontmatter_obj, + content="", + observations=[], + relations=[], + ) + + # --- Permalink Resolution --- + if self.app_config and self.app_config.disable_permalinks: + schema._permalink = "" + else: + if existing and not (content_markdown and content_markdown.frontmatter.permalink): + schema._permalink = existing.permalink or await self.resolve_permalink( + file_path, skip_conflict_check=True + ) + else: + schema._permalink = await self.resolve_permalink( + file_path, content_markdown, skip_conflict_check=True + ) + + # --- File Write --- + post = await schema_to_markdown(schema) + final_content = dump_frontmatter(post) + checksum = await self.file_service.write_file(file_path, final_content) + + # --- Minimal DB Upsert --- + metadata = post.metadata or {} + entity_metadata = {k: str(v) for k, v in metadata.items() if v is not None} + update_data = { + "title": schema.title, + "entity_type": schema.entity_type, + "file_path": file_path.as_posix(), + "content_type": schema.content_type, + "entity_metadata": entity_metadata or None, + "permalink": schema.permalink, + "checksum": checksum, + "updated_at": datetime.now().astimezone(), + } + + if existing: + updated = await self.repository.update(existing.id, update_data) + if not updated: + raise ValueError(f"Failed to update entity in database: {existing.id}") + return updated + + create_data = { + **update_data, + "external_id": external_id, + } + return await self.repository.create(create_data) + + async def fast_edit_entity( + self, + entity: EntityModel, + operation: str, + content: str, + section: Optional[str] = None, + find_text: Optional[str] = None, + expected_replacements: int = 1, + ) -> EntityModel: + """Edit an entity quickly and defer full indexing to background.""" + logger.debug(f"Fast editing entity: {entity.external_id}, operation: {operation}") + + # --- File Edit --- + file_path = Path(entity.file_path) + current_content, _ = await self.file_service.read_file(file_path) + new_content = self.apply_edit_operation( + current_content, operation, content, section, find_text, expected_replacements + ) + checksum = await self.file_service.write_file(file_path, new_content) + + # --- Frontmatter Overrides --- + update_data = { + "checksum": checksum, + "updated_at": datetime.now().astimezone(), + } + content_markdown = None + if has_frontmatter(new_content): + content_frontmatter = parse_frontmatter(new_content) + + if "title" in content_frontmatter: + update_data["title"] = content_frontmatter["title"] + if "type" in content_frontmatter: + update_data["entity_type"] = content_frontmatter["type"] + + if "permalink" in content_frontmatter: + from basic_memory.markdown.schemas import EntityFrontmatter + + frontmatter_metadata = { + "title": update_data.get("title", entity.title), + "type": update_data.get("entity_type", entity.entity_type), + "permalink": content_frontmatter["permalink"], + } + frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) + content_markdown = EntityMarkdown( + frontmatter=frontmatter_obj, + content="", + observations=[], + relations=[], + ) + + metadata = content_frontmatter or {} + update_data["entity_metadata"] = { + k: str(v) for k, v in metadata.items() if v is not None + } + + # --- Permalink Resolution --- + if self.app_config and self.app_config.disable_permalinks: + update_data["permalink"] = None + elif content_markdown and content_markdown.frontmatter.permalink: + update_data["permalink"] = await self.resolve_permalink( + file_path, content_markdown, skip_conflict_check=True + ) + + updated = await self.repository.update(entity.id, update_data) + if not updated: + raise ValueError(f"Failed to update entity in database: {entity.id}") + return updated + + async def reindex_entity(self, entity_id: int) -> None: + """Parse file content and rebuild observations/relations/search for an entity.""" + entity = await self.repository.find_by_id(entity_id) + if not entity: + raise EntityNotFoundError(f"Entity not found: {entity_id}") + + # --- Full Parse --- + file_path = Path(entity.file_path) + content = await self.file_service.read_file_content(file_path) + entity_markdown = await self.entity_parser.parse_markdown_content( + file_path=file_path, + content=content, + ) + + # --- DB Reindex --- + updated = await self.update_entity_and_observations(file_path, entity_markdown) + updated = await self.update_entity_relations(file_path.as_posix(), entity_markdown) + checksum = await self.file_service.compute_checksum(file_path) + updated = await self.repository.update(updated.id, {"checksum": checksum}) + if not updated: + raise ValueError(f"Failed to update entity in database: {entity.id}") + + # --- Search Reindex --- + if self.search_service: + await self.search_service.index_entity_data(updated, content=content) + async def delete_entity(self, permalink_or_id: str | int) -> bool: """Delete entity and its file.""" logger.debug(f"Deleting entity: {permalink_or_id}") diff --git a/tests/api/conftest.py b/tests/api/conftest.py deleted file mode 100644 index 87e5d933..00000000 --- a/tests/api/conftest.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Tests for knowledge graph API routes.""" - -from typing import AsyncGenerator - -import pytest -import pytest_asyncio -from fastapi import FastAPI -from httpx import AsyncClient, ASGITransport - -from basic_memory.deps import get_project_config, get_engine_factory, get_app_config -from basic_memory.models import Project - - -@pytest_asyncio.fixture -async def app(test_config, engine_factory, app_config) -> FastAPI: - """Create FastAPI test application.""" - from basic_memory.api.app import app - - app.dependency_overrides[get_app_config] = lambda: app_config - app.dependency_overrides[get_project_config] = lambda: test_config.project_config - app.dependency_overrides[get_engine_factory] = lambda: engine_factory - return app - - -@pytest_asyncio.fixture -async def client(app: FastAPI) -> AsyncGenerator[AsyncClient, None]: - """Create client using ASGI transport - same as CLI will use.""" - async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: - yield client - - -@pytest.fixture -def project_url(test_project: Project) -> str: - """Create a URL prefix for the project routes. - - This helps tests generate the correct URL for project-scoped routes. - """ - # Make sure this matches what's in tests/conftest.py for test_project creation - # The permalink should be generated from "Test Project Context" - return f"/{test_project.permalink}" diff --git a/tests/api/test_api_container.py b/tests/api/test_api_container.py deleted file mode 100644 index fb13d6c5..00000000 --- a/tests/api/test_api_container.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Tests for API container composition root.""" - -import pytest - -from basic_memory.api.container import ( - ApiContainer, - get_container, - set_container, -) -from basic_memory.runtime import RuntimeMode - - -class TestApiContainer: - """Tests for ApiContainer.""" - - def test_create_from_config(self, app_config): - """Container can be created from config manager.""" - container = ApiContainer(config=app_config, mode=RuntimeMode.LOCAL) - assert container.config == app_config - assert container.mode == RuntimeMode.LOCAL - - def test_should_sync_files_when_enabled_and_not_test(self, app_config): - """Sync should be enabled when config says so and not in test mode.""" - app_config.sync_changes = True - container = ApiContainer(config=app_config, mode=RuntimeMode.LOCAL) - assert container.should_sync_files is True - - def test_should_not_sync_files_when_disabled(self, app_config): - """Sync should be disabled when config says so.""" - app_config.sync_changes = False - container = ApiContainer(config=app_config, mode=RuntimeMode.LOCAL) - assert container.should_sync_files is False - - def test_should_not_sync_files_in_test_mode(self, app_config): - """Sync should be disabled in test mode regardless of config.""" - app_config.sync_changes = True - container = ApiContainer(config=app_config, mode=RuntimeMode.TEST) - assert container.should_sync_files is False - - -class TestContainerAccessors: - """Tests for container get/set functions.""" - - def test_get_container_raises_when_not_set(self, monkeypatch): - """get_container raises RuntimeError when container not initialized.""" - # Clear any existing container - import basic_memory.api.container as container_module - - monkeypatch.setattr(container_module, "_container", None) - - with pytest.raises(RuntimeError, match="API container not initialized"): - get_container() - - def test_set_and_get_container(self, app_config, monkeypatch): - """set_container allows get_container to return the container.""" - import basic_memory.api.container as container_module - - container = ApiContainer(config=app_config, mode=RuntimeMode.LOCAL) - monkeypatch.setattr(container_module, "_container", None) - - set_container(container) - assert get_container() is container diff --git a/tests/api/test_async_client.py b/tests/api/test_async_client.py deleted file mode 100644 index c63b2c5e..00000000 --- a/tests/api/test_async_client.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Tests for async_client configuration.""" - -from httpx import AsyncClient, ASGITransport, Timeout - -from basic_memory.mcp.async_client import create_client - - -def test_create_client_uses_asgi_when_no_remote_env(config_manager, monkeypatch): - """Test that create_client uses ASGI transport when cloud mode is disabled.""" - monkeypatch.delenv("BASIC_MEMORY_USE_REMOTE_API", raising=False) - monkeypatch.delenv("BASIC_MEMORY_CLOUD_MODE", raising=False) - - cfg = config_manager.load_config() - cfg.cloud_mode = False - config_manager.save_config(cfg) - - client = create_client() - - assert isinstance(client, AsyncClient) - assert isinstance(client._transport, ASGITransport) - assert str(client.base_url) == "http://test" - - -def test_create_client_uses_http_when_cloud_mode_env_set(config_manager, monkeypatch): - """Test that create_client uses HTTP transport when BASIC_MEMORY_CLOUD_MODE is set.""" - monkeypatch.setenv("BASIC_MEMORY_CLOUD_MODE", "True") - - config = config_manager.load_config() - client = create_client() - - assert isinstance(client, AsyncClient) - assert not isinstance(client._transport, ASGITransport) - # Cloud mode uses cloud_host/proxy as base_url - assert str(client.base_url) == f"{config.cloud_host}/proxy/" - - -def test_create_client_configures_extended_timeouts(config_manager, monkeypatch): - """Test that create_client configures 30-second timeouts for long operations.""" - monkeypatch.delenv("BASIC_MEMORY_USE_REMOTE_API", raising=False) - monkeypatch.delenv("BASIC_MEMORY_CLOUD_MODE", raising=False) - - cfg = config_manager.load_config() - cfg.cloud_mode = False - config_manager.save_config(cfg) - - client = create_client() - - # Verify timeout configuration - assert isinstance(client.timeout, Timeout) - assert client.timeout.connect == 10.0 # 10 seconds for connection - assert client.timeout.read == 30.0 # 30 seconds for reading - assert client.timeout.write == 30.0 # 30 seconds for writing - assert client.timeout.pool == 30.0 # 30 seconds for pool diff --git a/tests/api/test_continue_conversation_template.py b/tests/api/test_continue_conversation_template.py deleted file mode 100644 index 6f75b85c..00000000 --- a/tests/api/test_continue_conversation_template.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Tests for the continue_conversation template rendering.""" - -import datetime -import pytest - -from basic_memory.api.template_loader import TemplateLoader -from basic_memory.schemas.memory import EntitySummary -from basic_memory.schemas.search import SearchItemType - - -@pytest.fixture -def template_loader(): - """Return a TemplateLoader instance for testing.""" - return TemplateLoader() - - -@pytest.fixture -def entity_summary(): - """Create a sample EntitySummary for testing.""" - return EntitySummary( - entity_id=1, - title="Test Entity", - permalink="test/entity", - type=SearchItemType.ENTITY, - content="This is a test entity with some content.", - file_path="/path/to/test/entity.md", - created_at=datetime.datetime(2023, 1, 1, 12, 0), - ) - - -@pytest.fixture -def context_with_results(entity_summary): - """Create a sample context with results for testing.""" - from basic_memory.schemas.memory import ObservationSummary, ContextResult - - # Create an observation for the entity - observation = ObservationSummary( - observation_id=1, - entity_id=1, - title="Test Observation", - permalink="test/entity/observations/1", - category="test", - content="This is a test observation.", - file_path="/path/to/test/entity.md", - created_at=datetime.datetime(2023, 1, 1, 12, 0), - ) - - # Create a context result with primary_result, observations, and related_results - context_item = ContextResult( - primary_result=entity_summary, - observations=[observation], - related_results=[entity_summary], - ) - - return { - "topic": "Test Topic", - "timeframe": "7d", - "has_results": True, - "hierarchical_results": [context_item], - } - - -@pytest.fixture -def context_without_results(): - """Create a sample context without results for testing.""" - return { - "topic": "Empty Topic", - "timeframe": "1d", - "has_results": False, - "hierarchical_results": [], - } - - -@pytest.mark.asyncio -async def test_continue_conversation_with_results(template_loader, context_with_results): - """Test rendering the continue_conversation template with results.""" - result = await template_loader.render("prompts/continue_conversation.hbs", context_with_results) - - # Check that key elements are present - assert "Continuing conversation on: Test Topic" in result - assert "memory://test/entity" in result - assert "Test Entity" in result - assert "This is a test entity with some content." in result - assert "Related Context" in result - assert "read_note" in result - assert "Next Steps" in result - assert "Knowledge Capture Recommendation" in result - - -@pytest.mark.asyncio -async def test_continue_conversation_without_results(template_loader, context_without_results): - """Test rendering the continue_conversation template without results.""" - result = await template_loader.render( - "prompts/continue_conversation.hbs", context_without_results - ) - - # Check that key elements are present - assert "Continuing conversation on: Empty Topic" in result - assert "The supplied query did not return any information" in result - assert "Opportunity to Capture New Knowledge!" in result - assert 'title="Empty Topic"' in result - assert "Next Steps" in result - assert "Knowledge Capture Recommendation" in result - - -@pytest.mark.asyncio -async def test_next_steps_section(template_loader, context_with_results): - """Test that the next steps section is rendered correctly.""" - result = await template_loader.render("prompts/continue_conversation.hbs", context_with_results) - - assert "Next Steps" in result - assert 'Explore more with: `search_notes("Test Topic")`' in result - assert ( - f'See what\'s changed: `recent_activity(timeframe="{context_with_results["timeframe"]}")`' - in result - ) - assert "Record new learnings or decisions from this conversation" in result - - -@pytest.mark.asyncio -async def test_knowledge_capture_recommendation(template_loader, context_with_results): - """Test that the knowledge capture recommendation is rendered.""" - result = await template_loader.render("prompts/continue_conversation.hbs", context_with_results) - - assert "Knowledge Capture Recommendation" in result - assert "actively look for opportunities to:" in result - assert "Record key information, decisions, or insights" in result - assert "Link new knowledge to existing topics" in result - assert "Suggest capturing important context" in result - assert "one of the most valuable aspects of Basic Memory" in result - - -@pytest.mark.asyncio -async def test_timeframe_default_value(template_loader, context_with_results): - """Test that the timeframe uses the default value when not provided.""" - # Remove the timeframe from the context - context_without_timeframe = context_with_results.copy() - context_without_timeframe["timeframe"] = None - - result = await template_loader.render( - "prompts/continue_conversation.hbs", context_without_timeframe - ) - - # Check that the default value is used - assert 'recent_activity(timeframe="7d")' in result diff --git a/tests/api/test_directory_router.py b/tests/api/test_directory_router.py deleted file mode 100644 index f55d9249..00000000 --- a/tests/api/test_directory_router.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Tests for the directory router API endpoints.""" - -import pytest - - -@pytest.mark.asyncio -async def test_get_directory_tree_endpoint(test_graph, client, project_url): - """Test the get_directory_tree endpoint returns correctly structured data.""" - # Call the endpoint - response = await client.get(f"{project_url}/directory/tree") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check that the response is a valid directory tree - assert "name" in data - assert "directory_path" in data - assert "children" in data - assert "type" in data - - # The root node should have children - assert isinstance(data["children"], list) - - # Root name should be the project name or similar - assert data["name"] - - # Root directory_path should be a string - assert isinstance(data["directory_path"], str) - - -@pytest.mark.asyncio -async def test_get_directory_tree_structure(test_graph, client, project_url): - """Test the structure of the directory tree returned by the endpoint.""" - # Call the endpoint - response = await client.get(f"{project_url}/directory/tree") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Function to recursively check each node in the tree - def check_node_structure(node): - assert "name" in node - assert "directory_path" in node - assert "children" in node - assert "type" in node - assert isinstance(node["children"], list) - - # Check each child recursively - for child in node["children"]: - check_node_structure(child) - - # Check the entire tree structure - check_node_structure(data) - - -@pytest.mark.asyncio -async def test_list_directory_endpoint_default(test_graph, client, project_url): - """Test the list_directory endpoint with default parameters.""" - # Call the endpoint with default parameters - response = await client.get(f"{project_url}/directory/list") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Should return a list - assert isinstance(data, list) - - # With test_graph, should return the "test" directory - assert len(data) == 1 - assert data[0]["name"] == "test" - assert data[0]["type"] == "directory" - - -@pytest.mark.asyncio -async def test_list_directory_endpoint_specific_path(test_graph, client, project_url): - """Test the list_directory endpoint with specific directory path.""" - # Call the endpoint with /test directory - response = await client.get(f"{project_url}/directory/list?dir_name=/test") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Should return list of files in test directory - assert isinstance(data, list) - assert len(data) == 5 - - # All should be files (no subdirectories in test_graph) - for item in data: - assert item["type"] == "file" - assert item["name"].endswith(".md") - - -@pytest.mark.asyncio -async def test_list_directory_endpoint_with_glob(test_graph, client, project_url): - """Test the list_directory endpoint with glob filtering.""" - # Call the endpoint with glob filter - response = await client.get( - f"{project_url}/directory/list?dir_name=/test&file_name_glob=*Connected*" - ) - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Should return only Connected Entity files - assert isinstance(data, list) - assert len(data) == 2 - - file_names = {item["name"] for item in data} - assert file_names == {"Connected Entity 1.md", "Connected Entity 2.md"} - - -@pytest.mark.asyncio -async def test_list_directory_endpoint_with_depth(test_graph, client, project_url): - """Test the list_directory endpoint with depth control.""" - # Test depth=1 (default) - response_depth_1 = await client.get(f"{project_url}/directory/list?dir_name=/&depth=1") - assert response_depth_1.status_code == 200 - data_depth_1 = response_depth_1.json() - assert len(data_depth_1) == 1 # Just the test directory - - # Test depth=2 (should include files in test directory) - response_depth_2 = await client.get(f"{project_url}/directory/list?dir_name=/&depth=2") - assert response_depth_2.status_code == 200 - data_depth_2 = response_depth_2.json() - assert len(data_depth_2) == 6 # test directory + 5 files - - -@pytest.mark.asyncio -async def test_list_directory_endpoint_nonexistent_path(test_graph, client, project_url): - """Test the list_directory endpoint with nonexistent directory.""" - # Call the endpoint with nonexistent directory - response = await client.get(f"{project_url}/directory/list?dir_name=/nonexistent") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Should return empty list - assert isinstance(data, list) - assert len(data) == 0 - - -@pytest.mark.asyncio -async def test_list_directory_endpoint_validation_errors(client, project_url): - """Test the list_directory endpoint with invalid parameters.""" - # Test depth too low - response = await client.get(f"{project_url}/directory/list?depth=0") - assert response.status_code == 422 # Validation error - - # Test depth too high - response = await client.get(f"{project_url}/directory/list?depth=11") - assert response.status_code == 422 # Validation error - - -@pytest.mark.asyncio -async def test_get_directory_structure_endpoint(test_graph, client, project_url): - """Test the get_directory_structure endpoint returns folders only.""" - # Call the endpoint - response = await client.get(f"{project_url}/directory/structure") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check that the response is a valid directory tree - assert "name" in data - assert "directory_path" in data - assert "children" in data - assert "type" in data - assert data["type"] == "directory" - - # Root should be present - assert data["name"] == "Root" - assert data["directory_path"] == "/" - - # Should have the test directory - assert len(data["children"]) == 1 - test_dir = data["children"][0] - assert test_dir["name"] == "test" - assert test_dir["type"] == "directory" - assert test_dir["directory_path"] == "/test" - - # Should NOT have any files (test_graph has files but no subdirectories) - assert len(test_dir["children"]) == 0 - - # Verify no file metadata is present in directory nodes - assert test_dir.get("entity_id") is None - assert test_dir.get("content_type") is None - assert test_dir.get("title") is None - assert test_dir.get("permalink") is None - - -@pytest.mark.asyncio -async def test_get_directory_structure_empty(client, project_url): - """Test the get_directory_structure endpoint with empty database.""" - # Call the endpoint - response = await client.get(f"{project_url}/directory/structure") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Should return root with no children - assert data["name"] == "Root" - assert data["directory_path"] == "/" - assert data["type"] == "directory" - assert len(data["children"]) == 0 diff --git a/tests/api/test_importer_router.py b/tests/api/test_importer_router.py deleted file mode 100644 index ccf9800c..00000000 --- a/tests/api/test_importer_router.py +++ /dev/null @@ -1,465 +0,0 @@ -"""Tests for importer API routes.""" - -import json -from pathlib import Path - -import pytest -from httpx import AsyncClient - -from basic_memory.schemas.importer import ( - ChatImportResult, - EntityImportResult, - ProjectImportResult, -) - - -@pytest.fixture -def chatgpt_json_content(): - """Sample ChatGPT conversation data for testing.""" - return [ - { - "title": "Test Conversation", - "create_time": 1736616594.24054, # Example timestamp - "update_time": 1736616603.164995, - "mapping": { - "root": {"id": "root", "message": None, "parent": None, "children": ["msg1"]}, - "msg1": { - "id": "msg1", - "message": { - "id": "msg1", - "author": {"role": "user", "name": None, "metadata": {}}, - "create_time": 1736616594.24054, - "content": { - "content_type": "text", - "parts": ["Hello, this is a test message"], - }, - "status": "finished_successfully", - "metadata": {}, - }, - "parent": "root", - "children": ["msg2"], - }, - "msg2": { - "id": "msg2", - "message": { - "id": "msg2", - "author": {"role": "assistant", "name": None, "metadata": {}}, - "create_time": 1736616603.164995, - "content": {"content_type": "text", "parts": ["This is a test response"]}, - "status": "finished_successfully", - "metadata": {}, - }, - "parent": "msg1", - "children": [], - }, - }, - } - ] - - -@pytest.fixture -def claude_conversations_json_content(): - """Sample Claude conversations data for testing.""" - return [ - { - "uuid": "test-uuid", - "name": "Test Conversation", - "created_at": "2025-01-05T20:55:32.499880+00:00", - "updated_at": "2025-01-05T20:56:39.477600+00:00", - "chat_messages": [ - { - "uuid": "msg-1", - "text": "Hello, this is a test", - "sender": "human", - "created_at": "2025-01-05T20:55:32.499880+00:00", - "content": [{"type": "text", "text": "Hello, this is a test"}], - }, - { - "uuid": "msg-2", - "text": "Response to test", - "sender": "assistant", - "created_at": "2025-01-05T20:55:40.123456+00:00", - "content": [{"type": "text", "text": "Response to test"}], - }, - ], - } - ] - - -@pytest.fixture -def claude_projects_json_content(): - """Sample Claude projects data for testing.""" - return [ - { - "uuid": "test-uuid", - "name": "Test Project", - "created_at": "2025-01-05T20:55:32.499880+00:00", - "updated_at": "2025-01-05T20:56:39.477600+00:00", - "prompt_template": "# Test Prompt\n\nThis is a test prompt.", - "docs": [ - { - "uuid": "doc-uuid-1", - "filename": "Test Document", - "content": "# Test Document\n\nThis is test content.", - "created_at": "2025-01-05T20:56:39.477600+00:00", - }, - { - "uuid": "doc-uuid-2", - "filename": "Another Document", - "content": "# Another Document\n\nMore test content.", - "created_at": "2025-01-05T20:56:39.477600+00:00", - }, - ], - } - ] - - -@pytest.fixture -def memory_json_content(): - """Sample memory.json data for testing.""" - return [ - { - "type": "entity", - "name": "test_entity", - "entityType": "test", - "observations": ["Test observation 1", "Test observation 2"], - }, - { - "type": "relation", - "from": "test_entity", - "to": "related_entity", - "relationType": "test_relation", - }, - ] - - -async def create_test_upload_file(tmp_path, content): - """Create a test file for upload.""" - file_path = tmp_path / "test_import.json" - with open(file_path, "w", encoding="utf-8") as f: - json.dump(content, f) - - return file_path - - -@pytest.mark.asyncio -async def test_import_chatgpt( - project_config, client: AsyncClient, tmp_path, chatgpt_json_content, file_service, project_url -): - """Test importing ChatGPT conversations.""" - # Create a test file - file_path = await create_test_upload_file(tmp_path, chatgpt_json_content) - - # Create a multipart form with the file - with open(file_path, "rb") as f: - files = {"file": ("conversations.json", f, "application/json")} - data = {"directory": "test_chatgpt"} - - # Send request - response = await client.post(f"{project_url}/import/chatgpt", files=files, data=data) - - # Check response - assert response.status_code == 200 - result = ChatImportResult.model_validate(response.json()) - assert result.success is True - assert result.conversations == 1 - assert result.messages == 2 - - # Verify files were created - conv_path = Path("test_chatgpt") / "20250111-Test_Conversation.md" - assert await file_service.exists(conv_path) - - content, _ = await file_service.read_file(conv_path) - assert "# Test Conversation" in content - assert "Hello, this is a test message" in content - assert "This is a test response" in content - - -@pytest.mark.asyncio -async def test_import_chatgpt_invalid_file(client: AsyncClient, tmp_path, project_url): - """Test importing invalid ChatGPT file.""" - # Create invalid file - file_path = tmp_path / "invalid.json" - with open(file_path, "w") as f: - f.write("This is not JSON") - - # Create multipart form with invalid file - with open(file_path, "rb") as f: - files = {"file": ("invalid.json", f, "application/json")} - data = {"directory": "test_chatgpt"} - - # Send request - this should return an error - response = await client.post(f"{project_url}/import/chatgpt", files=files, data=data) - - # Check response - assert response.status_code == 500 - assert "Import failed" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_import_claude_conversations( - client: AsyncClient, tmp_path, claude_conversations_json_content, file_service, project_url -): - """Test importing Claude conversations.""" - # Create a test file - file_path = await create_test_upload_file(tmp_path, claude_conversations_json_content) - - # Create a multipart form with the file - with open(file_path, "rb") as f: - files = {"file": ("conversations.json", f, "application/json")} - data = {"directory": "test_claude_conversations"} - - # Send request - response = await client.post( - f"{project_url}/import/claude/conversations", files=files, data=data - ) - - # Check response - assert response.status_code == 200 - result = ChatImportResult.model_validate(response.json()) - assert result.success is True - assert result.conversations == 1 - assert result.messages == 2 - - # Verify files were created - conv_path = Path("test_claude_conversations") / "20250105-Test_Conversation.md" - assert await file_service.exists(conv_path) - - content, _ = await file_service.read_file(conv_path) - assert "# Test Conversation" in content - assert "Hello, this is a test" in content - assert "Response to test" in content - - -@pytest.mark.asyncio -async def test_import_claude_conversations_invalid_file(client: AsyncClient, tmp_path, project_url): - """Test importing invalid Claude conversations file.""" - # Create invalid file - file_path = tmp_path / "invalid.json" - with open(file_path, "w") as f: - f.write("This is not JSON") - - # Create multipart form with invalid file - with open(file_path, "rb") as f: - files = {"file": ("invalid.json", f, "application/json")} - data = {"directory": "test_claude_conversations"} - - # Send request - this should return an error - response = await client.post( - f"{project_url}/import/claude/conversations", files=files, data=data - ) - - # Check response - assert response.status_code == 500 - assert "Import failed" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_import_claude_projects( - client: AsyncClient, tmp_path, claude_projects_json_content, file_service, project_url -): - """Test importing Claude projects.""" - # Create a test file - file_path = await create_test_upload_file(tmp_path, claude_projects_json_content) - - # Create a multipart form with the file - with open(file_path, "rb") as f: - files = {"file": ("projects.json", f, "application/json")} - data = {"directory": "test_claude_projects"} - - # Send request - response = await client.post( - f"{project_url}/import/claude/projects", files=files, data=data - ) - - # Check response - assert response.status_code == 200 - result = ProjectImportResult.model_validate(response.json()) - assert result.success is True - assert result.documents == 2 - assert result.prompts == 1 - - # Verify files were created - project_dir = Path("test_claude_projects") / "Test_Project" - assert await file_service.exists(project_dir / "prompt-template.md") - assert await file_service.exists(project_dir / "docs" / "Test_Document.md") - assert await file_service.exists(project_dir / "docs" / "Another_Document.md") - - # Check content - prompt_content, _ = await file_service.read_file(project_dir / "prompt-template.md") - assert "# Test Prompt" in prompt_content - - doc_content, _ = await file_service.read_file(project_dir / "docs" / "Test_Document.md") - assert "# Test Document" in doc_content - assert "This is test content" in doc_content - - -@pytest.mark.asyncio -async def test_import_claude_projects_invalid_file(client: AsyncClient, tmp_path, project_url): - """Test importing invalid Claude projects file.""" - # Create invalid file - file_path = tmp_path / "invalid.json" - with open(file_path, "w") as f: - f.write("This is not JSON") - - # Create multipart form with invalid file - with open(file_path, "rb") as f: - files = {"file": ("invalid.json", f, "application/json")} - data = {"directory": "test_claude_projects"} - - # Send request - this should return an error - response = await client.post( - f"{project_url}/import/claude/projects", files=files, data=data - ) - - # Check response - assert response.status_code == 500 - assert "Import failed" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_import_memory_json( - client: AsyncClient, tmp_path, memory_json_content, file_service, project_url -): - """Test importing memory.json file.""" - # Create a test file - json_file = tmp_path / "memory.json" - with open(json_file, "w", encoding="utf-8") as f: - for entity in memory_json_content: - f.write(json.dumps(entity) + "\n") - - # Create a multipart form with the file - with open(json_file, "rb") as f: - files = {"file": ("memory.json", f, "application/json")} - data = {"directory": "test_memory_json"} - - # Send request - response = await client.post(f"{project_url}/import/memory-json", files=files, data=data) - - # Check response - assert response.status_code == 200 - result = EntityImportResult.model_validate(response.json()) - assert result.success is True - assert result.entities == 1 - assert result.relations == 1 - - # Verify files were created - entity_path = Path("test_memory_json") / "test" / "test_entity.md" - assert await file_service.exists(entity_path) - - # Check content - content, _ = await file_service.read_file(entity_path) - assert "Test observation 1" in content - assert "Test observation 2" in content - assert "test_relation [[related_entity]]" in content - - -@pytest.mark.asyncio -async def test_import_memory_json_without_folder( - client: AsyncClient, tmp_path, memory_json_content, file_service, project_url -): - """Test importing memory.json file without specifying a destination folder.""" - # Create a test file - json_file = tmp_path / "memory.json" - with open(json_file, "w", encoding="utf-8") as f: - for entity in memory_json_content: - f.write(json.dumps(entity) + "\n") - - # Create a multipart form with the file - with open(json_file, "rb") as f: - files = {"file": ("memory.json", f, "application/json")} - - # Send request without destination_folder - response = await client.post(f"{project_url}/import/memory-json", files=files) - - # Check response - assert response.status_code == 200 - result = EntityImportResult.model_validate(response.json()) - assert result.success is True - assert result.entities == 1 - assert result.relations == 1 - - # Verify files were created in the root directory - entity_path = Path("conversations") / "test" / "test_entity.md" - assert await file_service.exists(entity_path) - - -@pytest.mark.asyncio -async def test_import_memory_json_invalid_file(client: AsyncClient, tmp_path, project_url): - """Test importing invalid memory.json file.""" - # Create invalid file - file_path = tmp_path / "invalid.json" - with open(file_path, "w") as f: - f.write("This is not JSON") - - # Create multipart form with invalid file - with open(file_path, "rb") as f: - files = {"file": ("invalid.json", f, "application/json")} - data = {"destination_folder": "test_memory_json"} - - # Send request - this should return an error - response = await client.post(f"{project_url}/import/memory-json", files=files, data=data) - - # Check response - assert response.status_code == 500 - assert "Import failed" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_import_missing_file(client: AsyncClient, tmp_path, project_url): - """Test importing with missing file.""" - # Send a request without a file - response = await client.post(f"{project_url}/import/chatgpt", data={"directory": "test_folder"}) - - # Check that the request was rejected - assert response.status_code in [400, 422] # Either bad request or unprocessable entity - - -@pytest.mark.asyncio -async def test_import_empty_file(client: AsyncClient, tmp_path, project_url): - """Test importing an empty file.""" - # Create an empty file - file_path = tmp_path / "empty.json" - with open(file_path, "w") as f: - f.write("") - - # Create multipart form with empty file - with open(file_path, "rb") as f: - files = {"file": ("empty.json", f, "application/json")} - data = {"directory": "test_chatgpt"} - - # Send request - response = await client.post(f"{project_url}/import/chatgpt", files=files, data=data) - - # Check response - assert response.status_code == 500 - assert "Import failed" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_import_malformed_json(client: AsyncClient, tmp_path, project_url): - """Test importing malformed JSON for all import endpoints.""" - # Create malformed JSON file - file_path = tmp_path / "malformed.json" - with open(file_path, "w") as f: - f.write('{"incomplete": "json"') # Missing closing brace - - # Test all import endpoints - endpoints = [ - (f"{project_url}/import/chatgpt", {"directory": "test"}), - (f"{project_url}/import/claude/conversations", {"directory": "test"}), - (f"{project_url}/import/claude/projects", {"base_folder": "test"}), - (f"{project_url}/import/memory-json", {"destination_folder": "test"}), - ] - - for endpoint, data in endpoints: - # Create multipart form with malformed JSON - with open(file_path, "rb") as f: - files = {"file": ("malformed.json", f, "application/json")} - - # Send request - response = await client.post(endpoint, files=files, data=data) - - # Check response - assert response.status_code == 500 - assert "Import failed" in response.json()["detail"] diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.py deleted file mode 100644 index 4c4f273e..00000000 --- a/tests/api/test_knowledge_router.py +++ /dev/null @@ -1,1406 +0,0 @@ -"""Tests for knowledge graph API routes.""" - -from urllib.parse import quote - -import pytest -from httpx import AsyncClient - -from basic_memory.schemas import ( - Entity, - EntityResponse, -) -from basic_memory.schemas.response import DirectoryMoveResult -from basic_memory.schemas.search import SearchItemType, SearchResponse -from basic_memory.utils import normalize_newlines - - -@pytest.mark.asyncio -async def test_create_entity(client: AsyncClient, file_service, project_url): - """Should create entity successfully.""" - - data = { - "title": "TestEntity", - "directory": "test", - "entity_type": "test", - "content": "TestContent", - "project": "Test Project Context", - } - # Create an entity - print(f"Requesting with data: {data}") - # Use the permalink version of the project name in the path - response = await client.post(f"{project_url}/knowledge/entities", json=data) - # Print response for debugging - print(f"Response status: {response.status_code}") - print(f"Response content: {response.text}") - # Verify creation - assert response.status_code == 200 - entity = EntityResponse.model_validate(response.json()) - - assert entity.permalink == "test/test-entity" - assert entity.file_path == "test/TestEntity.md" - assert entity.entity_type == data["entity_type"] - assert entity.content_type == "text/markdown" - - # Verify file has new content but preserved metadata - file_path = file_service.get_entity_path(entity) - file_content, _ = await file_service.read_file(file_path) - - assert data["content"] in file_content - - -@pytest.mark.asyncio -async def test_create_entity_observations_relations(client: AsyncClient, file_service, project_url): - """Should create entity successfully.""" - - data = { - "title": "TestEntity", - "directory": "test", - "content": """ -# TestContent - -## Observations -- [note] This is notable #tag1 (testing) -- related to [[SomeOtherThing]] -""", - } - # Create an entity - response = await client.post(f"{project_url}/knowledge/entities", json=data) - # Verify creation - assert response.status_code == 200 - entity = EntityResponse.model_validate(response.json()) - - assert entity.permalink == "test/test-entity" - assert entity.file_path == "test/TestEntity.md" - assert entity.entity_type == "note" - assert entity.content_type == "text/markdown" - - assert len(entity.observations) == 1 - assert entity.observations[0].category == "note" - assert entity.observations[0].content == "This is notable #tag1" - assert entity.observations[0].tags == ["tag1"] - assert entity.observations[0].context == "testing" - - assert len(entity.relations) == 1 - assert entity.relations[0].relation_type == "related to" - assert entity.relations[0].from_id == "test/test-entity" - assert entity.relations[0].to_id is None - - # Verify file has new content but preserved metadata - file_path = file_service.get_entity_path(entity) - file_content, _ = await file_service.read_file(file_path) - - assert data["content"].strip() in file_content - - -@pytest.mark.asyncio -async def test_relation_resolution_after_creation(client: AsyncClient, project_url): - """Test that relation resolution works after creating entities and handles exceptions gracefully.""" - - # Create first entity with unresolved relation - entity1_data = { - "title": "EntityOne", - "directory": "test", - "entity_type": "test", - "content": "This entity references [[EntityTwo]]", - } - response1 = await client.put( - f"{project_url}/knowledge/entities/test/entity-one", json=entity1_data - ) - assert response1.status_code == 201 - entity1 = response1.json() - - # Verify relation exists but is unresolved - assert len(entity1["relations"]) == 1 - assert entity1["relations"][0]["to_id"] is None - assert entity1["relations"][0]["to_name"] == "EntityTwo" - - # Create the referenced entity - entity2_data = { - "title": "EntityTwo", - "directory": "test", - "entity_type": "test", - "content": "This is the referenced entity", - } - response2 = await client.put( - f"{project_url}/knowledge/entities/test/entity-two", json=entity2_data - ) - assert response2.status_code == 201 - - # Verify the original entity's relation was resolved - response_check = await client.get(f"{project_url}/knowledge/entities/test/entity-one") - assert response_check.status_code == 200 - updated_entity1 = response_check.json() - - # The relation should now be resolved via the automatic resolution after entity creation - resolved_relations = [r for r in updated_entity1["relations"] if r["to_id"] is not None] - assert ( - len(resolved_relations) >= 0 - ) # May or may not be resolved immediately depending on timing - - -@pytest.mark.asyncio -async def test_get_entity_by_permalink(client: AsyncClient, project_url): - """Should retrieve an entity by path ID.""" - # First create an entity - data = {"title": "TestEntity", "directory": "test", "entity_type": "test"} - response = await client.post(f"{project_url}/knowledge/entities", json=data) - assert response.status_code == 200 - data = response.json() - - # Now get it by permalink - permalink = data["permalink"] - response = await client.get(f"{project_url}/knowledge/entities/{permalink}") - - # Verify retrieval - assert response.status_code == 200 - entity = response.json() - assert entity["title"] == "TestEntity" - assert entity["file_path"] == "test/TestEntity.md" - assert entity["entity_type"] == "test" - assert entity["permalink"] == "test/test-entity" - - -@pytest.mark.asyncio -async def test_get_entity_by_file_path(client: AsyncClient, project_url): - """Should retrieve an entity by path ID.""" - # First create an entity - data = {"title": "TestEntity", "directory": "test", "entity_type": "test"} - response = await client.post(f"{project_url}/knowledge/entities", json=data) - assert response.status_code == 200 - data = response.json() - - # Now get it by path - file_path = data["file_path"] - response = await client.get(f"{project_url}/knowledge/entities/{file_path}") - - # Verify retrieval - assert response.status_code == 200 - entity = response.json() - assert entity["title"] == "TestEntity" - assert entity["file_path"] == "test/TestEntity.md" - assert entity["entity_type"] == "test" - assert entity["permalink"] == "test/test-entity" - - -@pytest.mark.asyncio -async def test_get_entities(client: AsyncClient, project_url): - """Should open multiple entities by path IDs.""" - # Create a few entities with different names - await client.post( - f"{project_url}/knowledge/entities", - json={"title": "AlphaTest", "directory": "", "entity_type": "test"}, - ) - await client.post( - f"{project_url}/knowledge/entities", - json={"title": "BetaTest", "directory": "", "entity_type": "test"}, - ) - - # Open nodes by path IDs - response = await client.get( - f"{project_url}/knowledge/entities?permalink=alpha-test&permalink=beta-test", - ) - - # Verify results - assert response.status_code == 200 - data = response.json() - assert len(data["entities"]) == 2 - - entity_0 = data["entities"][0] - assert entity_0["title"] == "AlphaTest" - assert entity_0["file_path"] == "AlphaTest.md" - assert entity_0["entity_type"] == "test" - assert entity_0["permalink"] == "alpha-test" - - entity_1 = data["entities"][1] - assert entity_1["title"] == "BetaTest" - assert entity_1["file_path"] == "BetaTest.md" - assert entity_1["entity_type"] == "test" - assert entity_1["permalink"] == "beta-test" - - -@pytest.mark.asyncio -async def test_delete_entity(client: AsyncClient, project_url): - """Test DELETE /knowledge/entities with path ID.""" - # Create test entity - entity_data = {"file_path": "TestEntity", "entity_type": "test"} - await client.post(f"{project_url}/knowledge/entities", json=entity_data) - - # Test deletion - response = await client.post( - f"{project_url}/knowledge/entities/delete", json={"permalinks": ["test-entity"]} - ) - assert response.status_code == 200 - assert response.json() == {"deleted": True} - - # Verify entity is gone - permalink = quote("test/TestEntity") - response = await client.get(f"{project_url}/knowledge/entities/{permalink}") - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_delete_single_entity(client: AsyncClient, project_url): - """Test DELETE /knowledge/entities with path ID.""" - # Create test entity - entity_data = {"title": "TestEntity", "directory": "", "entity_type": "test"} - await client.post(f"{project_url}/knowledge/entities", json=entity_data) - - # Test deletion - response = await client.delete(f"{project_url}/knowledge/entities/test-entity") - assert response.status_code == 200 - assert response.json() == {"deleted": True} - - # Verify entity is gone - permalink = quote("test/TestEntity") - response = await client.get(f"{project_url}/knowledge/entities/{permalink}") - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_delete_single_entity_by_title(client: AsyncClient, project_url): - """Test DELETE /knowledge/entities with file path.""" - # Create test entity - entity_data = {"title": "TestEntity", "directory": "", "entity_type": "test"} - response = await client.post(f"{project_url}/knowledge/entities", json=entity_data) - assert response.status_code == 200 - data = response.json() - - # Test deletion - response = await client.delete(f"{project_url}/knowledge/entities/TestEntity") - assert response.status_code == 200 - assert response.json() == {"deleted": True} - - # Verify entity is gone - file_path = quote(data["file_path"]) - response = await client.get(f"{project_url}/knowledge/entities/{file_path}") - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_delete_single_entity_not_found(client: AsyncClient, project_url): - """Test DELETE /knowledge/entities with path ID.""" - - # Test deletion - response = await client.delete(f"{project_url}/knowledge/entities/test-not-found") - assert response.status_code == 200 - assert response.json() == {"deleted": False} - - -@pytest.mark.asyncio -async def test_delete_entity_bulk(client: AsyncClient, project_url): - """Test bulk entity deletion using path IDs.""" - # Create test entities - await client.post( - f"{project_url}/knowledge/entities", json={"file_path": "Entity1", "entity_type": "test"} - ) - await client.post( - f"{project_url}/knowledge/entities", json={"file_path": "Entity2", "entity_type": "test"} - ) - - # Test deletion - response = await client.post( - f"{project_url}/knowledge/entities/delete", json={"permalinks": ["Entity1", "Entity2"]} - ) - assert response.status_code == 200 - assert response.json() == {"deleted": True} - - # Verify entities are gone - for name in ["Entity1", "Entity2"]: - permalink = quote(f"{name}") - response = await client.get(f"{project_url}/knowledge/entities/{permalink}") - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_delete_nonexistent_entity(client: AsyncClient, project_url): - """Test deleting a nonexistent entity by path ID.""" - response = await client.post( - f"{project_url}/knowledge/entities/delete", json={"permalinks": ["non_existent"]} - ) - assert response.status_code == 200 - assert response.json() == {"deleted": True} - - -@pytest.mark.asyncio -async def test_entity_indexing(client: AsyncClient, project_url): - """Test entity creation includes search indexing.""" - # Create entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "SearchTest", - "directory": "", - "entity_type": "test", - "observations": ["Unique searchable observation"], - }, - ) - assert response.status_code == 200 - - # Verify it's searchable - search_response = await client.post( - f"{project_url}/search/", - json={"text": "search", "entity_types": [SearchItemType.ENTITY.value]}, - ) - assert search_response.status_code == 200 - search_result = SearchResponse.model_validate(search_response.json()) - assert len(search_result.results) == 1 - assert search_result.results[0].permalink == "search-test" - assert search_result.results[0].type == SearchItemType.ENTITY.value - - -@pytest.mark.asyncio -async def test_entity_delete_indexing(client: AsyncClient, project_url): - """Test deleted entities are removed from search index.""" - - # Create entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "DeleteTest", - "directory": "", - "entity_type": "test", - "observations": ["Searchable observation that should be removed"], - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Verify it's initially searchable - search_response = await client.post( - f"{project_url}/search/", - json={"text": "delete", "entity_types": [SearchItemType.ENTITY.value]}, - ) - search_result = SearchResponse.model_validate(search_response.json()) - assert len(search_result.results) == 1 - - # Delete entity - delete_response = await client.post( - f"{project_url}/knowledge/entities/delete", json={"permalinks": [entity["permalink"]]} - ) - assert delete_response.status_code == 200 - - # Verify it's no longer searchable - search_response = await client.post( - f"{project_url}/search/", json={"text": "delete", "types": [SearchItemType.ENTITY.value]} - ) - search_result = SearchResponse.model_validate(search_response.json()) - assert len(search_result.results) == 0 - - -@pytest.mark.asyncio -async def test_update_entity_basic(client: AsyncClient, project_url): - """Test basic entity field updates.""" - # Create initial entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "test", - "directory": "", - "entity_type": "test", - "content": "Initial summary", - "entity_metadata": {"status": "draft"}, - }, - ) - entity_response = response.json() - - # Update fields - entity = Entity(**entity_response, directory="") - entity.entity_metadata["status"] = "final" - entity.content = "Updated summary" - - response = await client.put( - f"{project_url}/knowledge/entities/{entity.permalink}", json=entity.model_dump() - ) - assert response.status_code == 200 - updated = response.json() - - # Verify updates - assert updated["entity_metadata"]["status"] == "final" # Preserved - - response = await client.get(f"{project_url}/resource/{updated['permalink']}?content=true") - - # raw markdown content - fetched = response.text - assert "Updated summary" in fetched - - -@pytest.mark.asyncio -async def test_update_entity_content(client: AsyncClient, project_url): - """Test updating content for different entity types.""" - # Create a note entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={"title": "test-note", "directory": "", "entity_type": "note", "summary": "Test note"}, - ) - note = response.json() - - # Update fields - entity = Entity(**note, directory="") - entity.content = "# Updated Note\n\nNew content." - - response = await client.put( - f"{project_url}/knowledge/entities/{note['permalink']}", json=entity.model_dump() - ) - assert response.status_code == 200 - updated = response.json() - - # Verify through get request to check file - response = await client.get(f"{project_url}/resource/{updated['permalink']}?content=true") - - # raw markdown content - fetched = response.text - assert "# Updated Note" in fetched - assert "New content" in fetched - - -@pytest.mark.asyncio -async def test_update_entity_type_conversion(client: AsyncClient, project_url): - """Test converting between note and knowledge types.""" - # Create a note - note_data = { - "title": "test-note", - "directory": "", - "entity_type": "note", - "summary": "Test note", - "content": "# Test Note\n\nInitial content.", - } - response = await client.post(f"{project_url}/knowledge/entities", json=note_data) - note = response.json() - - # Update fields - entity = Entity(**note, directory="") - entity.entity_type = "test" - - response = await client.put( - f"{project_url}/knowledge/entities/{note['permalink']}", json=entity.model_dump() - ) - assert response.status_code == 200 - updated = response.json() - - # Verify conversion - assert updated["entity_type"] == "test" - - # Get latest to verify file format - response = await client.get(f"{project_url}/knowledge/entities/{updated['permalink']}") - knowledge = response.json() - assert knowledge.get("content") is None - - -@pytest.mark.asyncio -async def test_update_entity_metadata(client: AsyncClient, project_url): - """Test updating entity metadata.""" - # Create entity - data = { - "title": "test", - "directory": "", - "entity_type": "test", - "entity_metadata": {"status": "draft"}, - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - - # Update fields - entity = Entity(**entity_response, directory="") - entity.entity_metadata["status"] = "final" - entity.entity_metadata["reviewed"] = True - - # Update metadata - response = await client.put( - f"{project_url}/knowledge/entities/{entity.permalink}", json=entity.model_dump() - ) - assert response.status_code == 200 - updated = response.json() - - # Verify metadata was merged, not replaced - assert updated["entity_metadata"]["status"] == "final" - assert updated["entity_metadata"]["reviewed"] in (True, "True") - - -@pytest.mark.asyncio -async def test_update_entity_not_found_does_create(client: AsyncClient, project_url): - """Test updating non-existent entity does a create""" - - data = { - "title": "nonexistent", - "directory": "", - "entity_type": "test", - "observations": ["First observation", "Second observation"], - } - entity = Entity(**data) - response = await client.put( - f"{project_url}/knowledge/entities/nonexistent", json=entity.model_dump() - ) - assert response.status_code == 201 - - -@pytest.mark.asyncio -async def test_update_entity_incorrect_permalink(client: AsyncClient, project_url): - """Test updating non-existent entity does a create""" - - data = { - "title": "Test Entity", - "directory": "", - "entity_type": "test", - "observations": ["First observation", "Second observation"], - } - entity = Entity(**data) - response = await client.put( - f"{project_url}/knowledge/entities/nonexistent", json=entity.model_dump() - ) - assert response.status_code == 400 - - -@pytest.mark.asyncio -async def test_update_entity_search_index(client: AsyncClient, project_url): - """Test search index is updated after entity changes.""" - # Create entity - data = { - "title": "test", - "directory": "", - "entity_type": "test", - "content": "Initial searchable content", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - - # Update fields - entity = Entity(**entity_response, directory="") - entity.content = "Updated with unique sphinx marker" - - response = await client.put( - f"{project_url}/knowledge/entities/{entity.permalink}", json=entity.model_dump() - ) - assert response.status_code == 200 - - # Search should find new content - search_response = await client.post( - f"{project_url}/search/", - json={"text": "sphinx marker", "entity_types": [SearchItemType.ENTITY.value]}, - ) - results = search_response.json()["results"] - assert len(results) == 1 - assert results[0]["permalink"] == entity.permalink - - -# PATCH edit entity endpoint tests - - -@pytest.mark.asyncio -async def test_edit_entity_append(client: AsyncClient, project_url): - """Test appending content to an entity via PATCH endpoint.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Test Note", - "directory": "test", - "entity_type": "note", - "content": "Original content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Edit entity with append operation - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "append", "content": "Appended content"}, - ) - if response.status_code != 200: - print(f"PATCH failed with status {response.status_code}") - print(f"Response content: {response.text}") - assert response.status_code == 200 - updated = response.json() - - # Verify content was appended by reading the file - response = await client.get(f"{project_url}/resource/{updated['permalink']}?content=true") - file_content = response.text - assert "Original content" in file_content - assert "Appended content" in file_content - assert file_content.index("Original content") < file_content.index("Appended content") - - -@pytest.mark.asyncio -async def test_edit_entity_prepend(client: AsyncClient, project_url): - """Test prepending content to an entity via PATCH endpoint.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Test Note", - "directory": "test", - "entity_type": "note", - "content": "Original content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Edit entity with prepend operation - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "prepend", "content": "Prepended content"}, - ) - if response.status_code != 200: - print(f"PATCH prepend failed with status {response.status_code}") - print(f"Response content: {response.text}") - assert response.status_code == 200 - updated = response.json() - - # Verify the entire file content structure - response = await client.get(f"{project_url}/resource/{updated['permalink']}?content=true") - file_content = response.text - - # Expected content with frontmatter preserved and content prepended to body - expected_content = normalize_newlines("""--- -title: Test Note -type: note -permalink: test/test-note ---- - -Prepended content -Original content""") - - assert file_content.strip() == expected_content.strip() - - -@pytest.mark.asyncio -async def test_edit_entity_find_replace(client: AsyncClient, project_url): - """Test find and replace operation via PATCH endpoint.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Test Note", - "directory": "test", - "entity_type": "note", - "content": "This is old content that needs updating", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Edit entity with find_replace operation - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "find_replace", "content": "new content", "find_text": "old content"}, - ) - assert response.status_code == 200 - updated = response.json() - - # Verify content was replaced - response = await client.get(f"{project_url}/resource/{updated['permalink']}?content=true") - file_content = response.text - assert "old content" not in file_content - assert "This is new content that needs updating" in file_content - - -@pytest.mark.asyncio -async def test_edit_entity_find_replace_with_expected_replacements( - client: AsyncClient, project_url -): - """Test find and replace with expected_replacements parameter.""" - # Create test entity with repeated text - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Sample Note", - "directory": "docs", - "entity_type": "note", - "content": "The word banana appears here. Another banana word here.", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Edit entity with find_replace operation, expecting 2 replacements - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={ - "operation": "find_replace", - "content": "apple", - "find_text": "banana", - "expected_replacements": 2, - }, - ) - assert response.status_code == 200 - updated = response.json() - - # Verify both instances were replaced - response = await client.get(f"{project_url}/resource/{updated['permalink']}?content=true") - file_content = response.text - assert "The word apple appears here. Another apple word here." in file_content - - -@pytest.mark.asyncio -async def test_edit_entity_replace_section(client: AsyncClient, project_url): - """Test replacing a section via PATCH endpoint.""" - # Create test entity with sections - content = """# Main Title - -## Section 1 -Original section 1 content - -## Section 2 -Original section 2 content""" - - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Sample Note", - "directory": "docs", - "entity_type": "note", - "content": content, - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Edit entity with replace_section operation - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={ - "operation": "replace_section", - "content": "New section 1 content", - "section": "## Section 1", - }, - ) - assert response.status_code == 200 - updated = response.json() - - # Verify section was replaced - response = await client.get(f"{project_url}/resource/{updated['permalink']}?content=true") - file_content = response.text - assert "New section 1 content" in file_content - assert "Original section 1 content" not in file_content - assert "Original section 2 content" in file_content # Other sections preserved - - -@pytest.mark.asyncio -async def test_edit_entity_not_found(client: AsyncClient, project_url): - """Test editing a non-existent entity returns 400.""" - response = await client.patch( - f"{project_url}/knowledge/entities/non-existent", - json={"operation": "append", "content": "content"}, - ) - assert response.status_code == 400 - assert "Entity not found" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_edit_entity_invalid_operation(client: AsyncClient, project_url): - """Test editing with invalid operation returns 400.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Test Note", - "directory": "test", - "entity_type": "note", - "content": "Original content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Try invalid operation - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "invalid_operation", "content": "content"}, - ) - assert response.status_code == 422 - assert "invalid_operation" in response.json()["detail"][0]["input"] - - -@pytest.mark.asyncio -async def test_edit_entity_find_replace_missing_find_text(client: AsyncClient, project_url): - """Test find_replace without find_text returns 400.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Test Note", - "directory": "test", - "entity_type": "note", - "content": "Original content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Try find_replace without find_text - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "find_replace", "content": "new content"}, - ) - assert response.status_code == 400 - assert "find_text is required" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_edit_entity_replace_section_missing_section(client: AsyncClient, project_url): - """Test replace_section without section parameter returns 400.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Test Note", - "directory": "test", - "entity_type": "note", - "content": "Original content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Try replace_section without section - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "replace_section", "content": "new content"}, - ) - assert response.status_code == 400 - assert "section is required" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_edit_entity_find_replace_not_found(client: AsyncClient, project_url): - """Test find_replace when text is not found returns 400.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Test Note", - "directory": "test", - "entity_type": "note", - "content": "This is some content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Try to replace text that doesn't exist - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "find_replace", "content": "new content", "find_text": "nonexistent"}, - ) - assert response.status_code == 400 - assert "Text to replace not found" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_edit_entity_find_replace_wrong_expected_count(client: AsyncClient, project_url): - """Test find_replace with wrong expected_replacements count returns 400.""" - # Create test entity with repeated text - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Sample Note", - "directory": "docs", - "entity_type": "note", - "content": "The word banana appears here. Another banana word here.", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Try to replace with wrong expected count - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={ - "operation": "find_replace", - "content": "replacement", - "find_text": "banana", - "expected_replacements": 1, # Wrong - there are actually 2 - }, - ) - assert response.status_code == 400 - assert "Expected 1 occurrences" in response.json()["detail"] - assert "but found 2" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_edit_entity_search_reindex(client: AsyncClient, project_url): - """Test that edited entities are reindexed for search.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "Search Test", - "directory": "test", - "entity_type": "note", - "content": "Original searchable content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Edit the entity - response = await client.patch( - f"{project_url}/knowledge/entities/{entity['permalink']}", - json={"operation": "append", "content": " with unique zebra marker"}, - ) - assert response.status_code == 200 - - # Search should find the new content - search_response = await client.post( - f"{project_url}/search/", - json={"text": "zebra marker", "entity_types": ["entity"]}, - ) - results = search_response.json()["results"] - assert len(results) == 1 - assert results[0]["permalink"] == entity["permalink"] - - -# Move entity endpoint tests - - -@pytest.mark.asyncio -async def test_move_entity_success(client: AsyncClient, project_url): - """Test successfully moving an entity to a new location.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "TestNote", - "directory": "source", - "entity_type": "note", - "content": "Test content", - }, - ) - assert response.status_code == 200 - entity = response.json() - original_permalink = entity["permalink"] - - # Move entity - move_data = { - "identifier": original_permalink, - "destination_path": "target/MovedNote.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 200 - response_model = EntityResponse.model_validate(response.json()) - assert response_model.file_path == "target/MovedNote.md" - - # Verify original entity no longer exists - response = await client.get(f"{project_url}/knowledge/entities/{original_permalink}") - assert response.status_code == 404 - - # Verify entity exists at new location - response = await client.get(f"{project_url}/knowledge/entities/target/moved-note") - assert response.status_code == 200 - moved_entity = response.json() - assert moved_entity["file_path"] == "target/MovedNote.md" - assert moved_entity["permalink"] == "target/moved-note" - - # Verify file content using resource endpoint - response = await client.get(f"{project_url}/resource/target/moved-note?content=true") - assert response.status_code == 200 - file_content = response.text - assert "Test content" in file_content - - -@pytest.mark.asyncio -async def test_move_entity_with_folder_creation(client: AsyncClient, project_url): - """Test moving entity creates necessary folders.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "TestNote", - "directory": "", - "entity_type": "note", - "content": "Test content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Move to deeply nested path - move_data = { - "identifier": entity["permalink"], - "destination_path": "deeply/nested/folder/MovedNote.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 200 - - # Verify entity exists at new location - response = await client.get(f"{project_url}/knowledge/entities/deeply/nested/folder/moved-note") - assert response.status_code == 200 - moved_entity = response.json() - assert moved_entity["file_path"] == "deeply/nested/folder/MovedNote.md" - - -@pytest.mark.asyncio -async def test_move_entity_with_observations_and_relations(client: AsyncClient, project_url): - """Test moving entity preserves observations and relations.""" - # Create test entity with complex content - content = """# Complex Entity - -## Observations -- [note] Important observation #tag1 -- [feature] Key feature #feature -- relation to [[SomeOtherEntity]] -- depends on [[Dependency]] - -Some additional content.""" - - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "ComplexEntity", - "directory": "source", - "entity_type": "note", - "content": content, - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Verify original observations and relations - assert len(entity["observations"]) == 2 - assert len(entity["relations"]) == 2 - - # Move entity - move_data = { - "identifier": entity["permalink"], - "destination_path": "target/MovedComplex.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 200 - - # Verify moved entity preserves data - response = await client.get(f"{project_url}/knowledge/entities/target/moved-complex") - assert response.status_code == 200 - moved_entity = response.json() - - # Check observations preserved - assert len(moved_entity["observations"]) == 2 - obs_categories = {obs["category"] for obs in moved_entity["observations"]} - assert obs_categories == {"note", "feature"} - - # Check relations preserved - assert len(moved_entity["relations"]) == 2 - rel_types = {rel["relation_type"] for rel in moved_entity["relations"]} - assert rel_types == {"relation to", "depends on"} - - # Verify file content preserved - response = await client.get(f"{project_url}/resource/target/moved-complex?content=true") - assert response.status_code == 200 - file_content = response.text - assert "Important observation #tag1" in file_content - assert "[[SomeOtherEntity]]" in file_content - - -@pytest.mark.asyncio -async def test_move_entity_search_reindexing(client: AsyncClient, project_url): - """Test that moved entities are properly reindexed for search.""" - # Create searchable entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "SearchableNote", - "directory": "source", - "entity_type": "note", - "content": "Unique searchable elephant content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Move entity - move_data = { - "identifier": entity["permalink"], - "destination_path": "target/MovedSearchable.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 200 - - # Search should find entity at new location - search_response = await client.post( - f"{project_url}/search/", - json={"text": "elephant", "entity_types": [SearchItemType.ENTITY.value]}, - ) - results = search_response.json()["results"] - assert len(results) == 1 - assert results[0]["permalink"] == "target/moved-searchable" - - -@pytest.mark.asyncio -async def test_move_entity_not_found(client: AsyncClient, project_url): - """Test moving non-existent entity returns 400 error.""" - move_data = { - "identifier": "non-existent-entity", - "destination_path": "target/SomeFile.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 400 - assert "Entity not found" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_move_entity_invalid_destination_path(client: AsyncClient, project_url): - """Test moving entity with invalid destination path.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "TestNote", - "directory": "", - "entity_type": "note", - "content": "Test content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Test various invalid paths - invalid_paths = [ - "/absolute/path.md", # Absolute path - "../parent/path.md", # Parent directory - "", # Empty string - " ", # Whitespace only - ] - - for invalid_path in invalid_paths: - move_data = { - "identifier": entity["permalink"], - "destination_path": invalid_path, - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 422 # Validation error - - -@pytest.mark.asyncio -async def test_move_entity_destination_exists(client: AsyncClient, project_url): - """Test moving entity to existing destination returns error.""" - # Create source entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "SourceNote", - "directory": "source", - "entity_type": "note", - "content": "Source content", - }, - ) - assert response.status_code == 200 - source_entity = response.json() - - # Create destination entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "DestinationNote", - "directory": "target", - "entity_type": "note", - "content": "Destination content", - }, - ) - assert response.status_code == 200 - - # Try to move source to existing destination - move_data = { - "identifier": source_entity["permalink"], - "destination_path": "target/DestinationNote.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 400 - assert "already exists" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_move_entity_missing_identifier(client: AsyncClient, project_url): - """Test move request with missing identifier.""" - move_data = { - "destination_path": "target/SomeFile.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 422 # Validation error - - -@pytest.mark.asyncio -async def test_move_entity_missing_destination(client: AsyncClient, project_url): - """Test move request with missing destination path.""" - move_data = { - "identifier": "some-entity", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 422 # Validation error - - -@pytest.mark.asyncio -async def test_move_entity_by_file_path(client: AsyncClient, project_url): - """Test moving entity using file path as identifier.""" - # Create test entity - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "TestNote", - "directory": "source", - "entity_type": "note", - "content": "Test content", - }, - ) - assert response.status_code == 200 - entity = response.json() - - # Move using file path as identifier - move_data = { - "identifier": entity["file_path"], - "destination_path": "target/MovedByPath.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 200 - - # Verify entity exists at new location - response = await client.get(f"{project_url}/knowledge/entities/target/moved-by-path") - assert response.status_code == 200 - moved_entity = response.json() - assert moved_entity["file_path"] == "target/MovedByPath.md" - - -@pytest.mark.asyncio -async def test_move_entity_by_title(client: AsyncClient, project_url): - """Test moving entity using title as identifier.""" - # Create test entity with unique title - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": "UniqueTestTitle", - "directory": "source", - "entity_type": "note", - "content": "Test content", - }, - ) - assert response.status_code == 200 - - # Move using title as identifier - move_data = { - "identifier": "UniqueTestTitle", - "destination_path": "target/MovedByTitle.md", - } - response = await client.post(f"{project_url}/knowledge/move", json=move_data) - assert response.status_code == 200 - - # Verify entity exists at new location - response = await client.get(f"{project_url}/knowledge/entities/target/moved-by-title") - assert response.status_code == 200 - moved_entity = response.json() - assert moved_entity["file_path"] == "target/MovedByTitle.md" - assert moved_entity["title"] == "UniqueTestTitle" - - -# --- Move directory tests --- - - -@pytest.mark.asyncio -async def test_move_directory_success(client: AsyncClient, project_url): - """Test POST /move-directory endpoint successfully moves all files in a directory.""" - # Create multiple notes in a source directory - for i in range(3): - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": f"DirMoveDoc{i + 1}", - "directory": "move-source", - "entity_type": "note", - "content": f"Content for document {i + 1}", - }, - ) - assert response.status_code == 200 - - # Move the entire directory - move_data = { - "source_directory": "move-source", - "destination_directory": "move-dest", - } - response = await client.post(f"{project_url}/knowledge/move-directory", json=move_data) - assert response.status_code == 200 - - result = DirectoryMoveResult.model_validate(response.json()) - assert result.total_files == 3 - assert result.successful_moves == 3 - assert result.failed_moves == 0 - assert len(result.moved_files) == 3 - - # Verify notes are accessible at new location - for i in range(3): - response = await client.get( - f"{project_url}/knowledge/entities/move-dest/dir-move-doc{i + 1}" - ) - assert response.status_code == 200 - entity = response.json() - assert entity["file_path"].startswith("move-dest/") - - -@pytest.mark.asyncio -async def test_move_directory_empty_directory(client: AsyncClient, project_url): - """Test move_directory with no files in source returns zero counts.""" - move_data = { - "source_directory": "nonexistent-source-dir", - "destination_directory": "some-dest", - } - response = await client.post(f"{project_url}/knowledge/move-directory", json=move_data) - assert response.status_code == 200 - - result = DirectoryMoveResult.model_validate(response.json()) - assert result.total_files == 0 - assert result.successful_moves == 0 - assert result.failed_moves == 0 - assert len(result.moved_files) == 0 - - -@pytest.mark.asyncio -async def test_move_directory_validation_error(client: AsyncClient, project_url): - """Test move_directory with missing required fields returns validation error.""" - # Missing destination_directory - move_data = { - "source_directory": "some-source", - } - response = await client.post(f"{project_url}/knowledge/move-directory", json=move_data) - assert response.status_code == 422 - - # Missing source_directory - move_data = { - "destination_directory": "some-dest", - } - response = await client.post(f"{project_url}/knowledge/move-directory", json=move_data) - assert response.status_code == 422 - - -@pytest.mark.asyncio -async def test_move_directory_nested_structure(client: AsyncClient, project_url): - """Test move_directory preserves nested directory structure.""" - # Create notes in nested structure - directories = [ - "nested-move/2024", - "nested-move/2024/q1", - ] - - for dir_path in directories: - response = await client.post( - f"{project_url}/knowledge/entities", - json={ - "title": f"Note in {dir_path.split('/')[-1]}", - "directory": dir_path, - "entity_type": "note", - "content": f"Content in {dir_path}", - }, - ) - assert response.status_code == 200 - - # Move the parent directory - move_data = { - "source_directory": "nested-move/2024", - "destination_directory": "archive/2024", - } - response = await client.post(f"{project_url}/knowledge/move-directory", json=move_data) - assert response.status_code == 200 - - result = DirectoryMoveResult.model_validate(response.json()) - assert result.total_files == 2 - assert result.successful_moves == 2 - - # Verify nested note is at new location - response = await client.get(f"{project_url}/knowledge/entities/archive/2024/q1/note-in-q1") - assert response.status_code == 200 diff --git a/tests/api/test_management_router.py b/tests/api/test_management_router.py deleted file mode 100644 index 3cfe558b..00000000 --- a/tests/api/test_management_router.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Tests for management router API endpoints (minimal mocking). - -These endpoints are mostly simple state checks and wiring; we use stub objects -and pytest monkeypatch instead of standard-library mocks. -""" - -from __future__ import annotations - -import pytest -from fastapi import FastAPI - -from basic_memory.api.routers.management_router import ( - WatchStatusResponse, - get_watch_status, - start_watch_service, - stop_watch_service, -) - - -class _Request: - def __init__(self, app: FastAPI): - self.app = app - - -class _Task: - def __init__(self, *, done: bool): - self._done = done - self.cancel_called = False - - def done(self) -> bool: - return self._done - - def cancel(self) -> None: - self.cancel_called = True - - -@pytest.fixture -def app_with_state() -> FastAPI: - app = FastAPI() - app.state.watch_task = None - return app - - -@pytest.mark.asyncio -async def test_get_watch_status_not_running(app_with_state: FastAPI): - app_with_state.state.watch_task = None - resp = await get_watch_status(_Request(app_with_state)) - assert isinstance(resp, WatchStatusResponse) - assert resp.running is False - - -@pytest.mark.asyncio -async def test_get_watch_status_running(app_with_state: FastAPI): - app_with_state.state.watch_task = _Task(done=False) - resp = await get_watch_status(_Request(app_with_state)) - assert resp.running is True - - -@pytest.mark.asyncio -async def test_start_watch_service_when_not_running(monkeypatch, app_with_state: FastAPI): - app_with_state.state.watch_task = None - - created = {"watch_service": None, "task": None} - - class _StubWatchService: - def __init__(self, *, app_config, project_repository): - self.app_config = app_config - self.project_repository = project_repository - created["watch_service"] = self - - def _create_background_sync_task(sync_service, watch_service): - created["task"] = _Task(done=False) - return created["task"] - - # start_watch_service imports these inside the function, so patch at the source modules. - monkeypatch.setattr("basic_memory.sync.WatchService", _StubWatchService) - monkeypatch.setattr( - "basic_memory.sync.background_sync.create_background_sync_task", - _create_background_sync_task, - ) - - project_repository = object() - sync_service = object() - - resp = await start_watch_service(_Request(app_with_state), project_repository, sync_service) - assert resp.running is True - assert app_with_state.state.watch_task is created["task"] - assert created["watch_service"] is not None - assert created["watch_service"].project_repository is project_repository - - -@pytest.mark.asyncio -async def test_start_watch_service_already_running(monkeypatch, app_with_state: FastAPI): - existing = _Task(done=False) - app_with_state.state.watch_task = existing - - def _should_not_be_called(*_args, **_kwargs): - raise AssertionError("create_background_sync_task should not be called if already running") - - monkeypatch.setattr( - "basic_memory.sync.background_sync.create_background_sync_task", - _should_not_be_called, - ) - - resp = await start_watch_service(_Request(app_with_state), object(), object()) - assert resp.running is True - assert app_with_state.state.watch_task is existing - - -@pytest.mark.asyncio -async def test_stop_watch_service_not_running(app_with_state: FastAPI): - app_with_state.state.watch_task = None - resp = await stop_watch_service(_Request(app_with_state)) - assert resp.running is False - - -@pytest.mark.asyncio -async def test_stop_watch_service_already_done(app_with_state: FastAPI): - app_with_state.state.watch_task = _Task(done=True) - resp = await stop_watch_service(_Request(app_with_state)) - assert resp.running is False diff --git a/tests/api/test_memory_router.py b/tests/api/test_memory_router.py deleted file mode 100644 index b3af6c4d..00000000 --- a/tests/api/test_memory_router.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Tests for memory router endpoints.""" - -from datetime import datetime - -import pytest - -from basic_memory.schemas.memory import GraphContext - - -@pytest.mark.asyncio -async def test_get_memory_context(client, test_graph, project_url): - """Test getting context from memory URL.""" - response = await client.get(f"{project_url}/memory/test/root") - assert response.status_code == 200 - - context = GraphContext(**response.json()) - assert len(context.results) == 1 - assert context.results[0].primary_result.permalink == "test/root" - assert len(context.results[0].related_results) > 0 - - # Verify metadata - assert context.metadata.uri == "test/root" - assert context.metadata.depth == 1 # default depth - assert isinstance(context.metadata.generated_at, datetime) - assert context.metadata.primary_count + context.metadata.related_count > 0 - assert context.metadata.total_results is not None # Backwards compatibility field - - -@pytest.mark.asyncio -async def test_get_memory_context_pagination(client, test_graph, project_url): - """Test getting context from memory URL.""" - response = await client.get(f"{project_url}/memory/test/root?page=1&page_size=1") - assert response.status_code == 200 - - context = GraphContext(**response.json()) - assert len(context.results) == 1 - assert context.results[0].primary_result.permalink == "test/root" - assert len(context.results[0].related_results) > 0 - - # Verify metadata - assert context.metadata.uri == "test/root" - assert context.metadata.depth == 1 # default depth - assert isinstance(context.metadata.generated_at, datetime) - assert context.metadata.primary_count > 0 - - -@pytest.mark.asyncio -async def test_get_memory_context_pattern(client, test_graph, project_url): - """Test getting context with pattern matching.""" - response = await client.get(f"{project_url}/memory/test/*") - assert response.status_code == 200 - - context = GraphContext(**response.json()) - assert len(context.results) > 1 # Should match multiple test/* paths - assert all("test/" in item.primary_result.permalink for item in context.results) - - -@pytest.mark.asyncio -async def test_get_memory_context_depth(client, test_graph, project_url): - """Test depth parameter affects relation traversal.""" - # With depth=1, should only get immediate connections - response = await client.get(f"{project_url}/memory/test/root?depth=1&max_results=20") - assert response.status_code == 200 - context1 = GraphContext(**response.json()) - - # With depth=2, should get deeper connections - response = await client.get(f"{project_url}/memory/test/root?depth=3&max_results=20") - assert response.status_code == 200 - context2 = GraphContext(**response.json()) - - # Calculate total related items in all result items - total_related1 = sum(len(item.related_results) for item in context1.results) - total_related2 = sum(len(item.related_results) for item in context2.results) - - assert total_related2 > total_related1 - - -@pytest.mark.asyncio -async def test_get_memory_context_timeframe(client, test_graph, project_url): - """Test timeframe parameter filters by date.""" - # Recent timeframe - response = await client.get(f"{project_url}/memory/test/root?timeframe=1d") - assert response.status_code == 200 - recent = GraphContext(**response.json()) - - # Longer timeframe - response = await client.get(f"{project_url}/memory/test/root?timeframe=30d") - assert response.status_code == 200 - older = GraphContext(**response.json()) - - # Calculate total related items - total_recent_related = ( - sum(len(item.related_results) for item in recent.results) if recent.results else 0 - ) - total_older_related = ( - sum(len(item.related_results) for item in older.results) if older.results else 0 - ) - - assert total_older_related >= total_recent_related - - -@pytest.mark.asyncio -async def test_not_found(client, project_url): - """Test handling of non-existent paths.""" - response = await client.get(f"{project_url}/memory/test/does-not-exist") - assert response.status_code == 200 - - context = GraphContext(**response.json()) - assert len(context.results) == 0 - - -@pytest.mark.asyncio -async def test_recent_activity(client, test_graph, project_url): - """Test handling of recent activity.""" - response = await client.get(f"{project_url}/memory/recent") - assert response.status_code == 200 - - context = GraphContext(**response.json()) - assert len(context.results) > 0 - assert context.metadata.primary_count > 0 - - -@pytest.mark.asyncio -async def test_recent_activity_pagination(client, test_graph, project_url): - """Test pagination for recent activity.""" - response = await client.get(f"{project_url}/memory/recent?page=1&page_size=1") - assert response.status_code == 200 - - context = GraphContext(**response.json()) - assert len(context.results) == 1 - assert context.page == 1 - assert context.page_size == 1 - - -@pytest.mark.asyncio -async def test_recent_activity_by_type(client, test_graph, project_url): - """Test filtering recent activity by type.""" - response = await client.get(f"{project_url}/memory/recent?type=relation&type=observation") - assert response.status_code == 200 - - context = GraphContext(**response.json()) - assert len(context.results) > 0 - - # Check for relation and observation types in primary results - primary_types = [item.primary_result.type for item in context.results] - assert "relation" in primary_types or "observation" in primary_types diff --git a/tests/api/test_project_router.py b/tests/api/test_project_router.py deleted file mode 100644 index 3a07bc53..00000000 --- a/tests/api/test_project_router.py +++ /dev/null @@ -1,843 +0,0 @@ -"""Tests for the project router API endpoints.""" - -import tempfile -from pathlib import Path - -import pytest - -from basic_memory.schemas.project_info import ProjectItem - - -@pytest.mark.asyncio -async def test_get_project_item(test_graph, client, project_config, test_project, project_url): - """Test the project item endpoint returns correctly structured data.""" - # Set up some test data in the database - - # Call the endpoint - response = await client.get(f"{project_url}/project/item") - - # Verify response - assert response.status_code == 200 - project_info = ProjectItem.model_validate(response.json()) - assert project_info.name == test_project.name - assert project_info.path == test_project.path - assert project_info.is_default == test_project.is_default - - -@pytest.mark.asyncio -async def test_get_project_item_not_found( - test_graph, client, project_config, test_project, project_url -): - """Test the project item endpoint returns correctly structured data.""" - # Set up some test data in the database - - # Call the endpoint - response = await client.get("/not-found/project/item") - - # Verify response - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_get_default_project(test_graph, client, project_config, test_project, project_url): - """Test the default project item endpoint returns the default project.""" - # Set up some test data in the database - - # Call the endpoint - response = await client.get("/projects/default") - - # Verify response - assert response.status_code == 200 - project_info = ProjectItem.model_validate(response.json()) - assert project_info.name == test_project.name - assert project_info.path == test_project.path - assert project_info.is_default == test_project.is_default - - -@pytest.mark.asyncio -async def test_get_project_info_endpoint(test_graph, client, project_config, project_url): - """Test the project-info endpoint returns correctly structured data.""" - # Set up some test data in the database - - # Call the endpoint - response = await client.get(f"{project_url}/project/info") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check top-level keys - assert "project_name" in data - assert "project_path" in data - assert "available_projects" in data - assert "default_project" in data - assert "statistics" in data - assert "activity" in data - assert "system" in data - - # Check statistics - stats = data["statistics"] - assert "total_entities" in stats - assert stats["total_entities"] >= 0 - assert "total_observations" in stats - assert stats["total_observations"] >= 0 - assert "total_relations" in stats - assert stats["total_relations"] >= 0 - - # Check activity - activity = data["activity"] - assert "recently_created" in activity - assert "recently_updated" in activity - assert "monthly_growth" in activity - - # Check system - system = data["system"] - assert "version" in system - assert "database_path" in system - assert "database_size" in system - assert "timestamp" in system - - -@pytest.mark.asyncio -async def test_get_project_info_content(test_graph, client, project_config, project_url): - """Test that project-info contains actual data from the test database.""" - # Call the endpoint - response = await client.get(f"{project_url}/project/info") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check that test_graph content is reflected in statistics - stats = data["statistics"] - - # Our test graph should have at least a few entities - assert stats["total_entities"] > 0 - - # It should also have some observations - assert stats["total_observations"] > 0 - - # And relations - assert stats["total_relations"] > 0 - - # Check that entity types include 'test' - assert "test" in stats["entity_types"] or "entity" in stats["entity_types"] - - -@pytest.mark.asyncio -async def test_list_projects_endpoint(test_config, test_graph, client, project_config, project_url): - """Test the list projects endpoint returns correctly structured data.""" - # Call the endpoint - response = await client.get("/projects/projects") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check that the response contains expected fields - assert "projects" in data - assert "default_project" in data - - # Check that projects is a list - assert isinstance(data["projects"], list) - - # There should be at least one project (the test project) - assert len(data["projects"]) > 0 - - # Verify project item structure - if data["projects"]: - project = data["projects"][0] - assert "name" in project - assert "path" in project - assert "is_default" in project - - # Default project should be marked - default_project = next((p for p in data["projects"] if p["is_default"]), None) - assert default_project is not None - assert default_project["name"] == data["default_project"] - - -@pytest.mark.asyncio -async def test_remove_project_endpoint(test_config, client, project_service): - """Test the remove project endpoint.""" - # First create a test project to remove - test_project_name = "test-remove-project" - await project_service.add_project(test_project_name, "/tmp/test-remove-project") - - # Verify it exists - project = await project_service.get_project(test_project_name) - assert project is not None - - # Remove the project - response = await client.delete(f"/projects/{test_project_name}") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check response structure - assert "message" in data - assert "status" in data - assert data["status"] == "success" - assert "old_project" in data - assert data["old_project"]["name"] == test_project_name - - # Verify project is actually removed - removed_project = await project_service.get_project(test_project_name) - assert removed_project is None - - -@pytest.mark.asyncio -async def test_set_default_project_endpoint(test_config, client, project_service): - """Test the set default project endpoint.""" - # Create a test project to set as default - test_project_name = "test-default-project" - await project_service.add_project(test_project_name, "/tmp/test-default-project") - - # Set it as default - response = await client.put(f"/projects/{test_project_name}/default") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check response structure - assert "message" in data - assert "status" in data - assert data["status"] == "success" - assert "new_project" in data - assert data["new_project"]["name"] == test_project_name - - # Verify it's actually set as default - assert project_service.default_project == test_project_name - - -@pytest.mark.asyncio -async def test_update_project_path_endpoint(test_config, client, project_service, project_url): - """Test the update project endpoint for changing project path.""" - # Create a test project to update - test_project_name = "test-update-project" - with tempfile.TemporaryDirectory() as temp_dir: - test_root = Path(temp_dir) - old_path = test_root / "old-location" - new_path = test_root / "new-location" - - await project_service.add_project(test_project_name, str(old_path)) - - try: - # Verify initial state - project = await project_service.get_project(test_project_name) - assert project is not None - assert Path(project.path) == old_path - - # Update the project path - response = await client.patch( - f"{project_url}/project/{test_project_name}", json={"path": str(new_path)} - ) - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check response structure - assert "message" in data - assert "status" in data - assert data["status"] == "success" - assert "old_project" in data - assert "new_project" in data - - # Check old project data - assert data["old_project"]["name"] == test_project_name - assert Path(data["old_project"]["path"]) == old_path - - # Check new project data - assert data["new_project"]["name"] == test_project_name - assert Path(data["new_project"]["path"]) == new_path - - # Verify project was actually updated in database - updated_project = await project_service.get_project(test_project_name) - assert updated_project is not None - assert Path(updated_project.path) == new_path - - finally: - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_update_project_is_active_endpoint(test_config, client, project_service, project_url): - """Test the update project endpoint for changing is_active status.""" - # Create a test project to update - test_project_name = "test-update-active-project" - test_path = "/tmp/test-update-active" - - await project_service.add_project(test_project_name, test_path) - - try: - # Update the project is_active status - response = await client.patch( - f"{project_url}/project/{test_project_name}", json={"is_active": False} - ) - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check response structure - assert "message" in data - assert "status" in data - assert data["status"] == "success" - assert f"Project '{test_project_name}' updated successfully" == data["message"] - - finally: - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_update_project_both_params_endpoint( - test_config, client, project_service, project_url -): - """Test the update project endpoint with both path and is_active parameters.""" - # Create a test project to update - test_project_name = "test-update-both-project" - with tempfile.TemporaryDirectory() as temp_dir: - test_root = Path(temp_dir) - old_path = (test_root / "old-location").as_posix() - new_path = (test_root / "new-location").as_posix() - - await project_service.add_project(test_project_name, old_path) - - try: - # Update both path and is_active (path should take precedence) - response = await client.patch( - f"{project_url}/project/{test_project_name}", - json={"path": new_path, "is_active": False}, - ) - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check that path update was performed (takes precedence) - assert data["new_project"]["path"] == new_path - - # Verify project was actually updated in database - updated_project = await project_service.get_project(test_project_name) - assert updated_project is not None - assert updated_project.path == new_path - - finally: - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_update_project_nonexistent_endpoint(client, project_url, tmp_path): - """Test the update project endpoint with a nonexistent project.""" - # Try to update a project that doesn't exist - # Use tmp_path for cross-platform absolute path compatibility - new_path = str(tmp_path / "new-path") - response = await client.patch( - f"{project_url}/project/nonexistent-project", json={"path": new_path} - ) - - # Should return 400 error - assert response.status_code == 400 - data = response.json() - assert "detail" in data - assert "not found in configuration" in data["detail"] - - -@pytest.mark.asyncio -async def test_update_project_relative_path_error_endpoint( - test_config, client, project_service, project_url -): - """Test the update project endpoint with relative path (should fail).""" - # Create a test project to update - test_project_name = "test-update-relative-project" - test_path = "/tmp/test-update-relative" - - await project_service.add_project(test_project_name, test_path) - - try: - # Try to update with relative path - response = await client.patch( - f"{project_url}/project/{test_project_name}", json={"path": "./relative-path"} - ) - - # Should return 400 error - assert response.status_code == 400 - data = response.json() - assert "detail" in data - assert "Path must be absolute" in data["detail"] - - finally: - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_update_project_no_params_endpoint(test_config, client, project_service, project_url): - """Test the update project endpoint with no parameters (should fail).""" - # Create a test project to update - test_project_name = "test-update-no-params-project" - test_path = "/tmp/test-update-no-params" - - await project_service.add_project(test_project_name, test_path) - proj_info = await project_service.get_project(test_project_name) - assert proj_info.name == test_project_name - # On Windows the path is prepended with a drive letter - assert test_path in proj_info.path - - try: - # Try to update with no parameters - response = await client.patch(f"{project_url}/project/{test_project_name}", json={}) - - # Should return 200 (no-op) - assert response.status_code == 200 - proj_info = await project_service.get_project(test_project_name) - assert proj_info.name == test_project_name - # On Windows the path is prepended with a drive letter - assert test_path in proj_info.path - - finally: - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_update_project_empty_path_endpoint( - test_config, client, project_service, project_url -): - """Test the update project endpoint with empty path parameter.""" - # Create a test project to update - test_project_name = "test-update-empty-path-project" - test_path = "/tmp/test-update-empty-path" - - await project_service.add_project(test_project_name, test_path) - - try: - # Try to update with empty/null path - should be treated as no path update - response = await client.patch( - f"{project_url}/project/{test_project_name}", json={"path": None, "is_active": True} - ) - - # Should succeed and perform is_active update - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - - finally: - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_sync_project_endpoint(test_graph, client, project_url): - """Test the project sync endpoint initiates background sync.""" - # Call the sync endpoint - response = await client.post(f"{project_url}/project/sync") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check response structure - assert "status" in data - assert "message" in data - assert data["status"] == "sync_started" - assert "Filesystem sync initiated" in data["message"] - - -@pytest.mark.asyncio -async def test_sync_project_endpoint_with_force_full(test_graph, client, project_url): - """Test the project sync endpoint with force_full parameter.""" - # Call the sync endpoint with force_full=true - response = await client.post(f"{project_url}/project/sync?force_full=true") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check response structure - assert "status" in data - assert "message" in data - assert data["status"] == "sync_started" - assert "Filesystem sync initiated" in data["message"] - - -@pytest.mark.asyncio -async def test_sync_project_endpoint_with_force_full_false(test_graph, client, project_url): - """Test the project sync endpoint with force_full=false.""" - # Call the sync endpoint with force_full=false - response = await client.post(f"{project_url}/project/sync?force_full=false") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check response structure - assert "status" in data - assert "message" in data - assert data["status"] == "sync_started" - assert "Filesystem sync initiated" in data["message"] - - -@pytest.mark.asyncio -async def test_sync_project_endpoint_not_found(client): - """Test the project sync endpoint with nonexistent project.""" - # Call the sync endpoint for a project that doesn't exist - response = await client.post("/nonexistent-project/project/sync") - - # Should return 404 - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_sync_project_endpoint_foreground(test_graph, client, project_url): - """Test the project sync endpoint with run_in_background=false returns sync report.""" - # Call the sync endpoint with run_in_background=false - response = await client.post(f"{project_url}/project/sync?run_in_background=false") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check that we get a sync report instead of status message - assert "new" in data - assert "modified" in data - assert "deleted" in data - assert "moves" in data - assert "checksums" in data - assert "skipped_files" in data - assert "total" in data - - # Verify these are the right types - assert isinstance(data["new"], list) - assert isinstance(data["modified"], list) - assert isinstance(data["deleted"], list) - assert isinstance(data["moves"], dict) - assert isinstance(data["checksums"], dict) - assert isinstance(data["skipped_files"], list) - assert isinstance(data["total"], int) - - -@pytest.mark.asyncio -async def test_sync_project_endpoint_foreground_with_force_full(test_graph, client, project_url): - """Test the project sync endpoint with run_in_background=false and force_full=true.""" - # Call the sync endpoint with both parameters - response = await client.post( - f"{project_url}/project/sync?run_in_background=false&force_full=true" - ) - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check that we get a sync report with all expected fields - assert "new" in data - assert "modified" in data - assert "deleted" in data - assert "moves" in data - assert "checksums" in data - assert "skipped_files" in data - assert "total" in data - - -@pytest.mark.asyncio -async def test_sync_project_endpoint_foreground_with_changes( - test_graph, client, project_config, project_url, tmpdir -): - """Test foreground sync detects actual file changes.""" - # Create a new file in the project directory - import os - from pathlib import Path - - test_file = Path(project_config.home) / "new_test_file.md" - test_file.write_text("# New Test File\n\nThis is a test file for sync detection.") - - try: - # Call the sync endpoint with run_in_background=false - response = await client.post(f"{project_url}/project/sync?run_in_background=false") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # The sync report should show changes (the new file we created) - assert data["total"] >= 0 # Should have at least detected changes - assert "new" in data - assert "modified" in data - assert "deleted" in data - - # At least one of these should have changes - has_changes = len(data["new"]) > 0 or len(data["modified"]) > 0 or len(data["deleted"]) > 0 - assert has_changes or data["total"] >= 0 # Either changes detected or empty sync is valid - - finally: - # Clean up the test file - if test_file.exists(): - os.remove(test_file) - - -@pytest.mark.asyncio -async def test_remove_default_project_fails(test_config, client, project_service): - """Test that removing the default project returns an error.""" - # Get the current default project - default_project_name = project_service.default_project - - # Try to remove the default project - response = await client.delete(f"/projects/{default_project_name}") - - # Should return 400 with helpful error message - assert response.status_code == 400 - data = response.json() - assert "detail" in data - assert "Cannot delete default project" in data["detail"] - assert default_project_name in data["detail"] - - -@pytest.mark.asyncio -async def test_remove_default_project_with_alternatives(test_config, client, project_service): - """Test that error message includes alternative projects when trying to delete default.""" - # Get the current default project - default_project_name = project_service.default_project - - # Create another project so there are alternatives - test_project_name = "test-alternative-project" - await project_service.add_project(test_project_name, "/tmp/test-alternative") - - try: - # Try to remove the default project - response = await client.delete(f"/projects/{default_project_name}") - - # Should return 400 with helpful error message including alternatives - assert response.status_code == 400 - data = response.json() - assert "detail" in data - assert "Cannot delete default project" in data["detail"] - assert "Set another project as default first" in data["detail"] - assert test_project_name in data["detail"] - - finally: - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_remove_non_default_project_succeeds(test_config, client, project_service): - """Test that removing a non-default project succeeds.""" - # Create a test project to remove - test_project_name = "test-remove-non-default" - await project_service.add_project(test_project_name, "/tmp/test-remove-non-default") - - # Verify it's not the default - assert project_service.default_project != test_project_name - - # Remove the project - response = await client.delete(f"/projects/{test_project_name}") - - # Should succeed - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - - # Verify project is removed - removed_project = await project_service.get_project(test_project_name) - assert removed_project is None - - -@pytest.mark.asyncio -async def test_set_nonexistent_project_as_default_fails(test_config, client, project_service): - """Test that setting a non-existent project as default returns 404.""" - # Try to set a project that doesn't exist as default - response = await client.put("/projects/nonexistent-project/default") - - # Should return 404 - assert response.status_code == 404 - data = response.json() - assert "detail" in data - assert "does not exist" in data["detail"] - - -@pytest.mark.asyncio -async def test_create_project_idempotent_same_path(test_config, client, project_service): - """Test that creating a project with same name and same path is idempotent.""" - # Create a project with platform-independent path - test_project_name = "test-idempotent" - with tempfile.TemporaryDirectory() as temp_dir: - test_project_path = (Path(temp_dir) / "test-idempotent").as_posix() - - response1 = await client.post( - "/projects/projects", - json={"name": test_project_name, "path": test_project_path, "set_default": False}, - ) - - # Should succeed with 201 Created - assert response1.status_code == 201 - data1 = response1.json() - assert data1["status"] == "success" - assert data1["new_project"]["name"] == test_project_name - - # Try to create the same project again with same name and path - response2 = await client.post( - "/projects/projects", - json={"name": test_project_name, "path": test_project_path, "set_default": False}, - ) - - # Should also succeed (idempotent) - assert response2.status_code == 200 - data2 = response2.json() - assert data2["status"] == "success" - assert "already exists" in data2["message"] - assert data2["new_project"]["name"] == test_project_name - # Normalize paths for cross-platform comparison - assert Path(data2["new_project"]["path"]).resolve() == Path(test_project_path).resolve() - - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_create_project_fails_different_path(test_config, client, project_service): - """Test that creating a project with same name but different path fails.""" - # Create a project - test_project_name = "test-path-conflict" - test_project_path1 = "/tmp/test-path-conflict-1" - - response1 = await client.post( - "/projects/projects", - json={"name": test_project_name, "path": test_project_path1, "set_default": False}, - ) - - # Should succeed with 201 Created - assert response1.status_code == 201 - - # Try to create the same project with different path - test_project_path2 = "/tmp/test-path-conflict-2" - response2 = await client.post( - "/projects/projects", - json={"name": test_project_name, "path": test_project_path2, "set_default": False}, - ) - - # Should fail with 400 - assert response2.status_code == 400 - data2 = response2.json() - assert "detail" in data2 - assert "already exists with different path" in data2["detail"] - assert test_project_path1 in data2["detail"] - assert test_project_path2 in data2["detail"] - - # Clean up - try: - await project_service.remove_project(test_project_name) - except Exception: - pass - - -@pytest.mark.asyncio -async def test_remove_project_with_delete_notes_false(test_config, client, project_service): - """Test that removing a project with delete_notes=False leaves directory intact.""" - # Create a test project with actual directory - test_project_name = "test-remove-keep-files" - with tempfile.TemporaryDirectory() as temp_dir: - test_path = Path(temp_dir) / "test-project" - test_path.mkdir() - test_file = test_path / "test.md" - test_file.write_text("# Test Note") - - await project_service.add_project(test_project_name, str(test_path)) - - # Remove the project without deleting files (default) - response = await client.delete(f"/projects/{test_project_name}") - - # Verify response - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - - # Verify project is removed from config/db - removed_project = await project_service.get_project(test_project_name) - assert removed_project is None - - # Verify directory still exists - assert test_path.exists() - assert test_file.exists() - - -@pytest.mark.asyncio -async def test_remove_project_with_delete_notes_true(test_config, client, project_service): - """Test that removing a project with delete_notes=True deletes the directory.""" - # Create a test project with actual directory - test_project_name = "test-remove-delete-files" - with tempfile.TemporaryDirectory() as temp_dir: - test_path = Path(temp_dir) / "test-project" - test_path.mkdir() - test_file = test_path / "test.md" - test_file.write_text("# Test Note") - - await project_service.add_project(test_project_name, str(test_path)) - - # Remove the project with delete_notes=True - response = await client.delete(f"/projects/{test_project_name}?delete_notes=true") - - # Verify response - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - - # Verify project is removed from config/db - removed_project = await project_service.get_project(test_project_name) - assert removed_project is None - - # Verify directory is deleted - assert not test_path.exists() - - -@pytest.mark.asyncio -async def test_remove_project_delete_notes_nonexistent_directory( - test_config, client, project_service -): - """Test that removing a project with delete_notes=True handles missing directory gracefully.""" - # Create a project pointing to a non-existent path - test_project_name = "test-remove-missing-dir" - test_path = "/tmp/this-directory-does-not-exist-12345" - - await project_service.add_project(test_project_name, test_path) - - # Remove the project with delete_notes=True (should not fail even if dir doesn't exist) - response = await client.delete(f"/projects/{test_project_name}?delete_notes=true") - - # Should succeed - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - - # Verify project is removed - removed_project = await project_service.get_project(test_project_name) - assert removed_project is None diff --git a/tests/api/test_project_router_operations.py b/tests/api/test_project_router_operations.py deleted file mode 100644 index 01dc6702..00000000 --- a/tests/api/test_project_router_operations.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Tests for project router operation endpoints.""" - -import pytest - - -@pytest.mark.asyncio -async def test_get_project_info_additional(client, test_graph, project_url): - """Test additional fields in the project info endpoint.""" - # Call the endpoint - response = await client.get(f"{project_url}/project/info") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Check specific fields we're interested in - assert "available_projects" in data - assert isinstance(data["available_projects"], dict) - - # Get a project from the list - for project_name, project_info in data["available_projects"].items(): - # Verify project structure - assert "path" in project_info - assert "active" in project_info - assert "is_default" in project_info - break # Just check the first one for structure - - -@pytest.mark.asyncio -async def test_project_list_additional(client, project_url): - """Test additional fields in the project list endpoint.""" - # Call the endpoint - response = await client.get("/projects/projects") - - # Verify response - assert response.status_code == 200 - data = response.json() - - # Verify projects list structure in more detail - assert "projects" in data - assert len(data["projects"]) > 0 - - # Verify the default project is identified - default_project = data["default_project"] - assert default_project - - # Verify the default_project appears in the projects list and is marked as default - default_in_list = False - for project in data["projects"]: - if project["name"] == default_project: - assert project["is_default"] is True - default_in_list = True - break - - assert default_in_list, "Default project should appear in the projects list" diff --git a/tests/api/test_prompt_router.py b/tests/api/test_prompt_router.py deleted file mode 100644 index b85982f3..00000000 --- a/tests/api/test_prompt_router.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Tests for the prompt router endpoints.""" - -import pytest -import pytest_asyncio -from httpx import AsyncClient - -from basic_memory.services.context_service import ContextService - - -@pytest_asyncio.fixture -async def context_service(entity_repository, search_service, observation_repository): - """Create a real context service for testing.""" - return ContextService(entity_repository, search_service, observation_repository) - - -@pytest.mark.asyncio -async def test_continue_conversation_endpoint( - client: AsyncClient, - entity_service, - search_service, - context_service, - entity_repository, - test_graph, - project_url, -): - """Test the continue_conversation endpoint with real services.""" - # Create request data - request_data = { - "topic": "Root", # This should match our test entity in test_graph - "timeframe": "7d", - "depth": 1, - "related_items_limit": 2, - } - - # Call the endpoint - response = await client.post(f"{project_url}/prompt/continue-conversation", json=request_data) - - # Verify response - assert response.status_code == 200 - result = response.json() - assert "prompt" in result - assert "context" in result - - # Check content of context - context = result["context"] - assert context["topic"] == "Root" - assert context["timeframe"] == "7d" - assert context["has_results"] is True - assert len(context["hierarchical_results"]) > 0 - - # Check content of prompt - prompt = result["prompt"] - assert "Continuing conversation on: Root" in prompt - assert "memory retrieval session" in prompt - - # Test without topic - should use recent activity - request_data = {"timeframe": "1d", "depth": 1, "related_items_limit": 2} - - response = await client.post(f"{project_url}/prompt/continue-conversation", json=request_data) - - assert response.status_code == 200 - result = response.json() - assert "Recent Activity" in result["context"]["topic"] - - -@pytest.mark.asyncio -async def test_search_prompt_endpoint( - client: AsyncClient, entity_service, search_service, test_graph, project_url -): - """Test the search_prompt endpoint with real services.""" - # Create request data - request_data = { - "query": "Root", # This should match our test entity - "timeframe": "7d", - } - - # Call the endpoint - response = await client.post(f"{project_url}/prompt/search", json=request_data) - - # Verify response - assert response.status_code == 200 - result = response.json() - assert "prompt" in result - assert "context" in result - - # Check content of context - context = result["context"] - assert context["query"] == "Root" - assert context["timeframe"] == "7d" - assert context["has_results"] is True - assert len(context["results"]) > 0 - - # Check content of prompt - prompt = result["prompt"] - assert 'Search Results for: "Root"' in prompt - assert "This is a memory search session" in prompt - - -@pytest.mark.asyncio -async def test_search_prompt_no_results( - client: AsyncClient, entity_service, search_service, project_url -): - """Test the search_prompt endpoint with a query that returns no results.""" - # Create request data with a query that shouldn't match anything - request_data = {"query": "NonExistentQuery12345", "timeframe": "7d"} - - # Call the endpoint - response = await client.post(f"{project_url}/prompt/search", json=request_data) - - # Verify response - assert response.status_code == 200 - result = response.json() - - # Check content of context - context = result["context"] - assert context["query"] == "NonExistentQuery12345" - assert context["has_results"] is False - assert len(context["results"]) == 0 - - # Check content of prompt - prompt = result["prompt"] - assert 'Search Results for: "NonExistentQuery12345"' in prompt - assert "I couldn't find any results for this query" in prompt - assert "Opportunity to Capture Knowledge" in prompt - - -@pytest.mark.asyncio -async def test_error_handling(client: AsyncClient, monkeypatch, project_url): - """Test error handling in the endpoints by breaking the template loader.""" - - # Patch the template loader to raise an exception - def mock_render(*args, **kwargs): - raise Exception("Template error") - - # Apply the patch - monkeypatch.setattr("basic_memory.api.template_loader.TemplateLoader.render", mock_render) - - # Test continue_conversation error handling - response = await client.post( - f"{project_url}/prompt/continue-conversation", - json={"topic": "test error", "timeframe": "7d"}, - ) - - assert response.status_code == 500 - assert "detail" in response.json() - assert "Template error" in response.json()["detail"] - - # Test search_prompt error handling - response = await client.post( - f"{project_url}/prompt/search", json={"query": "test error", "timeframe": "7d"} - ) - - assert response.status_code == 500 - assert "detail" in response.json() - assert "Template error" in response.json()["detail"] diff --git a/tests/api/test_relation_background_resolution.py b/tests/api/test_relation_background_resolution.py deleted file mode 100644 index d643dc43..00000000 --- a/tests/api/test_relation_background_resolution.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Test that relation resolution happens in the background.""" - -import pytest - -from basic_memory.api.routers.knowledge_router import resolve_relations_background - - -@pytest.mark.asyncio -async def test_resolve_relations_background_success(): - """Test that background relation resolution calls sync service correctly.""" - - class StubSyncService: - def __init__(self) -> None: - self.calls: list[int] = [] - - async def resolve_relations(self, *, entity_id: int) -> None: - self.calls.append(entity_id) - - sync_service = StubSyncService() - - entity_id = 123 - entity_permalink = "test/entity" - - # Call the background function - await resolve_relations_background(sync_service, entity_id, entity_permalink) - - # Verify sync service was called with the entity_id - assert sync_service.calls == [entity_id] - - -@pytest.mark.asyncio -async def test_resolve_relations_background_handles_errors(): - """Test that background relation resolution handles errors gracefully.""" - - class StubSyncService: - def __init__(self) -> None: - self.calls: list[int] = [] - - async def resolve_relations(self, *, entity_id: int) -> None: - self.calls.append(entity_id) - raise Exception("Test error") - - sync_service = StubSyncService() - - entity_id = 123 - entity_permalink = "test/entity" - - # Call should not raise - errors are logged - await resolve_relations_background(sync_service, entity_id, entity_permalink) - - # Verify sync service was called - assert sync_service.calls == [entity_id] diff --git a/tests/api/test_resource_router.py b/tests/api/test_resource_router.py deleted file mode 100644 index 0ed79d32..00000000 --- a/tests/api/test_resource_router.py +++ /dev/null @@ -1,454 +0,0 @@ -"""Tests for resource router endpoints.""" - -import json -from datetime import datetime, timezone -from pathlib import Path - -import pytest - -from basic_memory.schemas import EntityResponse -from basic_memory.utils import normalize_newlines - - -@pytest.mark.asyncio -async def test_get_resource_content(client, project_config, entity_repository, project_url): - """Test getting content by permalink.""" - # Create a test file - content = "# Test Content\n\nThis is a test file." - test_file = Path(project_config.home) / "test" / "test.md" - test_file.parent.mkdir(parents=True, exist_ok=True) - test_file.write_text(content) - - # Create entity referencing the file - entity = await entity_repository.create( - { - "title": "Test Entity", - "entity_type": "test", - "permalink": "test/test", - "file_path": "test/test.md", # Relative to config.home - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - } - ) - - # Test getting the content - response = await client.get(f"{project_url}/resource/{entity.permalink}") - assert response.status_code == 200 - assert response.headers["content-type"] == "text/markdown; charset=utf-8" - assert response.text == normalize_newlines(content) - - -@pytest.mark.asyncio -async def test_get_resource_pagination(client, project_config, entity_repository, project_url): - """Test getting content by permalink with pagination.""" - # Create a test file - content = "# Test Content\n\nThis is a test file." - test_file = Path(project_config.home) / "test" / "test.md" - test_file.parent.mkdir(parents=True, exist_ok=True) - test_file.write_text(content) - - # Create entity referencing the file - entity = await entity_repository.create( - { - "title": "Test Entity", - "entity_type": "test", - "permalink": "test/test", - "file_path": "test/test.md", # Relative to config.home - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - } - ) - - # Test getting the content - response = await client.get( - f"{project_url}/resource/{entity.permalink}", params={"page": 1, "page_size": 1} - ) - assert response.status_code == 200 - assert response.headers["content-type"] == "text/markdown; charset=utf-8" - assert response.text == normalize_newlines(content) - - -@pytest.mark.asyncio -async def test_get_resource_by_title(client, project_config, entity_repository, project_url): - """Test getting content by permalink.""" - # Create a test file - content = "# Test Content\n\nThis is a test file." - test_file = Path(project_config.home) / "test" / "test.md" - test_file.parent.mkdir(parents=True, exist_ok=True) - test_file.write_text(content) - - # Create entity referencing the file - entity = await entity_repository.create( - { - "title": "Test Entity", - "entity_type": "test", - "permalink": "test/test", - "file_path": "test/test.md", # Relative to config.home - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - } - ) - - # Test getting the content - response = await client.get(f"{project_url}/resource/{entity.title}") - assert response.status_code == 200 - - -@pytest.mark.asyncio -async def test_get_resource_missing_entity(client, project_url): - """Test 404 when entity doesn't exist.""" - response = await client.get(f"{project_url}/resource/does/not/exist") - assert response.status_code == 404 - assert "Resource not found" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_get_resource_missing_file(client, project_config, entity_repository, project_url): - """Test 404 when file doesn't exist.""" - # Create entity referencing non-existent file - entity = await entity_repository.create( - { - "title": "Missing File", - "entity_type": "test", - "permalink": "test/missing", - "file_path": "test/missing.md", - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - } - ) - - response = await client.get(f"{project_url}/resource/{entity.permalink}") - assert response.status_code == 404 - assert "File not found" in response.json()["detail"] - - -@pytest.mark.asyncio -async def test_get_resource_observation(client, project_config, entity_repository, project_url): - """Test getting content by observation permalink.""" - # Create entity - content = "# Test Content\n\n- [note] an observation." - data = { - "title": "Test Entity", - "directory": "test", - "entity_type": "test", - "content": f"{content}", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - entity = EntityResponse(**entity_response) - - assert len(entity.observations) == 1 - observation = entity.observations[0] - - # Test getting the content via the observation - response = await client.get(f"{project_url}/resource/{observation.permalink}") - assert response.status_code == 200 - assert response.headers["content-type"] == "text/markdown; charset=utf-8" - assert ( - normalize_newlines( - """ ---- -title: Test Entity -type: test -permalink: test/test-entity ---- - -# Test Content - -- [note] an observation. - """.strip() - ) - in response.text - ) - - -@pytest.mark.asyncio -async def test_get_resource_entities(client, project_config, entity_repository, project_url): - """Test getting content by permalink match.""" - # Create entity - content1 = "# Test Content\n" - data = { - "title": "Test Entity", - "directory": "test", - "entity_type": "test", - "content": f"{content1}", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - entity1 = EntityResponse(**entity_response) - - content2 = "# Related Content\n- links to [[Test Entity]]" - data = { - "title": "Related Entity", - "directory": "test", - "entity_type": "test", - "content": f"{content2}", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - entity2 = EntityResponse(**entity_response) - - assert len(entity2.relations) == 1 - - # Test getting the content via the relation - response = await client.get(f"{project_url}/resource/test/*") - assert response.status_code == 200 - assert response.headers["content-type"] == "text/markdown; charset=utf-8" - assert ( - normalize_newlines( - f""" ---- memory://test/test-entity {entity1.updated_at.isoformat()} {entity1.checksum[:8]} - -# Test Content - ---- memory://test/related-entity {entity2.updated_at.isoformat()} {entity2.checksum[:8]} - -# Related Content -- links to [[Test Entity]] - - """.strip() - ) - in response.text - ) - - -@pytest.mark.asyncio -async def test_get_resource_entities_pagination( - client, project_config, entity_repository, project_url, db_backend -): - """Test getting content by permalink match.""" - if db_backend == "postgres": - pytest.skip( - "Pagination differs: relations expand to multiple entities, ordering is undefined" - ) - # Create entity - content1 = "# Test Content\n" - data = { - "title": "Test Entity", - "directory": "test", - "entity_type": "test", - "content": f"{content1}", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - entity1 = EntityResponse(**entity_response) - assert entity1 - - content2 = "# Related Content\n- links to [[Test Entity]]" - data = { - "title": "Related Entity", - "directory": "test", - "entity_type": "test", - "content": f"{content2}", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - entity2 = EntityResponse(**entity_response) - - assert len(entity2.relations) == 1 - - # Test getting second result - response = await client.get( - f"{project_url}/resource/test/*", params={"page": 2, "page_size": 1} - ) - assert response.status_code == 200 - assert response.headers["content-type"] == "text/markdown; charset=utf-8" - assert ( - normalize_newlines( - """ ---- -title: Related Entity -type: test -permalink: test/related-entity ---- - -# Related Content -- links to [[Test Entity]] -""".strip() - ) - in response.text - ) - - -@pytest.mark.asyncio -async def test_get_resource_relation(client, project_config, entity_repository, project_url): - """Test getting content by relation permalink.""" - # Create entity - content1 = "# Test Content\n" - data = { - "title": "Test Entity", - "directory": "test", - "entity_type": "test", - "content": f"{content1}", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - entity1 = EntityResponse(**entity_response) - - content2 = "# Related Content\n- links to [[Test Entity]]" - data = { - "title": "Related Entity", - "directory": "test", - "entity_type": "test", - "content": f"{content2}", - } - response = await client.post(f"{project_url}/knowledge/entities", json=data) - entity_response = response.json() - entity2 = EntityResponse(**entity_response) - - assert len(entity2.relations) == 1 - relation = entity2.relations[0] - - # Test getting the content via the relation - response = await client.get(f"{project_url}/resource/{relation.permalink}") - assert response.status_code == 200 - assert response.headers["content-type"] == "text/markdown; charset=utf-8" - assert ( - normalize_newlines( - f""" ---- memory://test/test-entity {entity1.updated_at.isoformat()} {entity1.checksum[:8]} - -# Test Content - ---- memory://test/related-entity {entity2.updated_at.isoformat()} {entity2.checksum[:8]} - -# Related Content -- links to [[Test Entity]] - - """.strip() - ) - in response.text - ) - - -@pytest.mark.asyncio -async def test_put_resource_new_file( - client, project_config, entity_repository, search_repository, project_url -): - """Test creating a new file via PUT.""" - # Test data - file_path = "visualizations/test.canvas" - canvas_data = { - "nodes": [ - { - "id": "node1", - "type": "text", - "text": "Test node content", - "x": 100, - "y": 200, - "width": 400, - "height": 300, - } - ], - "edges": [], - } - - # Make sure the file doesn't exist yet - full_path = Path(project_config.home) / file_path - if full_path.exists(): - full_path.unlink() - - # Execute PUT request - response = await client.put( - f"{project_url}/resource/{file_path}", json=json.dumps(canvas_data, indent=2) - ) - - # Verify response - assert response.status_code == 201 - response_data = response.json() - assert response_data["file_path"] == file_path - assert "checksum" in response_data - assert "size" in response_data - - # Verify file was created - full_path = Path(project_config.home) / file_path - assert full_path.exists() - - # Verify file content - file_content = full_path.read_text(encoding="utf-8") - assert json.loads(file_content) == canvas_data - - # Verify entity was created in DB - entity = await entity_repository.get_by_file_path(file_path) - assert entity is not None - assert entity.entity_type == "canvas" - assert entity.content_type == "application/json" - - # Verify entity was indexed for search - search_results = await search_repository.search(title="test.canvas") - assert len(search_results) > 0 - - -@pytest.mark.asyncio -async def test_put_resource_update_existing(client, project_config, entity_repository, project_url): - """Test updating an existing file via PUT.""" - # Create an initial file and entity - file_path = "visualizations/update-test.canvas" - full_path = Path(project_config.home) / file_path - full_path.parent.mkdir(parents=True, exist_ok=True) - - initial_data = { - "nodes": [ - { - "id": "initial", - "type": "text", - "text": "Initial content", - "x": 0, - "y": 0, - "width": 200, - "height": 100, - } - ], - "edges": [], - } - full_path.write_text(json.dumps(initial_data)) - - # Create the initial entity - initial_entity = await entity_repository.create( - { - "title": "update-test.canvas", - "entity_type": "canvas", - "file_path": file_path, - "content_type": "application/json", - "checksum": "initial123", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - } - ) - - # New data for update - updated_data = { - "nodes": [ - { - "id": "updated", - "type": "text", - "text": "Updated content", - "x": 100, - "y": 100, - "width": 300, - "height": 200, - } - ], - "edges": [], - } - - # Execute PUT request to update - response = await client.put( - f"{project_url}/resource/{file_path}", json=json.dumps(updated_data, indent=2) - ) - - # Verify response - assert response.status_code == 200 - - # Verify file was updated - updated_content = full_path.read_text(encoding="utf-8") - assert json.loads(updated_content) == updated_data - - # Verify entity was updated - updated_entity = await entity_repository.get_by_file_path(file_path) - assert updated_entity.id == initial_entity.id # Same entity, updated - assert updated_entity.checksum != initial_entity.checksum # Checksum changed diff --git a/tests/api/test_search_router.py b/tests/api/test_search_router.py deleted file mode 100644 index 2879ab73..00000000 --- a/tests/api/test_search_router.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Tests for search router.""" - -from datetime import datetime, timezone - -import pytest -import pytest_asyncio -from sqlalchemy import text - -from basic_memory import db -from basic_memory.schemas import Entity as EntitySchema -from basic_memory.schemas.search import SearchItemType, SearchResponse - - -@pytest_asyncio.fixture -async def indexed_entity(full_entity, search_service): - """Create an entity and index it.""" - await search_service.index_entity(full_entity) - return full_entity - - -@pytest.mark.asyncio -async def test_search_basic(client, indexed_entity, project_url): - """Test basic text search.""" - response = await client.post(f"{project_url}/search/", json={"text": "search"}) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) == 3 - - found = False - for r in search_results.results: - if r.type == SearchItemType.ENTITY.value: - assert r.permalink == indexed_entity.permalink - found = True - - assert found, "Expected to find indexed entity in results" - - -@pytest.mark.asyncio -async def test_search_basic_pagination(client, indexed_entity, project_url): - """Test basic text search.""" - response = await client.post( - f"{project_url}/search/?page=3&page_size=1", json={"text": "search"} - ) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) == 1 - - assert search_results.current_page == 3 - assert search_results.page_size == 1 - - -@pytest.mark.asyncio -async def test_search_with_entity_type_filter(client, indexed_entity, project_url): - """Test search with type filter.""" - # Should find with correct type - response = await client.post( - f"{project_url}/search/", - json={"text": "test", "entity_types": [SearchItemType.ENTITY.value]}, - ) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) > 0 - - # Should find with relation type - response = await client.post( - f"{project_url}/search/", - json={"text": "test", "entity_types": [SearchItemType.RELATION.value]}, - ) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) == 2 - - -@pytest.mark.asyncio -async def test_search_with_type_filter(client, indexed_entity, project_url): - """Test search with entity type filter.""" - # Should find with correct entity type - response = await client.post(f"{project_url}/search/", json={"text": "test", "types": ["test"]}) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) == 1 - - # Should not find with wrong entity type - response = await client.post(f"{project_url}/search/", json={"text": "test", "types": ["note"]}) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) == 0 - - -@pytest.mark.asyncio -async def test_search_with_date_filter(client, indexed_entity, project_url): - """Test search with date filter.""" - # Should find with past date - past_date = datetime(2020, 1, 1, tzinfo=timezone.utc) - response = await client.post( - f"{project_url}/search/", json={"text": "test", "after_date": past_date.isoformat()} - ) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - - # Should not find with future date - future_date = datetime(2030, 1, 1, tzinfo=timezone.utc) - response = await client.post( - f"{project_url}/search/", json={"text": "test", "after_date": future_date.isoformat()} - ) - assert response.status_code == 200 - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) == 0 - - -@pytest.mark.asyncio -async def test_search_empty(search_service, client, project_url): - """Test search with no matches.""" - response = await client.post(f"{project_url}/search/", json={"text": "nonexistent"}) - assert response.status_code == 200 - search_result = SearchResponse.model_validate(response.json()) - assert len(search_result.results) == 0 - - -@pytest.mark.asyncio -async def test_reindex( - client, search_service, entity_service, session_maker, project_url, app_config -): - """Test reindex endpoint.""" - # Skip for Postgres - needs investigation of database connection isolation - from basic_memory.config import DatabaseBackend - - if app_config.database_backend == DatabaseBackend.POSTGRES: - pytest.skip("Not yet supported for Postgres - database connection isolation issue") - - # Create test entity and document - await entity_service.create_entity( - EntitySchema( - title="TestEntity1", - directory="test", - entity_type="test", - ), - ) - - # Clear search index - async with db.scoped_session(session_maker) as session: - await session.execute(text("DELETE FROM search_index")) - await session.commit() - - # Verify nothing is searchable - response = await client.post(f"{project_url}/search/", json={"text": "test"}) - search_results = SearchResponse.model_validate(response.json()) - assert len(search_results.results) == 0 - - # Trigger reindex - reindex_response = await client.post(f"{project_url}/search/reindex") - assert reindex_response.status_code == 200 - assert reindex_response.json()["status"] == "ok" - - # Verify content is searchable again - search_response = await client.post(f"{project_url}/search/", json={"text": "test"}) - search_results = SearchResponse.model_validate(search_response.json()) - assert len(search_results.results) == 1 - - -@pytest.mark.asyncio -async def test_multiple_filters(client, indexed_entity, project_url): - """Test search with multiple filters combined.""" - response = await client.post( - f"{project_url}/search/", - json={ - "text": "test", - "entity_types": [SearchItemType.ENTITY.value], - "types": ["test"], - "after_date": datetime(2020, 1, 1, tzinfo=timezone.utc).isoformat(), - }, - ) - assert response.status_code == 200 - search_result = SearchResponse.model_validate(response.json()) - assert len(search_result.results) == 1 - result = search_result.results[0] - assert result.permalink == indexed_entity.permalink - assert result.type == SearchItemType.ENTITY.value - assert result.metadata["entity_type"] == "test" diff --git a/tests/api/test_search_template.py b/tests/api/test_search_template.py deleted file mode 100644 index 9d776889..00000000 --- a/tests/api/test_search_template.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Tests for the search template rendering.""" - -import datetime -import pytest - -from basic_memory.api.template_loader import TemplateLoader -from basic_memory.schemas.search import SearchItemType, SearchResult - - -@pytest.fixture -def template_loader(): - """Return a TemplateLoader instance for testing.""" - return TemplateLoader() - - -@pytest.fixture -def search_result(): - """Create a sample SearchResult for testing.""" - return SearchResult( - title="Test Search Result", - type=SearchItemType.ENTITY, - permalink="test/search-result", - score=0.95, - content="This is a test search result with some content.", - file_path="/path/to/test/search-result.md", - metadata={"created_at": datetime.datetime(2023, 2, 1, 12, 0)}, - ) - - -@pytest.fixture -def context_with_results(search_result): - """Create a sample context with search results.""" - return { - "query": "test query", - "timeframe": "30d", - "has_results": True, - "result_count": 1, - "results": [search_result], - } - - -@pytest.fixture -def context_without_results(): - """Create a sample context without search results.""" - return { - "query": "empty query", - "timeframe": None, - "has_results": False, - "result_count": 0, - "results": [], - } - - -@pytest.mark.asyncio -async def test_search_with_results(template_loader, context_with_results): - """Test rendering the search template with results.""" - result = await template_loader.render("prompts/search.hbs", context_with_results) - - # Check that key elements are present - assert 'Search Results for: "test query" (after 30d)' in result - assert "1.0. Test Search Result" in result - assert "Type**: entity" in result - assert "Relevance Score**: 0.95" in result - assert "This is a test search result with some content." in result - assert 'read_note("test/search-result")' in result - assert "Next Steps" in result - assert "Synthesize and Capture Knowledge" in result - - -@pytest.mark.asyncio -async def test_search_without_results(template_loader, context_without_results): - """Test rendering the search template without results.""" - result = await template_loader.render("prompts/search.hbs", context_without_results) - - # Check that key elements are present - assert 'Search Results for: "empty query"' in result - assert "I couldn't find any results for this query." in result - assert "Opportunity to Capture Knowledge!" in result - assert "write_note(" in result - assert 'title="Empty query"' in result - assert "Other Suggestions" in result - - -@pytest.mark.asyncio -async def test_multiple_search_results(template_loader): - """Test rendering the search template with multiple results.""" - # Create multiple search results - results = [] - for i in range(1, 6): # Create 5 results - results.append( - SearchResult( - title=f"Search Result {i}", - type=SearchItemType.ENTITY, - permalink=f"test/result-{i}", - score=1.0 - (i * 0.1), # Decreasing scores - content=f"Content for result {i}", - file_path=f"/path/to/result-{i}.md", - metadata={}, - ) - ) - - context = { - "query": "multiple results", - "timeframe": None, - "has_results": True, - "result_count": len(results), - "results": results, - } - - result = await template_loader.render("prompts/search.hbs", context) - - # Check that all results are rendered - for i in range(1, 6): - assert f"{i}.0. Search Result {i}" in result - assert f"Content for result {i}" in result - assert f'read_note("test/result-{i}")' in result - - -@pytest.mark.asyncio -async def test_capitalization_in_write_note_template(template_loader, context_with_results): - """Test that the query is capitalized in the write_note template.""" - result = await template_loader.render("prompts/search.hbs", context_with_results) - - # The query should be capitalized in the suggested write_note call - assert "Synthesis of Test query Information" in result - - -@pytest.mark.asyncio -async def test_timeframe_display(template_loader): - """Test that the timeframe is displayed correctly when present, and not when absent.""" - # Context with timeframe - context_with_timeframe = { - "query": "with timeframe", - "timeframe": "7d", - "has_results": True, - "result_count": 0, - "results": [], - } - - result_with_timeframe = await template_loader.render( - "prompts/search.hbs", context_with_timeframe - ) - assert 'Search Results for: "with timeframe" (after 7d)' in result_with_timeframe - - # Context without timeframe - context_without_timeframe = { - "query": "without timeframe", - "timeframe": None, - "has_results": True, - "result_count": 0, - "results": [], - } - - result_without_timeframe = await template_loader.render( - "prompts/search.hbs", context_without_timeframe - ) - assert 'Search Results for: "without timeframe"' in result_without_timeframe - assert 'Search Results for: "without timeframe" (after' not in result_without_timeframe diff --git a/tests/api/test_template_loader.py b/tests/api/test_template_loader.py deleted file mode 100644 index 93c4d244..00000000 --- a/tests/api/test_template_loader.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Tests for the template loader functionality.""" - -import datetime -import pytest -from pathlib import Path - -from basic_memory.api.template_loader import TemplateLoader - - -@pytest.fixture -def temp_template_dir(tmpdir): - """Create a temporary directory for test templates.""" - template_dir = tmpdir.mkdir("templates").mkdir("prompts") - return template_dir - - -@pytest.fixture -def custom_template_loader(temp_template_dir): - """Return a TemplateLoader instance with a custom template directory.""" - return TemplateLoader(str(temp_template_dir)) - - -@pytest.fixture -def simple_template(temp_template_dir): - """Create a simple test template.""" - template_path = temp_template_dir / "simple.hbs" - template_path.write_text("Hello, {{name}}!", encoding="utf-8") - return "simple.hbs" - - -@pytest.mark.asyncio -async def test_render_simple_template(custom_template_loader, simple_template): - """Test rendering a simple template.""" - context = {"name": "World"} - result = await custom_template_loader.render(simple_template, context) - assert result == "Hello, World!" - - -@pytest.mark.asyncio -async def test_template_cache(custom_template_loader, simple_template): - """Test that templates are cached.""" - context = {"name": "World"} - - # First render, should load template - await custom_template_loader.render(simple_template, context) - - # Check that template is in cache - assert simple_template in custom_template_loader.template_cache - - # Modify the template file - shouldn't affect the cached version - template_path = Path(custom_template_loader.template_dir) / simple_template - template_path.write_text("Goodbye, {{name}}!", encoding="utf-8") - - # Second render, should use cached template - result = await custom_template_loader.render(simple_template, context) - assert result == "Hello, World!" - - # Clear cache and render again - should use updated template - custom_template_loader.clear_cache() - assert simple_template not in custom_template_loader.template_cache - - result = await custom_template_loader.render(simple_template, context) - assert result == "Goodbye, World!" - - -@pytest.mark.asyncio -async def test_date_helper(custom_template_loader, temp_template_dir): - # Test date helper - date_path = temp_template_dir / "date.hbs" - date_path.write_text("{{date timestamp}}", encoding="utf-8") - date_result = await custom_template_loader.render( - "date.hbs", {"timestamp": datetime.datetime(2023, 1, 1, 12, 30)} - ) - assert "2023-01-01" in date_result - - -@pytest.mark.asyncio -async def test_default_helper(custom_template_loader, temp_template_dir): - # Test default helper - default_path = temp_template_dir / "default.hbs" - default_path.write_text("{{default null 'default-value'}}", encoding="utf-8") - default_result = await custom_template_loader.render("default.hbs", {"null": None}) - assert default_result == "default-value" - - -@pytest.mark.asyncio -async def test_capitalize_helper(custom_template_loader, temp_template_dir): - # Test capitalize helper - capitalize_path = temp_template_dir / "capitalize.hbs" - capitalize_path.write_text("{{capitalize 'test'}}", encoding="utf-8") - capitalize_result = await custom_template_loader.render("capitalize.hbs", {}) - assert capitalize_result == "Test" - - -@pytest.mark.asyncio -async def test_size_helper(custom_template_loader, temp_template_dir): - # Test size helper - size_path = temp_template_dir / "size.hbs" - size_path.write_text("{{size collection}}", encoding="utf-8") - size_result = await custom_template_loader.render("size.hbs", {"collection": [1, 2, 3]}) - assert size_result == "3" - - -@pytest.mark.asyncio -async def test_json_helper(custom_template_loader, temp_template_dir): - # Test json helper - json_path = temp_template_dir / "json.hbs" - json_path.write_text("{{json data}}", encoding="utf-8") - json_result = await custom_template_loader.render("json.hbs", {"data": {"key": "value"}}) - assert json_result == '{"key": "value"}' - - -@pytest.mark.asyncio -async def test_less_than_helper(custom_template_loader, temp_template_dir): - # Test lt (less than) helper - lt_path = temp_template_dir / "lt.hbs" - lt_path.write_text("{{#if_cond (lt 2 3)}}true{{else}}false{{/if_cond}}", encoding="utf-8") - lt_result = await custom_template_loader.render("lt.hbs", {}) - assert lt_result == "true" - - -@pytest.mark.asyncio -async def test_file_not_found(custom_template_loader): - """Test that FileNotFoundError is raised when a template doesn't exist.""" - with pytest.raises(FileNotFoundError): - await custom_template_loader.render("non_existent_template.hbs", {}) - - -@pytest.mark.asyncio -async def test_extension_handling(custom_template_loader, temp_template_dir): - """Test that template extensions are handled correctly.""" - # Create template with .hbs extension - template_path = temp_template_dir / "test_extension.hbs" - template_path.write_text("Template with extension: {{value}}", encoding="utf-8") - - # Test accessing with full extension - result = await custom_template_loader.render("test_extension.hbs", {"value": "works"}) - assert result == "Template with extension: works" - - # Test accessing without extension - result = await custom_template_loader.render("test_extension", {"value": "also works"}) - assert result == "Template with extension: also works" - - # Test accessing with wrong extension gets converted - template_path = temp_template_dir / "liquid_template.hbs" - template_path.write_text("Liquid template: {{value}}", encoding="utf-8") - - result = await custom_template_loader.render("liquid_template.liquid", {"value": "converted"}) - assert result == "Liquid template: converted" - - -@pytest.mark.asyncio -async def test_dedent_helper(custom_template_loader, temp_template_dir): - """Test the dedent helper for text blocks.""" - dedent_path = temp_template_dir / "dedent.hbs" - - # Create a template with indented text blocks - template_content = """Before - {{#dedent}} - This is indented text - with nested indentation - that should be dedented - while preserving relative indentation - {{/dedent}} -After""" - - dedent_path.write_text(template_content, encoding="utf-8") - - # Render the template - result = await custom_template_loader.render("dedent.hbs", {}) - - # Print the actual output for debugging - print(f"Dedent helper result: {repr(result)}") - - # Check that the indentation is properly removed - assert "This is indented text" in result - assert "with nested indentation" in result - assert "that should be dedented" in result - assert "while preserving relative indentation" in result - assert "Before" in result - assert "After" in result - - # Check that relative indentation is preserved - assert result.find("with nested indentation") > result.find("This is indented text") - - -@pytest.mark.asyncio -async def test_nested_dedent_helper(custom_template_loader, temp_template_dir): - """Test the dedent helper with nested content.""" - dedent_path = temp_template_dir / "nested_dedent.hbs" - - # Create a template with nested indented blocks - template_content = """ -{{#each items}} - {{#dedent}} - --- Item {{this}} - - Details for item {{this}} - - Indented detail 1 - - Indented detail 2 - {{/dedent}} -{{/each}}""" - - dedent_path.write_text(template_content, encoding="utf-8") - - # Render the template - result = await custom_template_loader.render("nested_dedent.hbs", {"items": [1, 2]}) - - # Print the actual output for debugging - print(f"Actual result: {repr(result)}") - - # Use a more flexible assertion that checks individual components - # instead of exact string matching - assert "--- Item 1" in result - assert "Details for item 1" in result - assert "- Indented detail 1" in result - assert "--- Item 2" in result - assert "Details for item 2" in result - assert "- Indented detail 2" in result diff --git a/tests/api/test_template_loader_helpers.py b/tests/api/test_template_loader_helpers.py deleted file mode 100644 index cf7f2fe8..00000000 --- a/tests/api/test_template_loader_helpers.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Tests for additional template loader helpers.""" - -import pytest -from datetime import datetime - -from basic_memory.api.template_loader import TemplateLoader - - -@pytest.fixture -def temp_template_dir(tmpdir): - """Create a temporary directory for test templates.""" - template_dir = tmpdir.mkdir("templates").mkdir("prompts") - return template_dir - - -@pytest.fixture -def custom_template_loader(temp_template_dir): - """Return a TemplateLoader instance with a custom template directory.""" - return TemplateLoader(str(temp_template_dir)) - - -@pytest.mark.asyncio -async def test_round_helper(custom_template_loader, temp_template_dir): - """Test the round helper for number formatting.""" - # Create template file - round_path = temp_template_dir / "round.hbs" - round_path.write_text( - "{{round number}} {{round number 0}} {{round number 3}}", - encoding="utf-8", - ) - - # Test with various values - result = await custom_template_loader.render("round.hbs", {"number": 3.14159}) - assert result == "3.14 3.0 3.142" or result == "3.14 3 3.142" - - # Test with non-numeric value - result = await custom_template_loader.render("round.hbs", {"number": "not-a-number"}) - assert "not-a-number" in result - - # Test with insufficient args - empty_path = temp_template_dir / "round_empty.hbs" - empty_path.write_text("{{round}}", encoding="utf-8") - result = await custom_template_loader.render("round_empty.hbs", {}) - assert result == "" - - -@pytest.mark.asyncio -async def test_date_helper_edge_cases(custom_template_loader, temp_template_dir): - """Test edge cases for the date helper.""" - # Create template file - date_path = temp_template_dir / "date_edge.hbs" - date_path.write_text( - "{{date timestamp}} {{date timestamp '%Y'}} {{date string_date}} {{date invalid_date}} {{date}}", - encoding="utf-8", - ) - - # Test with various values - result = await custom_template_loader.render( - "date_edge.hbs", - { - "timestamp": datetime(2023, 1, 1, 12, 30), - "string_date": "2023-01-01T12:30:00", - "invalid_date": "not-a-date", - }, - ) - - assert "2023-01-01" in result - assert "2023" in result # Custom format - assert "not-a-date" in result # Invalid date passed through - assert result.strip() != "" # Empty date case - - -@pytest.mark.asyncio -async def test_size_helper_edge_cases(custom_template_loader, temp_template_dir): - """Test edge cases for the size helper.""" - # Create template file - size_path = temp_template_dir / "size_edge.hbs" - size_path.write_text( - "{{size list}} {{size string}} {{size dict}} {{size null}} {{size}}", - encoding="utf-8", - ) - - # Test with various values - result = await custom_template_loader.render( - "size_edge.hbs", - { - "list": [1, 2, 3, 4, 5], - "string": "hello", - "dict": {"a": 1, "b": 2, "c": 3}, - "null": None, - }, - ) - - assert "5" in result # List size - assert "hello".find("5") == -1 # String size should be 5 - assert "3" in result # Dict size - assert "0" in result # Null size - assert result.count("0") >= 2 # At least two zeros (null and empty args) - - -@pytest.mark.asyncio -async def test_math_helper(custom_template_loader, temp_template_dir): - """Test the math helper for basic arithmetic.""" - # Create template file - math_path = temp_template_dir / "math.hbs" - math_path.write_text( - "{{math 5 '+' 3}} {{math 10 '-' 4}} {{math 6 '*' 7}} {{math 20 '/' 5}}", - encoding="utf-8", - ) - - # Test basic operations - result = await custom_template_loader.render("math.hbs", {}) - assert "8" in result # Addition - assert "6" in result # Subtraction - assert "42" in result # Multiplication - assert "4" in result # Division - - # Test with invalid operator - invalid_op_path = temp_template_dir / "math_invalid_op.hbs" - invalid_op_path.write_text("{{math 5 'invalid' 3}}", encoding="utf-8") - result = await custom_template_loader.render("math_invalid_op.hbs", {}) - assert "Unsupported operator" in result - - # Test with invalid numeric values - invalid_num_path = temp_template_dir / "math_invalid_num.hbs" - invalid_num_path.write_text("{{math 'not-a-number' '+' 3}}", encoding="utf-8") - result = await custom_template_loader.render("math_invalid_num.hbs", {}) - assert "Math error" in result - - # Test with insufficient arguments - insufficient_path = temp_template_dir / "math_insufficient.hbs" - insufficient_path.write_text("{{math 5 '+'}}", encoding="utf-8") - result = await custom_template_loader.render("math_insufficient.hbs", {}) - assert "Insufficient arguments" in result - - -@pytest.mark.asyncio -async def test_if_cond_helper(custom_template_loader, temp_template_dir): - """Test the if_cond helper for conditionals.""" - # Create template file with true condition - if_true_path = temp_template_dir / "if_true.hbs" - if_true_path.write_text( - "{{#if_cond (lt 5 10)}}True condition{{else}}False condition{{/if_cond}}", - encoding="utf-8", - ) - - # Create template file with false condition - if_false_path = temp_template_dir / "if_false.hbs" - if_false_path.write_text( - "{{#if_cond (lt 15 10)}}True condition{{else}}False condition{{/if_cond}}", - encoding="utf-8", - ) - - # Test true condition - result = await custom_template_loader.render("if_true.hbs", {}) - assert result == "True condition" - - # Test false condition - result = await custom_template_loader.render("if_false.hbs", {}) - assert result == "False condition" - - -@pytest.mark.asyncio -async def test_lt_helper_edge_cases(custom_template_loader, temp_template_dir): - """Test edge cases for the lt (less than) helper.""" - # Create template file - lt_path = temp_template_dir / "lt_edge.hbs" - lt_path.write_text( - "{{#if_cond (lt 'a' 'b')}}String LT True{{else}}String LT False{{/if_cond}} " - "{{#if_cond (lt 'z' 'a')}}String LT2 True{{else}}String LT2 False{{/if_cond}} " - "{{#if_cond (lt)}}Missing args True{{else}}Missing args False{{/if_cond}}", - encoding="utf-8", - ) - - # Test with string values and missing args - result = await custom_template_loader.render("lt_edge.hbs", {}) - assert "String LT True" in result # 'a' < 'b' is true - assert "String LT2 False" in result # 'z' < 'a' is false - assert "Missing args False" in result # Missing args should return false - - -@pytest.mark.asyncio -async def test_dedent_helper_edge_case(custom_template_loader, temp_template_dir): - """Test an edge case for the dedent helper.""" - # Create template with empty dedent block - empty_dedent_path = temp_template_dir / "empty_dedent.hbs" - empty_dedent_path.write_text("{{#dedent}}{{/dedent}}", encoding="utf-8") - - # Test empty block - result = await custom_template_loader.render("empty_dedent.hbs", {}) - assert result == "" - - # Test with complex content including lists - complex_dedent_path = temp_template_dir / "complex_dedent.hbs" - complex_dedent_path.write_text( - "{{#dedent}}\n {{#each items}}\n - {{this}}\n {{/each}}\n{{/dedent}}", - encoding="utf-8", - ) - - result = await custom_template_loader.render("complex_dedent.hbs", {"items": [1, 2, 3]}) - assert "- 1" in result - assert "- 2" in result - assert "- 3" in result diff --git a/tests/api/v2/conftest.py b/tests/api/v2/conftest.py index bbf91890..3743ab90 100644 --- a/tests/api/v2/conftest.py +++ b/tests/api/v2/conftest.py @@ -1,10 +1,33 @@ """Fixtures for V2 API tests.""" +from typing import AsyncGenerator + import pytest +import pytest_asyncio +from fastapi import FastAPI +from httpx import AsyncClient, ASGITransport +from basic_memory.deps import get_engine_factory, get_app_config from basic_memory.models import Project +@pytest_asyncio.fixture +async def app(test_config, engine_factory, app_config) -> FastAPI: + """Create FastAPI test application.""" + from basic_memory.api.app import app + + app.dependency_overrides[get_app_config] = lambda: app_config + app.dependency_overrides[get_engine_factory] = lambda: engine_factory + return app + + +@pytest_asyncio.fixture +async def client(app: FastAPI) -> AsyncGenerator[AsyncClient, None]: + """Create client using ASGI transport - same as CLI will use.""" + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + yield client + + @pytest.fixture def v2_project_url(test_project: Project) -> str: """Create a URL prefix for v2 project-scoped routes using project external_id. diff --git a/tests/api/v2/test_knowledge_router.py b/tests/api/v2/test_knowledge_router.py index 6bb1b266..aac7fa5b 100644 --- a/tests/api/v2/test_knowledge_router.py +++ b/tests/api/v2/test_knowledge_router.py @@ -1,5 +1,7 @@ """Tests for V2 knowledge graph API routes (ID-based endpoints).""" +import uuid + import pytest from httpx import AsyncClient @@ -101,7 +103,9 @@ async def test_create_entity(client: AsyncClient, file_service, v2_project_url): "content": "TestContent for V2", } - response = await client.post(f"{v2_project_url}/knowledge/entities", json=data) + response = await client.post( + f"{v2_project_url}/knowledge/entities", json=data, params={"fast": False} + ) assert response.status_code == 200 entity = EntityResponseV2.model_validate(response.json()) @@ -138,7 +142,9 @@ async def test_create_entity_with_observations_and_relations( """, } - response = await client.post(f"{v2_project_url}/knowledge/entities", json=data) + response = await client.post( + f"{v2_project_url}/knowledge/entities", json=data, params={"fast": False} + ) assert response.status_code == 200 entity = EntityResponseV2.model_validate(response.json()) @@ -201,6 +207,70 @@ async def test_update_entity_by_id( assert "Original content" not in file_content +@pytest.mark.asyncio +async def test_update_entity_by_id_fast_does_not_duplicate( + client: AsyncClient, v2_project_url, entity_repository +): + """Fast PUT updates the existing external_id without creating duplicates.""" + create_data = { + "title": "07 - Get Started", + "directory": "docs", + "content": "Original content", + } + response = await client.post(f"{v2_project_url}/knowledge/entities", json=create_data) + assert response.status_code == 200 + created_entity = EntityResponseV2.model_validate(response.json()) + + update_data = { + "title": "07 Get Started", + "directory": "docs", + "content": "Updated content", + } + response = await client.put( + f"{v2_project_url}/knowledge/entities/{created_entity.external_id}", + json=update_data, + ) + assert response.status_code == 200 + + entities = await entity_repository.find_all() + assert len(entities) == 1 + assert entities[0].external_id == created_entity.external_id + + +@pytest.mark.asyncio +async def test_put_entity_fast_returns_minimal_row( + client: AsyncClient, v2_project_url, entity_repository +): + """Fast PUT returns a minimal row and persists the external_id immediately.""" + external_id = str(uuid.uuid4()) + update_data = { + "title": "FastPutEntity", + "directory": "test", + "content": """ +# FastPutEntity + +## Observations +- [note] This should be deferred + +- related_to [[AnotherEntity]] +""", + } + response = await client.put( + f"{v2_project_url}/knowledge/entities/{external_id}", + json=update_data, + params={"fast": True}, + ) + + assert response.status_code == 201 + created_entity = EntityResponseV2.model_validate(response.json()) + assert created_entity.external_id == external_id + assert created_entity.observations == [] + assert created_entity.relations == [] + + db_entity = await entity_repository.get_by_external_id(external_id) + assert db_entity is not None + + @pytest.mark.asyncio async def test_edit_entity_by_id_append( client: AsyncClient, file_service, v2_project_url, entity_repository diff --git a/tests/cli/cloud/test_cloud_api_client_and_utils.py b/tests/cli/cloud/test_cloud_api_client_and_utils.py index 5f95c65c..8c509df6 100644 --- a/tests/cli/cloud/test_cloud_api_client_and_utils.py +++ b/tests/cli/cloud/test_cloud_api_client_and_utils.py @@ -110,7 +110,7 @@ async def test_cloud_utils_fetch_and_exists_and_create_project( seen = {"create_payload": None} async def handler(request: httpx.Request) -> httpx.Response: - if request.method == "GET" and request.url.path == "/proxy/projects/projects": + if request.method == "GET" and request.url.path == "/proxy/v2/projects/": return httpx.Response( 200, json={ @@ -121,7 +121,7 @@ async def handler(request: httpx.Request) -> httpx.Response: }, ) - if request.method == "POST" and request.url.path == "/proxy/projects/projects": + if request.method == "POST" and request.url.path == "/proxy/v2/projects/": # httpx.Request doesn't have .json(); parse bytes payload. seen["create_payload"] = json.loads(request.content.decode("utf-8")) return httpx.Response( diff --git a/tests/mcp/clients/test_clients.py b/tests/mcp/clients/test_clients.py index b3829bb2..265a50d3 100644 --- a/tests/mcp/clients/test_clients.py +++ b/tests/mcp/clients/test_clients.py @@ -298,7 +298,7 @@ async def test_list_projects(self, monkeypatch): } async def mock_call_get(client, url, **kwargs): - assert "/projects/projects" in url + assert "/v2/projects" in url return mock_response monkeypatch.setattr(project_mod, "call_get", mock_call_get) diff --git a/tests/services/test_entity_service.py b/tests/services/test_entity_service.py index 9a0a99d9..5d6a5300 100644 --- a/tests/services/test_entity_service.py +++ b/tests/services/test_entity_service.py @@ -1,5 +1,6 @@ """Tests for EntityService.""" +import uuid from pathlib import Path from textwrap import dedent @@ -347,6 +348,55 @@ async def test_update_note_entity_content(entity_service: EntityService, file_se assert metadata.get("status") == "draft" +@pytest.mark.asyncio +async def test_fast_write_and_reindex_entity( + entity_repository: EntityRepository, + observation_repository, + relation_repository, + entity_parser: EntityParser, + file_service: FileService, + link_resolver, + search_service: SearchService, + app_config: BasicMemoryConfig, +): + """Fast write should defer observations/relations until reindex.""" + service = EntityService( + entity_repository=entity_repository, + observation_repository=observation_repository, + relation_repository=relation_repository, + entity_parser=entity_parser, + file_service=file_service, + link_resolver=link_resolver, + search_service=search_service, + app_config=app_config, + ) + + schema = EntitySchema( + title="Reindex Target", + directory="test", + entity_type="note", + content=dedent(""" + # Reindex Target + + - [note] Deferred observation + - relates_to [[Other Entity]] + """).strip(), + ) + external_id = str(uuid.uuid4()) + fast_entity = await service.fast_write_entity(schema, external_id=external_id) + + assert fast_entity.external_id == external_id + assert len(fast_entity.observations) == 0 + assert len(fast_entity.relations) == 0 + + await service.reindex_entity(fast_entity.id) + reindexed = await entity_repository.get_by_external_id(external_id) + + assert reindexed is not None + assert len(reindexed.observations) == 1 + assert len(reindexed.relations) == 1 + + @pytest.mark.asyncio async def test_create_or_update_new(entity_service: EntityService, file_service: FileService): """Should create a new entity.""" From 51639fc5995421b74814a56ec0cca66906b3e4e3 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 29 Jan 2026 05:19:36 -0600 Subject: [PATCH 2/6] defer indexing to async task --- src/basic_memory/api/app.py | 3 + .../api/v2/routers/knowledge_router.py | 51 +++----- .../api/v2/routers/project_router.py | 14 ++- .../api/v2/routers/search_router.py | 19 +-- src/basic_memory/deps/__init__.py | 2 + src/basic_memory/deps/services.py | 85 ++++++++++++- src/basic_memory/schemas/search.py | 1 + src/basic_memory/services/entity_service.py | 117 ++++++++---------- src/basic_memory/sync/sync_service.py | 20 ++- tests/api/v2/conftest.py | 19 ++- tests/api/v2/test_knowledge_router.py | 23 ++++ tests/sync/test_sync_service.py | 5 +- 12 files changed, 227 insertions(+), 132 deletions(-) diff --git a/src/basic_memory/api/app.py b/src/basic_memory/api/app.py index 9dcf3f37..71641321 100644 --- a/src/basic_memory/api/app.py +++ b/src/basic_memory/api/app.py @@ -79,6 +79,9 @@ async def lifespan(app: FastAPI): # pragma: no cover app.include_router(v2_importer, prefix="/v2/projects/{project_id}") app.include_router(v2_project, prefix="/v2") +# Legacy web app proxy paths (compat with /proxy/projects/projects) +app.include_router(v2_project, prefix="/proxy/projects") + # V2 routers are the only public API surface diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 35a5ade2..09b3bf08 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -19,9 +19,9 @@ LinkResolverV2ExternalDep, ProjectConfigV2ExternalDep, AppConfigDep, - SyncServiceV2ExternalDep, EntityRepositoryV2ExternalDep, ProjectExternalIdPathDep, + TaskSchedulerDep, ) from basic_memory.schemas import DeleteEntitiesResponse from basic_memory.schemas.base import Entity @@ -38,26 +38,6 @@ router = APIRouter(prefix="/knowledge", tags=["knowledge-v2"]) - -async def resolve_relations_background(sync_service, entity_id: int, entity_permalink: str) -> None: - """Background task to resolve relations for a specific entity. - - This runs asynchronously after the API response is sent, preventing - long delays when creating entities with many relations. - """ - try: # pragma: no cover - # Only resolve relations for the newly created entity - await sync_service.resolve_relations(entity_id=entity_id) # pragma: no cover - logger.debug( # pragma: no cover - f"Background: Resolved relations for entity {entity_permalink} (id={entity_id})" - ) - except Exception as e: # pragma: no cover - # Log but don't fail - this is a background task - logger.warning( # pragma: no cover - f"Background: Failed to resolve relations for entity {entity_permalink}: {e}" - ) - - ## Resolution endpoint @@ -182,6 +162,7 @@ async def create_entity( background_tasks: BackgroundTasks, entity_service: EntityServiceV2ExternalDep, search_service: SearchServiceV2ExternalDep, + task_scheduler: TaskSchedulerDep, fast: bool = Query( True, description="If true, write quickly and defer indexing to background tasks." ), @@ -201,7 +182,11 @@ async def create_entity( if fast: entity = await entity_service.fast_write_entity(data) - background_tasks.add_task(entity_service.reindex_entity, entity.id) + task_scheduler.schedule( + "reindex_entity", + entity_id=entity.id, + project_id=project_id, + ) else: entity = await entity_service.create_entity(data) await search_service.index_entity(entity, background_tasks=background_tasks) @@ -227,8 +212,8 @@ async def update_entity_by_id( project_id: ProjectExternalIdPathDep, entity_service: EntityServiceV2ExternalDep, search_service: SearchServiceV2ExternalDep, - sync_service: SyncServiceV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, + task_scheduler: TaskSchedulerDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), fast: bool = Query( True, description="If true, write quickly and defer indexing to background tasks." @@ -255,7 +240,12 @@ async def update_entity_by_id( if fast: entity = await entity_service.fast_write_entity(data, external_id=entity_id) response.status_code = 200 if existing else 201 - background_tasks.add_task(entity_service.reindex_entity, entity.id) + task_scheduler.schedule( + "reindex_entity", + entity_id=entity.id, + project_id=project_id, + resolve_relations=created, + ) else: if existing: # Update the existing entity in-place to avoid path-based duplication @@ -278,12 +268,6 @@ async def update_entity_by_id( await search_service.index_entity(entity, background_tasks=background_tasks) - # Schedule relation resolution for new entities - if created: - background_tasks.add_task( # pragma: no cover - resolve_relations_background, sync_service, entity.id, entity.permalink or "" - ) - result = EntityResponseV2.model_validate(entity) if fast: result = result.model_copy(update={"observations": [], "relations": []}) @@ -302,6 +286,7 @@ async def edit_entity_by_id( entity_service: EntityServiceV2ExternalDep, search_service: SearchServiceV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, + task_scheduler: TaskSchedulerDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), fast: bool = Query( True, description="If true, write quickly and defer indexing to background tasks." @@ -341,7 +326,11 @@ async def edit_entity_by_id( find_text=data.find_text, expected_replacements=data.expected_replacements, ) - background_tasks.add_task(entity_service.reindex_entity, updated_entity.id) + task_scheduler.schedule( + "reindex_entity", + entity_id=updated_entity.id, + project_id=project_id, + ) else: # Edit using the entity's permalink or path identifier = entity.permalink or entity.file_path diff --git a/src/basic_memory/api/v2/routers/project_router.py b/src/basic_memory/api/v2/routers/project_router.py index 5cf885fd..0a42948f 100644 --- a/src/basic_memory/api/v2/routers/project_router.py +++ b/src/basic_memory/api/v2/routers/project_router.py @@ -13,7 +13,7 @@ import os from typing import Optional -from fastapi import APIRouter, HTTPException, Body, Query, Path, BackgroundTasks +from fastapi import APIRouter, HTTPException, Body, Query, Path from loguru import logger from basic_memory.deps import ( @@ -21,6 +21,8 @@ ProjectRepositoryDep, ProjectConfigV2ExternalDep, SyncServiceV2ExternalDep, + TaskSchedulerDep, + ProjectExternalIdPathDep, ) from basic_memory.schemas import SyncReportResponse from basic_memory.schemas.project_info import ( @@ -158,10 +160,10 @@ async def synchronize_projects( @router.post("/{project_id}/sync") async def sync_project( - background_tasks: BackgroundTasks, sync_service: SyncServiceV2ExternalDep, project_config: ProjectConfigV2ExternalDep, - project_id: str = Path(..., description="Project external ID (UUID)"), + task_scheduler: TaskSchedulerDep, + project_internal_id: ProjectExternalIdPathDep, force_full: bool = Query( False, description="Force full scan, bypassing watermark optimization" ), @@ -169,8 +171,10 @@ async def sync_project( ): """Force project filesystem sync to database.""" if run_in_background: - background_tasks.add_task( - sync_service.sync, project_config.home, project_config.name, force_full=force_full + task_scheduler.schedule( + "sync_project", + project_id=project_internal_id, + force_full=force_full, ) logger.info( f"Filesystem sync initiated for project: {project_config.name} (force_full={force_full})" diff --git a/src/basic_memory/api/v2/routers/search_router.py b/src/basic_memory/api/v2/routers/search_router.py index eae643ab..d484086d 100644 --- a/src/basic_memory/api/v2/routers/search_router.py +++ b/src/basic_memory/api/v2/routers/search_router.py @@ -4,11 +4,16 @@ V1 uses string-based project names which are less efficient and less stable. """ -from fastapi import APIRouter, BackgroundTasks, Path +from fastapi import APIRouter, Path from basic_memory.api.v2.utils import to_search_results from basic_memory.schemas.search import SearchQuery, SearchResponse -from basic_memory.deps import SearchServiceV2ExternalDep, EntityServiceV2ExternalDep +from basic_memory.deps import ( + SearchServiceV2ExternalDep, + EntityServiceV2ExternalDep, + TaskSchedulerDep, + ProjectExternalIdPathDep, +) # Note: No prefix here - it's added during registration as /v2/{project_id}/search router = APIRouter(tags=["search"]) @@ -51,9 +56,8 @@ async def search( @router.post("/search/reindex") async def reindex( - background_tasks: BackgroundTasks, - search_service: SearchServiceV2ExternalDep, - project_id: str = Path(..., description="Project external UUID"), + task_scheduler: TaskSchedulerDep, + project_id: ProjectExternalIdPathDep, ): """Recreate and populate the search index for a project. @@ -63,11 +67,10 @@ async def reindex( Args: project_id: Project external UUID from URL path - background_tasks: FastAPI background tasks handler - search_service: Search service scoped to project + task_scheduler: Task scheduler for background work Returns: Status message indicating reindex has been initiated """ - await search_service.reindex_all(background_tasks=background_tasks) + task_scheduler.schedule("reindex_project", project_id=project_id) return {"status": "ok", "message": "Reindex initiated"} diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index 9924e7b3..9b52edc1 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -91,6 +91,8 @@ FileServiceV2Dep, get_file_service_v2_external, FileServiceV2ExternalDep, + get_task_scheduler, + TaskSchedulerDep, get_search_service, SearchServiceDep, get_search_service_v2, diff --git a/src/basic_memory/deps/services.py b/src/basic_memory/deps/services.py index 1c866dd0..a9002f57 100644 --- a/src/basic_memory/deps/services.py +++ b/src/basic_memory/deps/services.py @@ -7,7 +7,8 @@ - SyncService, ProjectService, DirectoryService """ -from typing import Annotated +import asyncio +from typing import Annotated, Any, Callable, Coroutine, Mapping, Protocol from fastapi import Depends from loguru import logger @@ -43,7 +44,6 @@ from basic_memory.services.search_service import SearchService from basic_memory.sync import SyncService - # --- Entity Parser --- @@ -430,6 +430,87 @@ async def get_sync_service_v2_external( SyncServiceV2ExternalDep = Annotated[SyncService, Depends(get_sync_service_v2_external)] +# --- Background Task Scheduler --- + + +class TaskScheduler(Protocol): + def schedule(self, task_name: str, **payload: Any) -> None: + """Schedule a background task by name.""" + + +def _log_task_failure(completed: asyncio.Task) -> None: + try: + completed.result() + except Exception as exc: # pragma: no cover + logger.exception("Background task failed", error=str(exc)) + + +class LocalTaskScheduler: + """Default scheduler that runs tasks in-process via asyncio.create_task.""" + + def __init__( + self, + handlers: Mapping[str, Callable[..., Coroutine[Any, Any, None]]], + ) -> None: + self._handlers = handlers + + def schedule(self, task_name: str, **payload: Any) -> None: + handler = self._handlers.get(task_name) + # Trigger: task name is not registered + # Why: avoid silently dropping background work + # Outcome: fail fast to surface misconfiguration + if not handler: + raise ValueError(f"Unknown task name: {task_name}") + task = asyncio.create_task(handler(**payload)) + task.add_done_callback(_log_task_failure) + + +async def get_task_scheduler( + entity_service: EntityServiceV2ExternalDep, + sync_service: SyncServiceV2ExternalDep, + search_service: SearchServiceV2ExternalDep, + project_config: ProjectConfigV2ExternalDep, +) -> TaskScheduler: + """Create a scheduler that maps task specs to coroutines.""" + + async def _reindex_entity( + entity_id: int, + resolve_relations: bool = False, + **_: Any, + ) -> None: + await entity_service.reindex_entity(entity_id) + # Trigger: caller requests relation resolution + # Why: resolve forward references created before the entity existed + # Outcome: updates unresolved relations pointing to this entity + if resolve_relations: + await sync_service.resolve_relations(entity_id=entity_id) + + async def _resolve_relations(entity_id: int, **_: Any) -> None: + await sync_service.resolve_relations(entity_id=entity_id) + + async def _sync_project(force_full: bool = False, **_: Any) -> None: + await sync_service.sync( + project_config.home, + project_config.name, + force_full=force_full, + ) + + async def _reindex_project(**_: Any) -> None: + await search_service.reindex_all() + + return LocalTaskScheduler( + { + "reindex_entity": _reindex_entity, + "resolve_relations": _resolve_relations, + "sync_project": _sync_project, + "reindex_project": _reindex_project, + } + ) + + +TaskSchedulerDep = Annotated[TaskScheduler, Depends(get_task_scheduler)] + + # --- Project Service --- diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index e69be4db..929430bf 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -29,6 +29,7 @@ class SearchQuery(BaseModel): - permalink: Exact permalink match - permalink_match: Path pattern with * - text: Full-text search of title/content (supports boolean operators: AND, OR, NOT) + - title: Title only search Optionally filter results by: - types: Limit to specific item types diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 0e72d7a2..c67fe49a 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -168,6 +168,25 @@ async def resolve_permalink( return permalink + def _build_frontmatter_markdown( + self, title: str, entity_type: str, permalink: str + ) -> EntityMarkdown: + """Build a minimal EntityMarkdown object for permalink resolution.""" + from basic_memory.markdown.schemas import EntityFrontmatter + + frontmatter_metadata = { + "title": title, + "type": entity_type, + "permalink": permalink, + } + frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) + return EntityMarkdown( + frontmatter=frontmatter_obj, + content="", + observations=[], + relations=[], + ) + async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]: """Create new entity or update existing one. Returns: (entity, is_new) where is_new is True if a new entity was created @@ -211,20 +230,8 @@ async def create_entity(self, schema: EntitySchema) -> EntityModel: schema.entity_type = content_frontmatter["type"] if "permalink" in content_frontmatter: - # Create a minimal EntityMarkdown object for permalink resolution - from basic_memory.markdown.schemas import EntityFrontmatter - - frontmatter_metadata = { - "title": schema.title, - "type": schema.entity_type, - "permalink": content_frontmatter["permalink"], - } - frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) - content_markdown = EntityMarkdown( - frontmatter=frontmatter_obj, - content="", # content not needed for permalink resolution - observations=[], - relations=[], + content_markdown = self._build_frontmatter_markdown( + schema.title, schema.entity_type, content_frontmatter["permalink"] ) # Get unique permalink (prioritizing content frontmatter) unless disabled @@ -249,11 +256,8 @@ async def create_entity(self, schema: EntitySchema) -> EntityModel: content=final_content, ) - # create entity - created = await self.create_entity_from_markdown(file_path, entity_markdown) - - # add relations - entity = await self.update_entity_relations(created.file_path, entity_markdown) + # create entity and relations + entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=True) # Set final checksum to mark complete return await self.repository.update(entity.id, {"checksum": checksum}) @@ -284,20 +288,8 @@ async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> Enti schema.entity_type = content_frontmatter["type"] if "permalink" in content_frontmatter: - # Create a minimal EntityMarkdown object for permalink resolution - from basic_memory.markdown.schemas import EntityFrontmatter - - frontmatter_metadata = { - "title": schema.title, - "type": schema.entity_type, - "permalink": content_frontmatter["permalink"], - } - frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) - content_markdown = EntityMarkdown( - frontmatter=frontmatter_obj, - content="", # content not needed for permalink resolution - observations=[], - relations=[], + content_markdown = self._build_frontmatter_markdown( + schema.title, schema.entity_type, content_frontmatter["permalink"] ) # Check if we need to update the permalink based on content frontmatter (unless disabled) @@ -334,11 +326,8 @@ async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> Enti content=final_content, ) - # update entity in db - entity = await self.update_entity_and_observations(file_path, entity_markdown) - - # add relations - await self.update_entity_relations(file_path.as_posix(), entity_markdown) + # update entity and relations + entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False) # Set final checksum to match file entity = await self.repository.update(entity.id, {"checksum": checksum}) @@ -380,19 +369,8 @@ async def fast_write_entity( schema.entity_type = content_frontmatter["type"] if "permalink" in content_frontmatter: - from basic_memory.markdown.schemas import EntityFrontmatter - - frontmatter_metadata = { - "title": schema.title, - "type": schema.entity_type, - "permalink": content_frontmatter["permalink"], - } - frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) - content_markdown = EntityMarkdown( - frontmatter=frontmatter_obj, - content="", - observations=[], - relations=[], + content_markdown = self._build_frontmatter_markdown( + schema.title, schema.entity_type, content_frontmatter["permalink"] ) # --- Permalink Resolution --- @@ -474,19 +452,10 @@ async def fast_edit_entity( update_data["entity_type"] = content_frontmatter["type"] if "permalink" in content_frontmatter: - from basic_memory.markdown.schemas import EntityFrontmatter - - frontmatter_metadata = { - "title": update_data.get("title", entity.title), - "type": update_data.get("entity_type", entity.entity_type), - "permalink": content_frontmatter["permalink"], - } - frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) - content_markdown = EntityMarkdown( - frontmatter=frontmatter_obj, - content="", - observations=[], - relations=[], + content_markdown = self._build_frontmatter_markdown( + update_data.get("title", entity.title), + update_data.get("entity_type", entity.entity_type), + content_frontmatter["permalink"], ) metadata = content_frontmatter or {} @@ -522,8 +491,7 @@ async def reindex_entity(self, entity_id: int) -> None: ) # --- DB Reindex --- - updated = await self.update_entity_and_observations(file_path, entity_markdown) - updated = await self.update_entity_relations(file_path.as_posix(), entity_markdown) + updated = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False) checksum = await self.file_service.compute_checksum(file_path) updated = await self.repository.update(updated.id, {"checksum": checksum}) if not updated: @@ -654,6 +622,20 @@ async def update_entity_and_observations( db_entity, ) + async def upsert_entity_from_markdown( + self, + file_path: Path, + markdown: EntityMarkdown, + *, + is_new: bool, + ) -> EntityModel: + """Create/update entity and relations from parsed markdown.""" + if is_new: + created = await self.create_entity_from_markdown(file_path, markdown) + else: + created = await self.update_entity_and_observations(file_path, markdown) + return await self.update_entity_relations(created.file_path, markdown) + async def update_entity_relations( self, path: str, @@ -778,8 +760,7 @@ async def edit_entity( ) # Update entity and its relationships - entity = await self.update_entity_and_observations(file_path, entity_markdown) - await self.update_entity_relations(file_path.as_posix(), entity_markdown) + entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False) # Set final checksum to match file entity = await self.repository.update(entity.id, {"checksum": checksum}) diff --git a/src/basic_memory/sync/sync_service.py b/src/basic_memory/sync/sync_service.py index 5a019315..0ee04d07 100644 --- a/src/basic_memory/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -685,19 +685,13 @@ async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optiona entity_markdown.frontmatter.metadata["permalink"] = permalink await self.file_service.update_frontmatter(path, {"permalink": permalink}) - # if the file is new, create an entity - if new: - # Create entity with final permalink - logger.debug(f"Creating new entity from markdown, path={path}") - await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown) - - # otherwise we need to update the entity and observations - else: - logger.debug(f"Updating entity from markdown, path={path}") - await self.entity_service.update_entity_and_observations(Path(path), entity_markdown) - - # Update relations and search index - entity = await self.entity_service.update_entity_relations(path, entity_markdown) + # Create/update entity and relations in one path + logger.debug( + f"{'Creating' if new else 'Updating'} entity from markdown, path={path}" + ) + entity = await self.entity_service.upsert_entity_from_markdown( + Path(path), entity_markdown, is_new=new + ) # After updating relations, we need to compute the checksum again # This is necessary for files with wikilinks to ensure consistent checksums diff --git a/tests/api/v2/conftest.py b/tests/api/v2/conftest.py index 3743ab90..af6cedd8 100644 --- a/tests/api/v2/conftest.py +++ b/tests/api/v2/conftest.py @@ -1,13 +1,14 @@ """Fixtures for V2 API tests.""" -from typing import AsyncGenerator +from typing import Any, AsyncGenerator import pytest import pytest_asyncio from fastapi import FastAPI from httpx import AsyncClient, ASGITransport -from basic_memory.deps import get_engine_factory, get_app_config +from basic_memory.deps import get_app_config, get_engine_factory +from basic_memory.deps.services import get_task_scheduler from basic_memory.models import Project @@ -28,6 +29,20 @@ async def client(app: FastAPI) -> AsyncGenerator[AsyncClient, None]: yield client +@pytest.fixture(autouse=True) +def task_scheduler_spy(app: FastAPI) -> list[dict[str, Any]]: + """Capture scheduled task specs without executing them.""" + scheduled: list[dict[str, Any]] = [] + + class SchedulerSpy: + def schedule(self, task_name: str, **payload: Any) -> None: + scheduled.append({"task_name": task_name, "payload": payload}) + + app.dependency_overrides[get_task_scheduler] = lambda: SchedulerSpy() + yield scheduled + app.dependency_overrides.pop(get_task_scheduler, None) + + @pytest.fixture def v2_project_url(test_project: Project) -> str: """Create a URL prefix for v2 project-scoped routes using project external_id. diff --git a/tests/api/v2/test_knowledge_router.py b/tests/api/v2/test_knowledge_router.py index aac7fa5b..6a8c28b3 100644 --- a/tests/api/v2/test_knowledge_router.py +++ b/tests/api/v2/test_knowledge_router.py @@ -271,6 +271,29 @@ async def test_put_entity_fast_returns_minimal_row( assert db_entity is not None +@pytest.mark.asyncio +async def test_fast_create_schedules_reindex_task( + client: AsyncClient, v2_project_url, task_scheduler_spy +): + """Fast create should enqueue a background reindex task.""" + start_count = len(task_scheduler_spy) + response = await client.post( + f"{v2_project_url}/knowledge/entities", + json={ + "title": "TaskScheduledEntity", + "directory": "test", + "content": "Content for task scheduling", + }, + params={"fast": True}, + ) + assert response.status_code == 200 + assert len(task_scheduler_spy) == start_count + 1 + created_entity = EntityResponseV2.model_validate(response.json()) + scheduled = task_scheduler_spy[-1] + assert scheduled["task_name"] == "reindex_entity" + assert scheduled["payload"]["entity_id"] == created_entity.id + + @pytest.mark.asyncio async def test_edit_entity_by_id_append( client: AsyncClient, file_service, v2_project_url, entity_repository diff --git a/tests/sync/test_sync_service.py b/tests/sync/test_sync_service.py index 6453e50a..dab7f010 100644 --- a/tests/sync/test_sync_service.py +++ b/tests/sync/test_sync_service.py @@ -1,7 +1,6 @@ """Test general sync behavior.""" import asyncio -import os from datetime import datetime, timezone from pathlib import Path from textwrap import dedent @@ -836,8 +835,8 @@ async def test_sync_updates_timestamps_on_file_modification( entity_updated_epoch = entity_after.updated_at.timestamp() file_mtime = file_stats_after_modification.st_mtime - # Allow 2s difference on Windows due to filesystem timing precision - tolerance = 2 if os.name == "nt" else 1 + # Allow 2s difference due to filesystem timing precision and sync processing delays + tolerance = 2 assert abs(entity_updated_epoch - file_mtime) < tolerance, ( f"Entity updated_at ({entity_after.updated_at}) should match file mtime " f"({datetime.fromtimestamp(file_mtime)}) within {tolerance}s tolerance" From 54aa9d1dde78cb8cf372cdef994abd5d3b9e88ab Mon Sep 17 00:00:00 2001 From: phernandez Date: Fri, 30 Jan 2026 22:27:35 -0600 Subject: [PATCH 3/6] api changes for recent activity --- src/basic_memory/api/v2/utils.py | 37 +++++++++++++++++----- src/basic_memory/schemas/memory.py | 4 +++ tests/mcp/test_prompts.py | 1 + tests/mcp/test_tool_recent_activity.py | 4 +++ tests/schemas/test_memory_serialization.py | 7 ++++ 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/api/v2/utils.py b/src/basic_memory/api/v2/utils.py index b4211a67..766588f0 100644 --- a/src/basic_memory/api/v2/utils.py +++ b/src/basic_memory/api/v2/utils.py @@ -24,29 +24,42 @@ async def to_graph_context( page: Optional[int] = None, page_size: Optional[int] = None, ): - # First pass: collect all entity IDs needed for relations + # First pass: collect all entity IDs needed for external_id lookup + # This includes: entity primary results, observation parent entities, relation from/to entities entity_ids_needed: set[int] = set() for context_item in context_result.results: for item in ( [context_item.primary_result] + context_item.observations + context_item.related_results ): - if item.type == SearchItemType.RELATION: + if item.type == SearchItemType.ENTITY: + # Entity's own ID for its external_id + entity_ids_needed.add(item.id) + elif item.type == SearchItemType.OBSERVATION: + # Parent entity ID for entity_external_id + if item.entity_id: # pyright: ignore + entity_ids_needed.add(item.entity_id) # pyright: ignore + elif item.type == SearchItemType.RELATION: + # Source and target entity IDs for external_ids if item.from_id: # pyright: ignore entity_ids_needed.add(item.from_id) # pyright: ignore if item.to_id: entity_ids_needed.add(item.to_id) - # Batch fetch all entities at once - entity_lookup: dict[int, str] = {} + # Batch fetch all entities at once - get both title and external_id + entity_title_lookup: dict[int, str] = {} + entity_external_id_lookup: dict[int, str] = {} if entity_ids_needed: entities = await entity_repository.find_by_ids(list(entity_ids_needed)) - entity_lookup = {e.id: e.title for e in entities} + for e in entities: + entity_title_lookup[e.id] = e.title + entity_external_id_lookup[e.id] = e.external_id # Helper function to convert items to summaries def to_summary(item: SearchIndexRow | ContextResultRow): match item.type: case SearchItemType.ENTITY: return EntitySummary( + external_id=entity_external_id_lookup.get(item.id, ""), entity_id=item.id, title=item.title, # pyright: ignore permalink=item.permalink, @@ -55,10 +68,14 @@ def to_summary(item: SearchIndexRow | ContextResultRow): created_at=item.created_at, ) case SearchItemType.OBSERVATION: + entity_ext_id = None + if item.entity_id: # pyright: ignore + entity_ext_id = entity_external_id_lookup.get(item.entity_id) # pyright: ignore return ObservationSummary( observation_id=item.id, entity_id=item.entity_id, # pyright: ignore - title=item.title, # pyright: ignore + entity_external_id=entity_ext_id, + title=entity_title_lookup.get(item.entity_id) , # pyright: ignore file_path=item.file_path, category=item.category, # pyright: ignore content=item.content, # pyright: ignore @@ -66,8 +83,10 @@ def to_summary(item: SearchIndexRow | ContextResultRow): created_at=item.created_at, ) case SearchItemType.RELATION: - from_title = entity_lookup.get(item.from_id) if item.from_id else None # pyright: ignore - to_title = entity_lookup.get(item.to_id) if item.to_id else None + from_title = entity_title_lookup.get(item.from_id) if item.from_id else None # pyright: ignore + to_title = entity_title_lookup.get(item.to_id) if item.to_id else None + from_ext_id = entity_external_id_lookup.get(item.from_id) if item.from_id else None # pyright: ignore + to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None return RelationSummary( relation_id=item.id, entity_id=item.entity_id, # pyright: ignore @@ -77,8 +96,10 @@ def to_summary(item: SearchIndexRow | ContextResultRow): relation_type=item.relation_type, # pyright: ignore from_entity=from_title, from_entity_id=item.from_id, # pyright: ignore + from_entity_external_id=from_ext_id, to_entity=to_title, to_entity_id=item.to_id, + to_entity_external_id=to_ext_id, created_at=item.created_at, ) case _: # pragma: no cover diff --git a/src/basic_memory/schemas/memory.py b/src/basic_memory/schemas/memory.py index 83ebb8cb..bf8c92ba 100644 --- a/src/basic_memory/schemas/memory.py +++ b/src/basic_memory/schemas/memory.py @@ -124,6 +124,7 @@ class EntitySummary(BaseModel): """Simplified entity representation.""" type: Literal["entity"] = "entity" + external_id: str # UUID for v2 API routing entity_id: int # Database ID for v2 API consistency permalink: Optional[str] title: str @@ -150,8 +151,10 @@ class RelationSummary(BaseModel): relation_type: str from_entity: Optional[str] = None from_entity_id: Optional[int] = None # ID of source entity + from_entity_external_id: Optional[str] = None # UUID of source entity for v2 API routing to_entity: Optional[str] = None to_entity_id: Optional[int] = None # ID of target entity + to_entity_external_id: Optional[str] = None # UUID of target entity for v2 API routing created_at: Annotated[ datetime, Field(json_schema_extra={"type": "string", "format": "date-time"}) ] @@ -167,6 +170,7 @@ class ObservationSummary(BaseModel): type: Literal["observation"] = "observation" observation_id: int # Database ID for v2 API consistency entity_id: Optional[int] = None # ID of the entity this observation belongs to + entity_external_id: Optional[str] = None # UUID of parent entity for v2 API routing title: str file_path: str permalink: str diff --git a/tests/mcp/test_prompts.py b/tests/mcp/test_prompts.py index af9cf8bc..e4dd6440 100644 --- a/tests/mcp/test_prompts.py +++ b/tests/mcp/test_prompts.py @@ -116,6 +116,7 @@ def test_prompt_context_with_file_path_no_permalink(): # Create a mock context with a file that has no permalink (like a binary file) test_entity = EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440000", entity_id=1, type="entity", title="Test File", diff --git a/tests/mcp/test_tool_recent_activity.py b/tests/mcp/test_tool_recent_activity.py index 7c6a27e0..df9a6387 100644 --- a/tests/mcp/test_tool_recent_activity.py +++ b/tests/mcp/test_tool_recent_activity.py @@ -233,6 +233,7 @@ async def fake_call_get(client, url, params=None): { "primary_result": { "type": "entity", + "external_id": "550e8400-e29b-41d4-a716-446655440001", "entity_id": 1, "permalink": "notes/x", "title": "X", @@ -247,6 +248,7 @@ async def fake_call_get(client, url, params=None): { "primary_result": { "type": "entity", + "external_id": "550e8400-e29b-41d4-a716-446655440002", "entity_id": 2, "permalink": "notes/y", "title": "Y", @@ -341,6 +343,7 @@ def test_recent_activity_format_discovery_output_includes_other_active_projects_ results=[ ContextResult( primary_result=EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440001", entity_id=1, permalink="docs/complete-feature", title="Complete Feature Spec", @@ -358,6 +361,7 @@ def test_recent_activity_format_discovery_output_includes_other_active_projects_ results=[ ContextResult( primary_result=EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440002", entity_id=2, permalink="docs/other", title="Other Note", diff --git a/tests/schemas/test_memory_serialization.py b/tests/schemas/test_memory_serialization.py index 919ae874..78293130 100644 --- a/tests/schemas/test_memory_serialization.py +++ b/tests/schemas/test_memory_serialization.py @@ -22,6 +22,7 @@ def test_entity_summary_datetime_serialization(self): test_datetime = datetime(2023, 12, 8, 10, 30, 0) entity = EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440000", entity_id=1, permalink="test/entity", title="Test Entity", @@ -36,6 +37,7 @@ def test_entity_summary_datetime_serialization(self): assert data["created_at"] == "2023-12-08T10:30:00" assert data["type"] == "entity" assert data["title"] == "Test Entity" + assert data["external_id"] == "550e8400-e29b-41d4-a716-446655440000" def test_relation_summary_datetime_serialization(self): """Test RelationSummary serializes datetime as ISO format string.""" @@ -105,6 +107,7 @@ def test_context_result_with_datetime_serialization(self): test_datetime = datetime(2023, 12, 8, 9, 30, 15) entity = EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440000", entity_id=1, permalink="test/entity", title="Test Entity", @@ -139,6 +142,7 @@ def test_graph_context_full_serialization(self): test_datetime = datetime(2023, 12, 8, 14, 20, 10) entity = EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440000", entity_id=1, permalink="test/entity", title="Test Entity", @@ -168,6 +172,7 @@ def test_datetime_with_microseconds_serialization(self): test_datetime = datetime(2023, 12, 8, 10, 30, 0, 123456) entity = EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440000", entity_id=1, permalink="test/entity", title="Test Entity", @@ -186,6 +191,7 @@ def test_mcp_schema_validation_compatibility(self): test_datetime = datetime(2023, 12, 8, 10, 30, 0) entity = EntitySummary( + external_id="550e8400-e29b-41d4-a716-446655440000", entity_id=1, permalink="test/entity", title="Test Entity", @@ -223,6 +229,7 @@ def test_all_models_have_datetime_serializers_configured(self): if model_class == EntitySummary: instance = model_class( + external_id="550e8400-e29b-41d4-a716-446655440000", entity_id=1, permalink="test", title="Test", From 8b7f1d3e459563d3384b76e990255e8926210be0 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 31 Jan 2026 12:05:41 -0600 Subject: [PATCH 4/6] fixes for webapi recent activity Signed-off-by: phernandez --- .../api/v2/routers/knowledge_router.py | 22 +++++++++++++-- tests/api/v2/test_knowledge_router.py | 28 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index edb29ac7..ef9c0a3a 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -22,6 +22,7 @@ EntityRepositoryV2ExternalDep, ProjectExternalIdPathDep, TaskSchedulerDep, + FileServiceV2ExternalDep, ) from basic_memory.schemas import DeleteEntitiesResponse from basic_memory.schemas.base import Entity @@ -167,6 +168,7 @@ async def create_entity( entity_service: EntityServiceV2ExternalDep, search_service: SearchServiceV2ExternalDep, task_scheduler: TaskSchedulerDep, + file_service: FileServiceV2ExternalDep, fast: bool = Query( True, description="If true, write quickly and defer indexing to background tasks." ), @@ -178,7 +180,7 @@ async def create_entity( fast: If True, defer indexing to background tasks Returns: - Created entity with generated external_id (UUID) + Created entity with generated external_id (UUID) and file content """ logger.info( "API v2 request", endpoint="create_entity", entity_type=data.entity_type, title=data.title @@ -199,6 +201,10 @@ async def create_entity( if fast: result = result.model_copy(update={"observations": [], "relations": []}) + # Always read and return file content + content = await file_service.read_file_content(entity.file_path) + result = result.model_copy(update={"content": content}) + logger.info( f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201" ) @@ -218,6 +224,7 @@ async def update_entity_by_id( search_service: SearchServiceV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, task_scheduler: TaskSchedulerDep, + file_service: FileServiceV2ExternalDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), fast: bool = Query( True, description="If true, write quickly and defer indexing to background tasks." @@ -233,7 +240,7 @@ async def update_entity_by_id( fast: If True, defer indexing to background tasks Returns: - Updated entity + Updated entity with file content """ logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}") @@ -276,6 +283,10 @@ async def update_entity_by_id( if fast: result = result.model_copy(update={"observations": [], "relations": []}) + # Always read and return file content + content = await file_service.read_file_content(entity.file_path) + result = result.model_copy(update={"content": content}) + logger.info( f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}" ) @@ -291,6 +302,7 @@ async def edit_entity_by_id( search_service: SearchServiceV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, task_scheduler: TaskSchedulerDep, + file_service: FileServiceV2ExternalDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), fast: bool = Query( True, description="If true, write quickly and defer indexing to background tasks." @@ -304,7 +316,7 @@ async def edit_entity_by_id( fast: If True, defer indexing to background tasks Returns: - Updated entity + Updated entity with file content Raises: HTTPException: 404 if entity not found, 400 if edit fails @@ -353,6 +365,10 @@ async def edit_entity_by_id( if fast: result = result.model_copy(update={"observations": [], "relations": []}) + # Always read and return file content + content = await file_service.read_file_content(updated_entity.file_path) + result = result.model_copy(update={"content": content}) + logger.info( f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200" ) diff --git a/tests/api/v2/test_knowledge_router.py b/tests/api/v2/test_knowledge_router.py index e9119e90..c74bf951 100644 --- a/tests/api/v2/test_knowledge_router.py +++ b/tests/api/v2/test_knowledge_router.py @@ -179,6 +179,34 @@ async def test_create_entity(client: AsyncClient, file_service, v2_project_url): assert data["content"] in file_content +@pytest.mark.asyncio +async def test_create_entity_returns_content(client: AsyncClient, file_service, v2_project_url): + """Test creating an entity always returns file content with frontmatter.""" + data = { + "title": "TestContentReturn", + "directory": "test", + "entity_type": "note", + "content_type": "text/markdown", + "content": "Body content for return test", + } + + response = await client.post( + f"{v2_project_url}/knowledge/entities", + json=data, + params={"fast": False}, + ) + assert response.status_code == 200 + entity = EntityResponseV2.model_validate(response.json()) + + # Content should always be populated with frontmatter + assert entity.content is not None + assert "---" in entity.content # frontmatter markers + assert "title: TestContentReturn" in entity.content + assert "type: note" in entity.content + assert "permalink:" in entity.content + assert data["content"] in entity.content + + @pytest.mark.asyncio async def test_create_entity_with_observations_and_relations( client: AsyncClient, file_service, v2_project_url From 4d64aa09b7121d094f28010b00a868f9852f9657 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 31 Jan 2026 13:54:54 -0600 Subject: [PATCH 5/6] feat: enhanced search Signed-off-by: phernandez --- README.md | 2 + docs/ARCHITECTURE.md | 17 +- docs/ai-assistant-guide-extended.md | 50 +++++- ...9a0b1c2_add_structured_metadata_indexes.py | 153 +++++++++++++++++ src/basic_memory/cli/commands/tool.py | 72 +++++++- src/basic_memory/markdown/utils.py | 7 +- src/basic_memory/mcp/tools/__init__.py | 3 +- src/basic_memory/mcp/tools/search.py | 114 ++++++++++++- .../repository/metadata_filters.py | 134 +++++++++++++++ .../repository/postgres_search_repository.py | 134 +++++++++++---- .../repository/search_repository.py | 1 + .../repository/search_repository_base.py | 2 + .../repository/sqlite_search_repository.py | 154 +++++++++++++++--- src/basic_memory/schemas/search.py | 27 ++- src/basic_memory/services/entity_service.py | 12 +- src/basic_memory/services/search_service.py | 12 +- 16 files changed, 811 insertions(+), 83 deletions(-) create mode 100644 src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py create mode 100644 src/basic_memory/repository/metadata_filters.py diff --git a/README.md b/README.md index 3986f785..d5c27a32 100644 --- a/README.md +++ b/README.md @@ -396,6 +396,8 @@ list_directory(dir_name, depth) - Browse directory contents with filtering **Search & Discovery:** ``` search(query, page, page_size) - Search across your knowledge base +search_notes(query, page, page_size, search_type, types, entity_types, after_date, metadata_filters, tags, status, project) - Search with filters +search_by_metadata(filters, limit, offset, project) - Structured frontmatter search ``` **Project Management:** diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 313ab94a..125eeb58 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -214,15 +214,28 @@ Example tool using typed client: ```python @mcp.tool() -async def search_notes(query: str, project: str | None = None) -> SearchResponse: +async def search_notes( + query: str, + project: str | None = None, + metadata_filters: dict | None = None, + tags: list[str] | None = None, + status: str | None = None, +) -> SearchResponse: async with get_client() as client: active_project = await get_active_project(client, project) # Import client inside function to avoid circular imports from basic_memory.mcp.clients import SearchClient + from basic_memory.schemas.search import SearchQuery + search_query = SearchQuery( + text=query, + metadata_filters=metadata_filters, + tags=tags, + status=status, + ) search_client = SearchClient(client, active_project.external_id) - return await search_client.search(query) + return await search_client.search(search_query.model_dump()) ``` ## Sync Coordination diff --git a/docs/ai-assistant-guide-extended.md b/docs/ai-assistant-guide-extended.md index 9d2bd524..e0085565 100644 --- a/docs/ai-assistant-guide-extended.md +++ b/docs/ai-assistant-guide-extended.md @@ -1038,6 +1038,35 @@ recent_decisions = await search_notes( ) ``` +**Structured frontmatter filters**: + +```python +# Filter by tags and status +results = await search_notes( + query="authentication", + tags=["security"], + status="in-progress", + project="main" +) + +# Complex metadata filters (supports $in, $gt, $gte, $lt, $lte, $between) +results = await search_notes( + query="api design", + metadata_filters={ + "type": "spec", + "priority": {"$in": ["high", "critical"]}, + "tags": ["architecture"] + }, + project="main" +) + +# Metadata-only search +results = await search_by_metadata( + filters={"type": "spec", "status": "in-progress"}, + project="main" +) +``` + ### Search Types **Text search (default)**: @@ -2861,7 +2890,7 @@ contents = await list_directory( ### Search & Discovery -**search_notes(query, page, page_size, search_type, types, entity_types, after_date, project)** +**search_notes(query, page, page_size, search_type, types, entity_types, after_date, metadata_filters, tags, status, project)** - Search across knowledge base - Parameters: - `query` (required): Search query @@ -2871,6 +2900,9 @@ contents = await list_directory( - `types` (optional): Entity type filter - `entity_types` (optional): Observation category filter - `after_date` (optional): Date filter (ISO format) + - `metadata_filters` (optional): Structured frontmatter filters (dict) + - `tags` (optional): Frontmatter tags filter (list) + - `status` (optional): Frontmatter status filter (string) - `project` (required unless default_project_mode): Target project - Returns: Matching entities with scores - Example: @@ -2883,6 +2915,22 @@ results = await search_notes( ) ``` +**search_by_metadata(filters, limit, offset, project)** +- Metadata-only search using structured frontmatter +- Parameters: + - `filters` (required): Dict of field -> value (supports $in, $gt/$gte/$lt/$lte, $between) + - `limit` (optional): Max results (default: 20) + - `offset` (optional): Pagination offset (default: 0) + - `project` (required unless default_project_mode): Target project +- Returns: Matching entities +- Example: +```python +results = await search_by_metadata( + filters={"type": "spec", "status": "in-progress"}, + project="main" +) +``` + ### Project Management **list_memory_projects()** diff --git a/src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py b/src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py new file mode 100644 index 00000000..d4a745c6 --- /dev/null +++ b/src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py @@ -0,0 +1,153 @@ +"""Add structured metadata indexes for entity frontmatter + +Revision ID: d7e8f9a0b1c2 +Revises: g9a0b3c4d5e6 +Create Date: 2026-01-31 12:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import text + + +def column_exists(connection, table: str, column: str) -> bool: + """Check if a column exists in a table (idempotent migration support).""" + if connection.dialect.name == "postgresql": + result = connection.execute( + text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :table AND column_name = :column" + ), + {"table": table, "column": column}, + ) + return result.fetchone() is not None + # SQLite + result = connection.execute(text(f"PRAGMA table_info({table})")) + columns = [row[1] for row in result] + return column in columns + + +def index_exists(connection, index_name: str) -> bool: + """Check if an index exists (idempotent migration support).""" + if connection.dialect.name == "postgresql": + result = connection.execute( + text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"), + {"index_name": index_name}, + ) + return result.fetchone() is not None + # SQLite + result = connection.execute( + text("SELECT 1 FROM sqlite_master WHERE type='index' AND name = :index_name"), + {"index_name": index_name}, + ) + return result.fetchone() is not None + + +# revision identifiers, used by Alembic. +revision: str = "d7e8f9a0b1c2" +down_revision: Union[str, None] = "g9a0b3c4d5e6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add JSONB/GiN indexes for Postgres and generated columns for SQLite.""" + connection = op.get_bind() + dialect = connection.dialect.name + + if dialect == "postgresql": + # Ensure JSONB for efficient indexing + result = connection.execute( + text( + "SELECT data_type FROM information_schema.columns " + "WHERE table_name = 'entity' AND column_name = 'entity_metadata'" + ) + ).fetchone() + if result and result[0] != "jsonb": + op.execute( + "ALTER TABLE entity ALTER COLUMN entity_metadata " + "TYPE jsonb USING entity_metadata::jsonb" + ) + + # General JSONB GIN index + op.execute( + "CREATE INDEX IF NOT EXISTS idx_entity_metadata_gin " + "ON entity USING GIN (entity_metadata jsonb_path_ops)" + ) + + # Common field indexes + op.execute( + "CREATE INDEX IF NOT EXISTS idx_entity_tags_json " + "ON entity USING GIN ((entity_metadata -> 'tags'))" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS idx_entity_frontmatter_type " + "ON entity ((entity_metadata ->> 'type'))" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS idx_entity_frontmatter_status " + "ON entity ((entity_metadata ->> 'status'))" + ) + return + + # SQLite: add generated columns for common frontmatter fields + if not column_exists(connection, "entity", "tags_json"): + op.add_column( + "entity", + sa.Column( + "tags_json", + sa.Text(), + sa.Computed("json_extract(entity_metadata, '$.tags')", persisted=True), + ), + ) + if not column_exists(connection, "entity", "frontmatter_status"): + op.add_column( + "entity", + sa.Column( + "frontmatter_status", + sa.Text(), + sa.Computed("json_extract(entity_metadata, '$.status')", persisted=True), + ), + ) + if not column_exists(connection, "entity", "frontmatter_type"): + op.add_column( + "entity", + sa.Column( + "frontmatter_type", + sa.Text(), + sa.Computed("json_extract(entity_metadata, '$.type')", persisted=True), + ), + ) + + # Index generated columns + if not index_exists(connection, "idx_entity_tags_json"): + op.create_index("idx_entity_tags_json", "entity", ["tags_json"]) + if not index_exists(connection, "idx_entity_frontmatter_status"): + op.create_index("idx_entity_frontmatter_status", "entity", ["frontmatter_status"]) + if not index_exists(connection, "idx_entity_frontmatter_type"): + op.create_index("idx_entity_frontmatter_type", "entity", ["frontmatter_type"]) + + +def downgrade() -> None: + """Best-effort downgrade (drop indexes, revert JSONB on Postgres).""" + connection = op.get_bind() + dialect = connection.dialect.name + + if dialect == "postgresql": + op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_status") + op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_type") + op.execute("DROP INDEX IF EXISTS idx_entity_tags_json") + op.execute("DROP INDEX IF EXISTS idx_entity_metadata_gin") + op.execute( + "ALTER TABLE entity ALTER COLUMN entity_metadata " + "TYPE json USING entity_metadata::json" + ) + return + + # SQLite: drop indexes (dropping generated columns requires table rebuild) + op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_status") + op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_type") + op.execute("DROP INDEX IF EXISTS idx_entity_tags_json") diff --git a/src/basic_memory/cli/commands/tool.py b/src/basic_memory/cli/commands/tool.py index b1c387d6..88a9bd33 100644 --- a/src/basic_memory/cli/commands/tool.py +++ b/src/basic_memory/cli/commands/tool.py @@ -1,5 +1,6 @@ """CLI tool commands for Basic Memory.""" +import json import sys from typing import Annotated, List, Optional @@ -288,7 +289,10 @@ def recent_activity( @tool_app.command("search-notes") def search_notes( - query: str, + query: Annotated[ + Optional[str], + typer.Argument(help="Search query string (optional when using metadata filters)"), + ] = "", permalink: Annotated[bool, typer.Option("--permalink", help="Search permalink values")] = False, title: Annotated[bool, typer.Option("--title", help="Search title values")] = False, project: Annotated[ @@ -301,6 +305,26 @@ def search_notes( Optional[str], typer.Option("--after_date", help="Search results after date, eg. '2d', '1 week'"), ] = None, + tags: Annotated[ + Optional[List[str]], + typer.Option("--tag", help="Filter by frontmatter tag (repeatable)"), + ] = None, + status: Annotated[ + Optional[str], + typer.Option("--status", help="Filter by frontmatter status"), + ] = None, + note_types: Annotated[ + Optional[List[str]], + typer.Option("--type", help="Filter by frontmatter type (repeatable)"), + ] = None, + meta: Annotated[ + Optional[List[str]], + typer.Option("--meta", help="Filter by frontmatter key=value (repeatable)"), + ] = None, + filter_json: Annotated[ + Optional[str], + typer.Option("--filter", help="JSON metadata filter (advanced)"), + ] = None, page: int = 1, page_size: int = 10, local: bool = typer.Option( @@ -335,21 +359,57 @@ def search_notes( ) raise typer.Exit(1) + # Build metadata filters from --filter and --meta + metadata_filters = {} + if filter_json: + try: + metadata_filters = json.loads(filter_json) + if not isinstance(metadata_filters, dict): + raise ValueError("Metadata filter JSON must be an object") + except json.JSONDecodeError as e: + typer.echo(f"Invalid JSON for --filter: {e}", err=True) + raise typer.Exit(1) + + if meta: + for item in meta: + if "=" not in item: + typer.echo( + f"Invalid --meta entry '{item}'. Use key=value format.", + err=True, + ) + raise typer.Exit(1) + key, value = item.split("=", 1) + key = key.strip() + if not key: + typer.echo(f"Invalid --meta entry '{item}'.", err=True) + raise typer.Exit(1) + metadata_filters[key] = value + + if not metadata_filters: + metadata_filters = None + # set search type - search_type = ("permalink" if permalink else None,) - search_type = ("permalink_match" if permalink and "*" in query else None,) - search_type = ("title" if title else None,) - search_type = "text" if search_type is None else search_type + search_type = "text" + if permalink: + search_type = "permalink" + if query and "*" in query: + search_type = "permalink" + if title: + search_type = "title" with force_routing(local=local, cloud=cloud): results = run_with_cleanup( mcp_search.fn( - query, + query or "", project_name, search_type=search_type, page=page, after_date=after_date, page_size=page_size, + types=note_types, + metadata_filters=metadata_filters, + tags=tags, + status=status, ) ) # Use json module for more controlled serialization diff --git a/src/basic_memory/markdown/utils.py b/src/basic_memory/markdown/utils.py index 3f05a314..29381b2d 100644 --- a/src/basic_memory/markdown/utils.py +++ b/src/basic_memory/markdown/utils.py @@ -9,6 +9,7 @@ from basic_memory.file_utils import has_frontmatter, remove_frontmatter, parse_frontmatter from basic_memory.markdown import EntityMarkdown +from basic_memory.markdown.entity_parser import normalize_frontmatter_metadata from basic_memory.models import Entity from basic_memory.models import Observation as ObservationModel @@ -58,9 +59,9 @@ def entity_model_from_markdown( model.created_at = markdown.created model.updated_at = markdown.modified - # Handle metadata - ensure all values are strings and filter None - metadata = markdown.frontmatter.metadata or {} - model.entity_metadata = {k: str(v) for k, v in metadata.items() if v is not None} + # Handle metadata - normalize values and filter None (preserve structured data) + metadata = normalize_frontmatter_metadata(markdown.frontmatter.metadata or {}) + model.entity_metadata = {k: v for k, v in metadata.items() if v is not None} # Get project_id from entity if not provided obs_project_id = project_id or (model.project_id if hasattr(model, "project_id") else None) diff --git a/src/basic_memory/mcp/tools/__init__.py b/src/basic_memory/mcp/tools/__init__.py index f7844529..ea2a22da 100644 --- a/src/basic_memory/mcp/tools/__init__.py +++ b/src/basic_memory/mcp/tools/__init__.py @@ -13,7 +13,7 @@ from basic_memory.mcp.tools.read_note import read_note from basic_memory.mcp.tools.view_note import view_note from basic_memory.mcp.tools.write_note import write_note -from basic_memory.mcp.tools.search import search_notes +from basic_memory.mcp.tools.search import search_notes, search_by_metadata from basic_memory.mcp.tools.canvas import canvas from basic_memory.mcp.tools.list_directory import list_directory from basic_memory.mcp.tools.edit_note import edit_note @@ -42,6 +42,7 @@ "read_note", "recent_activity", "search", + "search_by_metadata", "search_notes", "view_note", "write_note", diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index 401771a8..cf1d6591 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -1,7 +1,7 @@ """Search tools for Basic Memory MCP server.""" from textwrap import dedent -from typing import List, Optional +from typing import List, Optional, Dict, Any from loguru import logger from fastmcp import Context @@ -207,6 +207,9 @@ async def search_notes( types: List[str] | None = None, entity_types: List[str] | None = None, after_date: Optional[str] = None, + metadata_filters: Optional[Dict[str, Any]] = None, + tags: Optional[List[str]] = None, + status: Optional[str] = None, context: Context | None = None, ) -> SearchResponse | str: """Search across all content in the knowledge base with comprehensive syntax support. @@ -248,6 +251,27 @@ async def search_notes( - `search_notes("research", "query", entity_types=["observation"])` - Filter by entity type - `search_notes("team-docs", "query", after_date="2024-01-01")` - Recent content only - `search_notes("my-project", "query", after_date="1 week")` - Relative date filtering + - `search_notes("my-project", "query", tags=["security"])` - Filter by frontmatter tags + - `search_notes("my-project", "query", status="in-progress")` - Filter by frontmatter status + - `search_notes("my-project", "query", metadata_filters={"priority": {"$in": ["high"]}})` + + ### Structured Metadata Filters + Filters are exact matches on frontmatter metadata. Supported forms: + - Equality: `{"status": "in-progress"}` + - Array contains (all): `{"tags": ["security", "oauth"]}` + - Operators: + - `$in`: `{"priority": {"$in": ["high", "critical"]}}` + - `$gt`, `$gte`, `$lt`, `$lte`: `{"schema.confidence": {"$gt": 0.7}}` + - `$between`: `{"schema.confidence": {"$between": [0.3, 0.6]}}` + - Nested keys use dot notation (e.g., `"schema.confidence"`). + + ### Filter-only Searches + You can pass an empty query string when only using structured filters: + - `search_notes("my-project", "", metadata_filters={"type": "spec"})` + + ### Convenience Filters + `tags` and `status` are shorthand for metadata_filters. If the same key exists in + metadata_filters, that value wins. ### Advanced Pattern Examples - `search_notes("work-project", "project AND (meeting OR discussion)")` - Complex boolean logic @@ -265,6 +289,9 @@ async def search_notes( types: Optional list of note types to search (e.g., ["note", "person"]) entity_types: Optional list of entity types to filter by (e.g., ["entity", "observation"]) after_date: Optional date filter for recent content (e.g., "1 week", "2d", "2024-01-01") + metadata_filters: Optional structured frontmatter filters (e.g., {"status": "in-progress"}) + tags: Optional tag filter (frontmatter tags); shorthand for metadata_filters["tags"] + status: Optional status filter (frontmatter status); shorthand for metadata_filters["status"] context: Optional FastMCP context for performance caching. Returns: @@ -355,6 +382,12 @@ async def search_notes( search_query.types = types if after_date: search_query.after_date = after_date + if metadata_filters: + search_query.metadata_filters = metadata_filters + if tags: + search_query.tags = tags + if status: + search_query.status = status async with get_client() as client: active_project = await get_active_project(client, project, context) @@ -387,3 +420,82 @@ async def search_notes( logger.error(f"Search failed for query '{query}': {e}, project: {active_project.name}") # Return formatted error message as string for better user experience return _format_search_error_response(active_project.name, str(e), query, search_type) + + +@mcp.tool( + description="Search entities by structured frontmatter metadata.", +) +async def search_by_metadata( + filters: Dict[str, Any], + project: Optional[str] = None, + limit: int = 20, + offset: int = 0, + context: Context | None = None, +) -> SearchResponse | str: + """Search entities by structured frontmatter metadata. + + Args: + filters: Dictionary of metadata filters (e.g., {"status": "in-progress"}) + project: Project name to search in. Optional - server will resolve using hierarchy. + limit: Maximum number of results to return + offset: Number of results to skip (for pagination) + context: Optional FastMCP context for performance caching. + + Returns: + SearchResponse with results, or helpful error guidance if search fails + """ + if limit <= 0: + return "# Error\n\n`limit` must be greater than 0." + + # Build a structured-only search query + search_query = SearchQuery() + search_query.metadata_filters = filters + search_query.entity_types = [SearchItemType.ENTITY] + + # Convert offset/limit to page/page_size (API uses paging) + page_size = limit + page = (offset // limit) + 1 + offset_within_page = offset % limit + + async with get_client() as client: + active_project = await get_active_project(client, project, context) + logger.info( + f"Structured search in project {active_project.name} filters={filters} limit={limit} offset={offset}" + ) + + try: + from basic_memory.mcp.clients import SearchClient + + search_client = SearchClient(client, active_project.external_id) + result = await search_client.search( + search_query.model_dump(), + page=page, + page_size=page_size, + ) + + # Apply offset within page, fetch next page if needed + if offset_within_page: + remaining = result.results[offset_within_page:] + if len(remaining) < limit: + next_page = page + 1 + extra = await search_client.search( + search_query.model_dump(), + page=next_page, + page_size=page_size, + ) + remaining.extend(extra.results[: max(0, limit - len(remaining))]) + result = SearchResponse( + results=remaining[:limit], + current_page=page, + page_size=page_size, + ) + + return result + + except Exception as e: + logger.error( + f"Metadata search failed for filters '{filters}': {e}, project: {active_project.name}" + ) + return _format_search_error_response( + active_project.name, str(e), str(filters), "metadata" + ) diff --git a/src/basic_memory/repository/metadata_filters.py b/src/basic_memory/repository/metadata_filters.py new file mode 100644 index 00000000..e53fab6a --- /dev/null +++ b/src/basic_memory/repository/metadata_filters.py @@ -0,0 +1,134 @@ +"""Helpers for parsing structured metadata filters for search.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date, datetime +import re +from typing import Any, Iterable, List + + +_KEY_RE = re.compile(r"^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$") +_NUMERIC_RE = re.compile(r"^-?\d+(\.\d+)?$") + + +@dataclass(frozen=True) +class ParsedMetadataFilter: + """Normalized metadata filter for SQL generation.""" + + path_parts: List[str] + op: str + value: Any + comparison: str | None = None # "numeric" or "text" for comparisons + + +def _is_numeric_value(value: Any) -> bool: + if isinstance(value, bool): + return False + if isinstance(value, (int, float)): + return True + if isinstance(value, str): + return bool(_NUMERIC_RE.match(value.strip())) + return False + + +def _is_numeric_collection(values: Iterable[Any]) -> bool: + return all(_is_numeric_value(v) for v in values) + + +def _normalize_scalar(value: Any) -> Any: + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, bool): + return str(value) + if isinstance(value, (int, float)): + return str(value) + return value + + +def parse_metadata_filters(filters: dict[str, Any]) -> List[ParsedMetadataFilter]: + """Parse metadata filters into normalized clauses. + + Supported forms: + - {"status": "in-progress"} + - {"tags": ["security", "oauth"]} # array contains all + - {"priority": {"$in": ["high", "critical"]}} + - {"schema.confidence": {"$gt": 0.7}} + - {"schema.confidence": {"$between": [0.3, 0.6]}} + """ + parsed: List[ParsedMetadataFilter] = [] + + for raw_key, raw_value in (filters or {}).items(): + if not isinstance(raw_key, str) or not raw_key.strip(): + raise ValueError("metadata filter keys must be non-empty strings") + key = raw_key.strip() + if not _KEY_RE.match(key): + raise ValueError(f"Unsupported metadata filter key: {raw_key}") + + path_parts = key.split(".") + + # Operator form + if isinstance(raw_value, dict): + if len(raw_value) != 1: + raise ValueError(f"Invalid metadata filter for '{raw_key}': {raw_value}") + op, value = next(iter(raw_value.items())) + + if op == "$in": + if not isinstance(value, list) or not value: + raise ValueError(f"$in requires a non-empty list for '{raw_key}'") + parsed.append( + ParsedMetadataFilter(path_parts, "in", [_normalize_scalar(v) for v in value]) + ) + continue + + if op in {"$gt", "$gte", "$lt", "$lte"}: + normalized = _normalize_scalar(value) + comparison = "numeric" if _is_numeric_value(normalized) else "text" + parsed.append( + ParsedMetadataFilter(path_parts, op.lstrip("$"), normalized, comparison) + ) + continue + + if op == "$between": + if ( + not isinstance(value, list) + or len(value) != 2 + ): + raise ValueError(f"$between requires [min, max] for '{raw_key}'") + normalized = [_normalize_scalar(v) for v in value] + comparison = "numeric" if _is_numeric_collection(normalized) else "text" + parsed.append( + ParsedMetadataFilter(path_parts, "between", normalized, comparison) + ) + continue + + raise ValueError(f"Unsupported operator '{op}' in metadata filter for '{raw_key}'") + + # Array contains (all) + if isinstance(raw_value, list): + if not raw_value: + raise ValueError(f"Empty list not allowed for metadata filter '{raw_key}'") + parsed.append( + ParsedMetadataFilter(path_parts, "contains", [_normalize_scalar(v) for v in raw_value]) + ) + continue + + # Simple equality + parsed.append(ParsedMetadataFilter(path_parts, "eq", _normalize_scalar(raw_value))) + + return parsed + + +def build_sqlite_json_path(parts: List[str]) -> str: + """Build a SQLite JSON path for json_extract/json_each.""" + path = "$" + for part in parts: + path += f'.\"{part}\"' + return path + + +def build_postgres_json_path(parts: List[str]) -> str: + """Build a Postgres JSON path for #>>/#> operators.""" + return "{" + ",".join(parts) + "}" diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 26ef3486..4dcd6bd0 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -12,6 +12,7 @@ from basic_memory import db from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_repository_base import SearchRepositoryBase +from basic_memory.repository.metadata_filters import parse_metadata_filters, build_postgres_json_path from basic_memory.schemas.search import SearchItemType @@ -215,6 +216,7 @@ async def search( types: Optional[List[str]] = None, after_date: Optional[datetime] = None, search_item_types: Optional[List[SearchItemType]] = None, + metadata_filters: Optional[dict] = None, limit: int = 10, offset: int = 0, ) -> List[SearchIndexRow]: @@ -222,6 +224,7 @@ async def search( conditions = [] params = {} order_by_clause = "" + from_clause = "search_index" # Handle text search for title and content using tsvector if search_text: @@ -233,18 +236,22 @@ async def search( processed_text = self._prepare_search_term(search_text.strip()) params["text"] = processed_text # Use @@ operator for tsvector matching - conditions.append("textsearchable_index_col @@ to_tsquery('english', :text)") + conditions.append( + "search_index.textsearchable_index_col @@ to_tsquery('english', :text)" + ) # Handle title search if title: title_text = self._prepare_search_term(title.strip(), is_prefix=False) params["title_text"] = title_text - conditions.append("to_tsvector('english', title) @@ to_tsquery('english', :title_text)") + conditions.append( + "to_tsvector('english', search_index.title) @@ to_tsquery('english', :title_text)" + ) # Handle permalink exact search if permalink: params["permalink"] = permalink - conditions.append("permalink = :permalink") + conditions.append("search_index.permalink = :permalink") # Handle permalink pattern match if permalink_match: @@ -255,14 +262,14 @@ async def search( # Convert * to % for SQL LIKE permalink_pattern = permalink_text.replace("*", "%") params["permalink"] = permalink_pattern - conditions.append("permalink LIKE :permalink") + conditions.append("search_index.permalink LIKE :permalink") else: - conditions.append("permalink = :permalink") + conditions.append("search_index.permalink = :permalink") # Handle search item type filter if search_item_types: type_list = ", ".join(f"'{t.value}'" for t in search_item_types) - conditions.append(f"type IN ({type_list})") + conditions.append(f"search_index.type IN ({type_list})") # Handle entity type filter using JSONB containment if types: @@ -270,19 +277,90 @@ async def search( type_conditions = [] for entity_type in types: # Create JSONB containment condition for each type - type_conditions.append(f'metadata @> \'{{"entity_type": "{entity_type}"}}\'') + type_conditions.append( + f'search_index.metadata @> \'{{"entity_type": "{entity_type}"}}\'' + ) conditions.append(f"({' OR '.join(type_conditions)})") # Handle date filter if after_date: params["after_date"] = after_date - conditions.append("created_at > :after_date") + conditions.append("search_index.created_at > :after_date") # order by most recent first - order_by_clause = ", updated_at DESC" + order_by_clause = ", search_index.updated_at DESC" + + # Handle structured metadata filters (frontmatter) + if metadata_filters: + parsed_filters = parse_metadata_filters(metadata_filters) + from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id" + metadata_expr = "entity.entity_metadata::jsonb" + + for idx, filt in enumerate(parsed_filters): + path = build_postgres_json_path(filt.path_parts) + text_expr = f"({metadata_expr} #>> '{path}')" + json_expr = f"({metadata_expr} #> '{path}')" + + if filt.op == "eq": + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + conditions.append(f"{text_expr} = :{value_param}") + continue + + if filt.op == "in": + placeholders = [] + for j, val in enumerate(filt.value): + value_param = f"meta_val_{idx}_{j}" + params[value_param] = val + placeholders.append(f":{value_param}") + conditions.append(f"{text_expr} IN ({', '.join(placeholders)})") + continue + + if filt.op == "contains": + import json as _json + + tag_conditions = [] + # Require all values to be present + for j, val in enumerate(filt.value): + tag_param = f"{value_param}_{j}" + params[tag_param] = _json.dumps([val]) + like_param = f"{value_param}_{j}_like" + params[like_param] = f'%"{val}"%' + like_param_single = f"{value_param}_{j}_like_single" + params[like_param_single] = f"%'{val}'%" + tag_conditions.append( + f"({json_expr} @> :{tag_param}::jsonb " + f"OR {text_expr} LIKE :{like_param} " + f"OR {text_expr} LIKE :{like_param_single})" + ) + conditions.append(" AND ".join(tag_conditions)) + continue + + if filt.op in {"gt", "gte", "lt", "lte", "between"}: + if filt.comparison == "numeric": + numeric_expr = ( + f"CASE WHEN ({text_expr}) ~ '^-?\\\\d+(\\\\.\\\\d+)?$' " + f"THEN ({text_expr})::double precision END" + ) + compare_expr = numeric_expr + else: + compare_expr = text_expr + + if filt.op == "between": + min_param = f"meta_val_{idx}_min" + max_param = f"meta_val_{idx}_max" + params[min_param] = filt.value[0] + params[max_param] = filt.value[1] + conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}") + else: + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op] + conditions.append(f"{compare_expr} {operator} :{value_param}") + continue # Always filter by project_id params["project_id"] = self.project_id - conditions.append("project_id = :project_id") + conditions.append("search_index.project_id = :project_id") # set limit and offset params["limit"] = limit @@ -294,31 +372,31 @@ async def search( # Build SQL with ts_rank() for scoring # Note: If no text search, score will be NULL, so we use COALESCE to default to 0 if search_text and search_text.strip() and search_text.strip() != "*": - score_expr = "ts_rank(textsearchable_index_col, to_tsquery('english', :text))" + score_expr = "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text))" else: score_expr = "0" sql = f""" SELECT - project_id, - id, - title, - permalink, - file_path, - type, - metadata, - from_id, - to_id, - relation_type, - entity_id, - content_snippet, - category, - created_at, - updated_at, + search_index.project_id, + search_index.id, + search_index.title, + search_index.permalink, + search_index.file_path, + search_index.type, + search_index.metadata, + search_index.from_id, + search_index.to_id, + search_index.relation_type, + search_index.entity_id, + search_index.content_snippet, + search_index.category, + search_index.created_at, + search_index.updated_at, {score_expr} as score - FROM search_index + FROM {from_clause} WHERE {where_clause} - ORDER BY score DESC, id ASC {order_by_clause} + ORDER BY score DESC, search_index.id ASC {order_by_clause} LIMIT :limit OFFSET :offset """ diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index 4be70aae..aca17b00 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -40,6 +40,7 @@ async def search( types: Optional[List[str]] = None, after_date: Optional[datetime] = None, search_item_types: Optional[List[SearchItemType]] = None, + metadata_filters: Optional[dict] = None, limit: int = 10, offset: int = 0, ) -> List[SearchIndexRow]: diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 3c7adc24..31505cf5 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -78,6 +78,7 @@ async def search( types: Optional[List[str]] = None, after_date: Optional[datetime] = None, search_item_types: Optional[List[SearchItemType]] = None, + metadata_filters: Optional[Dict[str, Any]] = None, limit: int = 10, offset: int = 0, ) -> List[SearchIndexRow]: @@ -91,6 +92,7 @@ async def search( types: Filter by entity types (from metadata.entity_type) after_date: Filter by created_at > after_date search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION) + metadata_filters: Structured frontmatter metadata filters limit: Maximum results to return offset: Number of results to skip diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 66a39ccf..51b1c167 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -13,6 +13,7 @@ from basic_memory.models.search import CREATE_SEARCH_INDEX from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_repository_base import SearchRepositoryBase +from basic_memory.repository.metadata_filters import parse_metadata_filters, build_sqlite_json_path from basic_memory.schemas.search import SearchItemType @@ -26,6 +27,17 @@ class SQLiteSearchRepository(SearchRepositoryBase): - Prefix wildcard matching with * """ + def __init__(self, session_maker, project_id: int): + super().__init__(session_maker, project_id) + self._entity_columns: set[str] | None = None + + async def _get_entity_columns(self) -> set[str]: + if self._entity_columns is None: + async with db.scoped_session(self.session_maker) as session: + result = await session.execute(text("PRAGMA table_info(entity)")) + self._entity_columns = {row[1] for row in result.fetchall()} + return self._entity_columns + async def init_search_index(self): """Create FTS5 virtual table for search if it doesn't exist. @@ -287,6 +299,7 @@ async def search( types: Optional[List[str]] = None, after_date: Optional[datetime] = None, search_item_types: Optional[List[SearchItemType]] = None, + metadata_filters: Optional[dict] = None, limit: int = 10, offset: int = 0, ) -> List[SearchIndexRow]: @@ -294,6 +307,7 @@ async def search( conditions = [] params = {} order_by_clause = "" + from_clause = "search_index" # Handle text search for title and content if search_text: @@ -305,18 +319,20 @@ async def search( # Use _prepare_search_term to handle both Boolean and non-Boolean queries processed_text = self._prepare_search_term(search_text.strip()) params["text"] = processed_text - conditions.append("(title MATCH :text OR content_stems MATCH :text)") + conditions.append( + "(search_index.title MATCH :text OR search_index.content_stems MATCH :text)" + ) # Handle title match search if title: title_text = self._prepare_search_term(title.strip(), is_prefix=False) params["title_text"] = title_text - conditions.append("title MATCH :title_text") + conditions.append("search_index.title MATCH :title_text") # Handle permalink exact search if permalink: params["permalink"] = permalink - conditions.append("permalink = :permalink") + conditions.append("search_index.permalink = :permalink") # Handle permalink match search, supports * if permalink_match: @@ -325,38 +341,122 @@ async def search( permalink_text = permalink_match.lower().strip() params["permalink"] = permalink_text if "*" in permalink_match: - conditions.append("permalink GLOB :permalink") + conditions.append("search_index.permalink GLOB :permalink") else: # For exact matches without *, we can use FTS5 MATCH # but only prepare the term if it doesn't look like a path if "/" in permalink_text: - conditions.append("permalink = :permalink") + conditions.append("search_index.permalink = :permalink") else: permalink_text = self._prepare_search_term(permalink_text, is_prefix=False) params["permalink"] = permalink_text - conditions.append("permalink MATCH :permalink") + conditions.append("search_index.permalink MATCH :permalink") # Handle entity type filter if search_item_types: type_list = ", ".join(f"'{t.value}'" for t in search_item_types) - conditions.append(f"type IN ({type_list})") + conditions.append(f"search_index.type IN ({type_list})") # Handle type filter if types: type_list = ", ".join(f"'{t}'" for t in types) - conditions.append(f"json_extract(metadata, '$.entity_type') IN ({type_list})") + conditions.append( + f"json_extract(search_index.metadata, '$.entity_type') IN ({type_list})" + ) # Handle date filter using datetime() for proper comparison if after_date: params["after_date"] = after_date - conditions.append("datetime(created_at) > datetime(:after_date)") + conditions.append("datetime(search_index.created_at) > datetime(:after_date)") # order by most recent first - order_by_clause = ", updated_at DESC" + order_by_clause = ", search_index.updated_at DESC" + + # Handle structured metadata filters (frontmatter) + if metadata_filters: + parsed_filters = parse_metadata_filters(metadata_filters) + from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id" + entity_columns = await self._get_entity_columns() + + for idx, filt in enumerate(parsed_filters): + path_param = f"meta_path_{idx}" + extract_expr = None + use_tags_column = False + + if filt.path_parts == ["status"] and "frontmatter_status" in entity_columns: + extract_expr = "entity.frontmatter_status" + elif filt.path_parts == ["type"] and "frontmatter_type" in entity_columns: + extract_expr = "entity.frontmatter_type" + elif filt.path_parts == ["tags"] and "tags_json" in entity_columns: + extract_expr = "entity.tags_json" + use_tags_column = True + + if extract_expr is None: + params[path_param] = build_sqlite_json_path(filt.path_parts) + extract_expr = f"json_extract(entity.entity_metadata, :{path_param})" + + if filt.op == "eq": + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + conditions.append(f"{extract_expr} = :{value_param}") + continue + + if filt.op == "in": + placeholders = [] + for j, val in enumerate(filt.value): + value_param = f"meta_val_{idx}_{j}" + params[value_param] = val + placeholders.append(f":{value_param}") + conditions.append(f"{extract_expr} IN ({', '.join(placeholders)})") + continue + + if filt.op == "contains": + tag_conditions = [] + for j, val in enumerate(filt.value): + value_param = f"meta_val_{idx}_{j}" + params[value_param] = val + like_param = f"{value_param}_like" + params[like_param] = f'%"{val}"%' + like_param_single = f"{value_param}_like_single" + params[like_param_single] = f"%'{val}'%" + json_each_expr = ( + "json_each(entity.tags_json)" + if use_tags_column + else f"json_each(entity.entity_metadata, :{path_param})" + ) + tag_conditions.append( + "(" + f"EXISTS (SELECT 1 FROM {json_each_expr} WHERE value = :{value_param}) " + f"OR {extract_expr} LIKE :{like_param} " + f"OR {extract_expr} LIKE :{like_param_single}" + ")" + ) + conditions.append(" AND ".join(tag_conditions)) + continue + + if filt.op in {"gt", "gte", "lt", "lte", "between"}: + compare_expr = ( + f"CAST({extract_expr} AS REAL)" + if filt.comparison == "numeric" + else extract_expr + ) + + if filt.op == "between": + min_param = f"meta_val_{idx}_min" + max_param = f"meta_val_{idx}_max" + params[min_param] = filt.value[0] + params[max_param] = filt.value[1] + conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}") + else: + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op] + conditions.append(f"{compare_expr} {operator} :{value_param}") + continue # Always filter by project_id params["project_id"] = self.project_id - conditions.append("project_id = :project_id") + conditions.append("search_index.project_id = :project_id") # set limit on search query params["limit"] = limit @@ -367,23 +467,23 @@ async def search( sql = f""" SELECT - project_id, - id, - title, - permalink, - file_path, - type, - metadata, - from_id, - to_id, - relation_type, - entity_id, - content_snippet, - category, - created_at, - updated_at, + search_index.project_id, + search_index.id, + search_index.title, + search_index.permalink, + search_index.file_path, + search_index.type, + search_index.metadata, + search_index.from_id, + search_index.to_id, + search_index.relation_type, + search_index.entity_id, + search_index.content_snippet, + search_index.category, + search_index.created_at, + search_index.updated_at, bm25(search_index) as score - FROM search_index + FROM {from_clause} WHERE {where_clause} ORDER BY score ASC {order_by_clause} LIMIT :limit diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index 929430bf..c47f8021 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -6,7 +6,7 @@ 3. Full-text search across content """ -from typing import Optional, List, Union +from typing import Optional, List, Union, Any from datetime import datetime from enum import Enum from pydantic import BaseModel, field_validator @@ -32,9 +32,12 @@ class SearchQuery(BaseModel): - title: Title only search Optionally filter results by: - - types: Limit to specific item types - - entity_types: Limit to specific entity types + - types: Limit to specific entity types (frontmatter "type") + - entity_types: Limit to search item types (entity/observation/relation) - after_date: Only items after date + - metadata_filters: Structured frontmatter filters (field -> value) + - tags: Convenience frontmatter tag filter + - status: Convenience frontmatter status filter Boolean search examples: - "python AND flask" - Find items with both terms @@ -53,6 +56,9 @@ class SearchQuery(BaseModel): types: Optional[List[str]] = None # Filter by type entity_types: Optional[List[SearchItemType]] = None # Filter by entity type after_date: Optional[Union[datetime, str]] = None # Time-based filter + metadata_filters: Optional[dict[str, Any]] = None # Structured frontmatter filters + tags: Optional[List[str]] = None # Convenience tag filter + status: Optional[str] = None # Convenience status filter @field_validator("after_date") @classmethod @@ -63,14 +69,23 @@ def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]: return v def no_criteria(self) -> bool: + text_is_empty = self.text is None or (isinstance(self.text, str) and not self.text.strip()) + metadata_is_empty = not self.metadata_filters + tags_is_empty = not self.tags + status_is_empty = self.status is None or (isinstance(self.status, str) and not self.status) + types_is_empty = not self.types + entity_types_is_empty = not self.entity_types return ( self.permalink is None and self.permalink_match is None and self.title is None - and self.text is None + and text_is_empty and self.after_date is None - and self.types is None - and self.entity_types is None + and types_is_empty + and entity_types_is_empty + and metadata_is_empty + and tags_is_empty + and status_is_empty ) def has_boolean_operators(self) -> bool: diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index c67fe49a..a929f1e3 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -18,7 +18,7 @@ dump_frontmatter, ) from basic_memory.markdown import EntityMarkdown -from basic_memory.markdown.entity_parser import EntityParser +from basic_memory.markdown.entity_parser import EntityParser, normalize_frontmatter_metadata from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown from basic_memory.models import Entity as EntityModel from basic_memory.models import Observation, Relation @@ -392,8 +392,8 @@ async def fast_write_entity( checksum = await self.file_service.write_file(file_path, final_content) # --- Minimal DB Upsert --- - metadata = post.metadata or {} - entity_metadata = {k: str(v) for k, v in metadata.items() if v is not None} + metadata = normalize_frontmatter_metadata(post.metadata or {}) + entity_metadata = {k: v for k, v in metadata.items() if v is not None} update_data = { "title": schema.title, "entity_type": schema.entity_type, @@ -458,10 +458,8 @@ async def fast_edit_entity( content_frontmatter["permalink"], ) - metadata = content_frontmatter or {} - update_data["entity_metadata"] = { - k: str(v) for k, v in metadata.items() if v is not None - } + metadata = normalize_frontmatter_metadata(content_frontmatter or {}) + update_data["entity_metadata"] = {k: v for k, v in metadata.items() if v is not None} # --- Permalink Resolution --- if self.app_config and self.app_config.disable_permalinks: diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index 2e982d28..8b100b32 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -2,7 +2,7 @@ import ast from datetime import datetime -from typing import List, Optional, Set +from typing import List, Optional, Set, Dict, Any from dateparser import parse @@ -95,6 +95,15 @@ async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchInd else None ) + # Merge structured metadata filters (explicit + convenience fields) + metadata_filters: Optional[Dict[str, Any]] = None + if query.metadata_filters or query.tags or query.status: + metadata_filters = dict(query.metadata_filters or {}) + if query.tags: + metadata_filters.setdefault("tags", query.tags) + if query.status: + metadata_filters.setdefault("status", query.status) + # search results = await self.repository.search( search_text=query.text, @@ -104,6 +113,7 @@ async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchInd types=query.types, search_item_types=query.entity_types, after_date=after_date, + metadata_filters=metadata_filters, limit=limit, offset=offset, ) From bcf8f46c3a19e8286f2ed15934ca897536f62470 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 31 Jan 2026 14:43:47 -0600 Subject: [PATCH 6/6] fixes for tests, formatting, migration Signed-off-by: phernandez --- ...9a0b1c2_add_structured_metadata_indexes.py | 5 +- src/basic_memory/api/v2/utils.py | 2 +- src/basic_memory/cli/commands/tool.py | 3 - src/basic_memory/deps/__init__.py | 2 + .../mcp/prompts/recent_activity.py | 6 +- src/basic_memory/mcp/tools/move_note.py | 4 +- .../repository/metadata_filters.py | 15 ++--- .../repository/postgres_search_repository.py | 16 ++++-- src/basic_memory/services/entity_service.py | 8 ++- src/basic_memory/sync/sync_service.py | 4 +- tests/api/v2/test_importer_router.py | 4 +- tests/api/v2/test_knowledge_router.py | 4 +- tests/mcp/test_tool_canvas.py | 4 +- tests/mcp/test_tool_read_note.py | 5 +- tests/repository/test_project_repository.py | 40 +++++++------ tests/services/test_link_resolver.py | 56 ++++++------------- 16 files changed, 88 insertions(+), 90 deletions(-) diff --git a/src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py b/src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py index d4a745c6..21131595 100644 --- a/src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py +++ b/src/basic_memory/alembic/versions/d7e8f9a0b1c2_add_structured_metadata_indexes.py @@ -48,7 +48,7 @@ def index_exists(connection, index_name: str) -> bool: # revision identifiers, used by Alembic. revision: str = "d7e8f9a0b1c2" -down_revision: Union[str, None] = "g9a0b3c4d5e6" +down_revision: Union[str, None] = "6830751f5fb6" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -142,8 +142,7 @@ def downgrade() -> None: op.execute("DROP INDEX IF EXISTS idx_entity_tags_json") op.execute("DROP INDEX IF EXISTS idx_entity_metadata_gin") op.execute( - "ALTER TABLE entity ALTER COLUMN entity_metadata " - "TYPE json USING entity_metadata::json" + "ALTER TABLE entity ALTER COLUMN entity_metadata TYPE json USING entity_metadata::json" ) return diff --git a/src/basic_memory/api/v2/utils.py b/src/basic_memory/api/v2/utils.py index 766588f0..0b48f991 100644 --- a/src/basic_memory/api/v2/utils.py +++ b/src/basic_memory/api/v2/utils.py @@ -75,7 +75,7 @@ def to_summary(item: SearchIndexRow | ContextResultRow): observation_id=item.id, entity_id=item.entity_id, # pyright: ignore entity_external_id=entity_ext_id, - title=entity_title_lookup.get(item.entity_id) , # pyright: ignore + title=entity_title_lookup.get(item.entity_id), # pyright: ignore file_path=item.file_path, category=item.category, # pyright: ignore content=item.content, # pyright: ignore diff --git a/src/basic_memory/cli/commands/tool.py b/src/basic_memory/cli/commands/tool.py index 88a9bd33..c0bb3e7e 100644 --- a/src/basic_memory/cli/commands/tool.py +++ b/src/basic_memory/cli/commands/tool.py @@ -412,9 +412,6 @@ def search_notes( status=status, ) ) - # Use json module for more controlled serialization - import json - results_dict = results.model_dump(exclude_none=True) print(json.dumps(results_dict, indent=2, ensure_ascii=True, default=str)) except ValueError as e: diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index 9b52edc1..b7614ce6 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -229,6 +229,8 @@ "FileServiceV2Dep", "get_file_service_v2_external", "FileServiceV2ExternalDep", + "get_task_scheduler", + "TaskSchedulerDep", "get_search_service", "SearchServiceDep", "get_search_service_v2", diff --git a/src/basic_memory/mcp/prompts/recent_activity.py b/src/basic_memory/mcp/prompts/recent_activity.py index ad598e65..10a07f7a 100644 --- a/src/basic_memory/mcp/prompts/recent_activity.py +++ b/src/basic_memory/mcp/prompts/recent_activity.py @@ -46,9 +46,7 @@ async def recent_activity_prompt( # Call the tool function - it returns a well-formatted string # Pass type as string values (not enum) to match the tool's expected input - activity_summary = await recent_activity.fn( - project=project, timeframe=timeframe, type="entity" - ) + activity_summary = await recent_activity.fn(project=project, timeframe=timeframe, type="entity") # Build the prompt response # The tool already returns formatted markdown, so we use it directly @@ -92,7 +90,7 @@ async def recent_activity_prompt( - summarizes [[Recent Work]] ''', folder="insights", - project="{project or 'default'}" + project="{project or "default"}" ) ``` """) diff --git a/src/basic_memory/mcp/tools/move_note.py b/src/basic_memory/mcp/tools/move_note.py index 48c0e01d..78de8be3 100644 --- a/src/basic_memory/mcp/tools/move_note.py +++ b/src/basic_memory/mcp/tools/move_note.py @@ -488,7 +488,9 @@ async def move_note( return "\n".join(result_lines) except Exception as e: # pragma: no cover - logger.error(f"Directory move failed for '{identifier}' to '{destination_path}': {e}") + logger.error( + f"Directory move failed for '{identifier}' to '{destination_path}': {e}" + ) return f"""# Directory Move Failed Error moving directory '{identifier}' to '{destination_path}': {str(e)} diff --git a/src/basic_memory/repository/metadata_filters.py b/src/basic_memory/repository/metadata_filters.py index e53fab6a..764e1669 100644 --- a/src/basic_memory/repository/metadata_filters.py +++ b/src/basic_memory/repository/metadata_filters.py @@ -92,16 +92,11 @@ def parse_metadata_filters(filters: dict[str, Any]) -> List[ParsedMetadataFilter continue if op == "$between": - if ( - not isinstance(value, list) - or len(value) != 2 - ): + if not isinstance(value, list) or len(value) != 2: raise ValueError(f"$between requires [min, max] for '{raw_key}'") normalized = [_normalize_scalar(v) for v in value] comparison = "numeric" if _is_numeric_collection(normalized) else "text" - parsed.append( - ParsedMetadataFilter(path_parts, "between", normalized, comparison) - ) + parsed.append(ParsedMetadataFilter(path_parts, "between", normalized, comparison)) continue raise ValueError(f"Unsupported operator '{op}' in metadata filter for '{raw_key}'") @@ -111,7 +106,9 @@ def parse_metadata_filters(filters: dict[str, Any]) -> List[ParsedMetadataFilter if not raw_value: raise ValueError(f"Empty list not allowed for metadata filter '{raw_key}'") parsed.append( - ParsedMetadataFilter(path_parts, "contains", [_normalize_scalar(v) for v in raw_value]) + ParsedMetadataFilter( + path_parts, "contains", [_normalize_scalar(v) for v in raw_value] + ) ) continue @@ -125,7 +122,7 @@ def build_sqlite_json_path(parts: List[str]) -> str: """Build a SQLite JSON path for json_extract/json_each.""" path = "$" for part in parts: - path += f'.\"{part}\"' + path += f'."{part}"' return path diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 4dcd6bd0..29a3d04b 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -12,7 +12,10 @@ from basic_memory import db from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_repository_base import SearchRepositoryBase -from basic_memory.repository.metadata_filters import parse_metadata_filters, build_postgres_json_path +from basic_memory.repository.metadata_filters import ( + parse_metadata_filters, + build_postgres_json_path, +) from basic_memory.schemas.search import SearchItemType @@ -318,14 +321,15 @@ async def search( if filt.op == "contains": import json as _json + base_param = f"meta_val_{idx}" tag_conditions = [] # Require all values to be present for j, val in enumerate(filt.value): - tag_param = f"{value_param}_{j}" + tag_param = f"{base_param}_{j}" params[tag_param] = _json.dumps([val]) - like_param = f"{value_param}_{j}_like" + like_param = f"{base_param}_{j}_like" params[like_param] = f'%"{val}"%' - like_param_single = f"{value_param}_{j}_like_single" + like_param_single = f"{base_param}_{j}_like_single" params[like_param_single] = f"%'{val}'%" tag_conditions.append( f"({json_expr} @> :{tag_param}::jsonb " @@ -372,7 +376,9 @@ async def search( # Build SQL with ts_rank() for scoring # Note: If no text search, score will be NULL, so we use COALESCE to default to 0 if search_text and search_text.strip() and search_text.strip() != "*": - score_expr = "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text))" + score_expr = ( + "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text))" + ) else: score_expr = "0" diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index a929f1e3..0a69affb 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1095,7 +1095,9 @@ async def move_directory( old_path = entity.file_path # Replace only the first occurrence of the source directory prefix if old_path.startswith(f"{source_directory}/"): - new_path = old_path.replace(f"{source_directory}/", f"{destination_directory}/", 1) + new_path = old_path.replace( + f"{source_directory}/", f"{destination_directory}/", 1 + ) else: # pragma: no cover # Entity is directly in the source directory (shouldn't happen with prefix match) new_path = f"{destination_directory}/{old_path}" @@ -1184,7 +1186,9 @@ async def delete_directory( logger.debug(f"Deleted entity: {file_path}") else: # pragma: no cover failed_deletes += 1 - errors.append(DirectoryDeleteError(path=file_path, error="Delete returned False")) + errors.append( + DirectoryDeleteError(path=file_path, error="Delete returned False") + ) logger.warning(f"Delete returned False for entity: {file_path}") except Exception as e: # pragma: no cover diff --git a/src/basic_memory/sync/sync_service.py b/src/basic_memory/sync/sync_service.py index 0ee04d07..0287650a 100644 --- a/src/basic_memory/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -686,9 +686,7 @@ async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optiona await self.file_service.update_frontmatter(path, {"permalink": permalink}) # Create/update entity and relations in one path - logger.debug( - f"{'Creating' if new else 'Updating'} entity from markdown, path={path}" - ) + logger.debug(f"{'Creating' if new else 'Updating'} entity from markdown, path={path}") entity = await self.entity_service.upsert_entity_from_markdown( Path(path), entity_markdown, is_new=new ) diff --git a/tests/api/v2/test_importer_router.py b/tests/api/v2/test_importer_router.py index ec6fdcb9..459631f4 100644 --- a/tests/api/v2/test_importer_router.py +++ b/tests/api/v2/test_importer_router.py @@ -474,7 +474,9 @@ async def test_import_invalid_project_id(client: AsyncClient, tmp_path, chatgpt_ async def test_import_missing_file(client: AsyncClient, v2_project_url: str): """Test importing with missing file via v2 endpoint.""" # Send a request without a file - response = await client.post(f"{v2_project_url}/import/chatgpt", data={"directory": "test_folder"}) + response = await client.post( + f"{v2_project_url}/import/chatgpt", data={"directory": "test_folder"} + ) # Check that the request was rejected assert response.status_code in [400, 422] # Either bad request or unprocessable entity diff --git a/tests/api/v2/test_knowledge_router.py b/tests/api/v2/test_knowledge_router.py index c74bf951..a58e018d 100644 --- a/tests/api/v2/test_knowledge_router.py +++ b/tests/api/v2/test_knowledge_router.py @@ -80,7 +80,9 @@ async def test_resolve_identifier_no_fuzzy_match(client: AsyncClient, v2_project @pytest.mark.asyncio -async def test_resolve_identifier_with_source_path_no_fuzzy_match(client: AsyncClient, v2_project_url): +async def test_resolve_identifier_with_source_path_no_fuzzy_match( + client: AsyncClient, v2_project_url +): """Test that context-aware resolution also uses strict mode. Even with source_path for context-aware resolution, nonexistent diff --git a/tests/mcp/test_tool_canvas.py b/tests/mcp/test_tool_canvas.py index ee1f0d9f..11bf9a99 100644 --- a/tests/mcp/test_tool_canvas.py +++ b/tests/mcp/test_tool_canvas.py @@ -109,7 +109,9 @@ async def test_update_existing_canvas(app, project_config, test_project): folder = "visualizations" # Create initial canvas - await canvas.fn(project=test_project.name, nodes=nodes, edges=edges, title=title, directory=folder) + await canvas.fn( + project=test_project.name, nodes=nodes, edges=edges, title=title, directory=folder + ) # Verify file exists file_path = Path(project_config.home) / folder / f"{title}.canvas" diff --git a/tests/mcp/test_tool_read_note.py b/tests/mcp/test_tool_read_note.py index 00d2769a..14e901de 100644 --- a/tests/mcp/test_tool_read_note.py +++ b/tests/mcp/test_tool_read_note.py @@ -14,7 +14,10 @@ async def test_read_note_by_title(app, test_project): """Test reading a note by its title.""" # First create a note await write_note.fn( - project=test_project.name, title="Special Note", directory="test", content="Note content here" + project=test_project.name, + title="Special Note", + directory="test", + content="Note content here", ) # Should be able to read it by title diff --git a/tests/repository/test_project_repository.py b/tests/repository/test_project_repository.py index 0a260ea4..0934344b 100644 --- a/tests/repository/test_project_repository.py +++ b/tests/repository/test_project_repository.py @@ -144,23 +144,29 @@ async def test_get_default_project_with_false_values(project_repository: Project causing MultipleResultsFound when multiple projects had different boolean values. """ # Create projects with explicit is_default values - project_true = await project_repository.create({ - "name": "Default Project", - "path": "/default/path", - "is_default": True, - }) - - await project_repository.create({ - "name": "Not Default Project", - "path": "/not-default/path", - "is_default": False, - }) - - await project_repository.create({ - "name": "Null Default Project", - "path": "/null/path", - "is_default": None, - }) + project_true = await project_repository.create( + { + "name": "Default Project", + "path": "/default/path", + "is_default": True, + } + ) + + await project_repository.create( + { + "name": "Not Default Project", + "path": "/not-default/path", + "is_default": False, + } + ) + + await project_repository.create( + { + "name": "Null Default Project", + "path": "/null/path", + "is_default": None, + } + ) # Should return only the project with is_default=True default = await project_repository.get_default_project() diff --git a/tests/services/test_link_resolver.py b/tests/services/test_link_resolver.py index 0c390dd2..bb8e2138 100644 --- a/tests/services/test_link_resolver.py +++ b/tests/services/test_link_resolver.py @@ -509,8 +509,7 @@ async def test_source_path_same_folder_preference(context_link_resolver): """Test that links prefer notes in the same folder as the source.""" # From main/testing/another-test.md, [[testing]] should find main/testing/testing.md result = await context_link_resolver.resolve_link( - "testing", - source_path="main/testing/another-test.md" + "testing", source_path="main/testing/another-test.md" ) assert result is not None assert result.file_path == "main/testing/testing.md" @@ -520,10 +519,7 @@ async def test_source_path_same_folder_preference(context_link_resolver): async def test_source_path_from_root_prefers_root(context_link_resolver): """Test that links from root-level notes prefer root-level matches.""" # From root-note.md, [[testing]] should find testing.md (root level) - result = await context_link_resolver.resolve_link( - "testing", - source_path="some-root-note.md" - ) + result = await context_link_resolver.resolve_link("testing", source_path="some-root-note.md") assert result is not None assert result.file_path == "testing.md" @@ -534,10 +530,7 @@ async def test_source_path_different_branch_prefers_closest(context_link_resolve # From other/testing.md, [[testing]] should find other/testing.md (same folder) # Wait, other/testing.md IS the testing note in that folder, so this tests self-reference # Let's test from a hypothetical other/different.md - result = await context_link_resolver.resolve_link( - "testing", - source_path="other/different.md" - ) + result = await context_link_resolver.resolve_link("testing", source_path="other/different.md") assert result is not None # Should find other/testing.md since it's in the same folder assert result.file_path == "other/testing.md" @@ -556,8 +549,7 @@ async def test_source_path_ancestor_preference(context_link_resolver): # 1. deep/nested/folder/note.md (same folder) - but that's the note itself # Let's say we're linking from a different file in that folder result = await context_link_resolver.resolve_link( - "note", - source_path="deep/nested/folder/other-file.md" + "note", source_path="deep/nested/folder/other-file.md" ) assert result is not None # Should find deep/nested/folder/note.md (same folder) @@ -573,8 +565,7 @@ async def test_source_path_parent_folder_preference(context_link_resolver): # For this test, let's check that from deep/nested/other/file.md, # [[note]] finds deep/note.md (ancestor) rather than note.md (root) result = await context_link_resolver.resolve_link( - "note", - source_path="deep/nested/other/file.md" + "note", source_path="deep/nested/other/file.md" ) assert result is not None # No note.md in deep/nested/other/, so should find deep/note.md (closest ancestor) @@ -602,7 +593,7 @@ async def test_source_path_unique_title_ignores_context(context_link_resolver): # "another-test" only exists in one place result = await context_link_resolver.resolve_link( "another-test", - source_path="other/some-file.md" # Different folder + source_path="other/some-file.md", # Different folder ) assert result is not None assert result.file_path == "main/testing/another-test.md" @@ -617,8 +608,7 @@ async def test_source_path_with_permalink_conflict(context_link_resolver): # even though there's a permalink match at root result = await context_link_resolver.resolve_link( - "testing", - source_path="main/testing/another-test.md" + "testing", source_path="main/testing/another-test.md" ) assert result is not None # Should prefer same-folder title match over root permalink match @@ -633,24 +623,22 @@ async def test_find_closest_entity_same_folder(context_link_resolver, context_aw assert len(testing_entities) == 3 # root, main/testing, other closest = context_link_resolver._find_closest_entity( - testing_entities, - "main/testing/another-test.md" + testing_entities, "main/testing/another-test.md" ) assert closest.file_path == "main/testing/testing.md" @pytest.mark.asyncio -async def test_find_closest_entity_ancestor_preference(context_link_resolver, context_aware_entities): +async def test_find_closest_entity_ancestor_preference( + context_link_resolver, context_aware_entities +): """Test _find_closest_entity prefers closer ancestors.""" # Get entities with title "note" note_entities = [e for e in context_aware_entities if e.title == "note"] assert len(note_entities) == 3 # deep/nested/folder, deep, root # From deep/nested/other/file.md, should prefer deep/note.md over note.md - closest = context_link_resolver._find_closest_entity( - note_entities, - "deep/nested/other/file.md" - ) + closest = context_link_resolver._find_closest_entity(note_entities, "deep/nested/other/file.md") assert closest.file_path == "deep/note.md" @@ -660,10 +648,7 @@ async def test_find_closest_entity_root_source(context_link_resolver, context_aw testing_entities = [e for e in context_aware_entities if e.title == "testing"] # From root level, should prefer root testing.md - closest = context_link_resolver._find_closest_entity( - testing_entities, - "some-root-file.md" - ) + closest = context_link_resolver._find_closest_entity(testing_entities, "some-root-file.md") assert closest.file_path == "testing.md" @@ -671,8 +656,7 @@ async def test_find_closest_entity_root_source(context_link_resolver, context_aw async def test_nonexistent_link_with_source_path(context_link_resolver): """Test that non-existent links return None even with source_path.""" result = await context_link_resolver.resolve_link( - "does-not-exist", - source_path="main/testing/another-test.md" + "does-not-exist", source_path="main/testing/another-test.md" ) assert result is None @@ -774,8 +758,7 @@ async def test_relative_path_resolution_from_subfolder(relative_path_resolver): """Test that [[nested/deep-note]] from testing/link-test.md resolves to testing/nested/deep-note.md.""" # From testing/link-test.md, [[nested/deep-note]] should resolve to testing/nested/deep-note.md result = await relative_path_resolver.resolve_link( - "nested/deep-note", - source_path="testing/link-test.md" + "nested/deep-note", source_path="testing/link-test.md" ) assert result is not None assert result.file_path == "testing/nested/deep-note.md" @@ -787,8 +770,7 @@ async def test_relative_path_falls_back_to_absolute(relative_path_resolver): # From other/file.md, [[nested/deep-note]] should resolve to nested/deep-note.md (absolute) # because other/nested/deep-note.md doesn't exist result = await relative_path_resolver.resolve_link( - "nested/deep-note", - source_path="other/file.md" + "nested/deep-note", source_path="other/file.md" ) assert result is not None assert result.file_path == "nested/deep-note.md" @@ -808,8 +790,7 @@ async def test_relative_path_from_root_falls_through(relative_path_resolver): """Test that paths from root-level files don't try relative resolution.""" # From root-file.md (no folder), [[nested/deep-note]] should resolve to nested/deep-note.md result = await relative_path_resolver.resolve_link( - "nested/deep-note", - source_path="root-file.md" + "nested/deep-note", source_path="root-file.md" ) assert result is not None assert result.file_path == "nested/deep-note.md" @@ -820,8 +801,7 @@ async def test_simple_link_no_slash_skips_relative_resolution(relative_path_reso """Test that links without '/' don't trigger relative path resolution.""" # [[deep-note]] should use context-aware title matching, not relative paths result = await relative_path_resolver.resolve_link( - "deep-note", - source_path="testing/link-test.md" + "deep-note", source_path="testing/link-test.md" ) assert result is not None # Should find testing/nested/deep-note.md via title match with same-folder preference