Generated with '
+ f'{_escape_html(str(summary_intro.get("model_deployment") or "configured model"))} on '
+ f'{_escape_html(str(summary_intro.get("generated_at", "")))}
'
+ )
+ if message.get('timestamp'):
+ parts.append(
+ f'
{_escape_html(str(message.get("timestamp")))}
'
+ )
+ parts.append('')
+ for thought in message.get('thoughts', []):
+ thought_label = (thought.get('step_type') or 'step').replace('_', ' ').title()
+ parts.append(
+ f'
{_escape_html(thought_label)}: '
+ f'{_escape_html(str(thought.get("content") or "No content recorded."))}'
+ )
+ if thought.get('duration_ms') is not None:
+ parts.append(
+ f' Duration: {thought.get("duration_ms")} ms'
+ )
+ if thought.get('timestamp'):
+ parts.append(
+ f' Timestamp: '
+ f'{_escape_html(str(thought.get("timestamp")))}'
+ )
+ if thought.get('detail'):
+ parts.append(' Detail:')
+ _append_html_code_block(parts, thought.get('detail'))
+ parts.append('
'
+ )
+ if message.get('timestamp'):
+ parts.append(
+ f'
{_escape_html(str(message.get("timestamp")))}
'
+ )
+ content = message.get('content_text', '') or 'No content recorded.'
+ content_html = markdown2.markdown(
+ content,
+ extras=['fenced-code-blocks', 'tables', 'break-on-newline']
+ )
+ parts.append(content_html)
+
+ return '\n'.join(parts)
+
+
+def _render_pdf_bytes(body_html: str) -> bytes:
+ """Render HTML body content to PDF bytes using PyMuPDF Story API."""
+ MEDIABOX = fitz.paper_rect("letter")
+ WHERE = MEDIABOX + (36, 36, -36, -36)
+
+ story = fitz.Story(html=body_html, user_css=_PDF_CSS)
+
+ tmp_path = None
+ try:
+ with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp:
+ tmp_path = tmp.name
+
+ writer = fitz.DocumentWriter(tmp_path)
+ more = True
+ while more:
+ device = writer.begin_page(MEDIABOX)
+ more, _ = story.place(WHERE)
+ story.draw(device)
+ writer.end_page()
+ writer.close()
+ del story
+ del writer
+
+ with open(tmp_path, 'rb') as f:
+ return f.read()
+ finally:
+ if tmp_path:
+ try:
+ os.unlink(tmp_path)
+ except OSError:
+ pass
+
+
+def _conversation_to_pdf_bytes(entry: Dict[str, Any]) -> bytes:
+ """Convert a conversation export entry to PDF bytes."""
+ body_html = _build_pdf_html_body(entry)
+ return _render_pdf_bytes(body_html)
+
+
+def _html_body_to_pdf_bytes(body_html: str) -> bytes:
+ """Convert raw HTML body content to PDF bytes."""
+ return _render_pdf_bytes(body_html)
+
+
+def _append_html_table(parts: List[str], mapping: Dict[str, Any]):
+ """Append a key-value mapping as an HTML table."""
+ if not isinstance(mapping, dict) or not mapping:
+ parts.append('
No data available.
')
+ return
+
+ parts.append('
')
+ parts.append('
Property
Value
')
+ for key, value in mapping.items():
+ label = _format_markdown_key(key)
+ if isinstance(value, dict):
+ formatted = _format_nested_html_value(value)
+ elif isinstance(value, list):
+ formatted = (
+ ', '.join(_escape_html(str(item)) for item in value)
+ if value else 'None'
+ )
+ elif isinstance(value, bool):
+ formatted = 'Yes' if value else 'No'
+ else:
+ formatted = _escape_html(str(value))
+ parts.append(f'
{_escape_html(label)}
{formatted}
')
+ parts.append('
')
+
+
+def _format_nested_html_value(mapping: Dict[str, Any], depth: int = 0) -> str:
+ """Format a nested dict as an HTML string for table cells."""
+ if not mapping:
+ return 'None'
+
+ items = []
+ for key, value in mapping.items():
+ label = _format_markdown_key(key)
+ if isinstance(value, dict):
+ nested = _format_nested_html_value(value, depth + 1)
+ items.append(f'{_escape_html(label)}: {nested}')
+ elif isinstance(value, list):
+ list_str = (
+ ', '.join(_escape_html(str(v)) for v in value)
+ if value else 'None'
+ )
+ items.append(f'{_escape_html(label)}: {list_str}')
+ elif isinstance(value, bool):
+ items.append(f'{_escape_html(label)}: {"Yes" if value else "No"}')
+ else:
+ items.append(f'{_escape_html(label)}: {_escape_html(str(value))}')
+ return ' '.join(items)
+
+
+def _append_html_citations(parts: List[str], message: Dict[str, Any]):
+ """Append citation data as HTML."""
+ citations = message.get('citations', [])
+ if not citations:
+ parts.append('
No citations were recorded for this message.
')
+ return
+
+ doc_citations = [c for c in citations if c.get('citation_type') == 'document']
+ web_citations = [c for c in citations if c.get('citation_type') == 'web']
+ agent_citations = [c for c in citations if c.get('citation_type') == 'agent_tool']
+ legacy_citations = [c for c in citations if c.get('citation_type') == 'legacy']
+
+ if doc_citations:
+ parts.append('
Document Sources
')
+ parts.append('')
+ for citation in doc_citations:
+ parts.append(
+ f'
')
+ parts.append('')
+
+ if web_citations:
+ parts.append('
Web Sources
')
+ parts.append('')
+ for citation in web_citations:
+ title = _escape_html(
+ str(citation.get('title') or citation.get('label') or 'Web source')
+ )
+ url = citation.get('url')
+ if url:
+ parts.append(f'
enable_text_to_speech: {{ 'true' if app_settings.enable_text_to_speech else 'false' }},
enable_speech_to_text_input: {{ 'true' if app_settings.enable_speech_to_text_input else 'false' }},
enable_web_search_user_notice: {{ 'true' if settings.enable_web_search_user_notice else 'false' }},
- enforce_workspace_scope_lock: {{ 'true' if settings.enforce_workspace_scope_lock else 'false' }}
+ enforce_workspace_scope_lock: {{ 'true' if settings.enforce_workspace_scope_lock else 'false' }},
+ enable_thoughts: {{ 'true' if settings.enable_thoughts else 'false' }}
};
// Layout related globals (can stay here or move entirely into chat-layout.js if preferred)
diff --git a/application/single_app/templates/group_workspaces.html b/application/single_app/templates/group_workspaces.html
index e14fe8a3..62937fc3 100644
--- a/application/single_app/templates/group_workspaces.html
+++ b/application/single_app/templates/group_workspaces.html
@@ -763,33 +763,42 @@
Group Workspace
You do not have permission to manage group agents.
-
+
+
+
+
+
+
+
-
-
-
-
Display Name
-
Description
-
Actions
-
-
-
-
-
-
- Loading...
-
- Select a group to load agents.
-
-
-
-
+
+
+
+
+
Display Name
+
Description
+
Actions
+
+
+
+
+
+
+ Loading...
+
+ Select a group to load agents.
+
+
+
+
+
+
@@ -813,33 +822,42 @@
Group Workspace
-
+
+
+
+
+
+
+
-
-
-
-
Display Name
-
Description
-
Actions
-
-
-
-
-
-
- Loading...
-
- Select a group to load actions.
-
-
-
-
+
+
+
+
+
Display Name
+
Description
+
Actions
+
+
+
+
+
+
+ Loading...
+
+ Select a group to load actions.
+
+
+
+
+
+
@@ -851,6 +869,22 @@
Group Workspace
+
+
+
+
+
+
View Item
+
+
+
+
+
+
+
+
Personal Workspace
+
+
+
+
+
+
-
-
Display Name
Description
Actions
-
-
-
-
Loading...
- Loading agents...
-
-
-
-
+
+
+
+
Display Name
Description
Actions
+
+
+
+
Loading...
+ Loading agents...
+
+
+
+
+
+
+
@@ -730,16 +741,27 @@
Personal Workspace
+
+
+
+
+
+
-
-
Display Name
Description
Actions
-
-
-
-
+
+
+
+
Display Name
Description
Actions
+
+
+
+
+
+
+
@@ -754,6 +776,24 @@
Personal Workspace
+
+
+
+
+
+
Details
+
+
+
+
+
+
+
+
+
+
/summary` with optional `{ "model_deployment": "..." }` body.
+
+### Reusable Helper
+- `generate_conversation_summary()` (in `route_backend_conversation_export.py`) is the shared LLM call function used by both the export pipeline and the API endpoint.
+- It handles transcript assembly, truncation, model role selection (developer vs system), and Cosmos persistence.
+
+## Security and Data-Shaping Rules
+- Route uses authenticated user checks before export
+- Only user-owned personal conversations are exported
+- Internal Cosmos metadata is not passed through
+- Deleted messages and inactive-thread retries are excluded
+- Exported message details are curated rather than raw metadata dumps
+- Raw settings are not exposed to the browser; the export wizard reuses the existing chat model selector already rendered from sanitized settings
+
+## Files Updated
+- `application/single_app/route_backend_conversation_export.py`
+- `application/single_app/route_backend_conversations.py`
+- `application/single_app/static/js/chat/chat-export.js`
+- `application/single_app/static/js/chat/chat-conversation-details.js`
+- `application/single_app/config.py`
+- `functional_tests/test_conversation_export.py`
+- `functional_tests/test_persistent_conversation_summary.py`
## Testing and Validation
-- **Functional test:** `functional_tests/test_conversation_export.py`
-- Tests cover:
- - Conversation sanitization (internal field stripping)
- - Message sanitization
- - Markdown generation (headings, metadata, citations)
- - JSON structure validation
- - ZIP packaging (correct entries, valid content)
- - Filename sanitization (special chars, truncation, empty input)
- - Active thread message filtering
+- **Functional tests:**
+ - `functional_tests/test_conversation_export.py` — export pipeline (8 tests)
+ - `functional_tests/test_persistent_conversation_summary.py` — summary caching (8 tests)
+
+Coverage includes:
+- deleted/inactive message filtering
+- normalized and raw citation export
+- thoughts attached to assistant messages
+- transcript-style Markdown appendices
+- summary-intro metadata shape
+- filename sanitization and ZIP naming
+- content normalization and citation-count helpers
+- PDF HTML body generation with chat-bubble classes
+- PDF byte output validation (%PDF- header)
## Known Limitations
-- Export is limited to conversations the authenticated user owns
-- Very large conversations (thousands of messages) may take longer to process
-- The wizard fetches conversation titles client-side; if a title lookup fails, it shows the conversation ID instead
+- Export remains limited to user-owned personal conversations
+- Optional intro summaries increase export time because each selected conversation is summarized individually
+- Very large conversations may be truncated for summary generation, though the full export content is still preserved in the output
+- Summary generation is best-effort: if model initialization or generation fails, the export still completes and records the summary error state
+- PDF rendering relies on PyMuPDF’s CSS subset which does not support flexbox, float, or full border-radius; bubble alignment is approximated using margin offsets
diff --git a/docs/explanation/features/v0.239.003/PROCESSING_THOUGHTS.md b/docs/explanation/features/v0.239.003/PROCESSING_THOUGHTS.md
new file mode 100644
index 00000000..5dad56d9
--- /dev/null
+++ b/docs/explanation/features/v0.239.003/PROCESSING_THOUGHTS.md
@@ -0,0 +1,153 @@
+---
+layout: libdoc/page
+title: Processing Thoughts
+order: 100
+category: Features
+---
+
+# Processing Thoughts
+
+## Overview
+The Processing Thoughts feature replaces the generic "AI is typing..." indicator with real-time processing step traces that show users what the system is doing during chat processing. Each step (document search, web search, agent invocation, content safety check, response generation) is persisted in Cosmos DB and can be reviewed later via a per-message collapsible section.
+
+**Version Implemented:** 0.239.003
+
+## Dependencies
+- Flask (backend routes)
+- Azure Cosmos DB (`thoughts` and `archive_thoughts` containers)
+- Bootstrap 5 (collapsible section, badges, icons)
+- ES modules (`chat-thoughts.js`)
+
+## Architecture Overview
+
+### Backend
+
+#### ThoughtTracker (`functions_thoughts.py`)
+Stateful per-request tracker that writes each thought step to Cosmos DB immediately so polling clients can see partial progress.
+
+```
+ThoughtTracker(conversation_id, message_id, thread_id, user_id)
+ .add_thought(step_type, content, detail=None) → thought_id
+ .complete_thought(thought_id, duration_ms) → updates duration
+ .enabled → checks settings['enable_thoughts']
+```
+
+Design rules:
+- Each `add_thought()` does an immediate `upsert_item()` to Cosmos DB.
+- All writes are wrapped in try/except — thought errors never crash the chat flow.
+- Auto-increments `step_index` per tracker instance.
+- Logs failures via `log_event()` at WARNING level.
+
+#### Thought Document Schema
+```json
+{
+ "id": "uuid",
+ "conversation_id": "str",
+ "message_id": "str (assistant message ID)",
+ "thread_id": "str",
+ "user_id": "str (partition key)",
+ "step_index": 0,
+ "step_type": "search | tabular_analysis | web_search | agent_tool_call | generation | content_safety",
+ "content": "Searching personal workspace documents for 'sales analysis'...",
+ "detail": "Optional technical detail",
+ "duration_ms": null,
+ "timestamp": "ISO-8601"
+}
+```
+
+#### API Endpoints (`route_backend_thoughts.py`)
+
+| Method | Endpoint | Purpose |
+|--------|----------|---------|
+| GET | `/api/conversations//messages//thoughts` | Fetch persisted thoughts for a specific assistant message (historical viewing) |
+| GET | `/api/conversations//thoughts/pending` | Fetch latest in-progress thoughts (polling while waiting for response) |
+
+Both endpoints return `{"thoughts": [...], "enabled": true/false}`. When `enable_thoughts` is off, they return `{"thoughts": [], "enabled": false}`.
+
+#### Instrumentation Points (`route_backend_chats.py`)
+
+| Step Type | Content Example | When |
+|-----------|----------------|------|
+| `content_safety` | "Checking content safety..." | Before content safety check |
+| `search` | "Searching personal documents for 'query'..." | Before hybrid search |
+| `search` | "Found 5 results from 3 documents" | After search results |
+| `tabular_analysis` | "Found tabular data — evaluating analysis..." | When tabular data detected |
+| `web_search` | "Searching the web for 'query'..." | Before web search |
+| `web_search` | "Got 8 web results" | After web search |
+| `agent_tool_call` | "Sending to agent 'Data Analyst'..." | Before agent invocation |
+| `generation` | "Generating response..." | Before GPT call |
+
+### Frontend
+
+#### Streaming Mode
+Thought events are embedded in the SSE stream as `{"type": "thought", ...}` JSON payloads. The streaming handler in `chat-streaming.js` passes these to `handleStreamingThought()` which updates the streaming placeholder badge.
+
+#### Non-Streaming Mode
+A polling mechanism in `chat-thoughts.js` fetches `/thoughts/pending` every 2 seconds while waiting for a response. The loading indicator text is updated with the latest thought step.
+
+#### Per-Message History
+Each assistant message footer includes a lightbulb toggle button (when thoughts exist). Clicking it opens a collapsible section that lazy-loads thoughts from the API. Each step shows an icon, content text, and optional duration.
+
+#### Icon Map
+| Step Type | Bootstrap Icon |
+|-----------|---------------|
+| `search` | `bi-search` |
+| `tabular_analysis` | `bi-table` |
+| `web_search` | `bi-globe` |
+| `agent_tool_call` | `bi-robot` |
+| `generation` | `bi-lightning` |
+| `content_safety` | `bi-shield-check` |
+
+## Configuration
+
+### Admin Settings
+- **Toggle**: `enable_thoughts` (default: `false`)
+- **Location**: Admin Settings > Optional Features tab > "Processing Thoughts" section
+- **Effect**: When disabled, no thoughts are recorded and no UI elements are shown
+
+### Cosmos DB Containers
+| Container | Partition Key | Purpose |
+|-----------|--------------|---------|
+| `thoughts` | `/user_id` | Active thought records |
+| `archive_thoughts` | `/user_id` | Archived thoughts from deleted conversations |
+
+## Archive and Cleanup
+
+When a conversation is deleted:
+- **Archiving enabled**: Thoughts are copied to `archive_thoughts` container, then deleted from `thoughts`
+- **Archiving disabled**: Thoughts are permanently deleted from `thoughts`
+
+This applies to both single conversation delete and bulk delete operations.
+
+## File Structure
+
+### Files Created
+| File | Purpose |
+|------|---------|
+| `functions_thoughts.py` | ThoughtTracker class, Cosmos CRUD helpers |
+| `route_backend_thoughts.py` | API endpoints for fetching thoughts |
+| `static/js/chat/chat-thoughts.js` | Frontend polling, rendering, toggle |
+
+### Files Modified
+| File | Change |
+|------|--------|
+| `config.py` | Added `thoughts` + `archive_thoughts` Cosmos containers, bumped VERSION |
+| `functions_settings.py` | Added `enable_thoughts` default setting |
+| `app.py` | Imported and registered thought routes |
+| `route_backend_chats.py` | Instrumented ~8 thought points per chat path |
+| `route_backend_conversations.py` | Added archive/delete thoughts on conversation delete |
+| `templates/admin_settings.html` | Added Processing Thoughts toggle card |
+| `static/js/admin/admin_settings.js` | Added `enable_thoughts` to settings collection |
+| `static/js/chat/chat-messages.js` | Integrated thoughts toggle in footer, polling start/stop |
+| `static/js/chat/chat-streaming.js` | Handle `type: "thought"` in SSE data |
+| `static/js/chat/chat-loading-indicator.js` | Added `updateLoadingIndicatorText()` for thought display |
+| `static/css/chats.css` | Added thought indicator, toggle, container, and dark mode styles |
+
+## Testing
+
+1. **Enable feature**: Set `enable_thoughts: True` in admin settings
+2. **Non-streaming**: Send a message with document search — verify loading indicator updates with thought steps, lightbulb icon appears after response
+3. **Streaming**: Send a message — verify streaming placeholder shows thought badges, lightbulb available after finalization
+4. **History**: Reload page, open old conversation — click lightbulb to verify lazy-loaded thoughts
+5. **Disabled**: Set `enable_thoughts: False` — verify no thoughts generated, no lightbulb icons
+6. **Archive**: Delete a conversation with archiving enabled — verify thoughts moved to `archive_thoughts`
diff --git a/docs/explanation/features/v0.239.022/CONVERSATION_EXPORT.md b/docs/explanation/features/v0.239.022/CONVERSATION_EXPORT.md
new file mode 100644
index 00000000..edb69c2e
--- /dev/null
+++ b/docs/explanation/features/v0.239.022/CONVERSATION_EXPORT.md
@@ -0,0 +1,58 @@
+# Conversation Export
+
+## Overview
+Snapshot of the Conversation Export feature as implemented in version **0.239.022**.
+
+This version updates export generation so JSON includes modern citation buckets, normalized citation summaries, and processing thoughts, while Markdown becomes a transcript-first report with appendix sections and optional AI-generated intro summaries.
+
+**Version Implemented:** 0.239.022
+**Dependencies:** Flask export route, Azure Cosmos DB conversations/messages/thoughts, Bootstrap modal workflow, chat-export.js, Azure OpenAI/APIM chat models
+
+## Technical Summary
+
+### Backend
+- Filters out deleted messages and inactive-thread retries
+- Reapplies thread-aware ordering to align with the live chat view
+- Includes both normalized and raw citations per message
+- Joins persisted processing thoughts by `message_id`
+- Supports optional per-conversation `summary_intro` generation using a selected model
+
+### Frontend
+- Adds a summary step to the export wizard
+- Lets users enable or disable intro summaries
+- Reuses the existing chat model selector options for summary model choice
+
+## Export Shape
+
+### JSON
+Each conversation entry contains:
+- `conversation`
+- `summary_intro`
+- `messages`
+
+Each message can include:
+- `content`
+- `content_text`
+- `details`
+- `citations`
+- `legacy_citations`
+- `hybrid_citations`
+- `web_search_citations`
+- `agent_citations`
+- `thoughts`
+
+### Markdown
+Markdown exports contain:
+- metadata header
+- optional abstract
+- transcript body
+- appendices for metadata, message details, references, thoughts, and supplemental messages
+
+## Files Updated
+- `application/single_app/route_backend_conversation_export.py`
+- `application/single_app/static/js/chat/chat-export.js`
+- `application/single_app/config.py`
+- `functional_tests/test_conversation_export.py`
+
+## Testing
+Validated by `functional_tests/test_conversation_export.py`.
diff --git a/docs/explanation/fixes/SQL_QUERY_PLUGIN_SCHEMA_AWARENESS_FIX.md b/docs/explanation/fixes/SQL_QUERY_PLUGIN_SCHEMA_AWARENESS_FIX.md
new file mode 100644
index 00000000..9713a62c
--- /dev/null
+++ b/docs/explanation/fixes/SQL_QUERY_PLUGIN_SCHEMA_AWARENESS_FIX.md
@@ -0,0 +1,160 @@
+# SQL Query Plugin Schema Awareness Fix
+
+## Fix Title
+SQL Query Plugin - Schema Awareness, Companion Plugin Auto-Creation, and Workflow Guidance
+
+## Issue Description
+When users asked database-related questions (e.g., "what is user1 licensed to use?"), agents connected to SQL databases would ask for clarification about table/column names instead of querying the database directly. The agent had no citations, meaning it never actually called any database tools.
+
+## Root Cause Analysis
+
+### Original Root Causes (v0.239.014)
+Three interconnected issues caused the initial failure:
+
+1. **Generic `@kernel_function` descriptions**: The SQL Query and SQL Schema plugin function descriptions were terse and generic (e.g., "Execute a SQL query and return results"). They provided no workflow guidance telling the LLM to discover the schema first before writing queries.
+
+2. **No schema context in agent instructions**: Agent instructions were passed through verbatim from configuration with no automatic injection of database schema information.
+
+3. **Independent, disconnected plugins**: The SQL Schema Plugin and SQL Query Plugin operated as completely independent plugins with no linkage.
+
+### Deeper Root Causes Discovered (v0.239.015)
+The v0.239.014 fix improved descriptions but actually made things **worse** because:
+
+4. **No companion schema plugin was ever loaded**: The ESAM agent only had ONE action configured (`sql_query` type). No `sql_schema` action existed in the agent's actions. The `_create_sql_plugin()` method creates exactly what the manifest requests — so only `SQLQueryPlugin` was loaded, never `SQLSchemaPlugin`.
+
+5. **Descriptions demanded non-existent functions**: The v0.239.014 descriptions said "you MUST first call get_database_schema or get_table_list from the SQL Schema plugin" — but those functions didn't exist in the kernel since no schema plugin was loaded. This created an **impossible dependency** that made the LLM ask for clarification instead.
+
+6. **Schema extraction found nothing**: `_extract_sql_schema_for_instructions()` only searched for `SQLSchemaPlugin` instances. Since none existed in the kernel, it returned an empty string, so no schema was injected into agent instructions.
+
+7. **SQLPluginFactory was disconnected**: The `SQLPluginFactory` class was designed to create `(SQLSchemaPlugin, SQLQueryPlugin)` pairs, but was never called by the `LoggedPluginLoader` pipeline.
+
+### Empty Schema Tables from INFORMATION_SCHEMA (v0.239.016)
+After the v0.239.015 fix, the agent could answer simple queries (e.g., "what is user1 licensed to use?" correctly returned Office 365 license data). However, complex multi-table JOIN queries (e.g., "which department is spending the most on licensing?") still failed because:
+
+8. **INFORMATION_SCHEMA views returned empty results on Azure SQL**: The `_get_tables_query()`, `_get_columns_query()`, and `_get_primary_keys_query()` methods used `INFORMATION_SCHEMA.TABLES`, `INFORMATION_SCHEMA.COLUMNS`, and `INFORMATION_SCHEMA.KEY_COLUMN_USAGE` respectively. These views returned **zero rows** in this Azure SQL environment, even though the database contained 5 user tables.
+
+9. **sys.\* catalog views worked correctly**: The `_get_relationships_data()` method used `sys.foreign_keys`, `sys.tables`, and `sys.columns` — and successfully returned 4 foreign key relationships. This proved the database connection and permissions were fine, but `INFORMATION_SCHEMA` access was restricted or misconfigured.
+
+10. **pyodbc.Row type mismatch**: The table iteration code used `isinstance(table, tuple)` to check row types, but `pyodbc.Row` objects may not pass this check depending on the pyodbc version. When `isinstance` returned `False`, the code fell into an `else` branch that assigned the entire Row object as the table name, causing subsequent SQL queries to fail silently in the exception handler.
+
+11. **Result**: `get_database_schema` returned `{'tables': {}, 'relationships': [4 items]}` — the agent had foreign key metadata but no table/column definitions, making it impossible to construct multi-table JOINs.
+
+## Version Implemented
+**Initial fix in version: 0.239.014**
+**Companion plugin fix in version: 0.239.015**
+**Schema catalog views fix in version: 0.239.016**
+
+## Files Modified
+
+| File | Change |
+|------|--------|
+| `application/single_app/semantic_kernel_plugins/sql_schema_plugin.py` | Rewrote all `@kernel_function` descriptions with prescriptive workflow guidance (v0.239.014); migrated all SQL Server queries from INFORMATION_SCHEMA to sys.\* catalog views and fixed pyodbc.Row handling (v0.239.016) |
+| `application/single_app/semantic_kernel_plugins/sql_query_plugin.py` | Rewrote all `@kernel_function` descriptions with resilient conditional guidance (v0.239.015); added `query_database` convenience function (v0.239.014); updated `metadata` property description |
+| `application/single_app/semantic_kernel_loader.py` | Added `_extract_sql_schema_for_instructions()` helper function; auto-injects database schema into agent instructions; added SQLQueryPlugin fallback detection (v0.239.015) |
+| `application/single_app/semantic_kernel_plugins/logged_plugin_loader.py` | Enabled SQL plugin creation path (v0.239.014); added `_auto_create_companion_schema_plugin()` method that auto-creates a SQLSchemaPlugin whenever a SQLQueryPlugin is loaded (v0.239.015) |
+| `application/single_app/config.py` | Version bump to 0.239.016 |
+
+## Code Changes Summary
+
+### v0.239.014 Changes
+
+#### 1. Prescriptive Function Descriptions (sql_schema_plugin.py)
+- `get_database_schema`: Now says "ALWAYS call this function FIRST before executing any SQL queries"
+- `get_table_list`: Now says "Use this function first to discover which tables are available"
+- `get_table_schema`: Now says "Call this after discovering tables via get_database_schema or get_table_list"
+- `get_relationships`: Now says "Use this to understand how tables connect via JOIN conditions"
+
+#### 2. New `query_database` Convenience Function (sql_query_plugin.py)
+- Accepts `question` (natural language) and `query` (SQL) parameters
+- Returns results with the original question context for better LLM response formatting
+
+#### 3. Auto Schema Injection (semantic_kernel_loader.py)
+- New `_extract_sql_schema_for_instructions()` function detects SQL Schema plugins in the kernel
+- Calls `get_database_schema()` at agent load time to fetch full schema
+- Formats schema as markdown tables (table names, columns, types, relationships)
+- Appends schema to agent instructions with directive: "Do NOT ask the user for table or column names"
+
+#### 4. Enabled SQL Plugin Creation Path (logged_plugin_loader.py)
+- Uncommented the `elif plugin_type in ['sql_schema', 'sql_query']` branch
+
+### v0.239.015 Changes (Complete Fix)
+
+#### 5. Auto-Create Companion Schema Plugin (logged_plugin_loader.py)
+- New `_auto_create_companion_schema_plugin()` method
+- When a `sql_query` plugin is loaded, automatically creates a companion `SQLSchemaPlugin` using the same connection details
+- Derives schema plugin name: `enterprise_software_asset_management_query` → `enterprise_software_asset_management_schema`
+- Checks if the companion already exists (idempotent)
+- Enables logging, wraps functions, registers with kernel
+- This is the **critical fix** — ensures schema discovery is always available even when only `sql_query` is configured
+
+#### 6. Resilient Function Descriptions (sql_query_plugin.py)
+- Changed from "you MUST first call get_database_schema" to "If the database schema is provided in your instructions, use those exact table and column names. If no schema is available, call get_database_schema"
+- This dual-path approach works whether schema is injected in instructions OR available via schema plugin functions
+
+#### 7. SQLQueryPlugin Fallback in Schema Extraction (semantic_kernel_loader.py)
+- Added fallback in `_extract_sql_schema_for_instructions()` that also detects `SQLQueryPlugin` instances
+- If no `SQLSchemaPlugin` is found, creates a temporary `SQLSchemaPlugin` from the query plugin's connection config
+- Belt-and-suspenders safety net in case companion auto-creation fails
+- Appends schema to agent instructions with directive: "Do NOT ask the user for table or column names"
+- This ensures the LLM ALWAYS has schema context even if it doesn't call the schema plugin
+
+### 4. Enabled SQL Plugin Creation Path (logged_plugin_loader.py)
+- Uncommented the `elif plugin_type in ['sql_schema', 'sql_query']` branch
+- SQL plugins now use the explicit `_create_sql_plugin()` method instead of generic discovery fallback
+
+### v0.239.016 Changes (Schema Catalog Views Fix)
+
+#### 8. Migrated SQL Server Queries to sys.\* Catalog Views (sql_schema_plugin.py)
+- **`_get_tables_query()`**: Replaced `INFORMATION_SCHEMA.TABLES` with `sys.tables t INNER JOIN sys.schemas s ON t.schema_id = s.schema_id WHERE t.type = 'U'`
+- **`_get_columns_query()`**: Replaced `INFORMATION_SCHEMA.COLUMNS` with `sys.columns c INNER JOIN sys.tables t ... LEFT JOIN sys.default_constraints dc ...` using `TYPE_NAME(c.user_type_id)` for data type resolution
+- **`_get_primary_keys_query()`**: Replaced `INFORMATION_SCHEMA.KEY_COLUMN_USAGE` with `sys.index_columns ic INNER JOIN sys.indexes i ... WHERE i.is_primary_key = 1`
+- This makes all SQL Server schema queries consistent with `_get_relationships_data()`, which already used sys.\* views successfully
+
+#### 9. Robust pyodbc.Row Handling (sql_schema_plugin.py)
+- **`get_database_schema()`**: Replaced `isinstance(table, tuple)` checks with try/except indexing; all row field values cast to `str()` before use as dict keys
+- **`get_table_list()`**: Same robust Row handling pattern applied to table row iteration
+- **`_get_table_schema_data()`**: Primary key list comprehension updated to use `str(pk[0])` without isinstance checks
+- This ensures the code works correctly regardless of whether `pyodbc.Row` inherits from `tuple` in the installed pyodbc version
+
+## Testing Approach
+- Functional test (v0.239.014): `functional_tests/test_sql_query_plugin_schema_awareness.py`
+- Functional test (v0.239.015): `functional_tests/test_sql_auto_schema_companion.py`
+- Validates `_auto_create_companion_schema_plugin` method exists with correct signature
+- Confirms companion creation is triggered in `load_plugin_from_manifest` for `sql_query` type
+- Verifies schema plugin name derivation logic (`_query` → `_schema` suffix swap)
+- Checks `@kernel_function` descriptions are resilient (no hard dependency on non-existent functions)
+- Validates `_extract_sql_schema_for_instructions` has SQLQueryPlugin fallback
+- Confirms version updated to 0.239.015
+- Functional test (v0.239.016): `functional_tests/test_sql_schema_sys_catalog_views.py`
+- Validates all SQL Server queries use `sys.tables`, `sys.columns`, `sys.indexes` instead of `INFORMATION_SCHEMA`
+- Confirms pyodbc.Row-safe iteration (no `isinstance(table, tuple)` checks)
+- Verifies primary key query uses `sys.index_columns` with `is_primary_key = 1`
+- Checks PostgreSQL/MySQL/SQLite queries remain unchanged
+- Confirms version updated to 0.239.016
+
+## Impact Analysis
+- **SQL-connected agents**: Will now automatically have BOTH a query plugin AND a companion schema plugin, even when only `sql_query` is configured. Schema is injected into agent instructions at load time.
+- **Non-SQL agents**: Completely unaffected (companion creation only triggers for `sql_query` type)
+- **LogAnalytics agents**: Unaffected (different plugin type)
+- **Performance**: One-time schema fetch at agent load time adds minimal latency; schema is cached in instructions for the session
+- **Backwards compatible**: If both `sql_query` and `sql_schema` actions are explicitly configured, the companion auto-creation is skipped (checks for existing plugin)
+
+## Before/After Comparison
+
+### Before (v0.239.014)
+- User: "What is user1 licensed to use?"
+- Agent: "I need the exact user identifier... Please provide the identifier..." (no database call, no citations)
+- Root cause: Descriptions demanded calling schema functions that didn't exist in the kernel
+
+### After v0.239.015
+- User: "What is user1 licensed to use?"
+- Agent: Correctly returns Office 365 license data with LicenseID 1, TotalQuantity 52 (simple single-table queries work)
+- User: "Which department is spending the most on licensing?"
+- Agent: Fails — says "I don't see a department dimension in the current schema" and calls `get_database_schema` which returns `{'tables': {}}` (empty)
+- Root cause: INFORMATION_SCHEMA views returned no results on Azure SQL
+
+### After v0.239.016
+- User: "What is user1 licensed to use?"
+- Agent: Correctly returns license data (still works)
+- User: "Which department is spending the most on licensing?"
+- Agent: Has full schema with all 5 tables and their columns → can construct multi-table JOINs (Licenses → Procurements for cost, Usage for department) → returns department-level spending analysis
diff --git a/docs/explanation/fixes/v0.239.008/CHAT_TABULAR_SK_TRIGGER_FIX.md b/docs/explanation/fixes/v0.239.008/CHAT_TABULAR_SK_TRIGGER_FIX.md
new file mode 100644
index 00000000..d525e2d1
--- /dev/null
+++ b/docs/explanation/fixes/v0.239.008/CHAT_TABULAR_SK_TRIGGER_FIX.md
@@ -0,0 +1,67 @@
+# Chat-Uploaded Tabular File SK Mini-Agent Trigger Fix
+
+## Issue Description
+
+When a user uploads a tabular file (CSV, XLSX, XLS, XLSM) directly to a chat conversation and asks a question in model-only mode (no agent selected), the SK mini-agent (`run_tabular_sk_analysis`) did not trigger. The model would see instructions to "use plugin functions" but could not call them without an agent, resulting in the model describing what it would do instead of providing actual analysis results.
+
+The full agent mode worked correctly because the agent has direct access to the `TabularProcessingPlugin` and can call its functions.
+
+## Root Cause
+
+Three gaps prevented the mini SK agent from activating for chat-uploaded tabular files:
+
+1. **Streaming path ignored `file` role messages**: The streaming conversation history loop (`/api/chat/stream`) only processed `user` and `assistant` roles, making chat-uploaded files completely invisible to the model.
+
+2. **Mini SK only triggered from search results**: Both streaming and non-streaming paths only invoked `run_tabular_sk_analysis()` when tabular files appeared in hybrid search results (`combined_documents`). Chat-uploaded files are stored in blob storage as `file` role messages and are not indexed in Azure AI Search, so they never appeared in search results.
+
+3. **Model-only mode can't call plugin functions**: The non-streaming path's file handler injected "Use the tabular_processing plugin functions" as a system message, but in model-only mode the model has no function-calling capability.
+
+## Technical Details
+
+### Files Modified
+
+- `application/single_app/route_backend_chats.py` — All code changes
+- `application/single_app/config.py` — Version bump to 0.239.008
+
+### Code Changes
+
+#### Non-streaming path (`/api/chat`)
+
+1. Added `chat_tabular_files = set()` tracker before the conversation history loop (~line 1896)
+2. Added `chat_tabular_files.add(filename)` inside the `if is_table and file_content_source == 'blob':` block (~line 1936)
+3. After the history loop, added a block that checks `chat_tabular_files` and calls `run_tabular_sk_analysis(source_hint="chat")`, injecting pre-computed results as a system message (~line 2027)
+
+#### Streaming path (`/api/chat/stream`)
+
+4. Replaced the simple 8-line history loop (which only handled `user`/`assistant`) with expanded logic that mirrors the non-streaming path's `file` role handling, including blob tabular file tracking (~line 3687)
+5. Added the same mini SK trigger block after the expanded loop (~line 3751)
+
+### How It Works After Fix
+
+1. User uploads `sales.xlsx` to chat, asks "analyze sales/profit"
+2. During conversation history building, the `file` role message with `is_table=True` and `file_content_source='blob'` is detected
+3. The filename is collected into `chat_tabular_files`
+4. After the history loop, `run_tabular_sk_analysis()` is called with `source_hint="chat"`, which resolves the file from the `personal-chat` blob container
+5. The mini SK agent pre-loads the file schema, calls plugin functions (aggregate, filter, etc.), and returns computed results
+6. Results are injected as a system message so the model can present accurate numbers
+7. Plugin invocation citations are collected for transparency
+
+## Testing
+
+1. Upload a tabular file (xlsx/csv) directly to chat
+2. With no agent selected, send a data analysis question
+3. Verify the response contains actual computed data (not just a description of steps)
+4. Check logs for `[Chat Tabular SK]` entries confirming the mini SK trigger
+5. Verify agent mode still works as before
+
+## Impact
+
+- Enables tabular data analysis in model-only chat mode for chat-uploaded files
+- No changes to existing search-result-based tabular detection
+- No changes to full agent mode behavior
+- Streaming and non-streaming paths both fixed
+
+## Version
+
+- **Version**: 0.239.008
+- **Implemented in**: 0.239.008
diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md
index 478242a6..e27c5551 100644
--- a/docs/explanation/release_notes.md
+++ b/docs/explanation/release_notes.md
@@ -2,6 +2,206 @@
# Feature Release
+### **(v0.239.031)**
+
+#### New Features
+
+* **Persistent Conversation Summaries**
+ * Summaries generated during conversation export are now saved to the conversation document in Cosmos DB for future reuse.
+ * Cached summaries include `message_time_start` and `message_time_end` — when a conversation has new messages beyond the cached range, a fresh summary is generated automatically.
+ * The conversation details modal now shows a **Summary** card at the top. If a summary exists it displays the content, generation date, and model used. If no summary exists a **Generate Summary** button with model selector lets users create one on demand.
+ * A **Regenerate** button is available on existing summaries to force a refresh with the currently selected model.
+ * New `POST /api/conversations//summary` endpoint accepts an optional `model_deployment` and returns the generated summary.
+ * The `GET /api/conversations//metadata` response now includes a `summary` field.
+ * Extracted `generate_conversation_summary()` as a shared helper used by both the export pipeline and the new API endpoint.
+ * (Ref: `route_backend_conversation_export.py`, `route_backend_conversations.py`, `chat-conversation-details.js`, `functions_conversation_metadata.py`)
+
+* **PDF Conversation Export**
+ * Added PDF as a third export format option alongside JSON and Markdown, giving users a print-ready, visually styled conversation archive.
+ * PDF output renders chat messages with colored bubbles that mirror the live chat UI: blue for user messages, gray for assistant messages, green for file messages, and amber for system messages.
+ * Message content is converted from Markdown to HTML for rich formatting (bold, italic, code blocks, lists, tables) inside the PDF.
+ * Full appendix structure is included (metadata, message details, references, processing thoughts, supplemental messages), matching the Markdown export layout.
+ * Rendering uses PyMuPDF's Story API on US Letter paper with 0.5-inch margins and automatic multi-page overflow.
+ * Works with both single-file and ZIP packaging; intro summaries are supported in PDF as well.
+ * Frontend format step updated to a 3-column card grid with a new PDF card using the `bi-filetype-pdf` icon.
+ * (Ref: `route_backend_conversation_export.py`, `chat-export.js`, PyMuPDF Story API, conversation export workflow)
+
+* **Conversation Export Intro Summaries**
+ * Added an optional AI-generated intro summary step to the conversation export workflow, so each exported chat can begin with a short abstract before the full transcript.
+ * Summary model selection now reuses the same model list shown in the chat composer, keeping the export flow aligned with the main chat experience.
+ * Works for both JSON and Markdown exports, including ZIP exports where each conversation keeps its own summary metadata.
+ * (Ref: `route_backend_conversation_export.py`, `chat-export.js`, conversation export workflow)
+
+* **Agent & Action User Tracking (created_by / modified_by)**
+ * All agent and action documents (personal, group, and global) now include `created_by`, `created_at`, `modified_by`, and `modified_at` fields that track which user created or last modified the entity.
+ * On updates, the original `created_by` and `created_at` values are preserved while `modified_by` and `modified_at` are refreshed with the current user and timestamp.
+ * New optional `user_id` parameter added to `save_group_agent`, `save_global_agent`, `save_group_action`, and `save_global_action` for caller-supplied user tracking (backward-compatible, defaults to `None`).
+ * (Ref: `functions_personal_agents.py`, `functions_group_agents.py`, `functions_global_agents.py`, `functions_personal_actions.py`, `functions_group_actions.py`, `functions_global_actions.py`)
+
+* **Activity Logging for Agent & Action CRUD Operations**
+ * Every create, update, and delete operation on agents and actions now generates an activity log record in the `activity_logs` Cosmos DB container and Application Insights.
+ * Six new logging functions: `log_agent_creation`, `log_agent_update`, `log_agent_deletion`, `log_action_creation`, `log_action_update`, `log_action_deletion`.
+ * Activity records include: `user_id`, `activity_type`, `entity_type` (agent/action), `operation` (create/update/delete), `workspace_type` (personal/group/global), and `workspace_context` (group_id when applicable).
+ * Logging is fire-and-forget — failures never break the CRUD operation.
+ * All personal, group, and admin routes for both agents and actions are wired up.
+ * (Ref: `functions_activity_logging.py`, `route_backend_agents.py`, `route_backend_plugins.py`)
+
+* **Tabular Data Analysis — SK Mini-Agent for Normal Chat**
+ * Tabular files (CSV, XLSX, XLS, XLSM) detected in search results now trigger a lightweight Semantic Kernel mini-agent that pre-computes data analysis before the main LLM response. This brings the same analytical depth previously only available in full agent mode to every normal chat conversation.
+ * **Automatic Detection**: When AI Search results include tabular files from any workspace (personal, group, or public) or chat-uploaded documents, the system automatically identifies them via the `TABULAR_EXTENSIONS` configuration and routes the query through the SK mini-agent pipeline.
+ * **Unified Workspace and Chat Handling**: Tabular files are processed identically regardless of their storage location. The plugin resolves blob paths across all four container types (`user-documents`, `group-documents`, `public-documents`, `personal-chat`) with automatic fallback resolution if the primary source lookup fails. A user asking about an Excel file in their personal workspace gets the same analytical treatment as one asking about a CSV uploaded directly to a chat.
+ * **Six Data Analysis Functions**: The `TabularProcessingPlugin` exposes `describe_tabular_file`, `aggregate_column` (sum, mean, count, min, max, median, std, nunique, value_counts), `filter_rows` (==, !=, >, <, >=, <=, contains, startswith, endswith), `query_tabular_data` (pandas query syntax), `group_by_aggregate`, and `list_tabular_files` — all registered as Semantic Kernel functions that the mini-agent orchestrates autonomously.
+ * **Pre-Computed Results Injected as Context**: The mini-agent's computed analysis (exact numerical results, aggregations, filtered data) is injected into the main LLM's system context so it can present accurate, citation-backed answers without hallucinating numbers.
+ * **Graceful Degradation**: If the mini-agent analysis fails for any reason, the system falls back to instructing the main LLM to use the tabular processing plugin functions directly, preserving full functionality.
+ * **Non-Streaming and Streaming Support**: Both chat modes are supported. The mini-agent runs synchronously before the main LLM call in both paths.
+ * **Requires Enhanced Citations**: The tabular processing plugin depends on the blob storage client initialized by the enhanced citations system. The `enable_enhanced_citations` admin setting must be enabled for tabular data analysis to activate.
+ * (Ref: `run_tabular_sk_analysis()`, `TabularProcessingPlugin`, `collect_tabular_sk_citations()`, `TABULAR_EXTENSIONS`)
+
+* **Tabular Tool Execution Citations**
+ * Every tool call made by the SK mini-agent during tabular analysis is captured and surfaced as an agent citation, providing full transparency into the data analysis pipeline.
+ * **Automatic Capture**: The existing `@plugin_function_logger` decorator on all `TabularProcessingPlugin` functions records each invocation including function name, input parameters, returned results, execution duration, and success/failure status.
+ * **Citation Format**: Tool execution citations appear in the same "Agent Tool Execution" modal used by full agent mode, showing `tool_name` (e.g., `TabularProcessingPlugin.aggregate_column`), `function_arguments` (the exact parameters passed), and `function_result` (the computed data returned).
+ * **End-to-End Auditability**: Users can verify exactly which aggregations, filters, or queries were run against their data, what parameters were used, and what raw results were returned — before the LLM summarized them into the final response.
+ * (Ref: `collect_tabular_sk_citations()`, `plugin_invocation_logger.py`)
+
+* **SK Mini-Agent Performance Optimization**
+ * Reduced typical tabular analysis time from ~74 seconds to an estimated ~30-33 seconds (55-60% reduction) through three complementary optimizations.
+ * **DataFrame Caching**: Per-request in-memory cache eliminates redundant blob downloads. Previously, each of the ~8 tool calls in a typical analysis downloaded and parsed the same file independently. Now the file is downloaded once and subsequent calls read from cache. Cache is automatically scoped to the request (new plugin instance per analysis) and garbage-collected afterward.
+ * **Pre-Dispatch Schema Injection**: File schemas (columns, data types, row counts, and a 3-row preview) are pre-loaded and injected into the SK mini-agent's system prompt before execution begins. This eliminates 2 LLM round-trips that were previously spent on file discovery (`list_tabular_files`) and schema inspection (`describe_tabular_file`), allowing the model to jump directly to analysis tool calls.
+ * **Async Plugin Functions**: All six `@kernel_function` methods converted to `async def` using `asyncio.to_thread()`. This enables Semantic Kernel's built-in `asyncio.gather()` to truly parallelize batched tool calls (e.g., 3 simultaneous `aggregate_column` calls) instead of executing them serially on the event loop.
+ * **Batching Instructions**: The system prompt now instructs the model to batch multiple independent function calls in a single response, reducing LLM round-trips further.
+ * (Ref: `_df_cache`, `asyncio.to_thread`, pre-dispatch schema injection in `run_tabular_sk_analysis()`)
+
+* **SQL Test Connection Button**
+ * Added a "Test Connection" button to the SQL Database Configuration section (Step 3) of the action wizard, allowing users to validate database connectivity before saving.
+ * Supports all database types: SQL Server, Azure SQL (with managed identity), PostgreSQL, MySQL, and SQLite.
+ * Shows inline success/failure alerts with a 15-second timeout cap and sanitized error messages.
+ * New backend endpoint: `POST /api/plugins/test-sql-connection`.
+ * (Ref: `route_backend_plugins.py`, `plugin_modal_stepper.js`, `_plugin_modal.html`)
+
+#### Bug Fixes
+
+* **On-Demand Summary Generation — Content Normalization Fix**
+ * Fixed the `POST /api/conversations//summary` endpoint failing with an error when generating summaries from the conversation details modal.
+ * Root cause: message `content` in Cosmos DB can be a list of content parts (e.g., `[{type: "text", text: "..."}]`) rather than a plain string. The endpoint was passing the raw list as `content_text`, which either stringified incorrectly or produced empty transcript text.
+ * Now uses `_normalize_content()` to properly flatten list/dict content into plain text, matching the export pipeline's behavior.
+ * (Ref: `route_backend_conversations.py`, `_normalize_content`, `generate_conversation_summary`)
+
+* **Export Summary Reasoning-Model Compatibility**
+ * Fixed export intro summary generation failing or returning empty content with reasoning-series models (gpt-5, o1, o3) through a series of incremental fixes: using `developer` role instead of `system` for instruction messages, removing all `max_tokens` / `max_completion_tokens` caps so the model decides output length naturally, and adding null-safe content extraction for `None` responses.
+ * Summary now includes ALL messages (user, assistant, system, file, image analysis) for full context, with a simplified prompt producing 1-2 factual paragraphs.
+ * Added detailed debug logging showing message count, character count, model name, role, and finish reason.
+ * (Ref: `route_backend_conversation_export.py`, `_build_summary_intro`, `generate_conversation_summary`)
+
+* **Conversation Export Schema and Markdown Refresh**
+ * Fixed conversation exports lagging behind the live chat schema. JSON exports now include processing thoughts, normalized citations, and the raw document/web/tool citation buckets stored with assistant messages.
+ * Fixed Markdown exports being too flat and text-heavy by reorganizing them into a transcript-first layout with appendices for metadata, message details, references, thoughts, and supplemental records.
+ * Fixed exported conversations including content that no longer matched the visible chat by filtering deleted messages and inactive-thread retries, then reapplying thread-aware ordering before export.
+ * (Ref: `route_backend_conversation_export.py`, `test_conversation_export.py`, conversation export rendering)
+
+* **Export Tag/Classification Rendering Fix**
+ * Fixed conversation tags and classifications rendering as raw Python dicts (e.g., `{'category': 'model', 'value': 'gpt-5'}`) in both Markdown and PDF exports.
+ * Tags now display as readable `category: value` strings, with smart handling for participant names, document titles, and generic category/value pairs.
+ * (Ref: `route_backend_conversation_export.py`, `_format_tag` helper, Markdown/PDF metadata rendering)
+
+* **Export Summary Error Visibility**
+ * Added `debug_print` and `log_event` logging to all summary generation error paths, including the empty-response path that previously failed silently.
+ * The actual error detail is now shown in both Markdown and PDF exports when summary generation fails, replacing the generic "could not be generated" message.
+ * (Ref: `route_backend_conversation_export.py`, `_build_summary_intro`, export error rendering)
+
+* **Content Safety for Streaming Chat Path**
+ * Added full Azure AI Content Safety checking to the streaming (`/api/chat/stream`) SSE path, matching the existing non-streaming (`/api/chat`) implementation.
+ * Previously, only the non-streaming path performed content safety analysis; streaming conversations bypassed safety checks entirely.
+ * Implementation includes: `AnalyzeTextOptions` analysis, severity threshold checking (severity ≥ 4 blocks the message), blocklist matching, persistence of blocked messages to `cosmos_safety_container`, creation of safety-role message documents, and proper SSE event delivery of blocked status to the client.
+ * On block, the streaming generator yields the safety message and `[DONE]` event, then stops — preventing any further LLM invocation.
+ * Errors in the content safety call are caught and logged without breaking the chat flow, consistent with the non-streaming behavior.
+ * (Ref: `route_backend_chats.py`, streaming SSE generator, `AnalyzeTextOptions`, `cosmos_safety_container`)
+
+* **SQL Schema Plugin — Eliminate Redundant Schema Calls**
+ * Fixed agent calling `get_database_schema` twice per query even though the full schema was already injected into the agent's instructions at load time.
+ * Root cause: The `@kernel_function` descriptions in `sql_schema_plugin.py` said "ALWAYS call this function FIRST," which overrode the schema context already available in the instructions.
+ * Updated all four function descriptions (`get_database_schema`, `get_table_schema`, `get_table_list`, `get_relationships`) to use the resilient pattern: "If the database schema is already provided in your instructions, use that directly and do NOT call this function."
+ * This eliminates ~400ms+ of unnecessary database round trips per query and aligns with the same pattern already used in `sql_query_plugin.py`.
+ * (Ref: `sql_schema_plugin.py`, `@kernel_function` descriptions, schema injection)
+
+* **SQL Schema Plugin — Empty Tables from INFORMATION_SCHEMA**
+ * Fixed `get_database_schema` returning `'tables': {}` (empty) despite the database having tables, while relationships were returned correctly.
+ * Root cause: SQL Server table/column enumeration used `INFORMATION_SCHEMA.TABLES` and `INFORMATION_SCHEMA.COLUMNS` views, which returned empty results in the Azure SQL environment. Meanwhile, the relationships query used `sys.foreign_keys`/`sys.tables`/`sys.columns` catalog views which worked perfectly.
+ * Migrated all SQL Server schema queries to use `sys.*` catalog views consistently: `sys.tables`/`sys.schemas` for table enumeration, `sys.columns` with `TYPE_NAME()` for column details, and `sys.indexes`/`sys.index_columns` for primary key detection.
+ * Fixed `pyodbc.Row` handling throughout the plugin — removed all `isinstance(table, tuple)` checks that could fail with pyodbc Row objects, replaced with robust try/except indexing.
+ * This enables the full schema (tables, columns, types, PKs, FKs) to be injected into agent instructions, allowing agents to construct complex multi-table JOINs for analytical queries.
+ * (Ref: `sql_schema_plugin.py`, `sys.tables`, `sys.columns`, `sys.indexes`, pyodbc.Row handling)
+
+* **SQL Query Plugin — Auto-Create Companion Schema Plugin**
+ * Fixed the remaining issue where SQL-connected agents still asked for clarification instead of querying the database, even after description improvements.
+ * Root cause: Agents configured with only a `sql_query` action never had a `SQLSchemaPlugin` loaded in the kernel. The descriptions demanded calling `get_database_schema` — a function that didn't exist — creating an impossible dependency that caused the LLM to ask for clarification.
+ * `LoggedPluginLoader` now automatically creates a companion `SQLSchemaPlugin` whenever a `SQLQueryPlugin` is loaded, using the same connection details. This ensures schema discovery is always available.
+ * Updated `@kernel_function` descriptions to be resilient: "If the database schema is provided in your instructions, use it directly. Otherwise, call get_database_schema." This dual-path approach works whether schema is injected via instructions or available via plugin functions.
+ * Added fallback in `_extract_sql_schema_for_instructions()` to also detect `SQLQueryPlugin` instances and create a temporary schema extractor if no `SQLSchemaPlugin` is found.
+ * (Ref: `logged_plugin_loader.py`, `sql_query_plugin.py`, `semantic_kernel_loader.py`)
+
+* **SQL Query Plugin Schema Awareness**
+ * Fixed agents connected to SQL databases asking users for clarification about table/column names instead of querying the database directly.
+ * Root cause: SQL Query and SQL Schema plugin `@kernel_function` descriptions were generic with no workflow guidance, agent instructions had no database schema context, and the two plugins operated independently with no linkage.
+ * Rewrote all `@kernel_function` descriptions in both SQL plugins to be prescriptive workflow guides (modeled after the working LogAnalyticsPlugin), explicitly instructing the LLM to discover schema first before generating queries.
+ * Added auto-injection of database schema into agent instructions at load time — when SQL Schema plugins are detected, the full schema (tables, columns, types, relationships) is fetched and appended to the agent's system prompt.
+ * Added new `query_database(question, query)` convenience function to `SQLQueryPlugin` for intent-aligned tool calling.
+ * Enabled the SQL-specific plugin creation path in `logged_plugin_loader.py` (was previously commented out).
+ * (Ref: `sql_query_plugin.py`, `sql_schema_plugin.py`, `semantic_kernel_loader.py`, `logged_plugin_loader.py`)
+
+* **Chat-Uploaded Tabular Files Now Trigger SK Mini-Agent in Model-Only Mode**
+ * Fixed an issue where tabular files (CSV, XLSX, XLS, XLSM) uploaded directly to a chat conversation were not analyzed by the SK mini-agent when no agent was selected. The model would describe what analysis it would perform instead of returning actual computed results.
+ * **Root Cause**: The mini SK agent only triggered from search results, but chat-uploaded files are stored in blob storage and not indexed in Azure AI Search. Additionally, the streaming path completely ignored `file` role messages in conversation history.
+ * **Fix**: Both streaming and non-streaming chat paths now detect chat-uploaded tabular files during conversation history building and trigger `run_tabular_sk_analysis(source_hint="chat")` to pre-compute results. The streaming path also now properly handles `file` role messages (tabular and non-tabular) matching the non-streaming path's behavior.
+ * (Ref: `route_backend_chats.py`, `run_tabular_sk_analysis()`, `collect_tabular_sk_citations()`)
+
+* **Group SQL Action/Plugin Save Failure**
+ * Fixed group SQL actions (sql_query and sql_schema types) failing to save correctly due to missing endpoint placeholder. Group routes now apply the same `sql://sql_query` / `sql://sql_schema` endpoint logic as personal action routes.
+ * Fixed Step 4 (Advanced) dynamic fields overwriting Step 3 (Configuration) SQL values with empty strings during form data collection. SQL types now skip the dynamic field merge entirely since Step 3 already provides all necessary configuration.
+ * Fixed auth type definition schemas (`sql_query.definition.json`, `sql_schema.definition.json`) only allowing `connection_string` auth type, blocking `user`, `identity`, and `servicePrincipal` types that the UI and runtime support.
+ * Fixed `__Secret` key suffix mismatch in additional settings schemas where `connection_string__Secret` and `password__Secret` didn't match the runtime's expected `connection_string` and `password` field names. Also removed duplicate `azuresql` enum value.
+ * (Ref: `route_backend_plugins.py`, `plugin_modal_stepper.js`, `sql_query.definition.json`, `sql_schema.definition.json`, `sql_query_plugin.additional_settings.schema.json`, `sql_schema_plugin.additional_settings.schema.json`)
+
+#### User Interface Enhancements
+
+* **Agent Responded Thought — Seconds & Total Duration**
+ * The "responded" thought now shows time in **seconds** instead of milliseconds, and clarifies it is the total time from the initial user message (e.g., `'gpt-5-nano' responded (16.3s from initial message)`).
+ * A `request_start_time` is now captured at the top of both the non-streaming and streaming chat handlers, so the duration reflects the full request lifecycle — including content safety, hybrid search, and agent invocation — not just the model response time.
+ * Applies to all three agent paths: local SK agents (non-streaming), Azure AI Foundry agents, and streaming SK agents.
+ * (Ref: `route_backend_chats.py`, `request_start_time`, agent responded thoughts)
+
+* **Enhanced Agent Execution Thoughts**
+ * Added detailed model-level status messages during agent execution, giving users full visibility into each stage of the AI pipeline.
+ * **Model Identification**: A new "Sending to '{deployment_name}'" thought appears immediately after "Sending to agent", showing the exact model deployment being used (e.g., `gpt-5-nano`).
+ * **Generating Response**: A "Generating response..." thought now appears before the agent begins its invocation loop, matching the existing behavior for non-agent GPT calls.
+ * **Model Responded with Duration**: A "'{deployment_name}' responded ({duration}ms)" thought appears after the agent completes, showing total wall-clock execution time.
+ * Applies to all three agent paths: local SK agents (streaming and non-streaming) and Azure AI Foundry agents.
+ * Uses the existing `generation` step type (lightning bolt icon) — no frontend changes required.
+ * (Ref: `route_backend_chats.py`, `ThoughtTracker`, agent execution pipeline)
+
+* **List/Grid View Toggle for Agents and Actions**
+ * Added a list/grid view toggle to all four workspace areas: personal agents, personal actions, group agents, and group actions.
+ * **Grid View**: Large cards with type icon, humanized name, truncated description, and action buttons (Chat, View, Edit, Delete as applicable).
+ * **List View**: Improved table layout with fixed column widths (28%/47%/25%), humanized display names, and truncated descriptions with hover tooltips for full text.
+ * **View Button**: New eye-icon button on every agent and action that opens a read-only detail modal with gradient-header summary cards (Basic Information, Model Configuration, Instructions for agents; Basic Information, Configuration for actions).
+ * **Name Humanization**: Display names are now automatically parsed — underscores and camelCase/PascalCase boundaries are converted to properly spaced, title-cased words (e.g., `myCustomAgent` → `My Custom Agent`).
+ * **Persistent Preference**: View mode selection (list/grid) is saved per area in localStorage and restored on page load.
+ * New shared utility module `view-utils.js` provides reusable functions for all four workspace areas.
+ * (Ref: `view-utils.js`, `workspace_agents.js`, `workspace_plugins.js`, `plugin_common.js`, `group_agents.js`, `group_plugins.js`, `workspace.html`, `group_workspaces.html`, `styles.css`)
+
+* **Chat with Agent Button for Group Agents**
+ * Added a "Chat" button to each group agent row, allowing users to quickly select a group agent and navigate to the chat page.
+ * (Ref: `group_agents.js`, `group_workspaces.html`)
+
+* **Hidden Deprecated Action Types**
+ * Deprecated action types (`sql_schema`, `ui_test`, `queue_storage`, `blob_storage`, `embedding_model`) are now hidden from the action creation wizard type selector. Existing actions of these types remain functional.
+ * (Ref: `plugin_modal_stepper.js`)
+
+* **Advanced Settings Collapse Toggle**
+ * Step 4 (Advanced) content is now hidden behind a collapsible toggle button ("Show Advanced Settings") instead of being displayed by default. Reduces visual noise for most users.
+ * For SQL action types, the redundant additional fields UI in Step 4 is hidden entirely since all SQL configuration is already handled in Step 3.
+ * Step 5 (Summary) no longer shows the raw additional fields JSON dump for SQL types, since that data is already shown in the SQL Database Configuration summary card.
+ * (Ref: `_plugin_modal.html`, `plugin_modal_stepper.js`)
### **(v0.239.005)**
#### New Features
@@ -139,19 +339,6 @@
* **Files Modified**: `chat-documents.js`, `chat-messages.js`, `functions_search.py`, `route_backend_chats.py`, `chats.html`.
* (Ref: Multi-document selection, tag filtering, OData search integration, `CHAT_DOCUMENT_AND_TAG_FILTERING.md`)
-#### New Features
-
-* **Conversation Export**
- * Export one or multiple conversations from the Chat page in JSON or Markdown format.
- * **Single Export**: Use the ellipsis menu on any conversation to quickly export it.
- * **Multi-Export**: Enter selection mode, check the conversations you want, and click the export button.
- * A guided 4-step wizard walks you through selection review, format choice, packaging options (single file or ZIP archive), and download.
- * Sensitive internal metadata is automatically stripped from exported data for security.
-
-* **Retention Policy UI for Groups and Public Workspaces**
- * Can now configure conversation and document retention periods directly from the workspace and group management page.
- * Choose from preset retention periods ranging from 7 days to 10 years, use the organization default, or disable automatic deletion entirely.
-
#### Bug Fixes
* **Citation Parsing Bug Fix**
diff --git a/functional_tests/test_chat_tabular_sk_trigger.py b/functional_tests/test_chat_tabular_sk_trigger.py
new file mode 100644
index 00000000..9de91b39
--- /dev/null
+++ b/functional_tests/test_chat_tabular_sk_trigger.py
@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+# test_chat_tabular_sk_trigger.py
+"""
+Functional test for chat-uploaded tabular file SK mini-agent trigger fix.
+Version: 0.239.008
+Implemented in: 0.239.008
+
+This test ensures that when tabular data files (CSV, XLSX) are uploaded directly
+to a chat conversation, the SK mini-agent (`run_tabular_sk_analysis`) is properly
+triggered in model-only mode (no agent selected) to pre-compute analysis results.
+
+Previously, the mini SK agent only triggered from search results, missing
+chat-uploaded files entirely. The streaming path also ignored file role messages.
+
+This test validates:
+1. Chat-uploaded tabular files are detected in conversation history (file role)
+2. Filenames are collected into chat_tabular_files set for both blob and inline cases
+3. The streaming path properly handles file role messages (not just user/assistant)
+4. The mini SK trigger block activates when chat_tabular_files is non-empty
+"""
+
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'application', 'single_app'))
+
+
+def test_chat_tabular_file_detection():
+ """Test that chat-uploaded tabular files are detected from file role messages."""
+ print("Testing chat-uploaded tabular file detection...")
+
+ try:
+ # Simulate conversation messages as they would appear from Cosmos DB
+ recent_messages = [
+ {
+ 'role': 'file',
+ 'filename': 'Sample_-_Superstore_1.xlsx',
+ 'file_content': '',
+ 'is_table': True,
+ 'file_content_source': 'blob',
+ 'metadata': {},
+ },
+ {
+ 'role': 'user',
+ 'content': 'analyze sales/profit',
+ 'metadata': {},
+ },
+ ]
+
+ # Simulate the conversation history building logic (streaming path)
+ conversation_history_for_api = []
+ allowed_roles_in_history = ['user', 'assistant']
+ max_file_content_length_in_history = 50000
+ max_tabular_content_length_in_history = 50000
+ chat_tabular_files = set()
+
+ for message in recent_messages:
+ role = message.get('role')
+ content = message.get('content', '')
+
+ if role in allowed_roles_in_history:
+ conversation_history_for_api.append({
+ 'role': role,
+ 'content': content
+ })
+ elif role == 'file':
+ filename = message.get('filename', 'uploaded_file')
+ file_content = message.get('file_content', '')
+ is_table = message.get('is_table', False)
+ file_content_source = message.get('file_content_source', '')
+
+ if is_table and file_content_source == 'blob':
+ chat_tabular_files.add(filename)
+ conversation_history_for_api.append({
+ 'role': 'system',
+ 'content': (
+ f"[User uploaded a tabular data file named '{filename}'. "
+ f"The file is stored in blob storage and available for analysis. "
+ f"The file source is 'chat'.]"
+ )
+ })
+
+ # Verify file was detected
+ assert 'Sample_-_Superstore_1.xlsx' in chat_tabular_files, \
+ f"Expected xlsx file in chat_tabular_files, got: {chat_tabular_files}"
+
+ # Verify system message was added for the file
+ system_msgs = [m for m in conversation_history_for_api if m['role'] == 'system']
+ assert len(system_msgs) == 1, f"Expected 1 system message, got {len(system_msgs)}"
+ assert 'Sample_-_Superstore_1.xlsx' in system_msgs[0]['content'], \
+ "System message should reference the uploaded file"
+
+ # Verify user message was included
+ user_msgs = [m for m in conversation_history_for_api if m['role'] == 'user']
+ assert len(user_msgs) == 1, f"Expected 1 user message, got {len(user_msgs)}"
+ assert user_msgs[0]['content'] == 'analyze sales/profit', \
+ "User message content should be preserved"
+
+ print("Test passed!")
+ return True
+
+ except Exception as e:
+ print(f"Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_non_tabular_file_handling():
+ """Test that non-tabular files are handled with content preview."""
+ print("Testing non-tabular file handling in streaming path...")
+
+ try:
+ recent_messages = [
+ {
+ 'role': 'file',
+ 'filename': 'report.txt',
+ 'file_content': 'This is the report content.',
+ 'is_table': False,
+ 'file_content_source': 'inline',
+ 'metadata': {},
+ },
+ {
+ 'role': 'user',
+ 'content': 'summarize the report',
+ 'metadata': {},
+ },
+ ]
+
+ conversation_history_for_api = []
+ allowed_roles_in_history = ['user', 'assistant']
+ max_file_content_length_in_history = 50000
+ max_tabular_content_length_in_history = 50000
+ chat_tabular_files = set()
+
+ for message in recent_messages:
+ role = message.get('role')
+ content = message.get('content', '')
+
+ if role in allowed_roles_in_history:
+ conversation_history_for_api.append({
+ 'role': role,
+ 'content': content
+ })
+ elif role == 'file':
+ filename = message.get('filename', 'uploaded_file')
+ file_content = message.get('file_content', '')
+ is_table = message.get('is_table', False)
+ file_content_source = message.get('file_content_source', '')
+
+ if is_table and file_content_source == 'blob':
+ chat_tabular_files.add(filename)
+ conversation_history_for_api.append({
+ 'role': 'system',
+ 'content': f"[User uploaded a tabular data file named '{filename}'.]"
+ })
+ else:
+ content_limit = (
+ max_tabular_content_length_in_history if is_table
+ else max_file_content_length_in_history
+ )
+ display_content = file_content[:content_limit]
+ if len(file_content) > content_limit:
+ display_content += "..."
+
+ conversation_history_for_api.append({
+ 'role': 'system',
+ 'content': (
+ f"[User uploaded a file named '{filename}'. "
+ f"Content preview:\n{display_content}]\n"
+ f"Use this file context if relevant."
+ )
+ })
+
+ # Non-tabular files should NOT be in chat_tabular_files
+ assert len(chat_tabular_files) == 0, \
+ f"Non-tabular files should not be tracked, got: {chat_tabular_files}"
+
+ # System message should contain file content preview
+ system_msgs = [m for m in conversation_history_for_api if m['role'] == 'system']
+ assert len(system_msgs) == 1, f"Expected 1 system message, got {len(system_msgs)}"
+ assert 'report.txt' in system_msgs[0]['content'], \
+ "System message should reference the file"
+ assert 'This is the report content.' in system_msgs[0]['content'], \
+ "System message should include file content preview"
+
+ print("Test passed!")
+ return True
+
+ except Exception as e:
+ print(f"Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_multiple_tabular_files():
+ """Test detection of multiple tabular files uploaded to chat."""
+ print("Testing multiple tabular file detection...")
+
+ try:
+ recent_messages = [
+ {
+ 'role': 'file',
+ 'filename': 'sales_2024.xlsx',
+ 'file_content': '',
+ 'is_table': True,
+ 'file_content_source': 'blob',
+ 'metadata': {},
+ },
+ {
+ 'role': 'file',
+ 'filename': 'inventory.csv',
+ 'file_content': 'item,count\nwidget,100',
+ 'is_table': True,
+ 'file_content_source': 'blob',
+ 'metadata': {},
+ },
+ {
+ 'role': 'user',
+ 'content': 'compare sales to inventory',
+ 'metadata': {},
+ },
+ ]
+
+ chat_tabular_files = set()
+ conversation_history_for_api = []
+ allowed_roles_in_history = ['user', 'assistant']
+
+ for message in recent_messages:
+ role = message.get('role')
+ content = message.get('content', '')
+
+ if role in allowed_roles_in_history:
+ conversation_history_for_api.append({'role': role, 'content': content})
+ elif role == 'file':
+ filename = message.get('filename', 'uploaded_file')
+ is_table = message.get('is_table', False)
+ file_content_source = message.get('file_content_source', '')
+
+ if is_table and file_content_source == 'blob':
+ chat_tabular_files.add(filename)
+ conversation_history_for_api.append({
+ 'role': 'system',
+ 'content': f"[Tabular file '{filename}' available for analysis.]"
+ })
+
+ assert len(chat_tabular_files) == 2, \
+ f"Expected 2 tabular files, got {len(chat_tabular_files)}"
+ assert 'sales_2024.xlsx' in chat_tabular_files, "Missing sales_2024.xlsx"
+ assert 'inventory.csv' in chat_tabular_files, "Missing inventory.csv"
+
+ print("Test passed!")
+ return True
+
+ except Exception as e:
+ print(f"Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_inline_tabular_not_tracked_for_sk():
+ """Test that inline tabular files (not blob) are NOT tracked for SK analysis."""
+ print("Testing inline tabular files are not tracked for SK analysis...")
+
+ try:
+ recent_messages = [
+ {
+ 'role': 'file',
+ 'filename': 'data.csv',
+ 'file_content': 'col1,col2\n1,2\n3,4',
+ 'is_table': True,
+ 'file_content_source': 'inline', # Not blob
+ 'metadata': {},
+ },
+ ]
+
+ chat_tabular_files = set()
+
+ for message in recent_messages:
+ role = message.get('role')
+ if role == 'file':
+ is_table = message.get('is_table', False)
+ file_content_source = message.get('file_content_source', '')
+ filename = message.get('filename', '')
+
+ if is_table and file_content_source == 'blob':
+ chat_tabular_files.add(filename)
+
+ assert len(chat_tabular_files) == 0, \
+ f"Inline tabular files should not be tracked for SK, got: {chat_tabular_files}"
+
+ print("Test passed!")
+ return True
+
+ except Exception as e:
+ print(f"Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+if __name__ == "__main__":
+ results = []
+ results.append(test_chat_tabular_file_detection())
+ results.append(test_non_tabular_file_handling())
+ results.append(test_multiple_tabular_files())
+ results.append(test_inline_tabular_not_tracked_for_sk())
+
+ print(f"\n{'='*50}")
+ print(f"Results: {sum(results)}/{len(results)} tests passed")
+ if all(results):
+ print("All tests passed!")
+ sys.exit(0)
+ else:
+ print("Some tests failed!")
+ sys.exit(1)
diff --git a/functional_tests/test_conversation_export.py b/functional_tests/test_conversation_export.py
index cb8e56d0..654059b1 100644
--- a/functional_tests/test_conversation_export.py
+++ b/functional_tests/test_conversation_export.py
@@ -1,365 +1,545 @@
#!/usr/bin/env python3
# test_conversation_export.py
"""
-Functional test for conversation export feature.
-Version: 0.237.050
-Implemented in: 0.237.050
-
-This test validates the conversation export backend endpoint
-and ensures JSON/Markdown formats and single/ZIP packaging work correctly.
+Functional test for conversation export enhancements.
+Version: 0.239.029
+Implemented in: 0.239.022 (base), 0.239.023 (PDF export), 0.239.025 (tag formatting fix), 0.239.028 (summary token budget fix)
+
+This test ensures conversation export now includes normalized and raw citations,
+processing thoughts, transcript-style Markdown appendices, deleted-message filtering,
+optional summary-intro metadata, PDF export with chat-bubble styling, and properly
+formatted tag/classification rendering.
"""
-import sys
-import os
+import io
import json
+import os
+import sys
import zipfile
-import io
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app'))
+from route_backend_conversation_export import ( # noqa: E402
+ _build_citation_counts,
+ _build_pdf_html_body,
+ _collect_raw_citation_buckets,
+ _conversation_to_markdown,
+ _conversation_to_pdf_bytes,
+ _filter_messages_for_export,
+ _format_tag,
+ _normalize_citations,
+ _normalize_content,
+ _pdf_bubble_class,
+ _safe_filename,
+ _sanitize_conversation,
+ _sanitize_message,
+ _truncate_for_summary,
+)
+
+
+def test_filter_messages_for_export():
+ """Test that deleted and inactive-thread messages are excluded."""
+ print("🔍 Testing export message filtering...")
-def test_sanitize_conversation():
- """Test that _sanitize_conversation strips internal fields."""
- print("🔍 Testing _sanitize_conversation...")
-
- raw_conversation = {
- 'id': 'conv-123',
- 'title': 'Test Conversation',
- 'last_updated': '2025-01-01T00:00:00Z',
- 'chat_type': 'personal',
- 'tags': ['test'],
- 'is_pinned': False,
- 'context': [],
- 'user_id': 'secret-user-id',
- '_rid': 'cosmos-internal-rid',
- '_self': 'cosmos-self-link',
- '_etag': 'some-etag',
- '_attachments': 'attachments',
- '_ts': 1234567890,
- 'partition_key': 'should-not-appear'
- }
+ messages = [
+ {'id': 'm1', 'role': 'user', 'content': 'Keep me', 'metadata': {}},
+ {'id': 'm2', 'role': 'assistant', 'content': 'Delete me', 'metadata': {'is_deleted': True}},
+ {'id': 'm3', 'role': 'assistant', 'content': 'Inactive retry', 'metadata': {'thread_info': {'active_thread': False}}},
+ {'id': 'm4', 'role': 'assistant', 'content': 'Active reply', 'metadata': {'thread_info': {'active_thread': True}}},
+ {'id': 'm5', 'role': 'user', 'content': 'Legacy message', 'metadata': {'thread_info': {}}},
+ ]
- # Import after path setup — may fail if dependencies aren't installed
- try:
- from route_backend_conversation_export import register_route_backend_conversation_export
- print(" Module imported successfully (dependencies available)")
- except ImportError as ie:
- print(f" Skipping import test (missing dependency: {ie})")
- print(" Verifying sanitization logic inline instead...")
-
- # We test the logic manually since inner functions are not directly accessible
- sanitized = {
- 'id': raw_conversation.get('id'),
- 'title': raw_conversation.get('title', 'Untitled'),
- 'last_updated': raw_conversation.get('last_updated', ''),
- 'chat_type': raw_conversation.get('chat_type', 'personal'),
- 'tags': raw_conversation.get('tags', []),
- 'is_pinned': raw_conversation.get('is_pinned', False),
- 'context': raw_conversation.get('context', [])
- }
+ filtered = _filter_messages_for_export(messages)
+ filtered_ids = [message['id'] for message in filtered]
- assert 'id' in sanitized, "Should retain id"
- assert 'title' in sanitized, "Should retain title"
- assert 'user_id' not in sanitized, "Should strip user_id"
- assert '_rid' not in sanitized, "Should strip Cosmos internal fields"
- assert '_etag' not in sanitized, "Should strip _etag"
- assert 'partition_key' not in sanitized, "Should strip partition_key"
+ assert filtered_ids == ['m1', 'm4', 'm5'], f"Unexpected filtered IDs: {filtered_ids}"
- print("✅ _sanitize_conversation test passed!")
+ print("✅ Export message filtering test passed!")
return True
-def test_sanitize_message():
- """Test that _sanitize_message strips internal fields."""
- print("🔍 Testing _sanitize_message...")
+def test_sanitize_message_with_citations_and_thoughts():
+ """Test that sanitized messages retain normalized/raw citations and thoughts."""
+ print("🔍 Testing message sanitization with citations and thoughts...")
- raw_message = {
- 'id': 'msg-456',
+ assistant_message = {
+ 'id': 'assistant-1',
'role': 'assistant',
- 'content': 'Hello, how can I help?',
- 'timestamp': '2025-01-01T00:00:01Z',
- 'citations': [{'title': 'Doc1', 'url': 'https://example.com'}],
- 'conversation_id': 'conv-123',
- 'user_id': 'secret-user-id',
- '_rid': 'cosmos-internal',
- 'metadata': {'thread_info': {'active_thread': True}},
+ 'content': [{'type': 'text', 'text': 'Answer with evidence.'}],
+ 'timestamp': '2026-03-06T12:00:02Z',
+ 'augmented': True,
+ 'hybrid_citations': [
+ {
+ 'file_name': 'paper.pdf',
+ 'page_number': 4,
+ 'citation_id': 'doc-1_4',
+ 'chunk_id': 'chunk-1',
+ 'metadata_type': 'abstract',
+ 'metadata_content': 'Study abstract text.'
+ }
+ ],
+ 'web_search_citations': [
+ {'title': 'Example Source', 'url': 'https://example.com/source'}
+ ],
+ 'agent_citations': [
+ {
+ 'tool_name': 'web_lookup',
+ 'function_name': 'azure_ai_foundry_web_search',
+ 'plugin_name': 'azure_ai_foundry',
+ 'function_arguments': {'query': 'test'},
+ 'function_result': {'answer': 'done'},
+ 'timestamp': '2026-03-06T12:00:01Z',
+ 'success': True
+ }
+ ],
+ 'metadata': {
+ 'reasoning_effort': 'medium',
+ 'token_usage': {
+ 'prompt_tokens': 10,
+ 'completion_tokens': 20,
+ 'total_tokens': 30
+ }
+ },
+ 'model_deployment_name': 'gpt-5-mini',
+ 'hybridsearch_query': 'refined search query'
}
+ thoughts = [
+ {
+ 'step_index': 0,
+ 'step_type': 'search',
+ 'content': 'Searching documents...',
+ 'detail': 'query=refined search query',
+ 'duration_ms': 123,
+ 'timestamp': '2026-03-06T12:00:01Z'
+ }
+ ]
- result = {
- 'role': raw_message.get('role', ''),
- 'content': raw_message.get('content', ''),
- 'timestamp': raw_message.get('timestamp', ''),
- }
- if raw_message.get('citations'):
- result['citations'] = raw_message['citations']
-
- assert result['role'] == 'assistant', "Should retain role"
- assert result['content'] == 'Hello, how can I help?', "Should retain content"
- assert 'citations' in result, "Should retain citations"
- assert 'user_id' not in result, "Should strip user_id"
- assert '_rid' not in result, "Should strip Cosmos internal fields"
- assert 'conversation_id' not in result, "Should strip conversation_id"
- assert 'metadata' not in result, "Should strip metadata"
-
- print("✅ _sanitize_message test passed!")
+ sanitized = _sanitize_message(assistant_message, sequence_index=2, transcript_index=2, thoughts=thoughts)
+
+ assert sanitized['content_text'] == 'Answer with evidence.', 'Content should normalize list-based text.'
+ assert sanitized['label'] == 'Turn 2', 'Transcript messages should use turn labels.'
+ assert sanitized['citation_counts']['document'] == 1, 'Document citation count should be preserved.'
+ assert sanitized['citation_counts']['web'] == 1, 'Web citation count should be preserved.'
+ assert sanitized['citation_counts']['agent_tool'] == 1, 'Agent citation count should be preserved.'
+ assert len(sanitized['citations']) == 3, 'Normalized citations should include all raw buckets.'
+ assert sanitized['hybrid_citations'][0]['file_name'] == 'paper.pdf', 'Raw hybrid citations should be preserved.'
+ assert sanitized['thoughts'][0]['step_type'] == 'search', 'Thoughts should be attached to the message.'
+ assert sanitized['details']['generation']['model_deployment'] == 'gpt-5-mini', 'Generation details should include the model.'
+
+ print("✅ Message sanitization test passed!")
+ return True
+
+
+def test_conversation_metadata_and_json_shape():
+ """Test that conversation-level metadata captures counts for export JSON."""
+ print("🔍 Testing conversation metadata and JSON shape...")
+
+ messages = [
+ {
+ 'id': 'u1',
+ 'role': 'user',
+ 'is_transcript_message': True,
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thoughts': []
+ },
+ {
+ 'id': 'a1',
+ 'role': 'assistant',
+ 'is_transcript_message': True,
+ 'citation_counts': {'document': 1, 'web': 1, 'agent_tool': 0, 'legacy': 0, 'total': 2},
+ 'thoughts': [{'step_type': 'search'}]
+ },
+ {
+ 'id': 'f1',
+ 'role': 'file',
+ 'is_transcript_message': False,
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thoughts': []
+ }
+ ]
+
+ conversation = _sanitize_conversation(
+ {
+ 'id': 'conv-123',
+ 'title': 'Export Test',
+ 'last_updated': '2026-03-06T12:00:00Z',
+ 'chat_type': 'personal',
+ 'tags': ['export', 'thoughts'],
+ 'context': ['workspace-a'],
+ 'classification': ['research'],
+ 'strict': False,
+ 'is_pinned': True,
+ 'scope_locked': True,
+ 'locked_contexts': ['workspace-a']
+ },
+ messages=messages,
+ role_counts={'user': 1, 'assistant': 1, 'file': 1},
+ citation_counts={'document': 1, 'web': 1, 'agent_tool': 0, 'legacy': 0, 'total': 2},
+ thought_count=1
+ )
+
+ exported = [{'conversation': conversation, 'summary_intro': {'enabled': False, 'generated': False}, 'messages': messages}]
+ parsed = json.loads(json.dumps(exported, indent=2, ensure_ascii=False, default=str))
+
+ assert parsed[0]['conversation']['transcript_message_count'] == 2, 'Transcript count should exclude supplemental messages.'
+ assert parsed[0]['conversation']['citation_counts']['total'] == 2, 'Conversation citation counts should be included.'
+ assert parsed[0]['summary_intro']['enabled'] is False, 'Summary intro status should be included in JSON.'
+
+ print("✅ Conversation metadata and JSON shape test passed!")
return True
-def test_conversation_to_markdown():
- """Test markdown generation from a conversation entry."""
- print("🔍 Testing markdown generation...")
+def test_markdown_export_structure():
+ """Test transcript-style Markdown with appendices, citations, and thoughts."""
+ print("🔍 Testing Markdown export structure...")
entry = {
'conversation': {
'id': 'conv-123',
- 'title': 'My Test Chat',
- 'last_updated': '2025-01-01T12:00:00Z',
+ 'title': 'My Exported Chat',
+ 'last_updated': '2026-03-06T12:00:00Z',
'chat_type': 'personal',
- 'tags': ['important', 'test'],
+ 'tags': ['science'],
+ 'classification': ['research'],
+ 'context': ['workspace-a'],
+ 'strict': False,
'is_pinned': False,
- 'context': []
+ 'scope_locked': True,
+ 'locked_contexts': ['workspace-a'],
+ 'message_count': 3,
+ 'message_counts_by_role': {'user': 1, 'assistant': 1, 'file': 1},
+ 'citation_counts': {'document': 1, 'web': 1, 'agent_tool': 1, 'legacy': 0, 'total': 3},
+ 'thought_count': 1
+ },
+ 'summary_intro': {
+ 'enabled': True,
+ 'generated': True,
+ 'model_deployment': 'gpt-5-mini',
+ 'generated_at': '2026-03-06T12:05:00Z',
+ 'content': 'A concise abstract.\n\n- Key point one\n- Key point two',
+ 'error': None
},
'messages': [
{
+ 'id': 'u1',
'role': 'user',
- 'content': 'Hello!',
- 'timestamp': '2025-01-01T12:00:01Z'
+ 'speaker_label': 'User',
+ 'label': 'Turn 1',
+ 'sequence_index': 1,
+ 'transcript_index': 1,
+ 'is_transcript_message': True,
+ 'timestamp': '2026-03-06T12:00:01Z',
+ 'content': 'What did the paper conclude?',
+ 'content_text': 'What did the paper conclude?',
+ 'details': {'interaction_mode': {'workspace_search': {'search_enabled': True, 'document_scope': 'personal'}}},
+ 'citations': [],
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thoughts': [],
+ 'legacy_citations': [],
+ 'hybrid_citations': [],
+ 'web_search_citations': [],
+ 'agent_citations': []
},
{
+ 'id': 'a1',
'role': 'assistant',
- 'content': 'Hi there! How can I help you?',
- 'timestamp': '2025-01-01T12:00:02Z',
- 'citations': [{'title': 'Doc1'}]
+ 'speaker_label': 'Assistant',
+ 'label': 'Turn 2',
+ 'sequence_index': 2,
+ 'transcript_index': 2,
+ 'is_transcript_message': True,
+ 'timestamp': '2026-03-06T12:00:02Z',
+ 'content': 'The paper concluded the intervention improved outcomes.',
+ 'content_text': 'The paper concluded the intervention improved outcomes.',
+ 'details': {'generation': {'model_deployment': 'gpt-5-mini', 'citation_counts': {'document': 1, 'web': 1, 'agent_tool': 1, 'legacy': 0, 'total': 3}}},
+ 'citations': [
+ {'citation_type': 'document', 'label': 'paper.pdf — Page 4', 'citation_id': 'doc-1_4', 'page_number': 4, 'classification': 'research', 'metadata_type': 'abstract', 'metadata_content': 'Study abstract text.'},
+ {'citation_type': 'web', 'label': 'Example Source', 'title': 'Example Source', 'url': 'https://example.com/source'},
+ {'citation_type': 'agent_tool', 'label': 'web_lookup'}
+ ],
+ 'citation_counts': {'document': 1, 'web': 1, 'agent_tool': 1, 'legacy': 0, 'total': 3},
+ 'thoughts': [{'step_type': 'search', 'content': 'Searching documents...', 'detail': 'query=paper outcome', 'duration_ms': 88, 'timestamp': '2026-03-06T12:00:01Z'}],
+ 'legacy_citations': [],
+ 'hybrid_citations': [{'file_name': 'paper.pdf'}],
+ 'web_search_citations': [{'title': 'Example Source', 'url': 'https://example.com/source'}],
+ 'agent_citations': [{'tool_name': 'web_lookup', 'function_name': 'azure_ai_foundry_web_search', 'plugin_name': 'azure_ai_foundry', 'function_arguments': {'query': 'paper outcome'}, 'function_result': {'answer': 'done'}, 'timestamp': '2026-03-06T12:00:01Z', 'success': True}]
+ },
+ {
+ 'id': 'f1',
+ 'role': 'file',
+ 'speaker_label': 'File',
+ 'label': 'Message 3',
+ 'sequence_index': 3,
+ 'transcript_index': None,
+ 'is_transcript_message': False,
+ 'timestamp': '2026-03-06T12:00:00Z',
+ 'content': 'paper.pdf',
+ 'content_text': 'paper.pdf',
+ 'details': {'message_context': {'filename': 'paper.pdf'}},
+ 'citations': [],
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thoughts': [],
+ 'legacy_citations': [],
+ 'hybrid_citations': [],
+ 'web_search_citations': [],
+ 'agent_citations': []
}
]
}
- # Replicate the markdown conversion logic
- conv = entry['conversation']
- messages = entry['messages']
- lines = []
- lines.append(f"# {conv['title']}")
- lines.append('')
- lines.append(f"**Last Updated:** {conv['last_updated']} ")
- lines.append(f"**Chat Type:** {conv['chat_type']} ")
- if conv.get('tags'):
- lines.append(f"**Tags:** {', '.join(conv['tags'])} ")
- lines.append(f"**Messages:** {len(messages)} ")
- lines.append('')
- lines.append('---')
- lines.append('')
-
- for msg in messages:
- role = msg.get('role', 'unknown')
- role_label = role.capitalize()
- if role == 'assistant':
- role_label = 'Assistant'
- elif role == 'user':
- role_label = 'User'
- lines.append(f"### {role_label}")
- if msg.get('timestamp'):
- lines.append(f"*{msg['timestamp']}*")
- lines.append('')
- lines.append(msg.get('content', ''))
- lines.append('')
- if msg.get('citations'):
- lines.append('**Citations:**')
- for cit in msg['citations']:
- if isinstance(cit, dict):
- source = cit.get('title') or cit.get('filepath') or cit.get('url', 'Unknown')
- lines.append(f"- {source}")
- lines.append('')
- lines.append('---')
- lines.append('')
-
- markdown = '\n'.join(lines)
-
- assert '# My Test Chat' in markdown, "Should have title as H1"
- assert '**Last Updated:**' in markdown, "Should have last updated"
- assert '**Tags:** important, test' in markdown, "Should list tags"
- assert '### User' in markdown, "Should have user heading"
- assert '### Assistant' in markdown, "Should have assistant heading"
- assert 'Hello!' in markdown, "Should contain user message"
- assert 'Hi there! How can I help you?' in markdown, "Should contain assistant reply"
- assert '**Citations:**' in markdown, "Should include citations section"
- assert '- Doc1' in markdown, "Should list citation title"
-
- print("✅ Markdown generation test passed!")
+ markdown = _conversation_to_markdown(entry)
+
+ assert '# My Exported Chat' in markdown, 'Markdown should include the title heading.'
+ assert '## Abstract' in markdown, 'Markdown should include the abstract section.'
+ assert '## Transcript' in markdown, 'Markdown should include the transcript section.'
+ assert '## Appendix B — Message Details' in markdown, 'Markdown should include the message-details appendix.'
+ assert '## Appendix C — References' in markdown, 'Markdown should include the references appendix.'
+ assert '## Appendix D — Processing Thoughts' in markdown, 'Markdown should include the thoughts appendix.'
+ assert '## Appendix E — Supplemental Messages' in markdown, 'Markdown should include supplemental messages.'
+ assert 'paper.pdf — Page 4' in markdown, 'Document citation labels should appear in references.'
+ assert 'Example Source' in markdown, 'Web citations should appear in references.'
+ assert 'Searching documents...' in markdown, 'Thought content should appear in the appendix.'
+
+ print("✅ Markdown export structure test passed!")
return True
-def test_json_export_structure():
- """Test that JSON export produces the expected structure."""
- print("🔍 Testing JSON export structure...")
+def test_safe_filename_and_zip_packaging():
+ """Test filename sanitization and ZIP naming consistency."""
+ print("🔍 Testing safe filename and ZIP packaging...")
+
+ assert _safe_filename('Normal Title') == 'Normal_Title', 'Spaces should become underscores.'
+ assert _safe_filename('File/With:Bad*Chars') == 'File_With_Bad_Chars', 'Unsafe characters should be replaced.'
+ assert _safe_filename('A' * 100) == 'A' * 50, 'Long names should be truncated.'
+ assert _safe_filename('') == 'Untitled', 'Empty titles should use Untitled.'
exported = [
- {
- 'conversation': {
- 'id': 'conv-abc',
- 'title': 'Test Convo',
- 'last_updated': '2025-01-01T00:00:00Z',
- 'chat_type': 'personal',
- 'tags': [],
- 'is_pinned': False,
- 'context': []
- },
- 'messages': [
- {'role': 'user', 'content': 'Hello', 'timestamp': '2025-01-01T00:00:01Z'},
- {'role': 'assistant', 'content': 'World', 'timestamp': '2025-01-01T00:00:02Z'}
- ]
- }
+ {'conversation': {'id': 'conv-001-extra', 'title': 'First Chat'}, 'messages': []},
+ {'conversation': {'id': 'conv-002-extra', 'title': 'Second Chat'}, 'messages': []}
]
- content = json.dumps(exported, indent=2, ensure_ascii=False, default=str)
- parsed = json.loads(content)
+ buffer = io.BytesIO()
+ with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as archive:
+ for entry in exported:
+ conversation = entry['conversation']
+ safe_title = _safe_filename(conversation.get('title', 'Untitled'))
+ conversation_id_short = conversation.get('id', 'unknown')[:8]
+ archive.writestr(f"{safe_title}_{conversation_id_short}.json", json.dumps(entry))
- assert isinstance(parsed, list), "Export should be a list"
- assert len(parsed) == 1, "Should have one conversation"
- assert 'conversation' in parsed[0], "Each entry should have conversation"
- assert 'messages' in parsed[0], "Each entry should have messages"
- assert len(parsed[0]['messages']) == 2, "Should have 2 messages"
- assert parsed[0]['conversation']['title'] == 'Test Convo', "Title should match"
+ buffer.seek(0)
+ with zipfile.ZipFile(buffer, 'r') as archive:
+ names = archive.namelist()
+ assert 'First_Chat_conv-001.json' in names, f"Unexpected ZIP entries: {names}"
+ assert 'Second_Chat_conv-002.json' in names, f"Unexpected ZIP entries: {names}"
- print("✅ JSON export structure test passed!")
+ print("✅ Safe filename and ZIP packaging test passed!")
return True
-def test_zip_packaging():
- """Test that ZIP packaging creates valid archive with correct entries."""
- print("🔍 Testing ZIP packaging...")
+def test_content_and_citation_helpers():
+ """Test content normalization, citation normalization, and summary truncation helpers."""
+ print("🔍 Testing helper utilities...")
- exported = [
- {
- 'conversation': {
- 'id': 'conv-001-abc-def',
- 'title': 'First Chat',
- 'last_updated': '2025-01-01',
- 'chat_type': 'personal',
- 'tags': [],
- 'is_pinned': False,
- 'context': []
- },
- 'messages': [
- {'role': 'user', 'content': 'Hello', 'timestamp': '2025-01-01'}
- ]
- },
- {
- 'conversation': {
- 'id': 'conv-002-xyz-ghi',
- 'title': 'Second Chat',
- 'last_updated': '2025-01-02',
- 'chat_type': 'personal',
- 'tags': [],
- 'is_pinned': False,
- 'context': []
- },
- 'messages': [
- {'role': 'user', 'content': 'Goodbye', 'timestamp': '2025-01-02'}
- ]
- }
- ]
+ content_text = _normalize_content([
+ {'type': 'text', 'text': 'Line one'},
+ {'type': 'image_url', 'image_url': {'url': 'https://example.com/image.png'}},
+ {'type': 'text', 'text': 'Line two'}
+ ])
+ assert content_text == 'Line one\n[Image]\nLine two', 'Content normalization should flatten list-based content.'
- import re
+ raw_buckets = _collect_raw_citation_buckets({
+ 'citations': [{'title': 'Legacy'}],
+ 'hybrid_citations': [{'file_name': 'doc.pdf', 'page_number': 2}],
+ 'web_search_citations': [{'title': 'Web', 'url': 'https://example.com'}],
+ 'agent_citations': [{'tool_name': 'lookup'}]
+ })
+ normalized = _normalize_citations(raw_buckets)
+ counts = _build_citation_counts(normalized)
- def safe_filename(title):
- safe = re.sub(r'[<>:"/\\|?*]', '_', title)
- safe = re.sub(r'\s+', '_', safe)
- safe = safe.strip('_. ')
- if len(safe) > 50:
- safe = safe[:50]
- return safe or 'Untitled'
+ assert counts == {'document': 1, 'web': 1, 'agent_tool': 1, 'legacy': 1, 'total': 4}, f"Unexpected citation counts: {counts}"
- buffer = io.BytesIO()
- with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
- for entry in exported:
- conv = entry['conversation']
- safe_title = safe_filename(conv.get('title', 'Untitled'))
- conv_id_short = conv.get('id', 'unknown')[:8]
- file_content = json.dumps(entry, indent=2, ensure_ascii=False, default=str)
- file_name = f"{safe_title}_{conv_id_short}.json"
- zf.writestr(file_name, file_content)
+ truncated = _truncate_for_summary('A' * 70000)
+ assert '[... transcript truncated for export summary generation ...]' in truncated, 'Summary truncation marker should be inserted for long transcripts.'
+ assert len(truncated) < 70000, 'Truncated summary source should be shorter than the original transcript.'
- buffer.seek(0)
+ print("✅ Helper utilities test passed!")
+ return True
- with zipfile.ZipFile(buffer, 'r') as zf:
- names = zf.namelist()
- assert len(names) == 2, f"ZIP should have 2 files, got {len(names)}"
- assert 'First_Chat_conv-001.json' in names, f"Expected First_Chat_conv-001.json, got {names}"
- assert 'Second_Chat_conv-002.json' in names, f"Expected Second_Chat_conv-002.json, got {names}"
- # Verify content
- first_content = json.loads(zf.read('First_Chat_conv-001.json'))
- assert first_content['conversation']['title'] == 'First Chat'
- assert len(first_content['messages']) == 1
+def test_pdf_export_generation():
+ """Test PDF export generates valid HTML body and PDF bytes."""
+ print("🔍 Testing PDF export generation...")
- print("✅ ZIP packaging test passed!")
+ entry = {
+ 'conversation': {
+ 'id': 'conv-pdf-001',
+ 'title': 'PDF Export Test',
+ 'last_updated': '2026-03-06T14:00:00Z',
+ 'chat_type': 'personal',
+ 'tags': ['pdf', 'test'],
+ 'classification': [],
+ 'context': [],
+ 'strict': False,
+ 'is_pinned': False,
+ 'scope_locked': False,
+ 'locked_contexts': [],
+ 'message_count': 2,
+ 'message_counts_by_role': {'user': 1, 'assistant': 1},
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thought_count': 0
+ },
+ 'summary_intro': {
+ 'enabled': False,
+ 'generated': False,
+ 'model_deployment': None,
+ 'generated_at': None,
+ 'content': '',
+ 'error': None
+ },
+ 'messages': [
+ {
+ 'id': 'u1',
+ 'role': 'user',
+ 'speaker_label': 'User',
+ 'label': 'Turn 1',
+ 'sequence_index': 1,
+ 'transcript_index': 1,
+ 'is_transcript_message': True,
+ 'timestamp': '2026-03-06T14:00:01Z',
+ 'content': 'Hello, can you help me?',
+ 'content_text': 'Hello, can you help me?',
+ 'details': {},
+ 'citations': [],
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thoughts': [],
+ 'legacy_citations': [],
+ 'hybrid_citations': [],
+ 'web_search_citations': [],
+ 'agent_citations': []
+ },
+ {
+ 'id': 'a1',
+ 'role': 'assistant',
+ 'speaker_label': 'Assistant',
+ 'label': 'Turn 2',
+ 'sequence_index': 2,
+ 'transcript_index': 2,
+ 'is_transcript_message': True,
+ 'timestamp': '2026-03-06T14:00:02Z',
+ 'content': 'Of course! How can I assist you today?',
+ 'content_text': 'Of course! How can I assist you today?',
+ 'details': {},
+ 'citations': [],
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thoughts': [],
+ 'legacy_citations': [],
+ 'hybrid_citations': [],
+ 'web_search_citations': [],
+ 'agent_citations': []
+ }
+ ]
+ }
+
+ # Test HTML body generation
+ html_body = _build_pdf_html_body(entry)
+ assert '
PDF Export Test
' in html_body, 'HTML should contain the conversation title.'
+ assert 'user-bubble' in html_body, 'HTML should contain user bubble CSS class.'
+ assert 'assistant-bubble' in html_body, 'HTML should contain assistant bubble CSS class.'
+ assert 'Transcript' in html_body, 'HTML should contain Transcript section.'
+ assert 'Appendix A' in html_body, 'HTML should contain Appendix A.'
+ assert 'Hello, can you help me?' in html_body, 'HTML should contain user message content.'
+ assert 'Of course! How can I assist you today?' in html_body, 'HTML should contain assistant message content.'
+
+ # Test bubble class helper
+ assert _pdf_bubble_class('user') == 'user-bubble', 'User role should map to user-bubble.'
+ assert _pdf_bubble_class('assistant') == 'assistant-bubble', 'Assistant role should map to assistant-bubble.'
+ assert _pdf_bubble_class('system') == 'system-bubble', 'System role should map to system-bubble.'
+ assert _pdf_bubble_class('file') == 'file-bubble', 'File role should map to file-bubble.'
+ assert _pdf_bubble_class('unknown') == 'other-bubble', 'Unknown roles should map to other-bubble.'
+
+ # Test PDF bytes generation
+ pdf_bytes = _conversation_to_pdf_bytes(entry)
+ assert isinstance(pdf_bytes, bytes), 'PDF output should be bytes.'
+ assert pdf_bytes[:5] == b'%PDF-', f'PDF should start with %PDF- header, got: {pdf_bytes[:10]}'
+ assert len(pdf_bytes) > 100, 'PDF should have a reasonable size.'
+
+ print("✅ PDF export generation test passed!")
return True
-def test_safe_filename():
- """Test filename sanitization."""
- print("🔍 Testing safe filename generation...")
+def test_tag_formatting():
+ """Test that dict-style tags and classifications render as readable strings."""
+ print("🔍 Testing tag/classification formatting...")
- import re
+ # Category/value tag
+ assert _format_tag({'category': 'model', 'value': 'gpt-5'}) == 'model: gpt-5'
- def safe_filename(title):
- safe = re.sub(r'[<>:"/\\|?*]', '_', title)
- safe = re.sub(r'\s+', '_', safe)
- safe = safe.strip('_. ')
- if len(safe) > 50:
- safe = safe[:50]
- return safe or 'Untitled'
+ # Participant tag with name
+ assert _format_tag({'category': 'participant', 'name': 'Alice', 'user_id': 'u1'}) == 'participant: Alice'
- assert safe_filename('Normal Title') == 'Normal_Title', "Spaces should become underscores"
- assert safe_filename('File/With:Bad*Chars') == 'File_With_Bad_Chars', "Bad chars should be replaced"
- assert safe_filename('A' * 100) == 'A' * 50, "Long names should be truncated"
- assert safe_filename('') == 'Untitled', "Empty should become Untitled"
- assert safe_filename(' ') == 'Untitled', "Whitespace-only should become Untitled"
+ # Participant tag with email fallback
+ assert _format_tag({'category': 'participant', 'email': 'bob@test.com'}) == 'participant: bob@test.com'
- print("✅ Safe filename test passed!")
- return True
+ # Document tag with title
+ assert _format_tag({'category': 'document', 'title': 'Study.pdf', 'document_id': 'd1'}) == 'document: Study.pdf'
+ # Document tag with only document_id
+ assert _format_tag({'category': 'document', 'document_id': 'd1'}) == 'document: d1'
-def test_active_thread_filter():
- """Test that only active thread messages are included."""
- print("🔍 Testing active thread message filtering...")
+ # Plain string tag (older data)
+ assert _format_tag('science') == 'science'
- messages = [
- {'role': 'user', 'content': 'Hello', 'metadata': {}},
- {'role': 'assistant', 'content': 'Reply 1', 'metadata': {'thread_info': {'active_thread': True}}},
- {'role': 'assistant', 'content': 'Reply 2 (inactive)', 'metadata': {'thread_info': {'active_thread': False}}},
- {'role': 'user', 'content': 'Follow up', 'metadata': {'thread_info': {}}},
- {'role': 'assistant', 'content': 'Final', 'metadata': {'thread_info': {'active_thread': None}}},
- ]
+ # Category only (no value)
+ assert _format_tag({'category': 'semantic'}) == 'semantic'
+
+ # Verify Markdown export uses formatted tags
+ entry = {
+ 'conversation': {
+ 'id': 'conv-tags-001',
+ 'title': 'Tag Format Test',
+ 'last_updated': '2026-03-08T00:00:00Z',
+ 'chat_type': 'personal',
+ 'tags': [
+ {'category': 'model', 'value': 'gpt-5'},
+ {'category': 'semantic', 'value': 'cubesats'}
+ ],
+ 'classification': [],
+ 'context': [],
+ 'strict': False,
+ 'is_pinned': False,
+ 'scope_locked': False,
+ 'locked_contexts': [],
+ 'message_count': 0,
+ 'message_counts_by_role': {},
+ 'citation_counts': {'document': 0, 'web': 0, 'agent_tool': 0, 'legacy': 0, 'total': 0},
+ 'thought_count': 0
+ },
+ 'summary_intro': {'enabled': False, 'generated': False},
+ 'messages': []
+ }
+
+ md = _conversation_to_markdown(entry)
+ assert 'model: gpt-5' in md, f'Markdown should contain formatted tag, got: {md[:500]}'
+ assert "{'category'" not in md, 'Markdown should not contain raw dict strings'
+
+ html = _build_pdf_html_body(entry)
+ assert 'model: gpt-5' in html, f'PDF HTML should contain formatted tag'
+ assert "{'category'" not in html, 'PDF HTML should not contain raw dict strings'
- filtered = []
- for msg in messages:
- thread_info = msg.get('metadata', {}).get('thread_info', {})
- active = thread_info.get('active_thread')
- if active is True or active is None or 'active_thread' not in thread_info:
- filtered.append(msg)
-
- assert len(filtered) == 4, f"Expected 4 active messages, got {len(filtered)}"
- contents = [m['content'] for m in filtered]
- assert 'Reply 2 (inactive)' not in contents, "Inactive thread message should be excluded"
- assert 'Hello' in contents, "Message without thread info should be included"
- assert 'Reply 1' in contents, "Active=True message should be included"
- assert 'Follow up' in contents, "Message with empty thread_info should be included"
- assert 'Final' in contents, "Message with active_thread=None should be included"
-
- print("✅ Active thread filter test passed!")
+ print("✅ Tag formatting test passed!")
return True
if __name__ == "__main__":
tests = [
- test_sanitize_conversation,
- test_sanitize_message,
- test_conversation_to_markdown,
- test_json_export_structure,
- test_zip_packaging,
- test_safe_filename,
- test_active_thread_filter
+ test_filter_messages_for_export,
+ test_sanitize_message_with_citations_and_thoughts,
+ test_conversation_metadata_and_json_shape,
+ test_markdown_export_structure,
+ test_safe_filename_and_zip_packaging,
+ test_content_and_citation_helpers,
+ test_pdf_export_generation,
+ test_tag_formatting,
]
results = []
@@ -367,8 +547,8 @@ def test_active_thread_filter():
print(f"\n🧪 Running {test.__name__}...")
try:
results.append(test())
- except Exception as e:
- print(f"❌ {test.__name__} failed: {e}")
+ except Exception as exc:
+ print(f"❌ {test.__name__} failed: {exc}")
import traceback
traceback.print_exc()
results.append(False)
diff --git a/functional_tests/test_persistent_conversation_summary.py b/functional_tests/test_persistent_conversation_summary.py
new file mode 100644
index 00000000..e8d95ffa
--- /dev/null
+++ b/functional_tests/test_persistent_conversation_summary.py
@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+# test_persistent_conversation_summary.py
+"""
+Functional test for persistent conversation summaries.
+Version: 0.239.030
+Implemented in: 0.239.030
+
+This test ensures that:
+1. generate_conversation_summary() returns properly structured summary data
+2. _build_summary_intro() uses cached summary when message_time_end hasn't changed
+3. _build_summary_intro() regenerates when messages are newer than cached summary
+4. Summary data includes message_time_start and message_time_end fields
+5. The truncation helper works correctly
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app'))
+
+from route_backend_conversation_export import ( # noqa: E402
+ _build_summary_intro,
+ _truncate_for_summary,
+ SUMMARY_SOURCE_CHAR_LIMIT,
+)
+
+
+def test_summary_intro_cache_hit():
+ """Test that _build_summary_intro returns cached summary when message_time_end matches."""
+ print("🔍 Testing summary cache hit (no new messages)...")
+
+ existing_summary = {
+ 'content': 'Cached summary text.',
+ 'model_deployment': 'gpt-4o',
+ 'generated_at': '2026-01-15T10:00:00',
+ 'message_time_start': '2026-01-15T09:00:00',
+ 'message_time_end': '2026-01-15T09:55:00'
+ }
+
+ conversation = {
+ 'id': 'conv-test-1',
+ 'title': 'Test Conversation',
+ 'summary': existing_summary
+ }
+
+ messages = [
+ {'role': 'user', 'content_text': 'Hello', 'speaker_label': 'USER'},
+ {'role': 'assistant', 'content_text': 'Hi there', 'speaker_label': 'ASSISTANT'}
+ ]
+
+ sanitized_conversation = {'title': 'Test Conversation'}
+
+ result = _build_summary_intro(
+ messages=messages,
+ conversation=conversation,
+ sanitized_conversation=sanitized_conversation,
+ settings={},
+ enabled=True,
+ summary_model_deployment='gpt-4o',
+ message_time_start='2026-01-15T09:00:00',
+ message_time_end='2026-01-15T09:55:00' # Same as cached
+ )
+
+ assert result['enabled'] is True, f"Expected enabled=True, got {result['enabled']}"
+ assert result['generated'] is True, f"Expected generated=True, got {result['generated']}"
+ assert result['content'] == 'Cached summary text.', f"Expected cached content, got: {result['content']}"
+ assert result['model_deployment'] == 'gpt-4o', f"Expected gpt-4o, got: {result['model_deployment']}"
+ assert result['generated_at'] == '2026-01-15T10:00:00', f"Expected cached timestamp, got: {result['generated_at']}"
+ assert result['error'] is None, f"Expected no error, got: {result['error']}"
+
+ print("✅ Summary cache hit test passed!")
+ return True
+
+
+def test_summary_intro_cache_hit_older_message():
+ """Test cache hit when cached message_time_end is NEWER than current (edge case)."""
+ print("🔍 Testing summary cache hit with older message_time_end...")
+
+ existing_summary = {
+ 'content': 'Summary covers more messages.',
+ 'model_deployment': 'gpt-4o',
+ 'generated_at': '2026-01-15T12:00:00',
+ 'message_time_start': '2026-01-15T09:00:00',
+ 'message_time_end': '2026-01-15T11:55:00'
+ }
+
+ conversation = {
+ 'id': 'conv-test-2',
+ 'summary': existing_summary
+ }
+
+ sanitized_conversation = {'title': 'Test'}
+
+ result = _build_summary_intro(
+ messages=[{'role': 'user', 'content_text': 'Hi', 'speaker_label': 'USER'}],
+ conversation=conversation,
+ sanitized_conversation=sanitized_conversation,
+ settings={},
+ enabled=True,
+ summary_model_deployment='gpt-4o',
+ message_time_start='2026-01-15T09:00:00',
+ message_time_end='2026-01-15T10:00:00' # Older than cached
+ )
+
+ assert result['generated'] is True, "Should use cache when cached end >= current end"
+ assert result['content'] == 'Summary covers more messages.', "Should return cached content"
+
+ print("✅ Summary cache hit (older message) test passed!")
+ return True
+
+
+def test_summary_intro_disabled():
+ """Test that disabled summary returns immediately."""
+ print("🔍 Testing summary disabled state...")
+
+ result = _build_summary_intro(
+ messages=[],
+ conversation={'id': 'conv-disabled'},
+ sanitized_conversation={'title': 'Test'},
+ settings={},
+ enabled=False,
+ summary_model_deployment=''
+ )
+
+ assert result['enabled'] is False, f"Expected enabled=False, got {result['enabled']}"
+ assert result['generated'] is False, f"Expected generated=False, got {result['generated']}"
+ assert result['content'] == '', f"Expected empty content, got: {result['content']}"
+
+ print("✅ Summary disabled state test passed!")
+ return True
+
+
+def test_summary_intro_stale_cache():
+ """Test that a stale cache (newer messages exist) triggers regeneration attempt."""
+ print("🔍 Testing summary stale cache detection...")
+
+ existing_summary = {
+ 'content': 'Old cached summary.',
+ 'model_deployment': 'gpt-4o',
+ 'generated_at': '2026-01-15T10:00:00',
+ 'message_time_start': '2026-01-15T09:00:00',
+ 'message_time_end': '2026-01-15T09:55:00'
+ }
+
+ conversation = {
+ 'id': 'conv-stale',
+ 'summary': existing_summary
+ }
+
+ sanitized_conversation = {'title': 'Test'}
+
+ # message_time_end is NEWER than cached — should try to regenerate
+ # Since we don't have a real OpenAI client, this will fail with an error,
+ # which confirms the cache was NOT used.
+ result = _build_summary_intro(
+ messages=[
+ {'role': 'user', 'content_text': 'New message', 'speaker_label': 'USER'}
+ ],
+ conversation=conversation,
+ sanitized_conversation=sanitized_conversation,
+ settings={},
+ enabled=True,
+ summary_model_deployment='gpt-4o',
+ message_time_start='2026-01-15T09:00:00',
+ message_time_end='2026-01-15T10:30:00' # NEWER than cached 09:55
+ )
+
+ # Without a real GPT client, generation should fail
+ assert result['generated'] is False, "Without GPT client, generation should fail"
+ assert result['error'] is not None, "Should have an error when no GPT client available"
+ # The key assertion: it DID NOT return the cached content
+ assert result['content'] != 'Old cached summary.', "Should not use stale cached summary"
+
+ print("✅ Summary stale cache detection test passed!")
+ return True
+
+
+def test_summary_data_structure():
+ """Test that the summary_intro structure contains all expected fields."""
+ print("🔍 Testing summary data structure...")
+
+ result = _build_summary_intro(
+ messages=[],
+ conversation={'id': 'conv-struct'},
+ sanitized_conversation={'title': 'Test'},
+ settings={},
+ enabled=True,
+ summary_model_deployment='gpt-4o',
+ message_time_start=None,
+ message_time_end=None
+ )
+
+ expected_keys = {'enabled', 'generated', 'model_deployment', 'generated_at', 'content', 'error'}
+ actual_keys = set(result.keys())
+ assert expected_keys.issubset(actual_keys), f"Missing keys: {expected_keys - actual_keys}"
+
+ print("✅ Summary data structure test passed!")
+ return True
+
+
+def test_truncation_short_text():
+ """Test that short text passes through unchanged."""
+ print("🔍 Testing truncation with short text...")
+
+ short_text = "This is a short transcript."
+ result = _truncate_for_summary(short_text)
+ assert result == short_text, "Short text should pass through unchanged"
+
+ print("✅ Truncation short text test passed!")
+ return True
+
+
+def test_truncation_long_text():
+ """Test that long text gets truncated with marker."""
+ print("🔍 Testing truncation with long text...")
+
+ long_text = "A" * (SUMMARY_SOURCE_CHAR_LIMIT + 1000)
+ result = _truncate_for_summary(long_text)
+
+ assert len(result) < len(long_text), "Truncated text should be shorter"
+ assert "transcript truncated" in result, "Should contain truncation marker"
+
+ print("✅ Truncation long text test passed!")
+ return True
+
+
+def test_summary_no_content():
+ """Test that summary with no message content returns an error."""
+ print("🔍 Testing summary with empty messages...")
+
+ result = _build_summary_intro(
+ messages=[
+ {'role': 'user', 'content_text': '', 'speaker_label': 'USER'},
+ {'role': 'assistant', 'content_text': '', 'speaker_label': 'ASSISTANT'}
+ ],
+ conversation={'id': 'conv-empty'},
+ sanitized_conversation={'title': 'Test'},
+ settings={},
+ enabled=True,
+ summary_model_deployment='gpt-4o',
+ message_time_start=None,
+ message_time_end=None
+ )
+
+ assert result['generated'] is False, "Should not generate with empty content"
+ assert result['error'] is not None, "Should have an error message"
+ assert 'No message content' in result['error'], f"Unexpected error: {result['error']}"
+
+ print("✅ Summary with empty messages test passed!")
+ return True
+
+
+if __name__ == "__main__":
+ tests = [
+ test_summary_intro_cache_hit,
+ test_summary_intro_cache_hit_older_message,
+ test_summary_intro_disabled,
+ test_summary_intro_stale_cache,
+ test_summary_data_structure,
+ test_truncation_short_text,
+ test_truncation_long_text,
+ test_summary_no_content,
+ ]
+ results = []
+
+ for test in tests:
+ print(f"\n🧪 Running {test.__name__}...")
+ try:
+ results.append(test())
+ except Exception as e:
+ print(f"❌ {test.__name__} failed with exception: {e}")
+ import traceback
+ traceback.print_exc()
+ results.append(False)
+
+ passed = sum(1 for r in results if r)
+ total = len(results)
+ print(f"\n📊 Results: {passed}/{total} tests passed")
+ sys.exit(0 if all(results) else 1)
diff --git a/functional_tests/test_sql_auto_schema_companion.py b/functional_tests/test_sql_auto_schema_companion.py
new file mode 100644
index 00000000..2f488c97
--- /dev/null
+++ b/functional_tests/test_sql_auto_schema_companion.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""
+Functional test for SQL Auto Schema Companion Plugin creation.
+Version: 0.239.015
+Implemented in: 0.239.015
+
+This test ensures that when a SQL Query plugin is loaded via LoggedPluginLoader,
+a companion SQL Schema plugin is automatically created and registered in the kernel.
+This fixes the issue where agents with only a sql_query action configured would have
+no schema awareness, causing the LLM to ask for clarification instead of querying.
+
+Additionally validates that:
+- SQLQueryPlugin @kernel_function descriptions are resilient (don't demand non-existent functions)
+- The _extract_sql_schema_for_instructions fallback detects SQLQueryPlugin instances
+"""
+
+import sys
+import os
+import inspect
+
+# Add the application directory to the path for imports
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app'))
+
+
+def test_companion_schema_plugin_method_exists():
+ """Test that LoggedPluginLoader has the _auto_create_companion_schema_plugin method."""
+ print("🔍 Testing _auto_create_companion_schema_plugin method exists...")
+
+ try:
+ from semantic_kernel_plugins.logged_plugin_loader import LoggedPluginLoader
+
+ assert hasattr(LoggedPluginLoader, '_auto_create_companion_schema_plugin'), \
+ "LoggedPluginLoader must have _auto_create_companion_schema_plugin method"
+
+ method = getattr(LoggedPluginLoader, '_auto_create_companion_schema_plugin')
+ assert callable(method), "_auto_create_companion_schema_plugin must be callable"
+
+ # Verify method signature includes query_manifest and query_plugin_name
+ sig = inspect.signature(method)
+ param_names = list(sig.parameters.keys())
+ assert 'query_manifest' in param_names, "Method must accept query_manifest parameter"
+ assert 'query_plugin_name' in param_names, "Method must accept query_plugin_name parameter"
+
+ print("✅ _auto_create_companion_schema_plugin method exists with correct signature!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_companion_creation_triggered_for_sql_query():
+ """Test that load_plugin_from_manifest triggers companion creation for sql_query type."""
+ print("🔍 Testing companion creation trigger in load_plugin_from_manifest...")
+
+ try:
+ from semantic_kernel_plugins.logged_plugin_loader import LoggedPluginLoader
+
+ source = inspect.getsource(LoggedPluginLoader.load_plugin_from_manifest)
+
+ # The method should check for sql_query type and call companion creation
+ assert "plugin_type == 'sql_query'" in source, \
+ "load_plugin_from_manifest must check for sql_query plugin type"
+ assert "_auto_create_companion_schema_plugin" in source, \
+ "load_plugin_from_manifest must call _auto_create_companion_schema_plugin"
+
+ print("✅ load_plugin_from_manifest triggers companion creation for sql_query!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_schema_plugin_name_derivation():
+ """Test that the companion schema plugin name is correctly derived."""
+ print("🔍 Testing schema plugin name derivation logic...")
+
+ try:
+ from semantic_kernel_plugins.logged_plugin_loader import LoggedPluginLoader
+
+ source = inspect.getsource(LoggedPluginLoader._auto_create_companion_schema_plugin)
+
+ # Should handle _query suffix replacement
+ assert "endswith('_query')" in source, \
+ "Method should check for _query suffix"
+ assert "'_schema'" in source, \
+ "Method should derive _schema suffix"
+
+ # Should check if schema plugin already exists
+ assert "self.kernel.plugins" in source, \
+ "Method should check kernel.plugins for existing schema plugin"
+
+ # Should create SQLSchemaPlugin
+ assert "SQLSchemaPlugin" in source, \
+ "Method should create a SQLSchemaPlugin instance"
+
+ # Should register with kernel
+ assert "_register_plugin_with_kernel" in source, \
+ "Method should register the companion plugin with the kernel"
+
+ print("✅ Schema plugin name derivation and creation logic is correct!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_resilient_query_plugin_descriptions():
+ """Test that SQLQueryPlugin descriptions don't demand non-existent schema functions."""
+ print("🔍 Testing SQLQueryPlugin @kernel_function descriptions are resilient...")
+
+ try:
+ from semantic_kernel_plugins.sql_query_plugin import SQLQueryPlugin
+
+ source = inspect.getsource(SQLQueryPlugin)
+
+ # Descriptions should NOT contain "you MUST first call" which creates a hard dependency
+ assert "you MUST first call" not in source, \
+ "Descriptions should not contain 'you MUST first call' (creates hard dependency on non-existent functions)"
+ assert "You MUST first discover" not in source, \
+ "Descriptions should not contain 'You MUST first discover' (creates hard dependency)"
+
+ # Descriptions SHOULD contain resilient language
+ assert "If the database schema is provided in your instructions" in source, \
+ "Descriptions should reference schema from instructions as primary source"
+
+ # Should still mention the schema plugin as a fallback
+ assert "get_database_schema" in source, \
+ "Descriptions should still reference get_database_schema as a fallback option"
+
+ print("✅ SQLQueryPlugin descriptions are resilient and don't demand non-existent functions!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_schema_extraction_fallback():
+ """Test that _extract_sql_schema_for_instructions has a fallback for SQLQueryPlugin."""
+ print("🔍 Testing _extract_sql_schema_for_instructions fallback...")
+
+ try:
+ from semantic_kernel_loader import _extract_sql_schema_for_instructions
+
+ source = inspect.getsource(_extract_sql_schema_for_instructions)
+
+ # Should have fallback logic
+ assert "Fallback" in source or "fallback" in source, \
+ "Function should contain fallback logic"
+
+ # Should check for SQLQueryPlugin
+ assert "SQLQueryPlugin" in source, \
+ "Fallback should check for SQLQueryPlugin instances"
+
+ # Should create temporary schema extractor
+ assert "temp_manifest" in source or "temp_schema" in source, \
+ "Fallback should create a temporary schema extractor"
+
+ print("✅ _extract_sql_schema_for_instructions has SQLQueryPlugin fallback!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_version_updated():
+ """Test that the version has been updated for this fix."""
+ print("🔍 Testing version update...")
+
+ try:
+ # Read config.py directly to check version
+ config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..', 'application', 'single_app', 'config.py')
+ with open(config_path, 'r') as f:
+ config_content = f.read()
+
+ assert 'VERSION = "0.239.015"' in config_content, \
+ f"config.py should contain VERSION = \"0.239.015\""
+
+ print("✅ Version correctly updated to 0.239.015!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+if __name__ == "__main__":
+ tests = [
+ test_companion_schema_plugin_method_exists,
+ test_companion_creation_triggered_for_sql_query,
+ test_schema_plugin_name_derivation,
+ test_resilient_query_plugin_descriptions,
+ test_schema_extraction_fallback,
+ test_version_updated,
+ ]
+ results = []
+
+ for test in tests:
+ print(f"\n🧪 Running {test.__name__}...")
+ results.append(test())
+
+ success = all(results)
+ passed = sum(results)
+ total = len(results)
+ print(f"\n📊 Results: {passed}/{total} tests passed")
+
+ if not success:
+ failed_tests = [t.__name__ for t, r in zip(tests, results) if not r]
+ print(f"❌ Failed tests: {', '.join(failed_tests)}")
+
+ sys.exit(0 if success else 1)
diff --git a/functional_tests/test_sql_query_plugin_schema_awareness.py b/functional_tests/test_sql_query_plugin_schema_awareness.py
new file mode 100644
index 00000000..b16f73bd
--- /dev/null
+++ b/functional_tests/test_sql_query_plugin_schema_awareness.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+"""
+Functional test for SQL Query Plugin Schema Awareness improvements.
+Version: 0.239.014
+Implemented in: 0.239.014
+
+This test ensures that the SQL Query and SQL Schema plugins have proper
+workflow-guiding descriptions in their @kernel_function decorators, metadata
+properties, and that the new query_database convenience function exists.
+These improvements ensure agents discover database schema before attempting
+to generate and execute SQL queries.
+"""
+
+import sys
+import os
+
+# Add the application directory to the path for imports
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app'))
+
+
+def test_sql_schema_plugin_descriptions():
+ """Test that SQL Schema Plugin has prescriptive workflow descriptions."""
+ print("🔍 Testing SQL Schema Plugin @kernel_function descriptions...")
+
+ try:
+ from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin
+ import inspect
+
+ source = inspect.getsource(SQLSchemaPlugin)
+
+ # Check get_database_schema has workflow guidance
+ assert "ALWAYS call this function FIRST" in source, \
+ "get_database_schema description should contain 'ALWAYS call this function FIRST'"
+
+ assert "before executing any SQL queries" in source, \
+ "get_database_schema description should guide calling before queries"
+
+ # Check get_table_list has workflow guidance
+ assert "Use this function first to discover which tables are available" in source, \
+ "get_table_list description should guide discovery workflow"
+
+ # Check get_table_schema has workflow guidance
+ assert "Call this after discovering tables" in source, \
+ "get_table_schema description should reference discovery step"
+
+ # Check get_relationships has workflow guidance
+ assert "JOIN conditions" in source, \
+ "get_relationships description should mention JOIN conditions"
+
+ print("✅ SQL Schema Plugin descriptions contain workflow guidance!")
+ return True
+
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_sql_query_plugin_descriptions():
+ """Test that SQL Query Plugin has prescriptive workflow descriptions."""
+ print("🔍 Testing SQL Query Plugin @kernel_function descriptions...")
+
+ try:
+ from semantic_kernel_plugins.sql_query_plugin import SQLQueryPlugin
+ import inspect
+
+ source = inspect.getsource(SQLQueryPlugin)
+
+ # Check execute_query has schema requirement
+ assert "you MUST first call get_database_schema or get_table_list" in source, \
+ "execute_query description should require schema discovery first"
+
+ assert "fully qualified table names" in source, \
+ "execute_query description should mention fully qualified table names"
+
+ # Check execute_scalar has schema requirement
+ assert "MUST first discover the database schema" in source, \
+ "execute_scalar description should require schema discovery"
+
+ # Check validate_query has useful guidance
+ assert "pre-check complex queries" in source, \
+ "validate_query description should guide pre-checking"
+
+ print("✅ SQL Query Plugin descriptions contain workflow guidance!")
+ return True
+
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_query_database_function_exists():
+ """Test that the new query_database convenience function exists."""
+ print("🔍 Testing query_database function existence...")
+
+ try:
+ from semantic_kernel_plugins.sql_query_plugin import SQLQueryPlugin
+
+ # Check the function exists on the class
+ assert hasattr(SQLQueryPlugin, 'query_database'), \
+ "SQLQueryPlugin should have a query_database method"
+
+ # Check it's in get_functions list
+ # We need to create a minimal instance to test, but we can check the class method
+ import inspect
+ source = inspect.getsource(SQLQueryPlugin.query_database)
+
+ assert "question" in source, \
+ "query_database should accept a 'question' parameter"
+
+ assert "query" in source, \
+ "query_database should accept a 'query' parameter"
+
+ assert "@kernel_function" in inspect.getsource(SQLQueryPlugin), \
+ "query_database should be decorated with @kernel_function"
+
+ print("✅ query_database function exists with correct parameters!")
+ return True
+
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_metadata_descriptions_updated():
+ """Test that metadata property descriptions include workflow guidance."""
+ print("🔍 Testing metadata property descriptions...")
+
+ try:
+ from semantic_kernel_plugins.sql_query_plugin import SQLQueryPlugin
+ from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin
+ import inspect
+
+ # Check SQL Query Plugin metadata
+ query_source = inspect.getsource(SQLQueryPlugin)
+ assert "WORKFLOW: Before executing any query" in query_source, \
+ "SQL Query Plugin metadata should contain workflow guidance"
+
+ # Check SQL Schema Plugin metadata
+ schema_source = inspect.getsource(SQLSchemaPlugin)
+ assert "ALWAYS call get_database_schema" in schema_source, \
+ "SQL Schema Plugin metadata should contain workflow guidance"
+
+ print("✅ Metadata descriptions include workflow guidance!")
+ return True
+
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_sql_plugin_creation_path_enabled():
+ """Test that the SQL plugin creation path is enabled in logged_plugin_loader."""
+ print("🔍 Testing SQL plugin creation path is enabled...")
+
+ try:
+ from semantic_kernel_plugins.logged_plugin_loader import LoggedPluginLoader
+ import inspect
+
+ source = inspect.getsource(LoggedPluginLoader._create_plugin_instance)
+
+ # Check the SQL path is NOT commented out
+ assert "elif plugin_type in ['sql_schema', 'sql_query']:" in source, \
+ "SQL plugin creation path should be enabled (not commented out)"
+
+ assert "return self._create_sql_plugin(manifest)" in source, \
+ "SQL plugin creation should call _create_sql_plugin"
+
+ # Make sure it's not still commented
+ lines = source.split('\n')
+ for line in lines:
+ stripped = line.strip()
+ if "sql_schema" in stripped and "sql_query" in stripped and "elif" in stripped:
+ assert not stripped.startswith('#'), \
+ "SQL plugin creation path should NOT be commented out"
+ break
+
+ print("✅ SQL plugin creation path is enabled!")
+ return True
+
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_schema_injection_function_exists():
+ """Test that the schema injection helper function exists in semantic_kernel_loader."""
+ print("🔍 Testing schema injection function exists...")
+
+ try:
+ # Read the file and check for the function
+ loader_path = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ '..', 'application', 'single_app', 'semantic_kernel_loader.py'
+ )
+
+ with open(loader_path, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ assert "def _extract_sql_schema_for_instructions(kernel)" in content, \
+ "_extract_sql_schema_for_instructions function should exist in semantic_kernel_loader.py"
+
+ assert "sql_schema_summary = _extract_sql_schema_for_instructions(kernel)" in content, \
+ "Schema injection should be called in the agent loading flow"
+
+ assert "Available Database Schema" in content, \
+ "Schema injection should add 'Available Database Schema' header to instructions"
+
+ assert "Do NOT ask the user for table or column names" in content, \
+ "Schema injection should instruct agent not to ask user for schema info"
+
+ print("✅ Schema injection function exists and is integrated!")
+ return True
+
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+if __name__ == "__main__":
+ tests = [
+ test_sql_schema_plugin_descriptions,
+ test_sql_query_plugin_descriptions,
+ test_query_database_function_exists,
+ test_metadata_descriptions_updated,
+ test_sql_plugin_creation_path_enabled,
+ test_schema_injection_function_exists,
+ ]
+ results = []
+
+ for test in tests:
+ print(f"\n🧪 Running {test.__name__}...")
+ results.append(test())
+
+ success = all(results)
+ print(f"\n📊 Results: {sum(results)}/{len(results)} tests passed")
+ sys.exit(0 if success else 1)
diff --git a/functional_tests/test_sql_schema_sys_catalog_views.py b/functional_tests/test_sql_schema_sys_catalog_views.py
new file mode 100644
index 00000000..dd1c5f05
--- /dev/null
+++ b/functional_tests/test_sql_schema_sys_catalog_views.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""
+Functional test for SQL Schema Plugin sys.* catalog view migration.
+Version: 0.239.016
+Implemented in: 0.239.016
+
+This test ensures that the SQL Schema Plugin uses sys.tables, sys.columns,
+sys.indexes (catalog views) instead of INFORMATION_SCHEMA views for SQL Server,
+fixing the issue where get_database_schema returned empty tables dict ('tables': {})
+despite the database having tables. The sys.* views are proven to work since the
+relationships query (which uses sys.foreign_keys) always returned valid data.
+
+Also validates robust pyodbc.Row handling (no isinstance(tuple) checks).
+"""
+
+import sys
+import os
+import inspect
+
+# Add the application directory to the path for imports
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'application', 'single_app'))
+
+
+def test_tables_query_uses_sys_views():
+ """Test that SQL Server tables query uses sys.tables instead of INFORMATION_SCHEMA."""
+ print("🔍 Testing SQL Server tables query uses sys.tables...")
+
+ try:
+ from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin
+
+ source = inspect.getsource(SQLSchemaPlugin._get_tables_query)
+
+ # Should use sys.tables
+ assert "sys.tables" in source, \
+ "_get_tables_query should reference sys.tables"
+ assert "sys.schemas" in source, \
+ "_get_tables_query should reference sys.schemas"
+
+ # Should NOT use INFORMATION_SCHEMA.TABLES for sqlserver
+ # (other DB types still use their own patterns)
+ assert "INFORMATION_SCHEMA.TABLES" not in source, \
+ "_get_tables_query should NOT use INFORMATION_SCHEMA.TABLES (use sys.tables instead)"
+
+ print("✅ Tables query uses sys.tables!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_columns_query_uses_sys_views():
+ """Test that SQL Server columns query uses sys.columns instead of INFORMATION_SCHEMA."""
+ print("🔍 Testing SQL Server columns query uses sys.columns...")
+
+ try:
+ from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin
+
+ source = inspect.getsource(SQLSchemaPlugin._get_columns_query)
+
+ # Should use sys.columns
+ assert "sys.columns" in source, \
+ "_get_columns_query should reference sys.columns"
+ assert "TYPE_NAME" in source, \
+ "_get_columns_query should use TYPE_NAME() for data type resolution"
+
+ # Should NOT use INFORMATION_SCHEMA.COLUMNS for sqlserver
+ assert "INFORMATION_SCHEMA.COLUMNS" not in source, \
+ "_get_columns_query should NOT use INFORMATION_SCHEMA.COLUMNS (use sys.columns instead)"
+
+ print("✅ Columns query uses sys.columns!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_primary_keys_query_uses_sys_views():
+ """Test that SQL Server PK query uses sys.indexes instead of INFORMATION_SCHEMA."""
+ print("🔍 Testing SQL Server primary keys query uses sys.indexes...")
+
+ try:
+ from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin
+
+ source = inspect.getsource(SQLSchemaPlugin._get_primary_keys_query)
+
+ # Should use sys.indexes with is_primary_key
+ assert "sys.indexes" in source, \
+ "_get_primary_keys_query should reference sys.indexes"
+ assert "is_primary_key" in source, \
+ "_get_primary_keys_query should filter on is_primary_key"
+ assert "sys.index_columns" in source, \
+ "_get_primary_keys_query should reference sys.index_columns"
+
+ # Should NOT use INFORMATION_SCHEMA.KEY_COLUMN_USAGE for sqlserver section
+ # Note: PostgreSQL/MySQL sections may still use INFORMATION_SCHEMA (that's fine)
+ # We check that the sqlserver branch uses sys.* by verifying sys.indexes is present
+ assert "sys.index_columns" in source, \
+ "_get_primary_keys_query should reference sys.index_columns for SQL Server"
+
+ print("✅ Primary keys query uses sys.indexes!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_robust_row_handling_no_isinstance_tuple():
+ """Test that table row parsing doesn't rely on isinstance(tuple) checks."""
+ print("🔍 Testing robust pyodbc.Row handling...")
+
+ try:
+ from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin
+
+ # Check get_database_schema table iteration
+ schema_source = inspect.getsource(SQLSchemaPlugin.get_database_schema)
+
+ # Should NOT use isinstance(table, tuple) pattern
+ assert "isinstance(table, tuple)" not in schema_source, \
+ "get_database_schema should not use isinstance(table, tuple) — pyodbc.Row may not be a tuple"
+
+ # Should use try/except for robust indexing
+ assert "except (TypeError, IndexError)" in schema_source, \
+ "get_database_schema should handle TypeError/IndexError for row parsing"
+
+ # Check get_table_list too
+ list_source = inspect.getsource(SQLSchemaPlugin.get_table_list)
+ assert "isinstance(table_row, (list, tuple))" not in list_source, \
+ "get_table_list should not use isinstance tuple check"
+
+ # Check primary keys list comprehension
+ schema_data_source = inspect.getsource(SQLSchemaPlugin._get_table_schema_data)
+ assert "isinstance(pk, (list, tuple))" not in schema_data_source, \
+ "Primary key parsing should not use isinstance tuple check"
+
+ print("✅ Robust Row handling — no isinstance(tuple) dependencies!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_consistency_with_relationships_query():
+ """Test that all SQL Server queries use sys.* views consistently (like relationships)."""
+ print("🔍 Testing consistency: all SQL Server queries use sys.* views...")
+
+ try:
+ from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin
+
+ # The relationships query already used sys.* — verify it still does
+ rel_source = inspect.getsource(SQLSchemaPlugin._get_relationships_data)
+ assert "sys.foreign_keys" in rel_source, \
+ "Relationships query should use sys.foreign_keys"
+ assert "sys.tables" in rel_source, \
+ "Relationships query should reference sys.tables"
+
+ # Tables query should also use sys.*
+ tables_source = inspect.getsource(SQLSchemaPlugin._get_tables_query)
+ assert "sys.tables" in tables_source
+
+ # Columns query should also use sys.*
+ cols_source = inspect.getsource(SQLSchemaPlugin._get_columns_query)
+ assert "sys.columns" in cols_source
+
+ # PK query should also use sys.*
+ pk_source = inspect.getsource(SQLSchemaPlugin._get_primary_keys_query)
+ assert "sys.indexes" in pk_source
+
+ print("✅ All SQL Server queries consistently use sys.* catalog views!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+def test_version_updated():
+ """Test that the version has been updated for this fix."""
+ print("🔍 Testing version update...")
+
+ try:
+ config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..', 'application', 'single_app', 'config.py')
+ with open(config_path, 'r') as f:
+ config_content = f.read()
+
+ assert 'VERSION = "0.239.016"' in config_content, \
+ f"config.py should contain VERSION = \"0.239.016\""
+
+ print("✅ Version correctly updated to 0.239.016!")
+ return True
+ except Exception as e:
+ print(f"❌ Test failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return False
+
+
+if __name__ == "__main__":
+ tests = [
+ test_tables_query_uses_sys_views,
+ test_columns_query_uses_sys_views,
+ test_primary_keys_query_uses_sys_views,
+ test_robust_row_handling_no_isinstance_tuple,
+ test_consistency_with_relationships_query,
+ test_version_updated,
+ ]
+ results = []
+
+ for test in tests:
+ print(f"\n🧪 Running {test.__name__}...")
+ results.append(test())
+
+ success = all(results)
+ passed = sum(results)
+ total = len(results)
+ print(f"\n📊 Results: {passed}/{total} tests passed")
+
+ if not success:
+ failed_tests = [t.__name__ for t, r in zip(tests, results) if not r]
+ print(f"❌ Failed tests: {', '.join(failed_tests)}")
+
+ sys.exit(0 if success else 1)
diff --git a/functional_tests/test_thoughts_feature.py b/functional_tests/test_thoughts_feature.py
new file mode 100644
index 00000000..c07df2da
--- /dev/null
+++ b/functional_tests/test_thoughts_feature.py
@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+"""
+Functional test for Processing Thoughts feature.
+Version: 0.239.003
+Implemented in: 0.239.003
+
+This test ensures that the Processing Thoughts feature is properly implemented
+across backend (ThoughtTracker, API endpoints, chat instrumentation) and
+frontend (polling, streaming, toggle, rendering) components.
+"""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'application', 'single_app'))
+
+
+def test_thoughts_backend_module():
+ """Test that functions_thoughts.py has proper ThoughtTracker and CRUD functions."""
+ print("Testing functions_thoughts.py module structure...")
+ try:
+ backend_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'functions_thoughts.py'
+ )
+
+ with open(backend_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'ThoughtTracker class': 'class ThoughtTracker' in content,
+ 'add_thought method': 'def add_thought' in content,
+ 'complete_thought method': 'def complete_thought' in content,
+ 'enabled property': 'def enabled' in content or 'enabled' in content,
+ 'get_thoughts_for_message': 'def get_thoughts_for_message' in content,
+ 'get_pending_thoughts': 'def get_pending_thoughts' in content,
+ 'archive_thoughts_for_conversation': 'def archive_thoughts_for_conversation' in content,
+ 'delete_thoughts_for_conversation': 'def delete_thoughts_for_conversation' in content,
+ 'delete_thoughts_for_message': 'def delete_thoughts_for_message' in content,
+ 'cosmos_thoughts_container import': 'cosmos_thoughts_container' in content,
+ 'step_index tracking': 'step_index' in content,
+ 'step_type field': 'step_type' in content,
+ 'error handling (try/except)': 'except Exception' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_route_module():
+ """Test that route_backend_thoughts.py has proper API endpoints."""
+ print("\nTesting route_backend_thoughts.py API endpoints...")
+ try:
+ route_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'route_backend_thoughts.py'
+ )
+
+ with open(route_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'register function': 'def register_route_backend_thoughts' in content,
+ 'message thoughts endpoint': '/messages/' in content and '/thoughts' in content,
+ 'pending thoughts endpoint': '/thoughts/pending' in content,
+ 'swagger_route decorator': 'swagger_route' in content,
+ 'login_required decorator': 'login_required' in content,
+ 'user_required decorator': 'user_required' in content,
+ 'get_auth_security import': 'get_auth_security' in content,
+ 'enable_thoughts check': 'enable_thoughts' in content,
+ 'returns enabled flag': "'enabled'" in content or '"enabled"' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_chat_instrumentation():
+ """Test that route_backend_chats.py has thought instrumentation points."""
+ print("\nTesting route_backend_chats.py thought instrumentation...")
+ try:
+ chats_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'route_backend_chats.py'
+ )
+
+ with open(chats_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'ThoughtTracker import': 'ThoughtTracker' in content,
+ 'thought_tracker instantiation': 'thought_tracker' in content and 'ThoughtTracker(' in content,
+ 'add_thought calls': 'thought_tracker.add_thought' in content,
+ 'content_safety thought': "'content_safety'" in content or '"content_safety"' in content,
+ 'search thought': "add_thought('search'" in content or 'add_thought("search"' in content,
+ 'web_search thought': "'web_search'" in content or '"web_search"' in content,
+ 'generation thought': "'generation'" in content or '"generation"' in content,
+ 'thoughts_enabled in response': 'thoughts_enabled' in content,
+ 'SSE thought event (streaming)': '"type": "thought"' in content or "'type': 'thought'" in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_frontend_module():
+ """Test that chat-thoughts.js has polling, streaming, and toggle logic."""
+ print("\nTesting chat-thoughts.js frontend module...")
+ try:
+ js_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'static', 'js', 'chat', 'chat-thoughts.js'
+ )
+
+ with open(js_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'startThoughtPolling export': 'export function startThoughtPolling' in content,
+ 'stopThoughtPolling export': 'export function stopThoughtPolling' in content,
+ 'handleStreamingThought export': 'export function handleStreamingThought' in content,
+ 'createThoughtsToggleHtml export': 'export function createThoughtsToggleHtml' in content,
+ 'attachThoughtsToggleListener export': 'export function attachThoughtsToggleListener' in content,
+ 'polling interval setup': 'setInterval' in content,
+ 'polling endpoint fetch': '/thoughts/pending' in content,
+ 'message thoughts fetch': '/thoughts' in content,
+ 'icon map (search)': 'bi-search' in content,
+ 'icon map (globe)': 'bi-globe' in content,
+ 'icon map (robot)': 'bi-robot' in content,
+ 'icon map (lightning)': 'bi-lightning' in content,
+ 'icon map (shield)': 'bi-shield-check' in content,
+ 'enable_thoughts check': 'enable_thoughts' in content,
+ 'd-none toggle': 'd-none' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_chat_messages_integration():
+ """Test that chat-messages.js integrates thought polling and toggle."""
+ print("\nTesting chat-messages.js thoughts integration...")
+ try:
+ messages_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'static', 'js', 'chat', 'chat-messages.js'
+ )
+
+ with open(messages_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'chat-thoughts import': 'chat-thoughts' in content,
+ 'startThoughtPolling import': 'startThoughtPolling' in content,
+ 'stopThoughtPolling import': 'stopThoughtPolling' in content,
+ 'createThoughtsToggleHtml import': 'createThoughtsToggleHtml' in content,
+ 'attachThoughtsToggleListener import': 'attachThoughtsToggleListener' in content,
+ 'startThoughtPolling call': 'startThoughtPolling(' in content,
+ 'stopThoughtPolling call': 'stopThoughtPolling(' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_streaming_integration():
+ """Test that chat-streaming.js handles thought SSE events."""
+ print("\nTesting chat-streaming.js thought event handling...")
+ try:
+ streaming_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'static', 'js', 'chat', 'chat-streaming.js'
+ )
+
+ with open(streaming_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'handleStreamingThought import': 'handleStreamingThought' in content,
+ 'thought type check': "'thought'" in content or '"thought"' in content,
+ 'handleStreamingThought call': 'handleStreamingThought(' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_loading_indicator():
+ """Test that chat-loading-indicator.js supports thought text updates."""
+ print("\nTesting chat-loading-indicator.js thought text support...")
+ try:
+ indicator_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'static', 'js', 'chat', 'chat-loading-indicator.js'
+ )
+
+ with open(indicator_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'updateLoadingIndicatorText export': 'export function updateLoadingIndicatorText' in content
+ or 'updateLoadingIndicatorText' in content,
+ 'text element update': 'thought-live-text' in content or 'loading-text' in content
+ or 'textContent' in content or 'innerHTML' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_admin_settings():
+ """Test that admin settings UI includes the thoughts toggle."""
+ print("\nTesting admin settings UI for thoughts toggle...")
+ try:
+ html_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'templates', 'admin_settings.html'
+ )
+
+ with open(html_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'enable_thoughts checkbox': 'id="enable_thoughts"' in content,
+ 'enable_thoughts name attr': 'name="enable_thoughts"' in content,
+ 'Processing Thoughts label': 'Processing Thoughts' in content,
+ 'lightbulb icon': 'bi-lightbulb' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_settings_default():
+ """Test that functions_settings.py includes enable_thoughts default."""
+ print("\nTesting functions_settings.py default setting...")
+ try:
+ settings_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'functions_settings.py'
+ )
+
+ with open(settings_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'enable_thoughts in defaults': 'enable_thoughts' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_cosmos_containers():
+ """Test that config.py defines thoughts Cosmos DB containers."""
+ print("\nTesting config.py Cosmos DB container definitions...")
+ try:
+ config_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'config.py'
+ )
+
+ with open(config_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'thoughts container name': 'cosmos_thoughts_container_name' in content,
+ 'thoughts container object': 'cosmos_thoughts_container' in content,
+ 'archive_thoughts container name': 'cosmos_archived_thoughts_container_name' in content,
+ 'archive_thoughts container object': 'cosmos_archived_thoughts_container' in content,
+ 'partition key /user_id': '/user_id' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_conversation_archive():
+ """Test that route_backend_conversations.py handles thought archive/delete."""
+ print("\nTesting route_backend_conversations.py thought archive support...")
+ try:
+ conv_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'route_backend_conversations.py'
+ )
+
+ with open(conv_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'archive_thoughts_for_conversation import': 'archive_thoughts_for_conversation' in content,
+ 'delete_thoughts_for_conversation import': 'delete_thoughts_for_conversation' in content,
+ 'archive_thoughts call': 'archive_thoughts_for_conversation(' in content,
+ 'delete_thoughts call': 'delete_thoughts_for_conversation(' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_css_styles():
+ """Test that chats.css includes thought-related styles."""
+ print("\nTesting chats.css thought styles...")
+ try:
+ css_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'static', 'css', 'chats.css'
+ )
+
+ with open(css_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'thoughts-toggle-btn style': '.thoughts-toggle-btn' in content,
+ 'thoughts-container style': '.thoughts-container' in content,
+ 'thought-step style': '.thought-step' in content,
+ 'animate-pulse animation': '.animate-pulse' in content,
+ 'thought-pulse keyframes': '@keyframes thought-pulse' in content,
+ 'dark mode thoughts toggle': '[data-bs-theme="dark"] .thoughts-toggle-btn' in content,
+ 'dark mode thought step': '[data-bs-theme="dark"] .thought-step' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+def test_thoughts_app_registration():
+ """Test that app.py registers the thoughts route blueprint."""
+ print("\nTesting app.py route registration...")
+ try:
+ app_file = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'application', 'single_app', 'app.py'
+ )
+
+ with open(app_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ checks = {
+ 'import register function': 'register_route_backend_thoughts' in content,
+ 'register call': 'register_route_backend_thoughts(app)' in content,
+ }
+
+ all_passed = True
+ for name, passed in checks.items():
+ status = 'PASS' if passed else 'FAIL'
+ print(f" [{status}] {name}")
+ if not passed:
+ all_passed = False
+
+ return all_passed
+
+ except Exception as e:
+ print(f" [FAIL] Exception: {e}")
+ return False
+
+
+if __name__ == "__main__":
+ print("=" * 60)
+ print("Processing Thoughts Feature - Functional Tests")
+ print("Version: 0.239.003")
+ print("=" * 60)
+
+ tests = [
+ test_thoughts_backend_module,
+ test_thoughts_route_module,
+ test_thoughts_chat_instrumentation,
+ test_thoughts_frontend_module,
+ test_thoughts_chat_messages_integration,
+ test_thoughts_streaming_integration,
+ test_thoughts_loading_indicator,
+ test_thoughts_admin_settings,
+ test_thoughts_settings_default,
+ test_thoughts_cosmos_containers,
+ test_thoughts_conversation_archive,
+ test_thoughts_css_styles,
+ test_thoughts_app_registration,
+ ]
+
+ results = []
+ for test_fn in tests:
+ try:
+ result = test_fn()
+ results.append(result)
+ except Exception as e:
+ print(f" [FAIL] Unhandled exception: {e}")
+ results.append(False)
+
+ print("\n" + "=" * 60)
+ passed = sum(1 for r in results if r)
+ total = len(results)
+ print(f"Results: {passed}/{total} test groups passed")
+
+ if passed == total:
+ print("All tests passed!")
+ else:
+ print("Some tests failed. Review output above.")
+
+ print("=" * 60)
+ sys.exit(0 if passed == total else 1)