diff --git a/design/vue/src/pages/ChatPage.vue b/design/vue/src/pages/ChatPage.vue index 709fbee..84182a6 100644 --- a/design/vue/src/pages/ChatPage.vue +++ b/design/vue/src/pages/ChatPage.vue @@ -171,79 +171,75 @@ - +
- -
-
-
- - - - 需要澄清 -
-
-

{{ pendingApproval.detail }}

- -
- -
- -
-
- +
+ + + +
diff --git a/internal/agent/prompt.go b/internal/agent/prompt.go index 670c0e5..6367d4e 100644 --- a/internal/agent/prompt.go +++ b/internal/agent/prompt.go @@ -60,11 +60,15 @@ func buildReActSystemPrompt(ctx context.Context, allTools []einoTool.BaseTool, i // ── 工作原则 ── sb.WriteString("## 工作原则\n") sb.WriteString("1. **先检索再回答**：第一步始终是 knowledge_search。即使你认为知道答案也必须先检索知识库\n") - sb.WriteString("2. **按需补充**：知识库结果不足时，再调用其他工具（grep_chunks 精准查找、get_document_info 查元数据等）\n") - sb.WriteString("3. **不重复调用**：已获得足够信息时，直接给出答案，不要为了'再确认'重复调用\n") - sb.WriteString("4. **工具上限 3 次**：工具调用总数不超过 3 次，用完必须收敛\n") - sb.WriteString("5. **强制收敛**：达到最大推理轮次或用完工具次数时，立即总结已有信息给出最终答案。禁止再规划'下一步应该'、'我还需要'等思考性输出\n") - sb.WriteString("6. **答案分层**：有 ToolCalls 的轮次 Message.Content 只写 1-2 句简短推理（不会展示给用户）；只有 ToolCalls 为空的轮次才是完整、可读、面向最终用户的答案正文\n") + sb.WriteString("2. **知识库内部工具各有分工**：knowledge_search 是语义搜索（找相关内容片段），list_knowledge_bases / list_knowledge_chunks 是列清单（看有哪些知识库/文档），grep_chunks 是关键词精准查找，get_document_info 是查文档详情。根据用户意图选择合适的工具——如果 knowledge_search 不满足需求（比如用户要'列出所有文档'而不是'找内容'），可以继续调用其他知识库内部工具\n") + sb.WriteString("3. **联网搜索谨慎使用**：knowledge_search 或其他知识库内部工具已经返回了相关内容时，**不要主动联网搜索补充细节**。联网搜索仅在以下情况使用：\n") + sb.WriteString(" - 用户明确要求'联网'、'最新'、'实时'、'当前'、'2024年后'等时效性内容\n") + sb.WriteString(" - 所有知识库工具都返回空结果或结果与问题完全不匹配\n") + sb.WriteString(" - 知识库内容有明显时间戳且已过时\n") + sb.WriteString("4. **不重复调用**：已获得足够信息时，直接给出答案，不要为了'再确认'重复调用\n") + sb.WriteString("5. **工具上限 3 次**：工具调用总数不超过 3 次，用完必须收敛\n") + sb.WriteString("6. **强制收敛**：达到最大推理轮次或用完工具次数时，立即总结已有信息给出最终答案。禁止再规划'下一步应该'、'我还需要'等思考性输出\n") + sb.WriteString("7. **答案分层**：有 ToolCalls 的轮次 Message.Content 只写 1-2 句简短推理（不会展示给用户）；只有 ToolCalls 为空的轮次才是完整、可读、面向最终用户的答案正文\n") // 危险工具补充说明 hasDangerous := false @@ -75,7 +79,7 @@ func buildReActSystemPrompt(ctx context.Context, allTools []einoTool.BaseTool, i } } if hasDangerous { - sb.WriteString("7. **危险工具审批**：delete_document 等危险工具会在执行前暂停并等待用户审批，调用后流程中断，用户确认后自动继续\n") + sb.WriteString("8. **危险工具审批**：delete_document 等危险工具会在执行前暂停并等待用户审批，调用后流程中断，用户确认后自动继续\n") sb.WriteString(" - ⚠️ **目标不明确先反问**：当用户说'删除那个文档'、'清理一下'、'把上面的删了'这类模糊指令，且从对话历史无法唯一确定目标时，**绝对不能编造参数调用工具**。先调用 ask_clarify 反问用户明确目标（例如：'你要删除的是《压力 - 07/13 16:03》那个文档吗？还是另一个？'）\n") sb.WriteString(" - ⚠️ **禁止猜测参数**：document_id 等关键参数必须来自可靠来源（用户明确提供、get_document_info 工具查询结果、历史对话中已确认的 ID）。严禁从模糊描述或'看起来像是'的文本中猜测或编造\n") sb.WriteString(" - 调用危险工具时务必在参数里写清楚目标和原因，便于用户决策\n") @@ -105,15 +109,15 @@ func buildReActSystemPrompt(ctx context.Context, allTools []einoTool.BaseTool, i } } if len(externals) > 0 { - sb.WriteString(fmt.Sprintf("8. **联网搜索**：当 knowledge_search 返回空结果或与问题不相关时，必须调用 %s 联网搜索。用户明确要求'联网'、'最新'时即使知识库有结果也应联网\n", - strings.Join(externals, " 或 "))) + sb.WriteString(fmt.Sprintf("\n## 联网搜索（%s）\n", strings.Join(externals, " 或 "))) + sb.WriteString("谨慎使用，触发条件见上面的工作原则第 3 条。**知识库已有相关内容或可以用其他内部工具解决时，不要主动联网补充。**\n") } sb.WriteString("\n") // ── 禁止 ── sb.WriteString("## 禁止\n") sb.WriteString("- 不检索知识库就用自身知识回答\n") - sb.WriteString("- 知识库检索失败不联网搜索就直接回答\n") + sb.WriteString("- 知识库已有相关结果时，为了'再确认'或'补充细节'而联网搜索\n") sb.WriteString("- 在有可用工具时直接给用户'请去界面手动操作'的建议——你应该调用工具来完成\n\n") // ── 回答要求 ── diff --git a/internal/app/app.go b/internal/app/app.go index c9eb512..155ec0d 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "os/signal" + "sync" "syscall" "time" @@ -16,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/redis/go-redis/v9" "go.uber.org/zap" + "golang.org/x/sync/singleflight" "gorm.io/gorm" "solvify-agent/internal/agent" @@ -127,25 +129,28 @@ func (a *App) initDatabase() error { return fmt.Errorf("初始化 PostgreSQL 失败: %w", err) } a.postgresqlDB = postgresqlDB - //自动迁移数据库表结构 - //if err := postgresqlDB.AutoMigrate( - // &entity.User{}, - // &entity.Model{}, - // &entity.UserModelConfig{}, - // &entity.KnowledgeBase{}, - // &entity.StorageQuota{}, - // &entity.Document{}, - // &entity.DocumentProcessingJob{}, - // &entity.DocumentVersion{}, - // &entity.DocumentChunk{}, - // &entity.ChatSession{}, - // &entity.ChatMessage{}, - // &entity.ToolType{}, - // &entity.ToolProvider{}, - // &entity.UserToolConfig{}, - //); err != nil { - // return fmt.Errorf("数据库自动迁移失败: %w", err) - //} + + // pgvector 索引健康检查（仅在 enable_pgvector 时执行） + if a.cfg.Database.Postgres.EnablePGVector { + if err := database.EnsurePGVectorIndex(postgresqlDB); err != nil { + logger.Warnf("pgvector 索引检查异常（不阻塞启动）: %v", err) + } + } + + // keywords GIN 索引健康检查（关键词检索核心加速，不依赖 pgvector 开关） + if err := database.EnsureKeywordsGINIndex(postgresqlDB); err != nil { + logger.Warnf("keywords GIN 索引检查异常（不阻塞启动）: %v", err) + } + + // 上下文加载链路高频索引（chat_messages / chat_sessions / user_memories） + if err := database.EnsureContextIndexes(postgresqlDB); err != nil { + logger.Warnf("上下文索引检查异常（不阻塞启动）: %v", err) + } + + // message_feedback 表 schema 补齐（早期 AutoMigrate 建表后 entity 新增列，AutoMigrate 不会 ADD COLUMN） + if err := database.EnsureMessageFeedbackSchema(postgresqlDB); err != nil { + logger.Warnf("message_feedback schema 补齐异常（不阻塞启动）: %v", err) + } // Redis 缓存连接 redisClient, err := database.OpenRedis(&a.cfg.Database.Redis) @@ -165,7 +170,19 @@ func (a *App) ensureStorageQuotaUniqueIndex(db *gorm.DB) error { return nil } -// initEmbedding 初始化 Embedding 客户端，返回带缓存的向量化函数 +const ( + embeddingInMemCacheSize = 2048 + embeddingRedisTTL = 24 * time.Hour +) + +// initEmbedding 初始化 Embedding 客户端，返回带两级缓存 + singleflight 去重的向量化函数 +// +// 缓存层级： +// 1. 进程内 sync.Map fast-path（零网络 IO，容量 embeddingInMemCacheSize，LRU 清理） +// 2. Redis 共享缓存（跨进程，24h TTL） +// 3. Embedding API 调用 +// +// singleflight 防止并发击穿：同一 key 的并发请求只打一次 API。 func (a *App) initEmbedding() rag.EmbeddingFunc { embeddingClient, err := llm.NewEmbeddingClientFromConfig(context.Background(), &a.cfg.Embedding) @@ -173,32 +190,86 @@ func (a *App) initEmbedding() rag.EmbeddingFunc { logger.Fatal("初始化 Embedding 客户端失败", zap.Error(err)) } - // Embedding 缓存（相同文本 → 相同向量，缓存 24 小时） - embeddingCache := cache.New(a.redis, "emb:", 24*time.Hour) + redisCache := cache.New(a.redis, "emb:", embeddingRedisTTL) + var inMem sync.Map // map[string][]float64 + var sf singleflight.Group + var inMemMu sync.Mutex + inMemCount := 0 return func(ctx context.Context, text string) ([]float64, error) { cacheKey := fmt.Sprintf("%x", sha256.Sum256([]byte(text))) - var vec []float64 - if found, _ := embeddingCache.Get(ctx, cacheKey, &vec); found { - logger.Infof("[Embedding] 缓存命中: key=%s text=%q dim=%d", cacheKey[:8], text, len(vec)) - return vec, nil - } - logger.Infof("[Embedding] 缓存未命中: key=%s text=%q, 调用 API...", cacheKey[:8], text) - vec, err := embeddingClient.Embed(ctx, text) - if err != nil { - logger.Errorf("[Embedding] API 调用失败: %v", err) - return nil, err + + // 层级 1：进程内 fast-path + if v, ok := inMem.Load(cacheKey); ok { + logger.Debugf("[Embedding] 内存缓存命中: key=%s", cacheKey[:8]) + return v.([]float64), nil } - logger.Infof("[Embedding] API 返回: dim=%d", len(vec)) - if err := embeddingCache.Set(ctx, cacheKey, vec, 0); err != nil { - logger.Warnf("[Embedding] 缓存写入失败: %v", err) - } else { - logger.Infof("[Embedding] 已写入缓存: key=%s", cacheKey[:8]) + + // singleflight 包裹：并发只调一次 + ch := sf.DoChan(cacheKey, func() (interface{}, error) { + // 层级 2：Redis + var vec []float64 + if found, _ := redisCache.Get(ctx, cacheKey, &vec); found { + logger.Infof("[Embedding] Redis 缓存命中: key=%s dim=%d", cacheKey[:8], len(vec)) + inMem.Store(cacheKey, vec) + return vec, nil + } + + // 层级 3：调 API + logger.Infof("[Embedding] 全缓存未命中: key=%s text=%q, 调用 API...", cacheKey[:8], truncateText(text, 60)) + vec, err := embeddingClient.Embed(ctx, text) + if err != nil { + logger.Errorf("[Embedding] API 调用失败: %v", err) + return nil, err + } + + if err := redisCache.Set(ctx, cacheKey, vec, 0); err != nil { + logger.Warnf("[Embedding] Redis 缓存写入失败: %v", err) + } + inMem.Store(cacheKey, vec) + + // 简单容量管控：超阈值时删除约 20% 条目 + inMemMu.Lock() + inMemCount++ + if inMemCount > embeddingInMemCacheSize { + cleaned := 0 + inMem.Range(func(key, _ any) bool { + if cleaned >= embeddingInMemCacheSize/5 { + return false + } + inMem.Delete(key) + cleaned++ + return true + }) + inMemCount -= cleaned + logger.Infof("[Embedding] 内存缓存清理: 删除 %d 条, 剩余约 %d", cleaned, inMemCount) + } + inMemMu.Unlock() + + logger.Infof("[Embedding] API 返回: dim=%d, 已写两级缓存", len(vec)) + return vec, nil + }) + + select { + case res := <-ch: + if res.Err != nil { + return nil, res.Err + } + return res.Val.([]float64), nil + case <-ctx.Done(): + return nil, ctx.Err() } - return vec, nil } } +func truncateText(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "..." +} + // initRetriever 初始化 RAG 检索器（混合检索 + 可选装饰器链） func (a *App) initRetriever(embeddingFunc rag.EmbeddingFunc) rag.Retriever { // 使用混合检索器（向量 + 关键词 + RRF 融合） diff --git a/internal/llm/factory.go b/internal/llm/factory.go index 8a45b58..71b0553 100644 --- a/internal/llm/factory.go +++ b/internal/llm/factory.go @@ -101,5 +101,6 @@ func NewEmbeddingClientFromConfig(ctx context.Context, cfg *config.EmbeddingConf BaseURL: cfg.BaseURL, Model: cfg.Model, Dimension: cfg.Dimension, + Timeout: cfg.Timeout, }) } diff --git a/internal/model/entity/chat_message.go b/internal/model/entity/chat_message.go index 109d756..88c4444 100644 --- a/internal/model/entity/chat_message.go +++ b/internal/model/entity/chat_message.go @@ -9,7 +9,7 @@ import ( // ChatMessage 映射聊天消息表 type ChatMessage struct { ID string `gorm:"column:id;type:uuid;default:gen_random_uuid();primaryKey"` - SessionID string `gorm:"column:session_id;type:uuid;not null"` + SessionID string `gorm:"column:session_id;type:uuid;not null;index:idx_chat_messages_session_created,priority:1"` Role string `gorm:"column:role;type:varchar(20);not null"` Content string `gorm:"column:content;type:text;not null;default:''"` ModelID string `gorm:"column:model_id;type:varchar(36)"` @@ -17,9 +17,8 @@ type ChatMessage struct { KnowledgeBaseIDs datatypes.JSON `gorm:"column:knowledge_base_ids;type:jsonb;not null;default:'[]'::jsonb"` Sources datatypes.JSON `gorm:"column:sources;type:jsonb"` Metadata datatypes.JSON `gorm:"column:metadata;type:jsonb;not null;default:'{}'::jsonb"` - // Embedding 消息内容的向量表示，用于语义相关历史检索 - Embedding FloatVector `gorm:"column:embedding;type:vector(1024)"` - CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` + Embedding FloatVector `gorm:"column:embedding;type:vector(1024)"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime;index:idx_chat_messages_session_created,priority:2"` } // TableName 返回聊天消息表名 diff --git a/internal/model/entity/chat_session.go b/internal/model/entity/chat_session.go index 808a256..4412a5c 100644 --- a/internal/model/entity/chat_session.go +++ b/internal/model/entity/chat_session.go @@ -10,10 +10,10 @@ import ( // ChatSession 映射聊天会话表 type ChatSession struct { ID string `gorm:"column:id;type:uuid;default:gen_random_uuid();primaryKey"` - UserID string `gorm:"column:user_id;type:uuid;not null"` + UserID string `gorm:"column:user_id;type:uuid;not null;index:idx_chat_sessions_user_status,priority:1"` Title string `gorm:"column:title;size:200;not null;default:''"` ModelID string `gorm:"column:model_id;type:varchar(36);not null"` - Status string `gorm:"column:status;type:varchar(20);not null;default:'active'"` + Status string `gorm:"column:status;type:varchar(20);not null;default:'active';index:idx_chat_sessions_user_status,priority:2"` PendingClarify datatypes.JSON `gorm:"column:pending_clarify;type:jsonb;not null;default:'{}'::jsonb"` PendingCheckpoint datatypes.JSON `gorm:"column:pending_checkpoint;type:jsonb;not null;default:'{}'::jsonb"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` diff --git a/internal/model/entity/user_memory.go b/internal/model/entity/user_memory.go index 0175314..7f608a2 100644 --- a/internal/model/entity/user_memory.go +++ b/internal/model/entity/user_memory.go @@ -10,12 +10,12 @@ import ( // UserMemory 用户长期记忆：保存跨会话的事实、偏好、约束和决策结论 type UserMemory struct { ID string `gorm:"type:uuid;primaryKey"` - UserID string `gorm:"type:uuid;not null"` - MemoryType string `gorm:"type:varchar(30);not null"` // fact / preference / constraint / decision + UserID string `gorm:"type:uuid;not null;index:idx_user_memories_user_active,priority:1"` + MemoryType string `gorm:"type:varchar(30);not null"` Content string `gorm:"type:text;not null"` SourceSession *string `gorm:"type:uuid"` Confidence float64 `gorm:"type:float;default:1.0"` - IsActive bool `gorm:"default:true"` + IsActive bool `gorm:"default:true;index:idx_user_memories_user_active,priority:2"` CreatedAt time.Time UpdatedAt time.Time } diff --git a/internal/observability/eino_callback.go b/internal/observability/eino_callback.go index 4c6bd3a..314db80 100644 --- a/internal/observability/eino_callback.go +++ b/internal/observability/eino_callback.go @@ -234,6 +234,9 @@ func einoOnEnd(rec Recorder) func(ctx context.Context, info *callbacks.RunInfo, rec.Incr(ctx, "eino_retriever_requests_total", rl, 1) rec.Observe(ctx, "eino_retriever_duration_seconds", rl, dur.Seconds()) rec.Observe(ctx, "eino_retriever_hit_count", rl, float64(hitN)) + if hitN == 0 { + rec.Incr(ctx, "eino_retriever_empty_results_total", rl, 1) + } case components.ComponentOfTool: tl := mergeToolEndAttrs(attrs, output, info, rec, baseLabels) rec.Incr(ctx, "eino_tool_calls_total", tl, 1) diff --git a/internal/rag/hybrid_retriever.go b/internal/rag/hybrid_retriever.go index 07b6745..96e7be0 100644 --- a/internal/rag/hybrid_retriever.go +++ b/internal/rag/hybrid_retriever.go @@ -10,8 +10,10 @@ import ( "github.com/go-ego/gse" "gorm.io/gorm" + "solvify-agent/internal/observability" "solvify-agent/pkg/config" "solvify-agent/pkg/logger" + "solvify-agent/pkg/stopwords" ) var ( @@ -156,21 +158,29 @@ func (r *HybridRetriever) Retrieve(ctx context.Context, query Query) (Result, er vr := <-vectorCh kr := <-keywordCh + rec := observability.RecorderFromContext(ctx) + // 向量检索失败时降级：仅用关键词结果，不阻断检索 if vr.err != nil { logger.Warnf("向量检索失败，降级为纯关键词检索: %v", vr.err) vr.docs = nil // 清空，后续只用关键词结果 + rec.Incr(ctx, "rag_retriever_degradation_total", map[string]string{"side": "vector", "reason": "search_error"}, 1) } if kr.err != nil { logger.Warnf("关键词检索失败，降级为纯向量检索: %v", kr.err) kr.docs = nil + rec.Incr(ctx, "rag_retriever_degradation_total", map[string]string{"side": "keyword", "reason": "search_error"}, 1) } // 两种检索都失败才报错 if vr.err != nil && kr.err != nil { + rec.Incr(ctx, "rag_retriever_degradation_total", map[string]string{"side": "both", "reason": "search_error"}, 1) return Result{}, fmt.Errorf("混合检索完全失败: 向量(%v), 关键词(%v)", vr.err, kr.err) } + observeStage(rec, ctx, "vector_raw", float64(len(vr.docs))) + observeStage(rec, ctx, "keyword_raw", float64(len(kr.docs))) + logger.Infof("向量检索命中: %d 条, 关键词检索命中: %d 条", len(vr.docs), len(kr.docs)) // ===== Step 1: 同源质量检查（各自独立过滤） ===== @@ -182,6 +192,7 @@ func (r *HybridRetriever) Retrieve(ctx context.Context, query Query) (Result, er filteredVector = append(filteredVector, doc) } } + observeStage(rec, ctx, "vector_filtered", float64(len(filteredVector))) // 1b. 关键词侧：陡峭度检测 + 最低匹配过滤 filteredKeyword := r.keywordSourceFilter(kr.docs) @@ -189,7 +200,9 @@ func (r *HybridRetriever) Retrieve(ctx context.Context, query Query) (Result, er // 1c. 向量全灭时，对关键词结果加最低匹配比例过滤 if len(filteredVector) == 0 && len(filteredKeyword) > 0 { filteredKeyword = filterByMinScore(filteredKeyword, r.keywordScoreThreshold, "关键词") + rec.Incr(ctx, "rag_retriever_degradation_total", map[string]string{"side": "keyword_only", "reason": "min_score_filter"}, 1) } + observeStage(rec, ctx, "keyword_filtered", float64(len(filteredKeyword))) // ===== Step 2: 同源内 Min-Max 归一化 ===== vectorNorm := minMaxNormalize(filteredVector) @@ -197,9 +210,11 @@ func (r *HybridRetriever) Retrieve(ctx context.Context, query Query) (Result, er // ===== Step 3: RRF 融合 ===== fused := r.reciprocalRankFusion(filteredVector, filteredKeyword) + observeStage(rec, ctx, "rrf_fused", float64(len(fused))) // ===== Step 4: 跨源交叉验证 ===== fused = r.crossSourceFilter(fused, filteredVector, filteredKeyword, vectorNorm, keywordNorm) + observeStage(rec, ctx, "cross_filtered", float64(len(fused))) // ===== Step 5: TopK 截取 ===== docs := make([]Document, 0, len(fused)) @@ -218,6 +233,7 @@ func (r *HybridRetriever) Retrieve(ctx context.Context, query Query) (Result, er Score: item.Score, }) } + observeStage(rec, ctx, "final", float64(len(docs))) logger.Infof("混合检索最终结果: %d 条 (向量过滤阈值=%.2f, TopK=%d, 向量候选=%d, 关键词候选=%d)", len(docs), r.scoreThreshold, topK, len(filteredVector), len(filteredKeyword)) @@ -229,6 +245,9 @@ func (r *HybridRetriever) Retrieve(ctx context.Context, query Query) (Result, er } // vectorSearch 执行向量检索 +// 优化：主查询只查 document_chunks 表（不 LEFT JOIN documents）， +// 向量距离排序在 chunks 表上直接跑，拿 topK*2 后再批量查 documents 表的 title。 +// 避免对所有候选 chunk 做额外 JOIN。 func (r *HybridRetriever) vectorSearch(ctx context.Context, query Query) ([]scoredChunk, error) { embedding, err := r.embeddingFunc(ctx, query.Question) if err != nil { @@ -236,130 +255,97 @@ func (r *HybridRetriever) vectorSearch(ctx context.Context, query Query) ([]scor } vectorStr := vectorToString(embedding) - topK := query.TopK * 2 // 多召回一些用于融合 + topK := query.TopK * 2 var results []scoredChunk err = r.db.WithContext(ctx).Raw(` - SELECT id, knowledge_base_id, document_id, version_id, chunk_index, title, content, score, keywords - FROM ( - SELECT - dc.id, - dc.knowledge_base_id, - dc.document_id, - dc.version_id, - dc.chunk_index, - COALESCE(d.title, '') as title, - dc.content, - 1 - (dc.embedding <=> ?::vector) AS score, - COALESCE(dc.keywords::text, '{}') as keywords - FROM document_chunks dc - LEFT JOIN documents d ON d.id = dc.document_id - WHERE dc.knowledge_base_id IN (?) - AND dc.embedding IS NOT NULL - AND dc.user_id = ? - ORDER BY dc.embedding <=> ?::vector - LIMIT ? - ) sub + SELECT + dc.id, + dc.knowledge_base_id, + dc.document_id, + dc.version_id, + dc.chunk_index, + dc.content, + 1 - (dc.embedding <=> ?::vector) AS score, + COALESCE(dc.keywords::text, '{}') as keywords + FROM document_chunks dc + WHERE dc.knowledge_base_id IN (?) + AND dc.embedding IS NOT NULL + AND dc.user_id = ? + ORDER BY dc.embedding <=> ?::vector + LIMIT ? `, vectorStr, query.KnowledgeBaseIDs, query.UserID, vectorStr, topK).Scan(&results).Error if err != nil { return nil, err } + // 批量填 title（只对 topK*2 条查，开销可忽略） + batchFillTitles(r.db, results) + logger.Infof("向量检索原始结果: %d 条", len(results)) return results, nil } // keywordSearch 执行关键词检索 // 优化：GIN 索引加速 && overlap 过滤（主收益），unnest 仅对过滤后的少量行计算分数 +// 也去掉了 LEFT JOIN documents，title 在主查询完成后批量填 func (r *HybridRetriever) keywordSearch(ctx context.Context, query Query) ([]scoredChunk, error) { - // 从问题中提取关键词（简单的分词策略） keywords := extractKeywords(query.Question) if len(keywords) == 0 { return nil, nil } - topK := query.TopK * 2 // 多召回一些用于融合 + topK := query.TopK * 2 var results []scoredChunk - // 构建关键词数组字面量 keywordArray := buildPostgresArray(keywords) err := r.db.WithContext(ctx).Raw(` - SELECT id, knowledge_base_id, document_id, version_id, chunk_index, title, content, score, keywords - FROM ( - SELECT - dc.id, - dc.knowledge_base_id, - dc.document_id, - dc.version_id, - dc.chunk_index, - COALESCE(d.title, '') as title, - dc.content, - -- 计算匹配关键词占比：匹配数 / 查询关键词总数 - -- unnest 仅在 && 过滤后的少量行上执行，GIN 索引保证过滤极快 - ( - SELECT COUNT(*)::float / GREATEST(cardinality(?::text[]), 1) - FROM unnest(dc.keywords) AS kw - WHERE kw = ANY(?::text[]) - ) AS score, - COALESCE(dc.keywords::text, '{}') as keywords - FROM document_chunks dc - LEFT JOIN documents d ON d.id = dc.document_id - WHERE dc.knowledge_base_id IN (?) - AND dc.keywords IS NOT NULL - AND dc.keywords && ?::text[] -- GIN 索引加速：先快速过滤有交集的 chunk - AND dc.user_id = ? - ORDER BY score DESC - LIMIT ? - ) sub - WHERE score > 0 + SELECT + dc.id, + dc.knowledge_base_id, + dc.document_id, + dc.version_id, + dc.chunk_index, + dc.content, + ( + SELECT COUNT(*)::float / GREATEST(cardinality(?::text[]), 1) + FROM unnest(dc.keywords) AS kw + WHERE kw = ANY(?::text[]) + ) AS score, + COALESCE(dc.keywords::text, '{}') as keywords + FROM document_chunks dc + WHERE dc.knowledge_base_id IN (?) + AND dc.keywords IS NOT NULL + AND dc.keywords && ?::text[] + AND dc.user_id = ? + AND dc.embedding IS NOT NULL + ORDER BY score DESC + LIMIT ? `, keywordArray, keywordArray, query.KnowledgeBaseIDs, keywordArray, query.UserID, topK).Scan(&results).Error if err != nil { return nil, err } - logger.Infof("关键词检索原始结果: %d 条, 关键词: %v", len(results), keywords) - return results, nil -} + batchFillTitles(r.db, results) -// extractKeywords 使用 gse 分词提取关键词 -func extractKeywords(question string) []string { - stopWords := map[string]bool{ - "的": true, "了": true, "在": true, "是": true, "我": true, - "有": true, "和": true, "就": true, "不": true, "人": true, - "都": true, "一": true, "一个": true, "上": true, "也": true, - "很": true, "到": true, "说": true, "要": true, "去": true, - "你": true, "会": true, "着": true, "没有": true, "看": true, - "好": true, "自己": true, "这": true, "他": true, "她": true, - "它": true, "们": true, "那": true, "什么": true, - "怎么": true, "如何": true, "为什么": true, "哪": true, "哪个": true, - "哪些": true, "吗": true, "呢": true, "吧": true, "啊": true, - "the": true, "a": true, "an": true, "is": true, "are": true, - "was": true, "were": true, "be": true, "been": true, "being": true, - "have": true, "has": true, "had": true, "do": true, "does": true, - "did": true, "will": true, "would": true, "could": true, "should": true, - "may": true, "might": true, "can": true, "shall": true, "must": true, - "to": true, "of": true, "in": true, "for": true, "on": true, - "with": true, "at": true, "by": true, "from": true, "as": true, - "into": true, "through": true, "during": true, "before": true, "after": true, - "above": true, "below": true, "between": true, "and": true, "but": true, - "or": true, "nor": true, "not": true, "so": true, "yet": true, - "both": true, "either": true, "neither": true, "each": true, "every": true, - "all": true, "any": true, "few": true, "more": true, "most": true, - "other": true, "some": true, "such": true, "no": true, "only": true, - "own": true, "same": true, "than": true, "too": true, "very": true, - "just": true, "because": true, "if": true, "when": true, "where": true, - "how": true, "what": true, "which": true, "who": true, "whom": true, - "this": true, "that": true, "these": true, "those": true, "i": true, - "me": true, "my": true, "we": true, "our": true, "you": true, - "your": true, "he": true, "him": true, "his": true, "she": true, - "her": true, "it": true, "its": true, "they": true, "them": true, - "their": true, + // 过滤零分结果 + filtered := results[:0] + for _, r := range results { + if r.Score > 0 { + filtered = append(filtered, r) + } } + logger.Infof("关键词检索原始结果: %d 条(有效), 关键词: %v", len(filtered), keywords) + return filtered, nil +} + +// extractKeywords 使用 gse 分词提取关键词，过滤停用词 +func extractKeywords(question string) []string { seg := getSegmenter() words := seg.Cut(question, true) @@ -370,7 +356,7 @@ func extractKeywords(question string) []string { if w == "" || len(w) < 2 { continue } - if stopWords[w] { + if stopwords.IsStopWord(w) { continue } if seen[w] { @@ -379,10 +365,50 @@ func extractKeywords(question string) []string { seen[w] = true keywords = append(keywords, w) } - return keywords } +// batchFillTitles 对检索结果批量填充文档标题。 +// 把 LEFT JOIN documents 从主查询里拆出来——主查询只跑 chunks 表排序取 topK， +// 然后对这少量结果的 document_id 做一次 IN 查询拿 title，JOIN 开销从 O(全量 chunks) 降到 O(topK)。 +func batchFillTitles(db *gorm.DB, chunks []scoredChunk) { + if db == nil || len(chunks) == 0 { + return + } + // 收集去重的 document_id + docIDs := make(map[string]struct{}) + for _, c := range chunks { + if c.DocumentID != "" { + docIDs[c.DocumentID] = struct{}{} + } + } + if len(docIDs) == 0 { + return + } + idList := make([]string, 0, len(docIDs)) + for id := range docIDs { + idList = append(idList, id) + } + + var rows []struct { + ID string + Title string + } + if err := db.Raw("SELECT id, title FROM documents WHERE id IN ?", idList).Scan(&rows).Error; err != nil { + logger.Warnf("batchFillTitles 查 documents 失败: %v", err) + return + } + titleMap := make(map[string]string, len(rows)) + for _, r := range rows { + titleMap[r.ID] = r.Title + } + for i := range chunks { + if t, ok := titleMap[chunks[i].DocumentID]; ok { + chunks[i].Title = t + } + } +} + // buildPostgresArray 构建 PostgreSQL 数组字面量 func buildPostgresArray(items []string) string { if len(items) == 0 { @@ -593,3 +619,12 @@ func (r *HybridRetriever) crossSourceFilter( return result } + +// observeStage 记录混合检索各阶段的候选条数，供 Prometheus 观测管线漏斗。 +// rec 为 nil 时静默跳过（单元测试或未注册 observability 的场景）。 +func observeStage(rec observability.Recorder, ctx context.Context, stage string, count float64) { + if rec == nil { + return + } + rec.Observe(ctx, "rag_retriever_stage_count", map[string]string{"stage": stage}, count) +} diff --git a/internal/rag/rerank_retriever.go b/internal/rag/rerank_retriever.go index 561d764..e1bceaf 100644 --- a/internal/rag/rerank_retriever.go +++ b/internal/rag/rerank_retriever.go @@ -24,6 +24,8 @@ type RerankRetriever struct { timeout time.Duration scoreThreshold float64 httpClient *http.Client + maxRetries int + baseBackoff time.Duration } // RerankRetrieverConfig 描述重排序检索器配置 @@ -37,31 +39,40 @@ type RerankRetrieverConfig struct { ScoreThreshold float64 } +const ( + defaultRerankTimeout = 5 + defaultRerankTopN = 3 + defaultRerankThreshold = 0.5 + defaultRerankMaxRetries = 3 + defaultRerankBaseBackoff = 100 * time.Millisecond +) + // NewRerankRetriever 创建重排序检索器 func NewRerankRetriever(cfg RerankRetrieverConfig) *RerankRetriever { timeout := cfg.Timeout if timeout <= 0 { - timeout = 5 + timeout = defaultRerankTimeout } topN := cfg.TopN if topN <= 0 { - topN = 3 + topN = defaultRerankTopN } threshold := cfg.ScoreThreshold if threshold <= 0 { - threshold = 0.5 + threshold = defaultRerankThreshold } + d := time.Duration(timeout) * time.Second return &RerankRetriever{ inner: cfg.Inner, endpoint: cfg.Endpoint, model: cfg.Model, apiKey: cfg.APIKey, topN: topN, - timeout: time.Duration(timeout) * time.Second, + timeout: d, scoreThreshold: threshold, - httpClient: &http.Client{ - Timeout: time.Duration(timeout) * time.Second, - }, + httpClient: &http.Client{Timeout: d}, + maxRetries: defaultRerankMaxRetries, + baseBackoff: defaultRerankBaseBackoff, } } @@ -103,13 +114,11 @@ type rerankResponse struct { func (r *RerankRetriever) Retrieve(ctx context.Context, query Query) (Result, error) { logger.Infof("[Rerank] 开始重排序, query=%q, endpoint=%s, model=%s", query.Question, r.endpoint, r.model) - // 先调用内层检索器 result, err := r.inner.Retrieve(ctx, query) if err != nil { return Result{}, err } - // 无结果则直接返回 if !result.Hit || len(result.Documents) == 0 { logger.Infof("[Rerank] 内层检索无结果，跳过重排序") return result, nil @@ -117,76 +126,100 @@ func (r *RerankRetriever) Retrieve(ctx context.Context, query Query) (Result, er logger.Infof("[Rerank] 内层检索返回 %d 条，开始调用 Rerank API", len(result.Documents)) for i, doc := range result.Documents { - logger.Infof("[Rerank] 输入#%d: [%s] score=%.4f chunk#%d title=%q content=%q", + logger.Debugf("[Rerank] 输入#%d: [%s] score=%.4f chunk#%d title=%q content=%q", i, doc.DocumentID, doc.Score, doc.ChunkIndex, doc.Title, truncate(doc.Content, 60)) } - // 调用 Rerank API - reranked, err := r.rerank(ctx, query.Question, result.Documents) + reranked, err := r.rerankWithRetry(ctx, query.Question, result.Documents) if err != nil { - // 优雅降级：Rerank 失败时返回原始结果 - logger.Warnf("[Rerank] Rerank API 调用失败，降级返回原始结果: %v", err) + logger.Warnf("[Rerank] Rerank API 调用失败（已重试 %d 次），降级返回原始结果: %v", r.maxRetries, err) return result, nil } - logger.Infof("[Rerank] API 返回 %d 条结果", len(reranked)) - for _, item := range reranked { - logger.Infof("[Rerank] API结果#%d: relevance_score=%.4f", item.Index, item.RelevanceScore) - } - - // 用重排序分数替换原始分数 for _, item := range reranked { if item.Index >= 0 && item.Index < len(result.Documents) { oldScore := result.Documents[item.Index].Score result.Documents[item.Index].Score = item.RelevanceScore - logger.Infof("[Rerank] 文档#%d [%s] 分数: %.4f → %.4f", + logger.Debugf("[Rerank] 文档#%d [%s] 分数: %.4f → %.4f", item.Index, result.Documents[item.Index].DocumentID, oldScore, item.RelevanceScore) } } - // 按新分数降序排序 sort.Slice(result.Documents, func(i, j int) bool { return result.Documents[i].Score > result.Documents[j].Score }) - // 过滤低分结果 filtered := make([]Document, 0, len(result.Documents)) for _, doc := range result.Documents { if doc.Score >= r.scoreThreshold { filtered = append(filtered, doc) } else { - logger.Infof("[Rerank] 过滤: [%s] score=%.4f < 阈值 %.2f", doc.DocumentID, doc.Score, r.scoreThreshold) + logger.Debugf("[Rerank] 过滤: [%s] score=%.4f < 阈值 %.2f", doc.DocumentID, doc.Score, r.scoreThreshold) } } - // 截取 TopN if len(filtered) > r.topN { logger.Infof("[Rerank] 截取 TopN: %d → %d", len(filtered), r.topN) filtered = filtered[:r.topN] } logger.Infof("[Rerank] 重排序完成: 原始 %d 条 → 过滤后 %d 条", len(result.Documents), len(filtered)) - for i, doc := range filtered { - logger.Infof("[Rerank] 最终#%d: [%s] score=%.4f chunk#%d title=%q content=%q", - i, doc.DocumentID, doc.Score, doc.ChunkIndex, doc.Title, truncate(doc.Content, 60)) - } - return Result{ Hit: len(filtered) > 0, Documents: filtered, }, nil } -// rerank 调用外部 Rerank API -func (r *RerankRetriever) rerank(ctx context.Context, query string, docs []Document) ([]rerankResult, error) { +// rerankWithRetry 调用 Rerank API，带指数退避重试 +// +// 重试策略： +// - 4xx（除 429）: 客户端错误，立即放弃（参数/鉴权问题，重试无意义） +// - 429 / 5xx / 网络错误: 重试，baseBackoff * 2^attempt，最多 maxRetries 次 +// - context 取消: 立即放弃 +func (r *RerankRetriever) rerankWithRetry(ctx context.Context, query string, docs []Document) ([]rerankResult, error) { + var lastErr error + + for attempt := 0; attempt <= r.maxRetries; attempt++ { + if attempt > 0 { + // 计算退避时间 + backoff := r.baseBackoff * time.Duration(1<= 500 { + // 5xx 服务端错误，可重试 + return nil, true, fmt.Errorf("rerank API 返回 %d (5xx): %s", resp.StatusCode, truncate(string(respBody), 200)) + } + // 其他 4xx（400, 401, 403, 404 等）：客户端错误，不可重试 + return nil, false, fmt.Errorf("rerank API 返回 %d (4xx, 不可重试): %s", resp.StatusCode, truncate(string(respBody), 200)) + } } diff --git a/internal/repository/chat_message_repository.go b/internal/repository/chat_message_repository.go index f46ae89..fe4bcde 100644 --- a/internal/repository/chat_message_repository.go +++ b/internal/repository/chat_message_repository.go @@ -108,15 +108,24 @@ func (r *chatMessageRepository) SearchRecentByKeywords(ctx context.Context, sess return r.FindRecent(ctx, sessionID, limit) } - query := r.db.WithContext(ctx). - Where("session_id = ?", sessionID) - - // 多个关键词用 OR 连接，任意匹配即可 - for _, kw := range keywords { - if kw == "" { - continue + // 关键：session_id 是硬过滤，多个关键词之间用 OR 连接，但必须整体包在 AND 里。 + // 错误写法：Where(session_id).Or(ILIKE)... → 实际 SQL 是 WHERE session_id = ? OR content ILIKE ? + // 会匹配到全库所有包含关键词的消息，再 ORDER BY + LIMIT，数据量上来必炸。 + // 正确 SQL：WHERE session_id = ? AND (content ILIKE ? OR content ILIKE ? ...) + query := r.db.WithContext(ctx).Where("session_id = ?", sessionID) + + if len(keywords) == 1 { + query = query.Where("content ILIKE ?", "%"+keywords[0]+"%") + } else { + sub := r.db.Session(&gorm.Session{NewDB: true}) + for i, kw := range keywords { + if i == 0 { + sub = sub.Where("content ILIKE ?", "%"+kw+"%") + } else { + sub = sub.Or("content ILIKE ?", "%"+kw+"%") + } } - query = query.Or("content ILIKE ?", "%"+kw+"%") + query = query.Where(sub) } var messages []entity.ChatMessage diff --git a/internal/repository/observability_repository.go b/internal/repository/observability_repository.go index f6f792b..8d0df7d 100644 --- a/internal/repository/observability_repository.go +++ b/internal/repository/observability_repository.go @@ -341,28 +341,40 @@ func AttrsFromRoot(root *observability.Span) map[string]any { } // stripInternalSpanAttrs 递归清理 span 树中 __ 开头的内部 attrs（写 span_tree JSON 前调用）。 +// +// 并发安全说明： +// - 此函数可能在 FlushTrace 的后台 goroutine 执行时，与主请求 goroutine 里尚未结束的 +// SetSpanAttrs 并发读写同一份 span.Attrs。Go 的原生 map 不支持并发读写，任何一边的 +// delete/写都会触发 fatal error: concurrent map iteration and map write。 +// - 修复方式：不原地 delete，而是先拷贝一份 attrs，在副本上过滤，再赋回 s.Attrs。 +// 这样迭代读的是独立副本，就算主 goroutine 同时在写，最坏情况是 strip 拿到的是 +// 旧快照，不会 crash——FlushTrace 完后 span 对象就丢了，不影响业务。 func stripInternalSpanAttrs(s *observability.Span) { if s == nil { return } - if s.Attrs != nil { - for k := range s.Attrs { - if strings.HasPrefix(k, "__") { - delete(s.Attrs, k) + if len(s.Attrs) > 0 { + clean := make(observability.Attrs, len(s.Attrs)) + for k, v := range s.Attrs { // 迭代期间无任何 delete/写，只是读 + if !strings.HasPrefix(k, "__") { + clean[k] = v } } + s.Attrs = clean } for i := range s.Children { stripInternalSpanAttrs(s.Children[i]) } for i := range s.Events { - if s.Events[i] == nil || s.Events[i].Attrs == nil { + if s.Events[i] == nil || len(s.Events[i].Attrs) == 0 { continue } - for k := range s.Events[i].Attrs { - if strings.HasPrefix(k, "__") { - delete(s.Events[i].Attrs, k) + clean := make(observability.Attrs, len(s.Events[i].Attrs)) + for k, v := range s.Events[i].Attrs { + if !strings.HasPrefix(k, "__") { + clean[k] = v } } + s.Events[i].Attrs = clean } } diff --git a/internal/service/chat_error_messages.go b/internal/service/chat_error_messages.go index aecb6b9..6bc0809 100644 --- a/internal/service/chat_error_messages.go +++ b/internal/service/chat_error_messages.go @@ -1,5 +1,7 @@ package service +import "strings" + // ErrorMessage 用户友好的错误消息 type ErrorMessage struct { Title string // 错误标题 @@ -8,8 +10,9 @@ type ErrorMessage struct { } // errorMessages 错误消息映射表 +// key 用于模糊匹配：getFriendlyError 会检查 err.Error() 和 rawError 是否包含 key var errorMessages = map[string]ErrorMessage{ - // 模型相关 + // 模型配置相关 "模型配置无效或无权访问": { Title: "模型加载失败", Detail: "请在设置中检查模型配置，或选择其他模型", @@ -43,7 +46,59 @@ var errorMessages = map[string]ErrorMessage{ Retryable: false, }, - // LLM 相关 + // LLM 服务端错误（HTTP 状态码 + 关键词） + "503": { + Title: "AI 服务暂时不可用", + Detail: "模型端点暂时无法响应，请稍后重试", + Retryable: true, + }, + "Service Unavailable": { + Title: "AI 服务暂时不可用", + Detail: "模型端点暂时无法响应，请稍后重试", + Retryable: true, + }, + "Service temporarily unavailable": { + Title: "AI 服务暂时不可用", + Detail: "模型端点暂时无法响应，请稍后重试", + Retryable: true, + }, + "429": { + Title: "AI 服务请求过于频繁", + Detail: "模型端点限流了，请稍等一会儿再试", + Retryable: true, + }, + "Too Many Requests": { + Title: "AI 服务请求过于频繁", + Detail: "模型端点限流了，请稍等一会儿再试", + Retryable: true, + }, + "context length": { + Title: "对话太长了", + Detail: "当前对话超出了模型的上下文限制，请开启新对话继续", + Retryable: false, + }, + "token limit": { + Title: "对话太长了", + Detail: "当前对话超出了模型的上下文限制，请开启新对话继续", + Retryable: false, + }, + "超时": { + Title: "AI 服务响应超时", + Detail: "模型端点响应过慢，请稍后重试或切换更快的模型", + Retryable: true, + }, + "timeout": { + Title: "AI 服务响应超时", + Detail: "模型端点响应过慢，请稍后重试或切换更快的模型", + Retryable: true, + }, + "context deadline exceeded": { + Title: "AI 服务响应超时", + Detail: "模型端点响应过慢，请稍后重试或切换更快的模型", + Retryable: true, + }, + + // LLM 调用通用错误 "LLM 调用失败": { Title: "AI 服务异常", Detail: "AI 服务暂时不可用，请稍后重试", @@ -72,6 +127,18 @@ var errorMessages = map[string]ErrorMessage{ Retryable: true, }, + // Graph 执行错误（快速模式） + "快速检索执行失败": { + Title: "快速检索链路异常", + Detail: "请稍后重试或切换到深度模式", + Retryable: true, + }, + "快速检索流式生成失败": { + Title: "AI 生成中断", + Detail: "回答生成过程中断，请重试", + Retryable: true, + }, + // 会话相关 "会话不存在": { Title: "会话已失效", @@ -118,20 +185,27 @@ var errorMessages = map[string]ErrorMessage{ } // getFriendlyError 获取用户友好的错误消息 -func getFriendlyError(errMsg string) ErrorMessage { - // 精确匹配 - if msg, ok := errorMessages[errMsg]; ok { +// 匹配顺序：先检查 err.Error()（底层错误详情，如 503/429），再检查 rawError（业务层自定义描述） +// 这样像 "快速检索执行失败" 这种通用描述不会掩盖掉底层真正的错误原因 +func getFriendlyError(err error, rawError string) ErrorMessage { + var combined strings.Builder + if err != nil { + combined.WriteString(err.Error()) + } + combined.WriteString("|") + combined.WriteString(rawError) + text := combined.String() + + if msg, ok := errorMessages[text]; ok { return msg } - // 模糊匹配 for key, msg := range errorMessages { - if contains(errMsg, key) { + if contains(text, key) { return msg } } - // 默认错误消息 return ErrorMessage{ Title: "操作失败", Detail: "请稍后重试，如问题持续请联系管理员", diff --git a/internal/service/chat_event_helpers.go b/internal/service/chat_event_helpers.go index cf603f9..25db296 100644 --- a/internal/service/chat_event_helpers.go +++ b/internal/service/chat_event_helpers.go @@ -6,8 +6,9 @@ import ( ) // sendErrorEvent 发送友好的错误事件 +// 匹配顺序：先查 err.Error()（包含底层错误详情如 503/429/timeout），再查 rawError（自定义描述） func sendErrorEvent(eventCh chan<- dto.StreamEvent, err error, rawError string) { - friendly := getFriendlyError(rawError) + friendly := getFriendlyError(err, rawError) logger.Errorf("错误事件: title=%s, raw=%s, err=%v", friendly.Title, rawError, err) diff --git a/internal/service/chat_service.go b/internal/service/chat_service.go index 6d7dd3a..934cf45 100644 --- a/internal/service/chat_service.go +++ b/internal/service/chat_service.go @@ -351,29 +351,76 @@ func (s *chatService) initContext(ctx context.Context, userID, sessionID, modelI } historyBudget, retrievalBudget, memoryBudget := calculateContextBudgets(maxCtx, toolsTokens) - // 3. 加载用户基本信息 - userCtx := s.loadUserContext(ctx, userID) + // 3. 并行加载：BuildContext / 用户信息 / 用户偏好 —— 三者无依赖，全部 goroutine + type userLoadResult struct { + entity *entity.User + err error + } + type prefLoadResult struct { + pref *entity.UserPreference + err error + } + type ctxLoadResult struct { + ctx *EnhancedContext + err error + } - // 4. 使用 ContextService 构建增强上下文 - var enhancedCtx *EnhancedContext - if s.contextSvc != nil { - t1 = time.Now() - enhancedCtx, err = s.contextSvc.BuildContext(ctx, userID, sessionID, currentQuery, BuildContextConfig{ - MaxTokens: historyBudget, - MaxMemories: 10, - MaxRecentMessages: 20, - RetrievalBudget: retrievalBudget, - MemoryBudget: memoryBudget, - ModelName: modelName, - ToolsTokens: toolsTokens, - }, client.ChatModel()) - if err != nil { - logger.Warnf("构建增强上下文失败，降级为传统方式: %v", err) + userCh := make(chan userLoadResult, 1) + prefCh := make(chan prefLoadResult, 1) + ctxCh := make(chan ctxLoadResult, 1) + + // goroutine A: 用户信息（只查一次，UserCtx 和 Profile 共用结果） + go func() { + var r userLoadResult + if s.userRepo != nil && userID != "" { + u, err := s.userRepo.FindByID(userID) + r = userLoadResult{entity: u, err: err} } - } + userCh <- r + }() - // 兜底：传统截断 + // goroutine B: 用户偏好 + go func() { + var r prefLoadResult + if s.prefSvc != nil && userID != "" { + p, err := s.prefSvc.GetByUserID(ctx, userID) + r = prefLoadResult{pref: p, err: err} + } + prefCh <- r + }() + + // goroutine C: BuildContext（内部自己已经是 4 路并行：summary/memories/recent/relevant） + go func() { + var r ctxLoadResult + if s.contextSvc != nil { + enhancedCtx, bErr := s.contextSvc.BuildContext(ctx, userID, sessionID, currentQuery, BuildContextConfig{ + MaxTokens: historyBudget, + MaxMemories: 10, + MaxRecentMessages: 20, + RetrievalBudget: retrievalBudget, + MemoryBudget: memoryBudget, + ModelName: modelName, + ToolsTokens: toolsTokens, + }, client.ChatModel()) + if bErr != nil { + logger.Warnf("构建增强上下文失败，降级为传统方式: %v", bErr) + } + r = ctxLoadResult{ctx: enhancedCtx, err: bErr} + } + ctxCh <- r + }() + + userRes := <-userCh + prefRes := <-prefCh + ctxRes := <-ctxCh + + // 填 enhancedCtx + var enhancedCtx *EnhancedContext + if ctxRes.err == nil && ctxRes.ctx != nil { + enhancedCtx = ctxRes.ctx + } if enhancedCtx == nil { + // 兜底：传统截断 msg, _ := s.messageRepo.FindRecentForContext(ctx, sessionID, 20) enhancedCtx = &EnhancedContext{ History: truncateHistoryByTokens(msg, historyBudget, modelName), @@ -381,22 +428,29 @@ func (s *chatService) initContext(ctx context.Context, userID, sessionID, modelI RetrievalBudget: retrievalBudget, } } - enhancedCtx.UserCtx = userCtx - // 填充阶段二用户画像、偏好（任何失败不阻断主流程） - if userEntity, err := s.userRepo.FindByID(userID); err == nil && userEntity != nil { - enhancedCtx.Profile = userEntity - if s.prefSvc != nil { - if p, e := s.prefSvc.GetByUserID(ctx, userID); e == nil { - enhancedCtx.Preference = p - } - } + // 用 goroutine A 的结果同时填 UserCtx 和 Profile（之前 loadUserContext + 后面 FindByID 查了两次，现在一次搞定） + if userRes.err != nil { + logger.Warnf("加载用户信息失败, userID=%s: %v", userID, userRes.err) + } + if userRes.entity != nil { + enhancedCtx.UserCtx = NewUserContext(*userRes.entity) + enhancedCtx.Profile = userRes.entity + } else { + enhancedCtx.UserCtx = NewUserContext(entity.User{}) + } + + if prefRes.err != nil { + logger.Warnf("加载用户偏好失败, userID=%s: %v", userID, prefRes.err) + } + if prefRes.pref != nil { + enhancedCtx.Preference = prefRes.pref } logger.Infof("增强上下文: 历史 %d 条(预算 %d), 记忆 %d 条(预算 %d), 检索预算 %d, 工具预留 %d, 摘要存在=%v, 模型窗口=%d, 用户=%s, 偏好=%v", len(enhancedCtx.History), enhancedCtx.HistoryBudget, len(enhancedCtx.Memories), memoryBudget, - enhancedCtx.RetrievalBudget, toolsTokens, enhancedCtx.Summary != nil, maxCtx, userCtx.Username, + enhancedCtx.RetrievalBudget, toolsTokens, enhancedCtx.Summary != nil, maxCtx, enhancedCtx.UserCtx.Username, enhancedCtx.Preference != nil) // P1-⑨：分块 token 指标（Prometheus /metrics 直接聚合可看"到底是哪一块把窗口撑爆了"） diff --git a/internal/service/chat_service_graph_quick.go b/internal/service/chat_service_graph_quick.go index bfc2dae..ce9d76e 100644 --- a/internal/service/chat_service_graph_quick.go +++ b/internal/service/chat_service_graph_quick.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "regexp" "sort" "strings" "time" @@ -66,6 +67,10 @@ type quickGraphState struct { ClarifyOptions []string // 追问选项(可选) Keywords []string // 改写时提取的关键词，可用于日志/调试 RetrievedDocs []*schema.Document + + // RewriteDone 后台 goroutine 完成 LLM 改写后关闭，Retrieve 阶段可选等待 + RewriteDone chan struct{} + RewriteErr error } // 查询改写意图类型 @@ -129,16 +134,18 @@ const ( graphQuickNodeGenerate = "generate" ) -// genState 每次 Graph 执行新建一个本地状态 -func genState(_ context.Context) *quickGraphState { - return &quickGraphState{} -} - // buildQuickGraph 构建 START → rewrite → retrieve → build_msgs → generate → END 流水线。 +// graphState 必须在 Invoke 前创建好并传入——它既是 eino compose 的 stateGenerator 返回值， +// 也是 Invoke 返回后外部读取 RetrievedDocs 的唯一入口。 func buildQuickGraph( - _ context.Context, + graphState *quickGraphState, einoRetriever *rag.EinoRetrieverAdapter, ) (*einoCompose.Graph[*quickGraphInput, *schema.StreamReader[*schema.Message]], error) { + // genState 是 buildQuickGraph 的局部闭包，始终返回同一个 graphState 实例。 + // eino compose 在 runCtx 里只是用 internalState 包装 graphState 指针， + // 所以 StatePostHandler 写入的字段和外部 graphState 是同一对象——Invoke 返回后还能读到。 + genState := func(_ context.Context) *quickGraphState { return graphState } + g := einoCompose.NewGraph[*quickGraphInput, *schema.StreamReader[*schema.Message]]( einoCompose.WithGenLocalState(genState), ) @@ -173,73 +180,168 @@ func addQuickRewriteNode(g *einoCompose.Graph[*quickGraphInput, *schema.StreamRe ) } -// quickRewriteFn 节点 1 实现：调 LLM 对用户问题做改写 + 意图识别。 -// 降级策略：LLM 改写失败 → fallback 原始 query，不阻塞主流程。 -// 短路优化：如果 graphInput.PreRewrittenQuery 已填（Graph 执行前已算过），直接复用不再调 LLM。 -// SkipRetrieve=true 时返回空串，Retriever 收到空串会快速返回空 docs。 +// quickRewriteFn 节点 1 实现：触发查询改写 + 意图识别，立即返回原始 query 让 Retrieve 先行。 +// 并行策略：LLM 改写放到后台 goroutine 异步执行，Retrieve 用 OriginalQuery 先跑， +// 改写结果通过 State.RewriteDone channel 通知下游，Retrieve StatePostHandler 可选等待并补检索。 +// 短路优化：如果 graphInput.PreRewrittenQuery 已填（Graph 执行前已算过），直接同步执行不再起 goroutine。 func quickRewriteFn(ctx context.Context, input *quickGraphInput) (string, error) { if input == nil { return "", apperrors.NewDefault(apperrors.CodeInvalidParam) } + + // 初始化 State：存 Input + 创建 RewriteDone channel if err := einoCompose.ProcessState(ctx, func(_ context.Context, state *quickGraphState) error { state.Input = input + state.RewriteDone = make(chan struct{}) return nil }); err != nil { return "", err } - startAt := time.Now() - var ( - rewritten string - intent string - keywords []string - skipRetrieve bool - needClarify bool - clarifyQuestion string - clarifyOptions []string - ) - - // 短路：Graph 执行前已算好，直接复用 + // 短路：Graph 执行前已算好 → 同步写 state，不走并行 if input.PreRewrittenQuery != "" { - rewritten = input.PreRewrittenQuery - intent = input.PreIntent - keywords = input.PreKeywords - skipRetrieve = input.PreSkipRetrieve - needClarify = input.PreNeedClarify - clarifyQuestion = input.PreClarifyQuestion - clarifyOptions = input.PreClarifyOptions - } else { - rewritten, intent, keywords, skipRetrieve, needClarify, clarifyQuestion, clarifyOptions = doRewriteWithLLM(ctx, input) - } - - _ = einoCompose.ProcessState(ctx, func(_ context.Context, state *quickGraphState) error { - state.RewrittenQuery = rewritten - state.Intent = intent - state.Keywords = keywords - state.SkipRetrieve = skipRetrieve - state.NeedClarify = needClarify - state.ClarifyQuestion = clarifyQuestion - state.ClarifyOptions = clarifyOptions - return nil - }) + rewritten, intent, keywords := input.PreRewrittenQuery, input.PreIntent, input.PreKeywords + skipRetrieve, needClarify := input.PreSkipRetrieve, input.PreNeedClarify + clarifyQ, clarifyO := input.PreClarifyQuestion, input.PreClarifyOptions + _ = einoCompose.ProcessState(ctx, func(_ context.Context, state *quickGraphState) error { + state.RewrittenQuery = rewritten + state.Intent = intent + state.Keywords = keywords + state.SkipRetrieve = skipRetrieve + state.NeedClarify = needClarify + state.ClarifyQuestion = clarifyQ + state.ClarifyOptions = clarifyO + close(state.RewriteDone) + return nil + }) + observability.SetSpanAttrs(ctx, observability.Attrs{ + "original_query": input.OriginalQuery, + "rewritten_query": rewritten, + "rewrite_mode": "precomputed", + }) + return rewritten, nil + } + + // 正常路径：后台 goroutine 异步跑 LLM 改写，立即返回 OriginalQuery 让 Retrieve 并行 + go func() { + startAt := time.Now() + rewritten, intent, keywords, skipRetrieve, needClarify, clarifyQ, clarifyO := doRewriteWithLLM(ctx, input) + _ = einoCompose.ProcessState(ctx, func(_ context.Context, state *quickGraphState) error { + state.RewrittenQuery = rewritten + state.Intent = intent + state.Keywords = keywords + state.SkipRetrieve = skipRetrieve + state.NeedClarify = needClarify + state.ClarifyQuestion = clarifyQ + state.ClarifyOptions = clarifyO + close(state.RewriteDone) + return nil + }) + observability.SetSpanAttrs(ctx, observability.Attrs{ + "rewrite_ms": time.Since(startAt).Milliseconds(), + "rewrite_mode": "async", + "original_query": input.OriginalQuery, + "rewritten_query": rewritten, + "intent": intent, + "skip_retrieve": fmt.Sprintf("%v", skipRetrieve), + "need_clarify": fmt.Sprintf("%v", needClarify), + }) + }() - durMs := time.Since(startAt).Milliseconds() observability.SetSpanAttrs(ctx, observability.Attrs{ - "original_query": input.OriginalQuery, - "rewritten_query": rewritten, - "intent": intent, - "skip_retrieve": fmt.Sprintf("%v", skipRetrieve), - "need_clarify": fmt.Sprintf("%v", needClarify), - "rewrite_ms": durMs, - "model_id": input.ModelName, + "original_query": input.OriginalQuery, + "rewrite_mode": "async_started", }) - return rewritten, nil + return input.OriginalQuery, nil +} + +// matchLocalIntent 本地快速意图匹配（纯正则 + 关键词，0ms）。 +// 返回 (intent, matched) —— matched=false 表示交给 LLM 判定。 +// +// 覆盖四类场景： +// greeting: 你好 / hi / 早上好 / 在吗 +// identity: 你是谁 / 你能做什么 / 介绍一下自己 +// chitchat: 今天星期几 / 讲个笑话 / 随便聊聊（含"今天/现在+时间查询"） +// meta: 我的历史 / 刚才说了什么 +// +// 不命中时返回 ("", false)，交给 LLM 做更精细的意图判定。 +func matchLocalIntent(raw string) (string, bool) { + q := strings.TrimSpace(strings.ToLower(raw)) + if q == "" { + return intentQuestion, true + } + + // ── greeting ── + greetingRegex := `^(你好|您好|hi+|hello+|嗨|哈喽|在吗|在不在|早|早上好|下午好|晚上好|晚安|早安|午安|晚安)$` + if matchRegex(greetingRegex, q) { + return intentGreeting, true + } + + // ── identity ── + identityRegex := `^(你是谁|你是谁呀|你叫什么|你叫什么名字|你能做什么|你能干什么|你是干什么的|介绍一下你自己|自我介绍|你是什么模型|你是什么)$` + if matchRegex(identityRegex, q) { + return intentIdentity, true + } + + // ── chitchat（闲聊 + 系统信息查询，LLM 容易误识别成 question 的场景）── + // 时间日期类 + timeRegex := `(今天|现在|当前|明天|后天)+(星期几|礼拜几|几号|多少号|日期|几号了|几点|几点钟|时间|日期是)` + // 纯闲聊类 + chitchatRegex := `^(讲个笑话|来个笑话|随便聊聊|聊聊呗|聊聊天|说点什么|有什么好玩的|今天天气怎么样|天气怎么样|心情不好|我心情不好|安慰一下我|夸夸我)$` + if matchRegex(timeRegex, q) || matchRegex(chitchatRegex, q) { + return intentChitchat, true + } + + // ── meta ── + metaRegex := `(我的历史|聊天记录|你刚才说了什么|刚才说的什么|上一个问题|前一个问题|回顾对话|我们聊了什么|你还记得|之前说的)` + if matchRegex(metaRegex, q) { + return intentMeta, true + } + + return "", false +} + +// matchRegex 简单的正则匹配封装，避免每次都 re.Compile +var ( + reGreeting = regexp.MustCompile(`^(你好|您好|hi+|hello+|嗨|哈喽|在吗|在不在|早|早上好|下午好|晚上好|晚安|早安|午安|晚安)$`) + reIdentity = regexp.MustCompile(`^(你是谁|你是谁呀|你叫什么|你叫什么名字|你能做什么|你能干什么|你是干什么的|介绍一下你自己|自我介绍|你是什么模型|你是什么)$`) + reTimeInfo = regexp.MustCompile(`(今天|现在|当前|明天|后天)+(星期几|礼拜几|几号|多少号|日期|几号了|几点|几点钟|时间|日期是)`) + reChitchat = regexp.MustCompile(`^(讲个笑话|来个笑话|随便聊聊|聊聊呗|聊聊天|说点什么|有什么好玩的|今天天气怎么样|天气怎么样|心情不好|我心情不好|安慰一下我|夸夸我)$`) + reMeta = regexp.MustCompile(`(我的历史|聊天记录|你刚才说了什么|刚才说的什么|上一个问题|前一个问题|回顾对话|我们聊了什么|你还记得|之前说的)`) +) + +func matchRegex(pattern string, q string) bool { + switch pattern { + case `^(你好|您好|hi+|hello+|嗨|哈喽|在吗|在不在|早|早上好|下午好|晚上好|晚安|早安|午安|晚安)$`: + return reGreeting.MatchString(q) + case `^(你是谁|你是谁呀|你叫什么|你叫什么名字|你能做什么|你能干什么|你是干什么的|介绍一下你自己|自我介绍|你是什么模型|你是什么)$`: + return reIdentity.MatchString(q) + case `(今天|现在|当前|明天|后天)+(星期几|礼拜几|几号|多少号|日期|几号了|几点|几点钟|时间|日期是)`: + return reTimeInfo.MatchString(q) + case `^(讲个笑话|来个笑话|随便聊聊|聊聊呗|聊聊天|说点什么|有什么好玩的|今天天气怎么样|天气怎么样|心情不好|我心情不好|安慰一下我|夸夸我)$`: + return reChitchat.MatchString(q) + case `(我的历史|聊天记录|你刚才说了什么|刚才说的什么|上一个问题|前一个问题|回顾对话|我们聊了什么|你还记得|之前说的)`: + return reMeta.MatchString(q) + default: + return false + } } // doRewriteWithLLM 调 LLM 做改写，失败时 fallback 原始 query。 // 返回 (rewritten, intent, keywords, skipRetrieve, needClarify, clarifyQuestion, clarifyOptions) +// +// 优化：先本地快速意图匹配（0ms，覆盖问候/身份/闲聊/系统查询等常见场景）， +// 命中后直接返回，省掉 LLM 调用。只有本地判定为 question 或不确定时才调 LLM。 func doRewriteWithLLM(ctx context.Context, input *quickGraphInput) (string, string, []string, bool, bool, string, []string) { - // 1. 从 context 拿 ChatModel + // ── Step 0: 本地快速意图匹配（0ms） ── + if localIntent, ok := matchLocalIntent(input.OriginalQuery); ok { + skip := localIntent == intentGreeting || localIntent == intentChitchat || localIntent == intentIdentity || localIntent == intentMeta + logger.Infof("[意图识别-本地] original=%q → intent=%s, skipRetrieve=%v, cost=0ms", + input.OriginalQuery, localIntent, skip) + return input.OriginalQuery, localIntent, nil, skip, false, "", nil + } + + // ── Step 1: 本地没命中 → 调 LLM ── cm, ok := graphChatModelFromContext(ctx) if !ok || cm == nil { logger.Warnf("quickRewriteFn: context 中没有 ChatModel，跳过改写") @@ -291,8 +393,14 @@ func doRewriteWithLLM(ctx context.Context, input *quickGraphInput) (string, stri result.Intent = intentQuestion } - // 7. 判定是否跳过检索（greeting/chitchat 不需要知识库） - skipRetrieve := result.Intent == intentGreeting || result.Intent == intentChitchat + // 7. 判定是否跳过检索 + // greeting/chitchat → 无需知识库，直接闲聊 + // identity → "你是谁/你能做什么"，System Prompt 里已定义，不需要检索 + // meta → "我的历史记录/你刚才说了什么"，属于会话层，不走知识检索 + skipRetrieve := result.Intent == intentGreeting || + result.Intent == intentChitchat || + result.Intent == intentIdentity || + result.Intent == intentMeta // 8. 澄清检查: need_clarify=true 且有 question 才生效 needClarify := result.NeedClarify && strings.TrimSpace(result.ClarifyQuestion) != "" @@ -347,17 +455,114 @@ func buildRewriteHistory(msgs []*schema.Message, currentUserMsgIdx, maxRounds in return strings.Join(pairs, "\n") } -// addQuickRetrieveNode 节点 2：Retrieve，PostHandler 把结果写回 State。 +// rewriteParallelMaxWait Retrieve 阶段等待后台 LLM 改写的最长时间。 +// 原始 query 检索通常 0.3-0.8s，LLM 改写通常 0.5-2s——等 500ms 给快模型一个机会， +// 超时就放弃，不阻塞主路径。 +const rewriteParallelMaxWait = 500 * time.Millisecond + +// addQuickRetrieveNode 节点 2：Retrieve +// 改进：用 LambdaNode 替代 AddRetrieverNode，在 Lambda 内部提前检查 SkipRetrieve / NeedClarify， +// 避免 EinoRetrieverAdapter 被实例化后才被 PostHandler 清空——那样知识库查询的开销已经花出去了。 func addQuickRetrieveNode(g *einoCompose.Graph[*quickGraphInput, *schema.StreamReader[*schema.Message]], einoRetriever *rag.EinoRetrieverAdapter) error { - return g.AddRetrieverNode(graphQuickNodeRetrieve, einoRetriever, - einoCompose.WithNodeName("KnowledgeRetrieve"), - einoCompose.WithStatePostHandler(func(_ context.Context, docs []*schema.Document, state *quickGraphState) ([]*schema.Document, error) { + return g.AddLambdaNode(graphQuickNodeRetrieve, + einoCompose.InvokableLambda(func(ctx context.Context, query string) ([]*schema.Document, error) { + var state *quickGraphState + if err := einoCompose.ProcessState(ctx, func(_ context.Context, s *quickGraphState) error { + state = s + return nil + }); err != nil || state == nil { + return nil, apperrors.NewDefault(apperrors.CodeInternalError) + } + + // 提前短路：Rewrite 阶段已判定不需要检索 → 不查知识库，直接返回空 docs + if state.SkipRetrieve || state.NeedClarify { + state.RetrievedDocs = nil + return nil, nil + } + + // 构造 retriever.Option（KBIDs / UserID / TopK） + opts := buildRetrieverOpts(state.Input) + + // 用当前 query（Rewrite 返回的原始或改写后 query）先查 + docs, err := einoRetriever.Retrieve(ctx, query, opts...) + if err != nil { + logger.Warnf("quickRetrieveFn: 检索失败，降级为空结果: %v", err) + state.RetrievedDocs = nil + return nil, nil + } state.RetrievedDocs = docs + + // 等后台 LLM 改写（最多 rewriteParallelMaxWait），改写完成后补一次检索 + 合并去重 + if state.RewriteDone != nil { + select { + case <-state.RewriteDone: + if state.RewrittenQuery != "" && state.RewrittenQuery != state.Input.OriginalQuery { + if !state.SkipRetrieve && !state.NeedClarify { + rewrittenDocs, rErr := einoRetriever.Retrieve(ctx, state.RewrittenQuery, opts...) + if rErr == nil && len(rewrittenDocs) > 0 { + docs = mergeDocsByScore(docs, rewrittenDocs) + state.RetrievedDocs = docs + } + } + } + case <-time.After(rewriteParallelMaxWait): + } + } return docs, nil }), + einoCompose.WithNodeName("KnowledgeRetrieve"), ) } +// buildRetrieverOpts 从 quickGraphInput 构造 retriever.Option 切片， +// 替代之前 quickRetrieverCallOpts 通过 einoCompose.WithRetrieverOption 注入的方式。 +func buildRetrieverOpts(input *quickGraphInput) []retriever.Option { + var opts []retriever.Option + if input != nil { + if len(input.KnowledgeBaseIDs) > 0 { + opts = append(opts, rag.WithKnowledgeBaseIDs(input.KnowledgeBaseIDs)) + } + if input.UserID != "" { + opts = append(opts, rag.WithUserID(input.UserID)) + } + } + if cfg := config.Get(); cfg != nil && cfg.RAG.TopK > 0 { + opts = append(opts, retriever.WithTopK(cfg.RAG.TopK)) + } + return opts +} + +// mergeDocsByScore 合并两组 docs，按 ID 去重取最高分，最后按分数降序。 +func mergeDocsByScore(a, b []*schema.Document) []*schema.Document { + scoreMap := make(map[string]*schema.Document, len(a)+len(b)) + for _, d := range a { + if d == nil { + continue + } + scoreMap[d.ID] = d + } + for _, d := range b { + if d == nil { + continue + } + if existing, ok := scoreMap[d.ID]; ok { + if d.Score() > existing.Score() { + scoreMap[d.ID] = d + } + } else { + scoreMap[d.ID] = d + } + } + merged := make([]*schema.Document, 0, len(scoreMap)) + for _, d := range scoreMap { + merged = append(merged, d) + } + sort.Slice(merged, func(i, j int) bool { + return merged[i].Score() > merged[j].Score() + }) + return merged +} + // addQuickBuildMsgsNode 节点 3：BuildPromptMessages。 // 从 State 拿 Input，在 userQuestionIndex 前插入检索上下文。 func addQuickBuildMsgsNode(g *einoCompose.Graph[*quickGraphInput, *schema.StreamReader[*schema.Message]]) error { @@ -708,7 +913,10 @@ func (s *chatService) processMessageGraphQuick( // 3.5) 预执行 Rewrite + 澄清检查：needClarify=true 时短路返回，不浪费后续节点 rewriteCheckCtx := withGraphChatModel(ctx, chatModel) + rewriteStart := time.Now() rewritten, intent, keywords, skipRetrieve, needClarify, clarifyQuestion, clarifyOptions := doRewriteWithLLM(rewriteCheckCtx, graphInput) + logger.Infof("[意图识别] original=%q → intent=%s, skipRetrieve=%v, needClarify=%v, rewritten=%q, keywords=%v, cost=%dms", + req.Content, intent, skipRetrieve, needClarify, rewritten, keywords, time.Since(rewriteStart).Milliseconds()) if needClarify { // 存 PendingClarify 到 session @@ -746,11 +954,15 @@ func (s *chatService) processMessageGraphQuick( graphInput.PreKeywords = keywords graphInput.PreSkipRetrieve = skipRetrieve - // 4) 构建并编译 compose.Graph（内部已经 push error 事件） + // 4) 提前创建 graphState：既是 eino stateGenerator 返回值，也是 Invoke 后外部读取 RetrievedDocs 的入口。 + // RewriteDone channel 在这里初始化——rewriteFn 后台 goroutine 完成后 close 它，retrieve StatePostHandler 等它。 + graphState := &quickGraphState{RewriteDone: make(chan struct{})} + + // 5) 构建并编译 compose.Graph（内部已经 push error 事件） sendProgressEvent(eventCh, "正在组装快速检索链路...") graphCtx, cancel := context.WithCancel(ctx) defer cancel() - runnable, err := compileQuickGraphLocal(graphCtx, s.einoRetriever, req.ModelID, s.obs, obsOk, eventCh) + runnable, err := compileQuickGraphLocal(graphState, s.einoRetriever, req.ModelID, s.obs, obsOk, eventCh) if err != nil { if obsOk { s.obs.MarkTraceError(ctx, err) @@ -758,11 +970,11 @@ func (s *chatService) processMessageGraphQuick( return } - // 5) 注入 per-request ChatModel + Retriever 节点选项 + // 6) 注入 per-request ChatModel + Retriever 节点选项 graphCtx = withGraphChatModel(graphCtx, chatModel) callOpts := quickRetrieverCallOpts(req, userID) - // 6) 生成助手消息 ID + 流式驱动 Graph 执行 + // 7) 生成助手消息 ID + 流式驱动 Graph 执行 assistantMsgID := uuid.New().String() if obsOk { s.obs.AddRootAttrs(ctx, observability.Attrs{"assistant_message_id": assistantMsgID}) @@ -781,8 +993,15 @@ func (s *chatService) processMessageGraphQuick( return } - // 7) 从 Graph State 取 RetrievedDocs → SourceInfo（引用展示用） - sources, docsCount := extractSourcesFromStateLocal(graphCtx) + // 8) 直接从 graphState 读 RetrievedDocs（genState 返回的就是这个对象，Invoke 内部 StatePostHandler 写的就是它） + var ( + sources []dto.SourceInfo + docsCount int + ) + if len(graphState.RetrievedDocs) > 0 { + sources = einoDocsToSourceInfos(graphState.RetrievedDocs) + docsCount = len(graphState.RetrievedDocs) + } if obsOk { s.obs.AddRootAttrs(ctx, observability.Attrs{ "assistant_chars": fmt.Sprintf("%d", len([]rune(fullContent))), @@ -842,23 +1061,25 @@ func buildQuickInput( } // compileQuickGraphLocal build + compile graph，失败时自动推 error 事件 +// graphState 在调用处提前创建好，此函数会把它传给 buildQuickGraph，使其成为 eino stateGenerator 的返回值。 +// 这样 Invoke 返回后外部直接读 graphState.RetrievedDocs 即可，不需要再从 context 里 ProcessState。 func compileQuickGraphLocal( - ctx context.Context, + graphState *quickGraphState, einoRetriever *rag.EinoRetrieverAdapter, modelID string, obs observability.Recorder, obsOk bool, eventCh chan<- dto.StreamEvent, ) (einoCompose.Runnable[*quickGraphInput, *schema.StreamReader[*schema.Message]], error) { - g, err := buildQuickGraph(ctx, einoRetriever) + g, err := buildQuickGraph(graphState, einoRetriever) if err != nil { - quickIncrError(ctx, obs, obsOk, "build_graph") + quickIncrError(nil, obs, obsOk, "build_graph") sendErrorEvent(eventCh, err, "快速检索链路初始化失败") return nil, err } - r, err := g.Compile(ctx, einoCompose.WithGraphName("quick_rag_pipeline")) + r, err := g.Compile(nil, einoCompose.WithGraphName("quick_rag_pipeline")) if err != nil { - quickIncrError(ctx, obs, obsOk, "compile_graph") + quickIncrError(nil, obs, obsOk, "compile_graph") sendErrorEvent(eventCh, err, "快速检索链路编译失败") return nil, err } @@ -894,25 +1115,6 @@ func runQuickStream( return consumeQuickGraphStream(graphCtx, reader, assistantMsgID, eventCh) } -// extractSourcesFromStateLocal 从 Graph Local State 取出 RetrievedDocs → SourceInfo -func extractSourcesFromStateLocal(graphCtx context.Context) ([]dto.SourceInfo, int) { - var ( - sources []dto.SourceInfo - docsCount int - ) - err := einoCompose.ProcessState(graphCtx, func(_ context.Context, state *quickGraphState) error { - if len(state.RetrievedDocs) > 0 { - sources = einoDocsToSourceInfos(state.RetrievedDocs) - docsCount = len(state.RetrievedDocs) - } - return nil - }) - if err != nil { - logger.Warnf("quick graph ProcessState 取 RetrievedDocs 失败: %v", err) - } - return sources, docsCount -} - // consumeQuickGraphStream 消费 ChatModel 输出的 StreamReader[*schema.Message] // 转成 dto.StreamEvent 推给前端，返回最终完整内容。 func consumeQuickGraphStream( @@ -984,22 +1186,9 @@ func quickIncrError(ctx context.Context, obs observability.Recorder, obsOk bool, obs.Incr(ctx, "chat_quick_graph_errors_total", map[string]string{"stage": stage}, 1) } -// quickRetrieverCallOpts 把 KBIDs/UserID/TopK 组合成只作用在 retrieve 节点的 compose.Option -func quickRetrieverCallOpts(req requestdto.SendMessageRequest, userID string) []einoCompose.Option { - opts := []retriever.Option{} - if len(req.KnowledgeBaseIDs) > 0 { - opts = append(opts, rag.WithKnowledgeBaseIDs(req.KnowledgeBaseIDs)) - } - if userID != "" { - opts = append(opts, rag.WithUserID(userID)) - } - if cfg := config.Get(); cfg != nil && cfg.RAG.TopK > 0 { - opts = append(opts, retriever.WithTopK(cfg.RAG.TopK)) - } - if len(opts) == 0 { - return nil - } - return []einoCompose.Option{ - einoCompose.WithRetrieverOption(opts...).DesignateNode(graphQuickNodeRetrieve), - } +// quickRetrieverCallOpts 现在返回 nil——Retrieve 节点已改为 LambdaNode， +// retriever.Option 通过 buildRetrieverOpts 在 Lambda 内部直接构造。 +// 保留函数签名以减少调用处改动。 +func quickRetrieverCallOpts(_ requestdto.SendMessageRequest, _ string) []einoCompose.Option { + return nil } diff --git a/internal/service/context_service.go b/internal/service/context_service.go index 47065ee..85ff20f 100644 --- a/internal/service/context_service.go +++ b/internal/service/context_service.go @@ -17,6 +17,7 @@ import ( "solvify-agent/internal/observability" "solvify-agent/internal/repository" "solvify-agent/pkg/logger" + "solvify-agent/pkg/stopwords" "solvify-agent/pkg/tokenutil" ) @@ -96,10 +97,20 @@ func (s *contextService) BuildContext(ctx context.Context, userID, sessionID, cu summary *entity.ChatSummary memories []entity.UserMemory recent []entity.ChatMessage + relevant []entity.ChatMessage err error } - resultCh := make(chan loadResult, 3) + // 提前算好关键词——纯计算，直接在主线程做 + var keywords []string + if currentQuery != "" { + keywords = cfg.PreExtractedKeywords + if len(keywords) == 0 { + keywords = extractKeywords(currentQuery) + } + } + + resultCh := make(chan loadResult, 4) go func() { summary, err := s.summaryRepo.GetBySessionID(ctx, sessionID) @@ -116,10 +127,25 @@ func (s *contextService) BuildContext(ctx context.Context, userID, sessionID, cu resultCh <- loadResult{recent: recent, err: err} }() - var summary *entity.ChatSummary - var memories []entity.UserMemory - var recent []entity.ChatMessage - for i := 0; i < 3; i++ { + go func() { + var relevant []entity.ChatMessage + if len(keywords) > 0 { + var err error + relevant, err = s.messageRepo.SearchRecentByKeywords(ctx, sessionID, keywords, 5) + if err != nil { + logger.Warnf("检索相关历史失败: %v", err) + } + } + resultCh <- loadResult{relevant: relevant} + }() + + var ( + summary *entity.ChatSummary + memories []entity.UserMemory + recent []entity.ChatMessage + relevant []entity.ChatMessage + ) + for i := 0; i < 4; i++ { r := <-resultCh if r.err != nil { logger.Warnf("加载上下文组件失败: %v", r.err) @@ -134,20 +160,8 @@ func (s *contextService) BuildContext(ctx context.Context, userID, sessionID, cu if r.recent != nil { recent = r.recent } - } - - var relevant []entity.ChatMessage - if currentQuery != "" { - keywords := cfg.PreExtractedKeywords - if len(keywords) == 0 { - keywords = extractKeywords(currentQuery) - } - if len(keywords) > 0 { - var err error - relevant, err = s.messageRepo.SearchRecentByKeywords(ctx, sessionID, keywords, 5) - if err != nil { - logger.Warnf("检索相关历史失败: %v", err) - } + if r.relevant != nil { + relevant = r.relevant } } @@ -504,24 +518,8 @@ func buildDialogueText(messages []entity.ChatMessage) string { return sb.String() } -// extractKeywords 从查询中提取关键词 +// extractKeywords 从查询中提取关键词，过滤停用词 func extractKeywords(query string) []string { - // 简单停用词表 - stopWords := map[string]struct{}{ - "的": {}, "了": {}, "是": {}, "我": {}, "你": {}, "他": {}, "她": {}, "它": {}, - "我们": {}, "你们": {}, "他们": {}, "这个": {}, "那个": {}, "什么": {}, "怎么": {}, - "为什么": {}, "如何": {}, "多少": {}, "哪些": {}, "谁": {}, "哪里": {}, "何时": {}, - "可以": {}, "能够": {}, "需要": {}, "想要": {}, "请": {}, "谢谢": {}, "你好": {}, - "这": {}, "那": {}, "什": {}, "么": {}, "怎": {}, "样": {}, - "a": {}, "an": {}, "the": {}, "is": {}, "are": {}, "was": {}, "were": {}, - "i": {}, "you": {}, "he": {}, "she": {}, "it": {}, "we": {}, "they": {}, - "this": {}, "that": {}, "these": {}, "those": {}, "what": {}, "how": {}, "why": {}, - "where": {}, "when": {}, "who": {}, "which": {}, "can": {}, "could": {}, "do": {}, - "does": {}, "did": {}, "will": {}, "would": {}, "should": {}, "may": {}, "might": {}, - "in": {}, "on": {}, "at": {}, "of": {}, "to": {}, "for": {}, "with": {}, - } - - // 按中文连续序列 或 英文/数字连续序列切分（复用包级已编译正则，避免每次重复编译） parts := tokenRegexp.FindAllString(query, -1) seen := make(map[string]struct{}) @@ -531,14 +529,13 @@ func extractKeywords(query string) []string { if p == "" { continue } - if _, ok := stopWords[p]; ok { + if stopwords.IsStopWord(p) { continue } if len([]rune(p)) < 2 { continue } - // 纯中文且每个字都是停用词，则跳过 - if isChineseString(p) && allRunesInSet(p, stopWords) { + if isChineseString(p) && allRunesAreStopWord(p) { continue } if _, ok := seen[p]; ok { @@ -565,10 +562,10 @@ func isChineseString(s string) bool { return true } -// allRunesInSet 判断字符串中每个 rune 是否都在集合中 -func allRunesInSet(s string, set map[string]struct{}) bool { +// allRunesAreStopWord 判断字符串中每个 rune（单字）是否都是停用词 +func allRunesAreStopWord(s string) bool { for _, r := range s { - if _, ok := set[string(r)]; !ok { + if !stopwords.IsStopWord(string(r)) { return false } } diff --git a/internal/tool/knowledge_search.go b/internal/tool/knowledge_search.go index 2462f09..7a5db23 100644 --- a/internal/tool/knowledge_search.go +++ b/internal/tool/knowledge_search.go @@ -102,6 +102,7 @@ func (t *KnowledgeSearchTool) InvokableRun(ctx context.Context, argumentsInJSON Content: doc.Content, }) } + contentBuilder.WriteString("以上为知识库检索结果，不需要联网搜索来补充。如果这些内容满足用户需求，直接组织答案；如果需要列出文档清单、关键词精准查找等其他操作，可以继续调用知识库内部工具。\n") // 记录来源（Agent 结束后从 CollectedSources 读取） t.CollectedSources = append(t.CollectedSources, sources...) diff --git a/pkg/config/config.go b/pkg/config/config.go index cd04038..d72377d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -91,6 +91,7 @@ type EmbeddingConfig struct { BaseURL string `mapstructure:"base_url"` Dimension int `mapstructure:"dimension"` BatchSize int `mapstructure:"batch_size"` + Timeout int `mapstructure:"timeout"` } // RAGConfig 描述检索增强配置 @@ -313,6 +314,7 @@ func Default() *Config { Model: "text-embedding-v4", Dimension: 1024, BatchSize: 10, + Timeout: 15, }, RAG: RAGConfig{ Enabled: true, @@ -530,6 +532,9 @@ func applyEnv(cfg *Config) { if value := os.Getenv("EMBEDDING_BATCH_SIZE"); value != "" { cfg.Embedding.BatchSize = parseInt(value, cfg.Embedding.BatchSize) } + if value := os.Getenv("EMBEDDING_TIMEOUT"); value != "" { + cfg.Embedding.Timeout = parseInt(value, cfg.Embedding.Timeout) + } // 数据库配置 cfg.Database.Postgres.Host = getEnv("POSTGRES_HOST", cfg.Database.Postgres.Host) diff --git a/pkg/database/postgresql.go b/pkg/database/postgresql.go index ab7820e..f312c19 100644 --- a/pkg/database/postgresql.go +++ b/pkg/database/postgresql.go @@ -86,3 +86,241 @@ func enablePGVector(db *gorm.DB) error { logger.Info("pgvector 扩展检查完成") return nil } + +// EnsurePGVectorIndex 检查并确保 document_chunks 表的 pgvector 向量索引存在。 +// +// 逻辑： +// 1. 查 pg_indexes 看 idx_document_chunks_embedding 是否已存在 +// 2. 不存在 → 自动创建 ivfflat 索引（lists=100, cosine, partial WHERE embedding IS NOT NULL） +// 3. 存在但类型不是 ivfflat/hnsw → 打警告（可能失效或全表扫描） +// 4. 表不存在或无 embedding 列 → 跳过（AutoMigrate 或 schema 会补上） +func EnsurePGVectorIndex(db *gorm.DB) error { + // 先检查表是否存在 + var tableExists bool + if err := db.Raw( + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'document_chunks')`, + ).Scan(&tableExists).Error; err != nil { + return fmt.Errorf("检查 document_chunks 表存在性失败: %w", err) + } + if !tableExists { + logger.Warn("[pgvector] document_chunks 表不存在，跳过向量索引检查") + return nil + } + + // 检查 embedding 列是否存在 + var colExists bool + if err := db.Raw( + `SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'document_chunks' AND column_name = 'embedding' + )`, + ).Scan(&colExists).Error; err != nil { + return fmt.Errorf("检查 embedding 列存在性失败: %w", err) + } + if !colExists { + logger.Warn("[pgvector] document_chunks.embedding 列不存在，跳过向量索引检查") + return nil + } + + // 查询索引信息 + type indexInfo struct { + IndexName string `gorm:"column:indexname"` + IndexType string `gorm:"column:indexdef"` + } + var existing []indexInfo + if err := db.Raw( + `SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'document_chunks' AND indexname = 'idx_document_chunks_embedding'`, + ).Scan(&existing).Error; err != nil { + return fmt.Errorf("查询向量索引状态失败: %w", err) + } + + if len(existing) > 0 { + idxDef := existing[0].IndexType + isValidType := contains(idxDef, "ivfflat") || contains(idxDef, "hnsw") || contains(idxDef, "pgvector") + if !isValidType { + logger.Warnf("[pgvector] 向量索引 idx_document_chunks_embedding 存在但类型异常，建议重建。当前定义: %s", idxDef) + } else { + logger.Infof("[pgvector] 向量索引 idx_document_chunks_embedding 已就绪: %s", idxDef) + } + return nil + } + + // 自动创建 ivfflat 索引 + logger.Info("[pgvector] 向量索引 idx_document_chunks_embedding 不存在，正在创建 ivfflat 索引...") + err := db.Exec(` + CREATE INDEX IF NOT EXISTS idx_document_chunks_embedding + ON document_chunks + USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100) + WHERE embedding IS NOT NULL + `).Error + if err != nil { + logger.Warnf("[pgvector] 自动创建 ivfflat 索引失败: %v（低流量环境可能需要先执行 VACUUM ANALYZE document_chunks）", err) + return nil // 不阻塞启动，只是警告 + } + logger.Info("[pgvector] 向量索引 idx_document_chunks_embedding 创建完成") + return nil +} + +// EnsureKeywordsGINIndex 检查并确保 document_chunks 表的 keywords 列有 GIN 索引。 +// keywords && ?::text[] 这种数组重叠操作必须用 GIN 索引加速，否则每次关键词检索都是全表扫描。 +func EnsureKeywordsGINIndex(db *gorm.DB) error { + var tableExists bool + if err := db.Raw( + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'document_chunks')`, + ).Scan(&tableExists).Error; err != nil { + return fmt.Errorf("检查 document_chunks 表存在性失败: %w", err) + } + if !tableExists { + logger.Warn("[pgvector] document_chunks 表不存在，跳过 keywords GIN 索引检查") + return nil + } + + var colExists bool + if err := db.Raw( + `SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'document_chunks' AND column_name = 'keywords' + )`, + ).Scan(&colExists).Error; err != nil { + return fmt.Errorf("检查 keywords 列存在性失败: %w", err) + } + if !colExists { + logger.Warn("[pgvector] document_chunks.keywords 列不存在，跳过 GIN 索引检查") + return nil + } + + var exists bool + if err := db.Raw( + `SELECT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'document_chunks' AND indexname = 'idx_document_chunks_keywords' + )`, + ).Scan(&exists).Error; err != nil { + return fmt.Errorf("查询 keywords GIN 索引状态失败: %w", err) + } + if exists { + logger.Info("[pgvector] keywords GIN 索引 idx_document_chunks_keywords 已就绪") + return nil + } + + logger.Info("[pgvector] keywords GIN 索引不存在，正在创建...") + err := db.Exec(` + CREATE INDEX IF NOT EXISTS idx_document_chunks_keywords + ON document_chunks + USING gin (keywords) + WHERE keywords IS NOT NULL + `).Error + if err != nil { + logger.Warnf("[pgvector] 自动创建 keywords GIN 索引失败: %v", err) + return nil + } + logger.Info("[pgvector] keywords GIN 索引 idx_document_chunks_keywords 创建完成") + return nil +} + +// EnsureContextIndexes 检查并确保 RAG 上下文加载链路高频查询涉及的三张表有正确索引。 +// chat_messages 的 (session_id, created_at) 复合索引是 initContext → BuildContext 里 FindRecent / SearchRecentByKeywords 的核心加速。 +// chat_sessions 和 user_memories 同理，ListActive / FindByUserID 都是高频操作。 +func EnsureContextIndexes(db *gorm.DB) error { + type ctxIndex struct { + table string + index string + sql string + } + idxs := []ctxIndex{ + { + table: "chat_messages", index: "idx_chat_messages_session_created", + sql: `CREATE INDEX IF NOT EXISTS idx_chat_messages_session_created ON chat_messages (session_id, created_at DESC)`, + }, + { + table: "chat_sessions", index: "idx_chat_sessions_user_status", + sql: `CREATE INDEX IF NOT EXISTS idx_chat_sessions_user_status ON chat_sessions (user_id, status)`, + }, + { + table: "user_memories", index: "idx_user_memories_user_active", + sql: `CREATE INDEX IF NOT EXISTS idx_user_memories_user_active ON user_memories (user_id, is_active) WHERE is_active = true`, + }, + } + for _, it := range idxs { + var exists bool + if err := db.Raw(`SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE tablename = ? AND indexname = ?)`, it.table, it.index).Scan(&exists).Error; err != nil { + logger.Warnf("[context] 查询索引状态失败 table=%s index=%s: %v", it.table, it.index, err) + continue + } + if exists { + continue + } + logger.Infof("[context] 创建索引 table=%s index=%s", it.table, it.index) + if err := db.Exec(it.sql).Error; err != nil { + logger.Warnf("[context] 自动创建索引失败 table=%s index=%s: %v", it.table, it.index, err) + } else { + logger.Infof("[context] 索引已就绪 index=%s", it.index) + } + } + return nil +} + +// EnsureMessageFeedbackSchema 补齐 message_feedback 表缺失的列。 +// 这张表是早期 AutoMigrate 创建的，后来 entity 加了 reasons / is_quick / trace_id 等列， +// 但 AutoMigrate 不会给已存在的表 ADD COLUMN，导致 INSERT 时报 column "xxx" does not exist。 +func EnsureMessageFeedbackSchema(db *gorm.DB) error { + var tableExists bool + if err := db.Raw( + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'message_feedback')`, + ).Scan(&tableExists).Error; err != nil { + return fmt.Errorf("检查 message_feedback 表存在性失败: %w", err) + } + if !tableExists { + return nil + } + + type colDef struct { + name string + ddl string // ALTER TABLE ... ADD COLUMN ... 的列定义（不含列名） + } + missingCols := []colDef{ + {name: "reasons", ddl: "jsonb"}, + {name: "is_quick", ddl: "boolean NOT NULL DEFAULT false"}, + {name: "trace_id", ddl: "varchar(128)"}, + {name: "reason_tag", ddl: "varchar(64)"}, + } + + for _, mc := range missingCols { + var exists bool + if err := db.Raw( + `SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'message_feedback' AND column_name = ? + )`, mc.name, + ).Scan(&exists).Error; err != nil { + logger.Warnf("[feedback] 检查列 %s 失败: %v", mc.name, err) + continue + } + if exists { + continue + } + logger.Infof("[feedback] 列 %s 不存在，正在 ALTER TABLE ADD COLUMN", mc.name) + if err := db.Exec( + fmt.Sprintf("ALTER TABLE message_feedback ADD COLUMN IF NOT EXISTS %s %s", mc.name, mc.ddl), + ).Error; err != nil { + logger.Warnf("[feedback] 自动补列 %s 失败: %v", mc.name, err) + } else { + logger.Infof("[feedback] 列 %s 已补齐", mc.name) + } + } + return nil +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && searchSubstring(s, substr) +} + +func searchSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 0fe2376..9564153 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -23,6 +23,11 @@ var ( sugar *zap.SugaredLogger ) +func init() { + log = zap.NewNop() + sugar = log.Sugar() +} + // CustomLevelEncoder 自定义日志级别编码器，支持控制台颜色显示 func CustomLevelEncoder(level zapcore.Level, enc zapcore.PrimitiveArrayEncoder) { var color string diff --git a/pkg/stopwords/chinese.go b/pkg/stopwords/chinese.go new file mode 100644 index 0000000..9acb900 --- /dev/null +++ b/pkg/stopwords/chinese.go @@ -0,0 +1,52 @@ +// Package stopwords 提供中英文停用词统一来源。 +// 所有需要过滤停用词的模块（检索、查询改写、上下文构建）都应引用本包，避免重复定义和不一致。 +package stopwords + +var stopWords = map[string]struct{}{ + // 中文 + "的": {}, "了": {}, "在": {}, "是": {}, "我": {}, + "有": {}, "和": {}, "就": {}, "不": {}, "人": {}, + "都": {}, "一": {}, "一个": {}, "上": {}, "也": {}, + "很": {}, "到": {}, "说": {}, "要": {}, "去": {}, + "你": {}, "会": {}, "着": {}, "没有": {}, "看": {}, + "好": {}, "自己": {}, "这": {}, "他": {}, "她": {}, + "它": {}, "们": {}, "那": {}, "什么": {}, + "怎么": {}, "如何": {}, "为什么": {}, "哪": {}, "哪个": {}, + "哪些": {}, "吗": {}, "呢": {}, "吧": {}, "啊": {}, + "哈哈": {}, "哦": {}, "嗯": {}, "然后": {}, "就是": {}, + "我们": {}, "你们": {}, "他们": {}, "这个": {}, "那个": {}, + "多少": {}, "谁": {}, "哪里": {}, "何时": {}, + "可以": {}, "能够": {}, "需要": {}, "想要": {}, + "请": {}, "谢谢": {}, "你好": {}, + "什": {}, "么": {}, "怎": {}, "样": {}, + + // 英文 + "the": {}, "a": {}, "an": {}, "is": {}, "are": {}, + "was": {}, "were": {}, "be": {}, "been": {}, "being": {}, + "have": {}, "has": {}, "had": {}, "do": {}, "does": {}, + "did": {}, "will": {}, "would": {}, "could": {}, "should": {}, + "may": {}, "might": {}, "can": {}, "shall": {}, "must": {}, + "to": {}, "of": {}, "in": {}, "for": {}, "on": {}, + "with": {}, "at": {}, "by": {}, "from": {}, "as": {}, + "into": {}, "through": {}, "during": {}, "before": {}, "after": {}, + "above": {}, "below": {}, "between": {}, "and": {}, "but": {}, + "or": {}, "nor": {}, "not": {}, "so": {}, "yet": {}, + "both": {}, "either": {}, "neither": {}, "each": {}, "every": {}, + "all": {}, "any": {}, "few": {}, "more": {}, "most": {}, + "other": {}, "some": {}, "such": {}, "no": {}, "only": {}, + "own": {}, "same": {}, "than": {}, "too": {}, "very": {}, + "just": {}, "because": {}, "if": {}, "when": {}, "where": {}, + "how": {}, "what": {}, "which": {}, "who": {}, "whom": {}, + "why": {}, + "this": {}, "that": {}, "these": {}, "those": {}, + "i": {}, "me": {}, "my": {}, "we": {}, "our": {}, + "you": {}, "your": {}, "he": {}, "him": {}, "his": {}, + "she": {}, "her": {}, "it": {}, "its": {}, + "they": {}, "them": {}, "their": {}, +} + +// IsStopWord 判断一个词是否为停用词。输入应为小写且已 trim 空格。 +func IsStopWord(word string) bool { + _, ok := stopWords[word] + return ok +} diff --git a/scripts/init_knowledge_schema.sql b/scripts/init_knowledge_schema.sql index c730294..87b659b 100644 --- a/scripts/init_knowledge_schema.sql +++ b/scripts/init_knowledge_schema.sql @@ -1,424 +1,1129 @@ -CREATE -EXTENSION IF NOT EXISTS pgcrypto; -CREATE -EXTENSION IF NOT EXISTS vector; - --- 用户表作为当前阶段的数据隔离边界 -CREATE TABLE IF NOT EXISTS users -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 用户 ID - username VARCHAR(64) NOT NULL, -- 用户名 - email VARCHAR(128) NOT NULL, -- 邮箱 - password TEXT NOT NULL DEFAULT '', -- 密码哈希 - status INT NOT NULL DEFAULT 1, -- 用户状态，1 正常，2 禁用，3 注销，4 待验证 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 更新时间 - - CONSTRAINT users_username_unique UNIQUE (username), - CONSTRAINT users_email_unique UNIQUE (email) -); - -COMMENT ON TABLE users IS '用户基础表，用于隔离每个用户自己的知识库和文档'; -COMMENT ON COLUMN users.id IS '用户 ID'; -COMMENT ON COLUMN users.username IS '用户名'; -COMMENT ON COLUMN users.email IS '邮箱'; -COMMENT ON COLUMN users.password IS '密码哈希'; -COMMENT ON COLUMN users.status IS '用户状态，1 正常，2 禁用，3 注销，4 待验证'; -COMMENT ON COLUMN users.created_at IS '创建时间'; -COMMENT ON COLUMN users.updated_at IS '更新时间'; - --- 知识库表保存用户自建、同步和联网搜索知识库 -CREATE TABLE IF NOT EXISTS knowledge_bases -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 知识库 ID - user_id UUID NOT NULL, -- 所属用户 ID - name VARCHAR(128) NOT NULL, -- 知识库名称 - category VARCHAR(64) NOT NULL DEFAULT '', -- 知识库分类 - description TEXT DEFAULT '', -- 知识库描述 - source_type VARCHAR(32) NOT NULL DEFAULT 'local', -- 知识库来源类型，local 自建，sync 同步，web_search 联网搜索 - source_platform VARCHAR(32) NOT NULL DEFAULT '', -- 同步来源平台 - document_count INT NOT NULL DEFAULT 0, -- 文档数量 - storage_bytes BIGINT NOT NULL DEFAULT 0, -- 已占用存储字节数 - status INT NOT NULL DEFAULT 1, -- 知识库状态，1 正常，2 已删除 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 更新时间 - deleted_at TIMESTAMPTZ, -- 删除时间 - delete_expired_at TIMESTAMPTZ -- 删除保留到期时间 -); - -COMMENT ON TABLE knowledge_bases IS '知识库主表，所有知识库按用户隔离'; -COMMENT ON COLUMN knowledge_bases.id IS '知识库 ID'; -COMMENT ON COLUMN knowledge_bases.user_id IS '所属用户 ID'; -COMMENT ON COLUMN knowledge_bases.name IS '知识库名称'; -COMMENT ON COLUMN knowledge_bases.category IS '知识库分类'; -COMMENT ON COLUMN knowledge_bases.description IS '知识库描述'; -COMMENT ON COLUMN knowledge_bases.source_type IS '知识库来源类型，local 自建，sync 同步，web_search 联网搜索'; -COMMENT ON COLUMN knowledge_bases.source_platform IS '同步来源平台'; -COMMENT ON COLUMN knowledge_bases.document_count IS '文档数量'; -COMMENT ON COLUMN knowledge_bases.storage_bytes IS '已占用存储字节数'; -COMMENT ON COLUMN knowledge_bases.status IS '知识库状态，1 正常，2 已删除'; -COMMENT ON COLUMN knowledge_bases.created_at IS '创建时间'; -COMMENT ON COLUMN knowledge_bases.updated_at IS '更新时间'; -COMMENT ON COLUMN knowledge_bases.deleted_at IS '删除时间'; -COMMENT ON COLUMN knowledge_bases.delete_expired_at IS '删除保留到期时间'; - --- 文档表记录上传文件和处理状态 -CREATE TABLE IF NOT EXISTS documents -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 文档 ID - user_id UUID NOT NULL, -- 所属用户 ID - knowledge_base_id UUID NOT NULL, -- 所属知识库 ID - title VARCHAR(255) NOT NULL, -- 文档标题 - file_name VARCHAR(255) NOT NULL, -- 原始文件名 - file_type VARCHAR(32) NOT NULL DEFAULT '', -- 文件类型 - file_size BIGINT NOT NULL DEFAULT 0, -- 文件大小字节数 - storage_path TEXT NOT NULL DEFAULT '', -- 文件存储路径 - file_hash VARCHAR(128) NOT NULL DEFAULT '', -- 原始文件内容指纹 - source_type VARCHAR(32) NOT NULL DEFAULT 'upload', -- 文档来源类型，upload 上传，edit 编辑，sync 同步，web_search 联网搜索 - external_id VARCHAR(255) NOT NULL DEFAULT '', -- 外部平台文档 ID - external_url TEXT NOT NULL DEFAULT '', -- 外部平台文档链接 - source_updated_at TIMESTAMPTZ, -- 外部平台更新时间 - status INT NOT NULL DEFAULT 1, -- 文档状态，1 已上传，2 处理中，3 已就绪，4 处理失败，5 已删除 - error_message TEXT NOT NULL DEFAULT '', -- 处理失败原因 - ready_at TIMESTAMPTZ, -- 文档就绪时间 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 更新时间 - deleted_at TIMESTAMPTZ, -- 删除时间 - delete_expired_at TIMESTAMPTZ -- 删除保留到期时间 - -); - -COMMENT ON TABLE documents IS '文档主表，记录知识库下的文件和处理状态'; -COMMENT ON COLUMN documents.id IS '文档 ID'; -COMMENT ON COLUMN documents.user_id IS '所属用户 ID'; -COMMENT ON COLUMN documents.knowledge_base_id IS '所属知识库 ID'; -COMMENT ON COLUMN documents.title IS '文档标题'; -COMMENT ON COLUMN documents.file_name IS '原始文件名'; -COMMENT ON COLUMN documents.file_type IS '文件类型'; -COMMENT ON COLUMN documents.file_size IS '文件大小字节数'; -COMMENT ON COLUMN documents.storage_path IS '文件存储路径'; -COMMENT ON COLUMN documents.file_hash IS '原始文件内容指纹'; -COMMENT ON COLUMN documents.source_type IS '文档来源类型，upload 上传，edit 编辑，sync 同步，web_search 联网搜索'; -COMMENT ON COLUMN documents.external_id IS '外部平台文档 ID'; -COMMENT ON COLUMN documents.external_url IS '外部平台文档链接'; -COMMENT ON COLUMN documents.source_updated_at IS '外部平台更新时间'; -COMMENT ON COLUMN documents.status IS '文档状态，1 已上传，2 处理中，3 已就绪，4 处理失败，5 已删除'; -COMMENT ON COLUMN documents.error_message IS '处理失败原因'; -COMMENT ON COLUMN documents.ready_at IS '文档就绪时间'; -COMMENT ON COLUMN documents.deleted_at IS '删除时间'; -COMMENT ON COLUMN documents.delete_expired_at IS '删除保留到期时间'; -COMMENT ON COLUMN documents.created_at IS '创建时间'; -COMMENT ON COLUMN documents.updated_at IS '更新时间'; - -ALTER TABLE documents - ADD COLUMN IF NOT EXISTS external_id VARCHAR(255) NOT NULL DEFAULT '', - ADD COLUMN IF NOT EXISTS external_url TEXT NOT NULL DEFAULT '', - ADD COLUMN IF NOT EXISTS source_updated_at TIMESTAMPTZ; - --- 文档版本表支持在线编辑和重新向量化 -CREATE TABLE IF NOT EXISTS document_versions -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 版本 ID - user_id UUID NOT NULL, -- 所属用户 ID - document_id UUID NOT NULL, -- 所属文档 ID - version_no INT NOT NULL DEFAULT 1, -- 版本号 - content TEXT NOT NULL DEFAULT '', -- 版本正文内容 - content_hash VARCHAR(128) NOT NULL DEFAULT '', -- 版本内容哈希 - change_summary TEXT NOT NULL DEFAULT '', -- 变更摘要 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - - CONSTRAINT document_versions_document_version_unique UNIQUE (document_id, version_no) -); - -COMMENT ON TABLE document_versions IS '文档版本表，用于记录原始解析内容和在线编辑历史'; -COMMENT ON COLUMN document_versions.id IS '版本 ID'; -COMMENT ON COLUMN document_versions.user_id IS '所属用户 ID'; -COMMENT ON COLUMN document_versions.document_id IS '所属文档 ID'; -COMMENT ON COLUMN document_versions.version_no IS '版本号'; -COMMENT ON COLUMN document_versions.content IS '版本正文内容'; -COMMENT ON COLUMN document_versions.content_hash IS '版本内容哈希'; -COMMENT ON COLUMN document_versions.change_summary IS '变更摘要'; -COMMENT ON COLUMN document_versions.created_at IS '创建时间'; - --- 文档分块表保存文本分块和向量数据 -CREATE TABLE IF NOT EXISTS document_chunks -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 分块 ID - user_id UUID NOT NULL, -- 所属用户 ID - knowledge_base_id UUID NOT NULL, -- 所属知识库 ID - document_id UUID NOT NULL, -- 所属文档 ID - version_id UUID NOT NULL, -- 所属文档版本 ID - chunk_index INT NOT NULL, -- 分块序号 - section_title TEXT NOT NULL DEFAULT '', -- 分块所属章节标题 - content TEXT NOT NULL, -- 分块文本内容 - token_count INT NOT NULL DEFAULT 0, -- 分块 token 数 - page_number INT, -- 来源页码 - embedding_model VARCHAR(128) NOT NULL DEFAULT '', -- 向量模型名称 - embedding vector(1024), -- 分块向量数据 - keywords TEXT[] NOT NULL DEFAULT '{}'::text[], -- 分块关键词 - metadata JSONB NOT NULL DEFAULT '{}'::jsonb, -- 分块扩展元数据 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - - CONSTRAINT document_chunks_version_index_unique UNIQUE (version_id, chunk_index) -); - -COMMENT ON TABLE document_chunks IS '文档分块表，保存可检索文本片段和 embedding 向量'; -COMMENT ON COLUMN document_chunks.id IS '分块 ID'; -COMMENT ON COLUMN document_chunks.user_id IS '所属用户 ID'; -COMMENT ON COLUMN document_chunks.knowledge_base_id IS '所属知识库 ID'; -COMMENT ON COLUMN document_chunks.document_id IS '所属文档 ID'; -COMMENT ON COLUMN document_chunks.version_id IS '所属文档版本 ID'; -COMMENT ON COLUMN document_chunks.chunk_index IS '分块序号'; -COMMENT ON COLUMN document_chunks.section_title IS '分块所属章节标题'; -COMMENT ON COLUMN document_chunks.content IS '分块文本内容'; -COMMENT ON COLUMN document_chunks.token_count IS '分块 token 数'; -COMMENT ON COLUMN document_chunks.page_number IS '来源页码'; -COMMENT ON COLUMN document_chunks.embedding_model IS '向量模型名称'; -COMMENT ON COLUMN document_chunks.embedding IS '分块向量数据'; -COMMENT ON COLUMN document_chunks.keywords IS '分块关键词'; -COMMENT ON COLUMN document_chunks.metadata IS '分块扩展元数据'; -COMMENT ON COLUMN document_chunks.created_at IS '创建时间'; - --- 文档处理任务表记录解析、分块、向量化和重建索引状态 -CREATE TABLE IF NOT EXISTS document_processing_jobs -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 任务 ID - user_id UUID NOT NULL, -- 所属用户 ID - document_id UUID NOT NULL, -- 所属文档 ID - job_type VARCHAR(32) NOT NULL, -- 任务类型，parse 解析，chunk 分块，embed 向量化，reindex 重建索引 - status INT NOT NULL DEFAULT 1, -- 任务状态，1 待处理，2 运行中，3 成功，4 失败 - error_message TEXT NOT NULL DEFAULT '', -- 任务失败原因 - started_at TIMESTAMPTZ, -- 开始时间 - finished_at TIMESTAMPTZ, -- 完成时间 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -- 更新时间 - -); - -COMMENT ON TABLE document_processing_jobs IS '文档处理任务表，记录解析、分块、向量化和重建索引状态'; -COMMENT ON COLUMN document_processing_jobs.id IS '任务 ID'; -COMMENT ON COLUMN document_processing_jobs.user_id IS '所属用户 ID'; -COMMENT ON COLUMN document_processing_jobs.document_id IS '所属文档 ID'; -COMMENT ON COLUMN document_processing_jobs.job_type IS '任务类型，parse 解析，chunk 分块，embed 向量化，reindex 重建索引'; -COMMENT ON COLUMN document_processing_jobs.status IS '任务状态，1 待处理，2 运行中，3 成功，4 失败'; -COMMENT ON COLUMN document_processing_jobs.error_message IS '任务失败原因'; -COMMENT ON COLUMN document_processing_jobs.started_at IS '开始时间'; -COMMENT ON COLUMN document_processing_jobs.finished_at IS '完成时间'; -COMMENT ON COLUMN document_processing_jobs.created_at IS '创建时间'; -COMMENT ON COLUMN document_processing_jobs.updated_at IS '更新时间'; - --- 同步源表记录钉钉等外部平台同步配置 -CREATE TABLE IF NOT EXISTS sync_sources -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 同步源 ID - user_id UUID NOT NULL, -- 所属用户 ID - knowledge_base_id UUID NOT NULL, -- 绑定知识库 ID - name VARCHAR(128) NOT NULL, -- 同步源名称 - platform VARCHAR(32) NOT NULL, -- 同步平台，当前支持 dingtalk - source_config JSONB NOT NULL DEFAULT '{}'::jsonb, -- 非敏感同步配置 - status INT NOT NULL DEFAULT 1, -- 同步源状态，1 正常，2 禁用，3 已删除 - last_sync_at TIMESTAMPTZ, -- 最近同步时间 - last_error_message TEXT NOT NULL DEFAULT '', -- 最近同步失败原因 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 更新时间 - deleted_at TIMESTAMPTZ -- 删除时间 -); - -COMMENT ON TABLE sync_sources IS '同步源配置表，记录钉钉等外部平台同步入口'; -COMMENT ON COLUMN sync_sources.id IS '同步源 ID'; -COMMENT ON COLUMN sync_sources.user_id IS '所属用户 ID'; -COMMENT ON COLUMN sync_sources.knowledge_base_id IS '绑定知识库 ID'; -COMMENT ON COLUMN sync_sources.name IS '同步源名称'; -COMMENT ON COLUMN sync_sources.platform IS '同步平台，当前支持 dingtalk'; -COMMENT ON COLUMN sync_sources.source_config IS '非敏感同步配置'; -COMMENT ON COLUMN sync_sources.status IS '同步源状态，1 正常，2 禁用，3 已删除'; -COMMENT ON COLUMN sync_sources.last_sync_at IS '最近同步时间'; -COMMENT ON COLUMN sync_sources.last_error_message IS '最近同步失败原因'; -COMMENT ON COLUMN sync_sources.created_at IS '创建时间'; -COMMENT ON COLUMN sync_sources.updated_at IS '更新时间'; -COMMENT ON COLUMN sync_sources.deleted_at IS '删除时间'; - --- 同步任务表记录每次手动触发同步的执行状态 -CREATE TABLE IF NOT EXISTS sync_jobs -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 同步任务 ID - user_id UUID NOT NULL, -- 所属用户 ID - sync_source_id UUID NOT NULL, -- 同步源 ID - knowledge_base_id UUID NOT NULL, -- 绑定知识库 ID - job_type VARCHAR(32) NOT NULL, -- 任务类型，manual 手动同步 - status INT NOT NULL DEFAULT 1, -- 任务状态，1 待同步，2 同步中，3 成功，4 失败 - total_count INT NOT NULL DEFAULT 0, -- 同步总数 - success_count INT NOT NULL DEFAULT 0, -- 同步成功数 - failed_count INT NOT NULL DEFAULT 0, -- 同步失败数 - error_message TEXT NOT NULL DEFAULT '', -- 任务失败原因 - started_at TIMESTAMPTZ, -- 开始时间 - finished_at TIMESTAMPTZ, -- 完成时间 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -- 更新时间 -); - -COMMENT ON TABLE sync_jobs IS '同步任务表，记录外部平台同步执行状态'; -COMMENT ON COLUMN sync_jobs.id IS '同步任务 ID'; -COMMENT ON COLUMN sync_jobs.user_id IS '所属用户 ID'; -COMMENT ON COLUMN sync_jobs.sync_source_id IS '同步源 ID'; -COMMENT ON COLUMN sync_jobs.knowledge_base_id IS '绑定知识库 ID'; -COMMENT ON COLUMN sync_jobs.job_type IS '任务类型，manual 手动同步'; -COMMENT ON COLUMN sync_jobs.status IS '任务状态，1 待同步，2 同步中，3 成功，4 失败'; -COMMENT ON COLUMN sync_jobs.total_count IS '同步总数'; -COMMENT ON COLUMN sync_jobs.success_count IS '同步成功数'; -COMMENT ON COLUMN sync_jobs.failed_count IS '同步失败数'; -COMMENT ON COLUMN sync_jobs.error_message IS '任务失败原因'; -COMMENT ON COLUMN sync_jobs.started_at IS '开始时间'; -COMMENT ON COLUMN sync_jobs.finished_at IS '完成时间'; -COMMENT ON COLUMN sync_jobs.created_at IS '创建时间'; -COMMENT ON COLUMN sync_jobs.updated_at IS '更新时间'; - --- 同步目录项表记录外部知识库中的目录和文件元数据 -CREATE TABLE IF NOT EXISTS sync_items -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 同步目录项 ID - user_id UUID NOT NULL, -- 所属用户 ID - sync_source_id UUID NOT NULL, -- 同步源 ID - knowledge_base_id UUID NOT NULL, -- 绑定知识库 ID - external_id VARCHAR(255) NOT NULL, -- 外部节点 ID - parent_external_id VARCHAR(255) NOT NULL DEFAULT '', -- 外部父节点 ID - name VARCHAR(255) NOT NULL, -- 节点名称 - item_type VARCHAR(32) NOT NULL, -- 节点类型，FILE 或 FOLDER - category VARCHAR(64) NOT NULL DEFAULT '', -- 钉钉节点分类 - extension VARCHAR(32) NOT NULL DEFAULT '', -- 文件扩展名 - external_url TEXT NOT NULL DEFAULT '', -- 外部原文链接 - file_size BIGINT NOT NULL DEFAULT 0, -- 文件大小 - has_children BOOLEAN NOT NULL DEFAULT FALSE, -- 是否有子节点 - source_updated_at TIMESTAMPTZ, -- 外部更新时间 - local_document_id UUID, -- 已导入本地文档 ID - import_status INT NOT NULL DEFAULT 1, -- 导入状态，1 未导入，2 导入中，3 已导入，4 导入失败 - error_message TEXT NOT NULL DEFAULT '', -- 导入失败原因 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 更新时间 - - CONSTRAINT sync_items_user_source_external_unique UNIQUE (user_id, sync_source_id, external_id) -); - -COMMENT ON TABLE sync_items IS '同步目录项表，记录外部知识库中的目录和文件元数据'; -COMMENT ON COLUMN sync_items.id IS '同步目录项 ID'; -COMMENT ON COLUMN sync_items.user_id IS '所属用户 ID'; -COMMENT ON COLUMN sync_items.sync_source_id IS '同步源 ID'; -COMMENT ON COLUMN sync_items.knowledge_base_id IS '绑定知识库 ID'; -COMMENT ON COLUMN sync_items.external_id IS '外部节点 ID'; -COMMENT ON COLUMN sync_items.parent_external_id IS '外部父节点 ID'; -COMMENT ON COLUMN sync_items.name IS '节点名称'; -COMMENT ON COLUMN sync_items.item_type IS '节点类型，FILE 或 FOLDER'; -COMMENT ON COLUMN sync_items.category IS '钉钉节点分类'; -COMMENT ON COLUMN sync_items.extension IS '文件扩展名'; -COMMENT ON COLUMN sync_items.external_url IS '外部原文链接'; -COMMENT ON COLUMN sync_items.file_size IS '文件大小'; -COMMENT ON COLUMN sync_items.has_children IS '是否有子节点'; -COMMENT ON COLUMN sync_items.source_updated_at IS '外部更新时间'; -COMMENT ON COLUMN sync_items.local_document_id IS '已导入本地文档 ID'; -COMMENT ON COLUMN sync_items.import_status IS '导入状态，1 未导入，2 导入中，3 已导入，4 导入失败'; -COMMENT ON COLUMN sync_items.error_message IS '导入失败原因'; -COMMENT ON COLUMN sync_items.created_at IS '创建时间'; -COMMENT ON COLUMN sync_items.updated_at IS '更新时间'; - --- 钉钉用户绑定表保存系统用户与钉钉身份的对应关系 -CREATE TABLE IF NOT EXISTS dingtalk_user_bindings -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 绑定 ID - user_id UUID NOT NULL, -- 系统用户 ID - ding_open_id VARCHAR(128) NOT NULL DEFAULT '', -- 钉钉 openid - ding_union_id VARCHAR(128) NOT NULL, -- 钉钉 unionId - corp_id VARCHAR(128) NOT NULL DEFAULT '', -- 钉钉企业 ID - nickname VARCHAR(128) NOT NULL DEFAULT '', -- 钉钉用户昵称 - avatar TEXT NOT NULL DEFAULT '', -- 钉钉用户头像 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 更新时间 - - CONSTRAINT dingtalk_user_bindings_user_unique UNIQUE (user_id), - CONSTRAINT dingtalk_user_bindings_union_unique UNIQUE (ding_union_id) -); - -COMMENT ON TABLE dingtalk_user_bindings IS '钉钉用户绑定表，用于保存系统用户与钉钉身份的对应关系'; -COMMENT ON COLUMN dingtalk_user_bindings.id IS '绑定 ID'; -COMMENT ON COLUMN dingtalk_user_bindings.user_id IS '系统用户 ID'; -COMMENT ON COLUMN dingtalk_user_bindings.ding_open_id IS '钉钉 openid'; -COMMENT ON COLUMN dingtalk_user_bindings.ding_union_id IS '钉钉 unionId'; -COMMENT ON COLUMN dingtalk_user_bindings.corp_id IS '钉钉企业 ID'; -COMMENT ON COLUMN dingtalk_user_bindings.nickname IS '钉钉用户昵称'; -COMMENT ON COLUMN dingtalk_user_bindings.avatar IS '钉钉用户头像'; -COMMENT ON COLUMN dingtalk_user_bindings.created_at IS '创建时间'; -COMMENT ON COLUMN dingtalk_user_bindings.updated_at IS '更新时间'; - --- 存储配额表记录用户存储上限和已用容量 -CREATE TABLE IF NOT EXISTS storage_quotas -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), -- 配额记录 ID - user_id UUID NOT NULL, -- 所属用户 ID - max_storage_bytes BIGINT NOT NULL DEFAULT 10737418240, -- 最大可用存储字节数 - used_storage_bytes BIGINT NOT NULL DEFAULT 0, -- 已用存储字节数 - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 创建时间 - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- 更新时间 - - CONSTRAINT storage_quotas_user_unique UNIQUE (user_id) -); - -COMMENT ON TABLE storage_quotas IS '用户存储配额表，用于限制单用户知识库容量'; -COMMENT ON COLUMN storage_quotas.id IS '配额记录 ID'; -COMMENT ON COLUMN storage_quotas.user_id IS '所属用户 ID'; -COMMENT ON COLUMN storage_quotas.max_storage_bytes IS '最大可用存储字节数'; -COMMENT ON COLUMN storage_quotas.used_storage_bytes IS '已用存储字节数'; -COMMENT ON COLUMN storage_quotas.created_at IS '创建时间'; -COMMENT ON COLUMN storage_quotas.updated_at IS '更新时间'; - --- 常用查询索引 -CREATE INDEX IF NOT EXISTS idx_knowledge_bases_user_id - ON knowledge_bases(user_id); - -CREATE UNIQUE INDEX IF NOT EXISTS knowledge_bases_user_name_normal_unique - ON knowledge_bases(user_id, name) - WHERE status = 1; - -CREATE INDEX IF NOT EXISTS idx_documents_user_kb - ON documents(user_id, knowledge_base_id); - -CREATE INDEX IF NOT EXISTS idx_documents_user_external - ON documents(user_id, source_type, external_id) - WHERE external_id <> ''; - -CREATE INDEX IF NOT EXISTS idx_document_versions_document_id - ON document_versions(document_id); - -CREATE INDEX IF NOT EXISTS idx_document_chunks_user_kb - ON document_chunks(user_id, knowledge_base_id); - -CREATE INDEX IF NOT EXISTS idx_document_chunks_document_id - ON document_chunks(document_id); - -CREATE INDEX IF NOT EXISTS idx_document_processing_jobs_document_id - ON document_processing_jobs(document_id); - -CREATE INDEX IF NOT EXISTS idx_sync_sources_user_kb - ON sync_sources(user_id, knowledge_base_id); - -CREATE INDEX IF NOT EXISTS idx_sync_jobs_source_id - ON sync_jobs(sync_source_id); - -CREATE INDEX IF NOT EXISTS idx_sync_jobs_user_kb - ON sync_jobs(user_id, knowledge_base_id); - -CREATE INDEX IF NOT EXISTS idx_sync_items_source_parent - ON sync_items(sync_source_id, parent_external_id); - -CREATE INDEX IF NOT EXISTS idx_sync_items_user_kb - ON sync_items(user_id, knowledge_base_id); - -CREATE INDEX IF NOT EXISTS idx_dingtalk_user_bindings_user_id - ON dingtalk_user_bindings(user_id); - -CREATE INDEX IF NOT EXISTS idx_dingtalk_user_bindings_corp_id - ON dingtalk_user_bindings(corp_id); - -CREATE INDEX IF NOT EXISTS idx_document_chunks_embedding - ON document_chunks - USING ivfflat (embedding vector_cosine_ops) - WITH (lists = 100) - WHERE embedding IS NOT NULL; +-- Solvify-Agent 生产数据库初始化基线 +-- 仅在空 PostgreSQL 数据卷首次启动时执行 + +\set ON_ERROR_STOP on + +BEGIN; + +CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS vector; + +-- ---------------------------- +-- Table structure for agent_task_steps +-- ---------------------------- +CREATE TABLE "public"."agent_task_steps" ( + "id" varchar(64) COLLATE "pg_catalog"."default" NOT NULL, + "task_id" varchar(64) COLLATE "pg_catalog"."default" NOT NULL, + "step_index" int4 NOT NULL DEFAULT 0, + "started_at" timestamptz(6) NOT NULL, + "ended_at" timestamptz(6), + "thinking_summary" text COLLATE "pg_catalog"."default", + "tool_name" varchar(128) COLLATE "pg_catalog"."default", + "tool_input_masked" text COLLATE "pg_catalog"."default", + "tool_result_summary" text COLLATE "pg_catalog"."default", + "tool_status" varchar(32) COLLATE "pg_catalog"."default", + "tool_error" text COLLATE "pg_catalog"."default", + "latency_ms" int8 NOT NULL DEFAULT 0, + "tokens_delta" int4 NOT NULL DEFAULT 0, + "attrs" jsonb +) +; + +-- ---------------------------- +-- Table structure for agent_tasks +-- ---------------------------- +CREATE TABLE "public"."agent_tasks" ( + "id" varchar(64) COLLATE "pg_catalog"."default" NOT NULL, + "trace_id" varchar(128) COLLATE "pg_catalog"."default", + "session_id" varchar(64) COLLATE "pg_catalog"."default", + "user_id" varchar(64) COLLATE "pg_catalog"."default", + "model_id" varchar(128) COLLATE "pg_catalog"."default", + "search_mode" varchar(32) COLLATE "pg_catalog"."default", + "started_at" timestamptz(6) NOT NULL, + "ended_at" timestamptz(6), + "total_steps" int4 NOT NULL DEFAULT 0, + "tool_calls" int4 NOT NULL DEFAULT 0, + "status" varchar(32) COLLATE "pg_catalog"."default", + "abort_reason" varchar(128) COLLATE "pg_catalog"."default", + "tokens_prompt" int4 NOT NULL DEFAULT 0, + "tokens_completion" int4 NOT NULL DEFAULT 0, + "total_cost" float8 NOT NULL DEFAULT 0, + "error_summary" text COLLATE "pg_catalog"."default", + "feedback_rating" int4 +) +; + +-- ---------------------------- +-- Table structure for chat_messages +-- ---------------------------- +CREATE TABLE "public"."chat_messages" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "session_id" uuid NOT NULL, + "role" varchar(20) COLLATE "pg_catalog"."default" NOT NULL, + "content" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "model_id" varchar(36) COLLATE "pg_catalog"."default", + "search_mode" varchar(20) COLLATE "pg_catalog"."default" NOT NULL DEFAULT 'quick'::character varying, + "knowledge_base_ids" jsonb NOT NULL DEFAULT '[]'::jsonb, + "sources" jsonb, + "metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "created_at" timestamptz(6) +) +; + +-- ---------------------------- +-- Table structure for chat_sessions +-- ---------------------------- +CREATE TABLE "public"."chat_sessions" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "title" varchar(200) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "model_id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "status" varchar(20) COLLATE "pg_catalog"."default" NOT NULL DEFAULT 'active'::character varying, + "message_count" int8 NOT NULL DEFAULT 0, + "created_at" timestamptz(6), + "updated_at" timestamptz(6) +) +; + +-- ---------------------------- +-- Table structure for chat_summaries +-- ---------------------------- +CREATE TABLE "public"."chat_summaries" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "session_id" uuid NOT NULL, + "summary" text COLLATE "pg_catalog"."default" NOT NULL, + "covered_count" int4 NOT NULL DEFAULT 0, + "last_message_id" uuid, + "created_at" timestamptz(6) NOT NULL DEFAULT now(), + "updated_at" timestamptz(6) NOT NULL DEFAULT now() +) +; +COMMENT ON COLUMN "public"."chat_summaries"."session_id" IS '会话 ID'; +COMMENT ON COLUMN "public"."chat_summaries"."summary" IS '摘要文本'; +COMMENT ON COLUMN "public"."chat_summaries"."covered_count" IS '覆盖消息数'; +COMMENT ON COLUMN "public"."chat_summaries"."last_message_id" IS '摘要覆盖到的最后一条消息 ID'; +COMMENT ON TABLE "public"."chat_summaries" IS '会话摘要表，保存单一会话的摘要以替代早期原始消息'; + +-- ---------------------------- +-- Table structure for chat_traces +-- ---------------------------- +CREATE TABLE "public"."chat_traces" ( + "id" varchar(128) COLLATE "pg_catalog"."default" NOT NULL, + "request_id" varchar(128) COLLATE "pg_catalog"."default", + "user_id" varchar(64) COLLATE "pg_catalog"."default", + "session_id" varchar(64) COLLATE "pg_catalog"."default", + "sample_rate" float8 NOT NULL DEFAULT 0, + "sampled" bool NOT NULL DEFAULT false, + "duration_ms" int8 NOT NULL DEFAULT 0, + "status" varchar(32) COLLATE "pg_catalog"."default", + "error" text COLLATE "pg_catalog"."default", + "attrs" jsonb, + "span_tree" jsonb, + "created_at" timestamptz(6) NOT NULL DEFAULT now() +) +; + +-- ---------------------------- +-- Table structure for dingtalk_user_bindings +-- ---------------------------- +CREATE TABLE "public"."dingtalk_user_bindings" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "ding_open_id" varchar(128) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "ding_union_id" varchar(128) COLLATE "pg_catalog"."default" NOT NULL, + "corp_id" varchar(128) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "nickname" varchar(128) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "avatar" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "created_at" timestamptz(6) NOT NULL DEFAULT now(), + "updated_at" timestamptz(6) NOT NULL DEFAULT now() +) +; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."id" IS '绑定 ID'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."user_id" IS '系统用户 ID'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."ding_open_id" IS '钉钉 openid'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."ding_union_id" IS '钉钉 unionId'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."corp_id" IS '钉钉企业 ID'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."nickname" IS '钉钉用户昵称'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."avatar" IS '钉钉用户头像'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."dingtalk_user_bindings"."updated_at" IS '更新时间'; +COMMENT ON TABLE "public"."dingtalk_user_bindings" IS '钉钉用户绑定表，用于保存系统用户与钉钉身份的对应关系'; + +-- ---------------------------- +-- Table structure for document_chunks +-- ---------------------------- +CREATE TABLE "public"."document_chunks" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "knowledge_base_id" uuid NOT NULL, + "document_id" uuid NOT NULL, + "version_id" uuid NOT NULL, + "chunk_index" int8 NOT NULL, + "section_title" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "content" text COLLATE "pg_catalog"."default" NOT NULL, + "token_count" int8 NOT NULL DEFAULT 0, + "page_number" int8, + "embedding_model" varchar(128) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "embedding" "public"."vector"(1024), + "metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "created_at" timestamptz(6), + "keywords" text[] COLLATE "pg_catalog"."default" DEFAULT '{}'::text[] +) +; +COMMENT ON COLUMN "public"."document_chunks"."id" IS '分块 ID'; +COMMENT ON COLUMN "public"."document_chunks"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."document_chunks"."knowledge_base_id" IS '所属知识库 ID'; +COMMENT ON COLUMN "public"."document_chunks"."document_id" IS '所属文档 ID'; +COMMENT ON COLUMN "public"."document_chunks"."version_id" IS '所属文档版本 ID'; +COMMENT ON COLUMN "public"."document_chunks"."chunk_index" IS '分块序号'; +COMMENT ON COLUMN "public"."document_chunks"."section_title" IS '分块所属章节标题'; +COMMENT ON COLUMN "public"."document_chunks"."content" IS '分块文本内容'; +COMMENT ON COLUMN "public"."document_chunks"."token_count" IS '分块 token 数'; +COMMENT ON COLUMN "public"."document_chunks"."page_number" IS '来源页码'; +COMMENT ON COLUMN "public"."document_chunks"."embedding_model" IS '向量模型名称'; +COMMENT ON COLUMN "public"."document_chunks"."embedding" IS '分块向量数据'; +COMMENT ON COLUMN "public"."document_chunks"."metadata" IS '分块扩展元数据'; +COMMENT ON COLUMN "public"."document_chunks"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."document_chunks"."keywords" IS '分块关键词'; +COMMENT ON TABLE "public"."document_chunks" IS '文档分块表，保存可检索文本片段和 embedding 向量'; + +-- ---------------------------- +-- Table structure for document_processing_jobs +-- ---------------------------- +CREATE TABLE "public"."document_processing_jobs" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "document_id" uuid NOT NULL, + "job_type" varchar(32) COLLATE "pg_catalog"."default" NOT NULL, + "status" int8 NOT NULL DEFAULT 1, + "error_message" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "started_at" timestamptz(6), + "finished_at" timestamptz(6), + "created_at" timestamptz(6), + "updated_at" timestamptz(6) +) +; +COMMENT ON COLUMN "public"."document_processing_jobs"."id" IS '任务 ID'; +COMMENT ON COLUMN "public"."document_processing_jobs"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."document_processing_jobs"."document_id" IS '所属文档 ID'; +COMMENT ON COLUMN "public"."document_processing_jobs"."job_type" IS '任务类型，parse 解析，chunk 分块，embed 向量化，reindex 重建索引'; +COMMENT ON COLUMN "public"."document_processing_jobs"."status" IS '任务状态，1 待处理，2 运行中，3 成功，4 失败'; +COMMENT ON COLUMN "public"."document_processing_jobs"."error_message" IS '任务失败原因'; +COMMENT ON COLUMN "public"."document_processing_jobs"."started_at" IS '开始时间'; +COMMENT ON COLUMN "public"."document_processing_jobs"."finished_at" IS '完成时间'; +COMMENT ON COLUMN "public"."document_processing_jobs"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."document_processing_jobs"."updated_at" IS '更新时间'; +COMMENT ON TABLE "public"."document_processing_jobs" IS '文档处理任务表，记录解析、分块、向量化和重建索引状态'; + +-- ---------------------------- +-- Table structure for document_versions +-- ---------------------------- +CREATE TABLE "public"."document_versions" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "document_id" uuid NOT NULL, + "version_no" int8 NOT NULL DEFAULT 1, + "content" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "content_hash" varchar(128) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "change_summary" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "created_at" timestamptz(6) +) +; +COMMENT ON COLUMN "public"."document_versions"."id" IS '版本 ID'; +COMMENT ON COLUMN "public"."document_versions"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."document_versions"."document_id" IS '所属文档 ID'; +COMMENT ON COLUMN "public"."document_versions"."version_no" IS '版本号'; +COMMENT ON COLUMN "public"."document_versions"."content" IS '版本正文内容'; +COMMENT ON COLUMN "public"."document_versions"."content_hash" IS '版本内容哈希'; +COMMENT ON COLUMN "public"."document_versions"."change_summary" IS '变更摘要'; +COMMENT ON COLUMN "public"."document_versions"."created_at" IS '创建时间'; +COMMENT ON TABLE "public"."document_versions" IS '文档版本表，用于记录原始解析内容和在线编辑历史'; + +-- ---------------------------- +-- Table structure for documents +-- ---------------------------- +CREATE TABLE "public"."documents" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "knowledge_base_id" uuid NOT NULL, + "title" varchar(255) COLLATE "pg_catalog"."default" NOT NULL, + "file_name" varchar(255) COLLATE "pg_catalog"."default" NOT NULL, + "file_type" varchar(32) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "file_size" int8 NOT NULL DEFAULT 0, + "storage_path" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "file_hash" varchar(128) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "source_type" varchar(32) COLLATE "pg_catalog"."default" NOT NULL DEFAULT 'upload'::character varying, + "status" int8 NOT NULL DEFAULT 1, + "error_message" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "ready_at" timestamptz(6), + "created_at" timestamptz(6), + "updated_at" timestamptz(6), + "deleted_at" timestamptz(6), + "delete_expired_at" timestamptz(6), + "external_id" varchar(255) COLLATE "pg_catalog"."default" DEFAULT ''::character varying, + "external_url" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "source_updated_at" timestamptz(6) +) +; +COMMENT ON COLUMN "public"."documents"."id" IS '文档 ID'; +COMMENT ON COLUMN "public"."documents"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."documents"."knowledge_base_id" IS '所属知识库 ID'; +COMMENT ON COLUMN "public"."documents"."title" IS '文档标题'; +COMMENT ON COLUMN "public"."documents"."file_name" IS '原始文件名'; +COMMENT ON COLUMN "public"."documents"."file_type" IS '文件类型'; +COMMENT ON COLUMN "public"."documents"."file_size" IS '文件大小字节数'; +COMMENT ON COLUMN "public"."documents"."storage_path" IS '文件存储路径'; +COMMENT ON COLUMN "public"."documents"."file_hash" IS '原始文件内容指纹'; +COMMENT ON COLUMN "public"."documents"."source_type" IS '文档来源类型，upload 上传，edit 编辑，sync 同步，web_search 联网搜索'; +COMMENT ON COLUMN "public"."documents"."status" IS '文档状态，1 已上传，2 处理中，3 已就绪，4 处理失败，5 已删除'; +COMMENT ON COLUMN "public"."documents"."error_message" IS '处理失败原因'; +COMMENT ON COLUMN "public"."documents"."ready_at" IS '文档就绪时间'; +COMMENT ON COLUMN "public"."documents"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."documents"."updated_at" IS '更新时间'; +COMMENT ON COLUMN "public"."documents"."deleted_at" IS '删除时间'; +COMMENT ON COLUMN "public"."documents"."delete_expired_at" IS '删除保留到期时间'; +COMMENT ON COLUMN "public"."documents"."external_id" IS '外部平台文档 ID'; +COMMENT ON COLUMN "public"."documents"."external_url" IS '外部平台文档链接'; +COMMENT ON COLUMN "public"."documents"."source_updated_at" IS '外部平台更新时间'; +COMMENT ON TABLE "public"."documents" IS '文档主表，记录知识库下的文件和处理状态'; + +-- ---------------------------- +-- Table structure for knowledge_bases +-- ---------------------------- +CREATE TABLE "public"."knowledge_bases" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "name" varchar(128) COLLATE "pg_catalog"."default" NOT NULL, + "category" varchar(128) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "description" text COLLATE "pg_catalog"."default", + "source_type" varchar(32) COLLATE "pg_catalog"."default" NOT NULL DEFAULT 'local'::character varying, + "source_platform" varchar(32) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "document_count" int8 NOT NULL DEFAULT 0, + "storage_bytes" int8 NOT NULL DEFAULT 0, + "created_at" timestamptz(6), + "updated_at" timestamptz(6), + "status" int8 NOT NULL DEFAULT 1, + "deleted_at" timestamptz(6), + "delete_expired_at" timestamptz(6) +) +; +COMMENT ON COLUMN "public"."knowledge_bases"."id" IS '知识库 ID'; +COMMENT ON COLUMN "public"."knowledge_bases"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."knowledge_bases"."name" IS '知识库名称'; +COMMENT ON COLUMN "public"."knowledge_bases"."category" IS '知识库分类'; +COMMENT ON COLUMN "public"."knowledge_bases"."description" IS '知识库描述'; +COMMENT ON COLUMN "public"."knowledge_bases"."source_type" IS '知识库来源类型，local 自建，sync 同步，web_search 联网搜索'; +COMMENT ON COLUMN "public"."knowledge_bases"."source_platform" IS '同步来源平台'; +COMMENT ON COLUMN "public"."knowledge_bases"."document_count" IS '文档数量'; +COMMENT ON COLUMN "public"."knowledge_bases"."storage_bytes" IS '已占用存储字节数'; +COMMENT ON COLUMN "public"."knowledge_bases"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."knowledge_bases"."updated_at" IS '更新时间'; +COMMENT ON COLUMN "public"."knowledge_bases"."status" IS '知识库状态，1 正常，2 已删除'; +COMMENT ON COLUMN "public"."knowledge_bases"."deleted_at" IS '删除时间'; +COMMENT ON COLUMN "public"."knowledge_bases"."delete_expired_at" IS '删除保留到期时间'; +COMMENT ON TABLE "public"."knowledge_bases" IS '知识库主表，所有知识库按用户隔离'; + +-- ---------------------------- +-- Table structure for message_feedback +-- ---------------------------- +CREATE TABLE "public"."message_feedback" ( + "id" varchar(64) COLLATE "pg_catalog"."default" NOT NULL, + "message_id" varchar(64) COLLATE "pg_catalog"."default" NOT NULL, + "user_id" varchar(64) COLLATE "pg_catalog"."default" NOT NULL, + "session_id" varchar(64) COLLATE "pg_catalog"."default", + "rating" int4 NOT NULL DEFAULT 0, + "reason_tag" varchar(64) COLLATE "pg_catalog"."default", + "comment" text COLLATE "pg_catalog"."default", + "trace_id" varchar(128) COLLATE "pg_catalog"."default", + "created_at" timestamptz(6) NOT NULL DEFAULT now() +) +; + +-- ---------------------------- +-- Table structure for models +-- ---------------------------- +CREATE TABLE "public"."models" ( + "id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "name" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "provider" varchar(50) COLLATE "pg_catalog"."default" NOT NULL, + "model_id" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "is_enabled" bool DEFAULT true, + "config" jsonb, + "created_at" timestamptz(6), + "updated_at" timestamptz(6), + "base_url" varchar(500) COLLATE "pg_catalog"."default", + "api_key" varchar(500) COLLATE "pg_catalog"."default", + "max_context_length" int4 NOT NULL DEFAULT 8192 +) +; +COMMENT ON COLUMN "public"."models"."max_context_length" IS '模型最大上下文 token 长度，用于计算历史消息和检索预算'; +COMMENT ON TABLE "public"."models" IS '系统预置模型配置表'; + +-- ---------------------------- +-- Table structure for role_templates +-- ---------------------------- +CREATE TABLE "public"."role_templates" ( + "id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "builtin_key" varchar(50) COLLATE "pg_catalog"."default", + "name" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "description" varchar(255) COLLATE "pg_catalog"."default", + "target_role" int2 NOT NULL DEFAULT 0, + "department" varchar(100) COLLATE "pg_catalog"."default", + "is_default" bool NOT NULL DEFAULT false, + "system_prompt" text COLLATE "pg_catalog"."default" NOT NULL, + "is_enabled" bool NOT NULL DEFAULT true, + "display_order" int4 NOT NULL DEFAULT 0, + "created_at" timestamp(6) NOT NULL DEFAULT now(), + "updated_at" timestamp(6) NOT NULL DEFAULT now() +) +; +COMMENT ON COLUMN "public"."role_templates"."builtin_key" IS '内置模板唯一标识：default/engineer/hr/finance；自定义为空'; +COMMENT ON COLUMN "public"."role_templates"."target_role" IS '适用角色：0通用/1普通用户/2管理员'; +COMMENT ON COLUMN "public"."role_templates"."department" IS '适用部门；空=全部'; +COMMENT ON COLUMN "public"."role_templates"."is_default" IS '是否为缺省模板；仅 1 条为 true'; +COMMENT ON COLUMN "public"."role_templates"."system_prompt" IS '额外追加到默认系统提示词的角色专属内容'; +COMMENT ON COLUMN "public"."role_templates"."is_enabled" IS '是否启用'; +COMMENT ON COLUMN "public"."role_templates"."display_order" IS '展示顺序'; +COMMENT ON TABLE "public"."role_templates" IS '角色模板：按用户角色/部门加载不同的 System Prompt'; + +-- ---------------------------- +-- Table structure for storage_quotas +-- ---------------------------- +CREATE TABLE "public"."storage_quotas" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "max_storage_bytes" int8 NOT NULL DEFAULT '10737418240'::bigint, + "used_storage_bytes" int8 NOT NULL DEFAULT 0, + "created_at" timestamptz(6), + "updated_at" timestamptz(6) +) +; +COMMENT ON COLUMN "public"."storage_quotas"."id" IS '配额记录 ID'; +COMMENT ON COLUMN "public"."storage_quotas"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."storage_quotas"."max_storage_bytes" IS '最大可用存储字节数'; +COMMENT ON COLUMN "public"."storage_quotas"."used_storage_bytes" IS '已用存储字节数'; +COMMENT ON COLUMN "public"."storage_quotas"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."storage_quotas"."updated_at" IS '更新时间'; +COMMENT ON TABLE "public"."storage_quotas" IS '用户存储配额表，用于限制单用户知识库容量'; + +-- ---------------------------- +-- Table structure for sync_items +-- ---------------------------- +CREATE TABLE "public"."sync_items" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "sync_source_id" uuid NOT NULL, + "knowledge_base_id" uuid NOT NULL, + "external_id" varchar(255) COLLATE "pg_catalog"."default" NOT NULL, + "parent_external_id" varchar(255) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "name" varchar(255) COLLATE "pg_catalog"."default" NOT NULL, + "item_type" varchar(32) COLLATE "pg_catalog"."default" NOT NULL, + "category" varchar(64) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "extension" varchar(32) COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::character varying, + "external_url" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "file_size" int8 NOT NULL DEFAULT 0, + "has_children" bool NOT NULL DEFAULT false, + "source_updated_at" timestamptz(6), + "local_document_id" uuid, + "import_status" int4 NOT NULL DEFAULT 1, + "error_message" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "created_at" timestamptz(6) NOT NULL DEFAULT now(), + "updated_at" timestamptz(6) NOT NULL DEFAULT now() +) +; +COMMENT ON COLUMN "public"."sync_items"."id" IS '同步目录项 ID'; +COMMENT ON COLUMN "public"."sync_items"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."sync_items"."sync_source_id" IS '同步源 ID'; +COMMENT ON COLUMN "public"."sync_items"."knowledge_base_id" IS '绑定知识库 ID'; +COMMENT ON COLUMN "public"."sync_items"."external_id" IS '外部节点 ID'; +COMMENT ON COLUMN "public"."sync_items"."parent_external_id" IS '外部父节点 ID'; +COMMENT ON COLUMN "public"."sync_items"."name" IS '节点名称'; +COMMENT ON COLUMN "public"."sync_items"."item_type" IS '节点类型，FILE 或 FOLDER'; +COMMENT ON COLUMN "public"."sync_items"."category" IS '钉钉节点分类'; +COMMENT ON COLUMN "public"."sync_items"."extension" IS '文件扩展名'; +COMMENT ON COLUMN "public"."sync_items"."external_url" IS '外部原文链接'; +COMMENT ON COLUMN "public"."sync_items"."file_size" IS '文件大小'; +COMMENT ON COLUMN "public"."sync_items"."has_children" IS '是否有子节点'; +COMMENT ON COLUMN "public"."sync_items"."source_updated_at" IS '外部更新时间'; +COMMENT ON COLUMN "public"."sync_items"."local_document_id" IS '已导入本地文档 ID'; +COMMENT ON COLUMN "public"."sync_items"."import_status" IS '导入状态，1 未导入，2 导入中，3 已导入，4 导入失败'; +COMMENT ON COLUMN "public"."sync_items"."error_message" IS '导入失败原因'; +COMMENT ON COLUMN "public"."sync_items"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."sync_items"."updated_at" IS '更新时间'; +COMMENT ON TABLE "public"."sync_items" IS '同步目录项表，记录外部知识库中的目录和文件元数据'; + +-- ---------------------------- +-- Table structure for sync_jobs +-- ---------------------------- +CREATE TABLE "public"."sync_jobs" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "sync_source_id" uuid NOT NULL, + "knowledge_base_id" uuid NOT NULL, + "job_type" varchar(32) COLLATE "pg_catalog"."default" NOT NULL, + "status" int8 NOT NULL DEFAULT 1, + "total_count" int8 NOT NULL DEFAULT 0, + "success_count" int8 NOT NULL DEFAULT 0, + "failed_count" int8 NOT NULL DEFAULT 0, + "error_message" text COLLATE "pg_catalog"."default" NOT NULL DEFAULT ''::text, + "started_at" timestamptz(6), + "finished_at" timestamptz(6), + "created_at" timestamptz(6), + "updated_at" timestamptz(6) +) +; +COMMENT ON COLUMN "public"."sync_jobs"."id" IS '同步任务 ID'; +COMMENT ON COLUMN "public"."sync_jobs"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."sync_jobs"."sync_source_id" IS '同步源 ID'; +COMMENT ON COLUMN "public"."sync_jobs"."knowledge_base_id" IS '绑定知识库 ID'; +COMMENT ON COLUMN "public"."sync_jobs"."job_type" IS '任务类型，manual 手动同步'; +COMMENT ON COLUMN "public"."sync_jobs"."status" IS '任务状态，1 待同步，2 同步中，3 成功，4 失败'; +COMMENT ON COLUMN "public"."sync_jobs"."total_count" IS '同步总数'; +COMMENT ON COLUMN "public"."sync_jobs"."success_count" IS '同步成功数'; +COMMENT ON COLUMN "public"."sync_jobs"."failed_count" IS '同步失败数'; +COMMENT ON COLUMN "public"."sync_jobs"."error_message" IS '任务失败原因'; +COMMENT ON COLUMN "public"."sync_jobs"."started_at" IS '开始时间'; +COMMENT ON COLUMN "public"."sync_jobs"."finished_at" IS '完成时间'; +COMMENT ON COLUMN "public"."sync_jobs"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."sync_jobs"."updated_at" IS '更新时间'; +COMMENT ON TABLE "public"."sync_jobs" IS '同步任务表，记录外部平台同步执行状态'; + +-- ---------------------------- +-- Table structure for sync_sources +-- ---------------------------- +CREATE TABLE "public"."sync_sources" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "knowledge_base_id" uuid NOT NULL, + "name" varchar(128) COLLATE "pg_catalog"."default" NOT NULL, + "platform" varchar(32) COLLATE "pg_catalog"."default" NOT NULL, + "source_config" jsonb NOT NULL DEFAULT '{}'::jsonb, + "status" int8 NOT NULL DEFAULT 1, + "last_sync_at" timestamptz(6), + "last_error_message" text COLLATE "pg_catalog"."default" DEFAULT ''::text, + "created_at" timestamptz(6), + "updated_at" timestamptz(6), + "deleted_at" timestamptz(6) +) +; +COMMENT ON COLUMN "public"."sync_sources"."id" IS '同步源 ID'; +COMMENT ON COLUMN "public"."sync_sources"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."sync_sources"."knowledge_base_id" IS '绑定知识库 ID'; +COMMENT ON COLUMN "public"."sync_sources"."name" IS '同步源名称'; +COMMENT ON COLUMN "public"."sync_sources"."platform" IS '同步平台，当前支持 dingtalk'; +COMMENT ON COLUMN "public"."sync_sources"."source_config" IS '非敏感同步配置'; +COMMENT ON COLUMN "public"."sync_sources"."status" IS '同步源状态，1 正常，2 禁用，3 已删除'; +COMMENT ON COLUMN "public"."sync_sources"."last_sync_at" IS '最近同步时间'; +COMMENT ON COLUMN "public"."sync_sources"."last_error_message" IS '最近同步失败原因'; +COMMENT ON COLUMN "public"."sync_sources"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."sync_sources"."updated_at" IS '更新时间'; +COMMENT ON COLUMN "public"."sync_sources"."deleted_at" IS '删除时间'; +COMMENT ON TABLE "public"."sync_sources" IS '同步源配置表，记录钉钉等外部平台同步入口'; + +-- ---------------------------- +-- Table structure for tool_providers +-- ---------------------------- +CREATE TABLE "public"."tool_providers" ( + "id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "tool_type_id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "provider_key" varchar(50) COLLATE "pg_catalog"."default" NOT NULL, + "name" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "description" text COLLATE "pg_catalog"."default", + "provider_type" varchar(20) COLLATE "pg_catalog"."default" NOT NULL DEFAULT 'http'::character varying, + "config_schema" jsonb, + "input_schema" jsonb, + "provider_config" jsonb, + "admin_config" jsonb, + "rate_limit" jsonb, + "is_enabled" bool DEFAULT true, + "display_order" int8 DEFAULT 0, + "created_at" timestamptz(6), + "updated_at" timestamptz(6) +) +; + +-- ---------------------------- +-- Table structure for tool_types +-- ---------------------------- +CREATE TABLE "public"."tool_types" ( + "id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "name" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "tool_key" varchar(50) COLLATE "pg_catalog"."default" NOT NULL, + "description" text COLLATE "pg_catalog"."default", + "execution_mode" varchar(20) COLLATE "pg_catalog"."default" DEFAULT 'sync'::character varying, + "input_schema" jsonb, + "is_enabled" bool DEFAULT true, + "created_at" timestamptz(6), + "updated_at" timestamptz(6) +) +; + +-- ---------------------------- +-- Table structure for user_memories +-- ---------------------------- +CREATE TABLE "public"."user_memories" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "memory_type" varchar(30) COLLATE "pg_catalog"."default" NOT NULL, + "content" text COLLATE "pg_catalog"."default" NOT NULL, + "source_session" uuid, + "confidence" float8 NOT NULL DEFAULT 1.0, + "is_active" bool NOT NULL DEFAULT true, + "created_at" timestamptz(6) NOT NULL DEFAULT now(), + "updated_at" timestamptz(6) NOT NULL DEFAULT now() +) +; +COMMENT ON COLUMN "public"."user_memories"."user_id" IS '所属用户 ID'; +COMMENT ON COLUMN "public"."user_memories"."memory_type" IS '记忆类型：fact / preference / constraint / decision'; +COMMENT ON COLUMN "public"."user_memories"."content" IS '记忆内容'; +COMMENT ON COLUMN "public"."user_memories"."source_session" IS '来源会话 ID'; +COMMENT ON COLUMN "public"."user_memories"."confidence" IS '置信度 0-1'; +COMMENT ON COLUMN "public"."user_memories"."is_active" IS '是否有效'; +COMMENT ON TABLE "public"."user_memories" IS '用户长期记忆表，保存跨会话的事实、偏好、约束和决策结论'; + +-- ---------------------------- +-- Table structure for user_model_configs +-- ---------------------------- +CREATE TABLE "public"."user_model_configs" ( + "id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "user_id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "display_name" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "api_format" varchar(20) COLLATE "pg_catalog"."default" NOT NULL, + "base_url" varchar(500) COLLATE "pg_catalog"."default" NOT NULL, + "model_id" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "api_key" varchar(500) COLLATE "pg_catalog"."default", + "config" jsonb, + "created_at" timestamptz(6), + "updated_at" timestamptz(6), + "max_context_length" int4 NOT NULL DEFAULT 8192 +) +; +COMMENT ON COLUMN "public"."user_model_configs"."max_context_length" IS '用户自定义模型的最大上下文 token 长度'; +COMMENT ON TABLE "public"."user_model_configs" IS '用户自定义模型配置表'; + +-- ---------------------------- +-- Table structure for user_preferences +-- ---------------------------- +CREATE TABLE "public"."user_preferences" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "user_id" uuid NOT NULL, + "default_model_id" varchar(36) COLLATE "pg_catalog"."default", + "preferred_kb_ids" jsonb, + "answer_style" varchar(20) COLLATE "pg_catalog"."default" NOT NULL DEFAULT 'balanced'::character varying, + "auto_deep_mode" bool NOT NULL DEFAULT false, + "auto_deep_threshold" int4 NOT NULL DEFAULT 2, + "use_markdown_table" bool NOT NULL DEFAULT true, + "citation_style" varchar(20) COLLATE "pg_catalog"."default" NOT NULL DEFAULT 'section_title'::character varying, + "created_at" timestamp(6) NOT NULL DEFAULT now(), + "updated_at" timestamp(6) NOT NULL DEFAULT now() +) +; +COMMENT ON COLUMN "public"."user_preferences"."default_model_id" IS '默认使用的模型ID（系统模型表ID，可空）'; +COMMENT ON COLUMN "public"."user_preferences"."preferred_kb_ids" IS '常用知识库ID列表，JSON array[string]'; +COMMENT ON COLUMN "public"."user_preferences"."answer_style" IS '回答风格：concise(简洁)/balanced(平衡)/detailed(详细)/step_by_step(分步)'; +COMMENT ON COLUMN "public"."user_preferences"."auto_deep_mode" IS '是否自动切深度模式（true=复杂问题自动切；false=始终用户手动）'; +COMMENT ON COLUMN "public"."user_preferences"."auto_deep_threshold" IS '自动切深度模式的信号阈值 1~5，越大越不容易切'; +COMMENT ON COLUMN "public"."user_preferences"."use_markdown_table" IS '回答尽量用表格呈现结构化数据'; +COMMENT ON COLUMN "public"."user_preferences"."citation_style" IS '引用格式：none(不标)/section_title(章节标题)/doc_title_only(仅文档名)'; +COMMENT ON TABLE "public"."user_preferences" IS '用户偏好：常用模型、常用知识库、回答风格、是否自动深度模式'; + +-- ---------------------------- +-- Table structure for user_tool_configs +-- ---------------------------- +CREATE TABLE "public"."user_tool_configs" ( + "id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "user_id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "tool_type_id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "provider_id" varchar(36) COLLATE "pg_catalog"."default" NOT NULL, + "display_name" varchar(100) COLLATE "pg_catalog"."default", + "config" jsonb NOT NULL, + "is_enabled" bool DEFAULT true, + "created_at" timestamptz(6), + "updated_at" timestamptz(6) +) +; + +-- ---------------------------- +-- Table structure for users +-- ---------------------------- +CREATE TABLE "public"."users" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "username" varchar(50) COLLATE "pg_catalog"."default" NOT NULL, + "password" varchar(255) COLLATE "pg_catalog"."default" NOT NULL, + "email" varchar(100) COLLATE "pg_catalog"."default", + "avatar" varchar(255) COLLATE "pg_catalog"."default", + "status" int2 DEFAULT 1, + "role" int2 DEFAULT 1, + "last_model" varchar(255) COLLATE "pg_catalog"."default", + "created_at" timestamptz(6), + "updated_at" timestamptz(6), + "department" varchar(100) COLLATE "pg_catalog"."default", + "position" varchar(100) COLLATE "pg_catalog"."default", + "expertise" varchar(255) COLLATE "pg_catalog"."default", + "preferred_language" varchar(20) COLLATE "pg_catalog"."default" DEFAULT 'zh-CN'::character varying, + "timezone" varchar(50) COLLATE "pg_catalog"."default" DEFAULT 'Asia/Shanghai'::character varying, + "role_template_id" varchar(36) COLLATE "pg_catalog"."default" +) +; +COMMENT ON COLUMN "public"."users"."id" IS '用户 ID'; +COMMENT ON COLUMN "public"."users"."username" IS '用户名'; +COMMENT ON COLUMN "public"."users"."password" IS '密码哈希'; +COMMENT ON COLUMN "public"."users"."email" IS '邮箱'; +COMMENT ON COLUMN "public"."users"."avatar" IS '头像'; +COMMENT ON COLUMN "public"."users"."status" IS '用户状态，1 正常，2 禁用，3 注销，4 待验证'; +COMMENT ON COLUMN "public"."users"."role" IS '角色:1普通用户, 2管理员'; +COMMENT ON COLUMN "public"."users"."last_model" IS '上次使用的模型'; +COMMENT ON COLUMN "public"."users"."created_at" IS '创建时间'; +COMMENT ON COLUMN "public"."users"."updated_at" IS '更新时间'; +COMMENT ON COLUMN "public"."users"."department" IS '部门'; +COMMENT ON COLUMN "public"."users"."position" IS '职位'; +COMMENT ON COLUMN "public"."users"."expertise" IS '擅长领域/业务方向，逗号分隔'; +COMMENT ON COLUMN "public"."users"."preferred_language" IS '偏好回答语言：zh-CN/en-US/ja-JP 等'; +COMMENT ON COLUMN "public"."users"."timezone" IS '时区 IANA，如 Asia/Shanghai'; +COMMENT ON COLUMN "public"."users"."role_template_id" IS '关联角色模板ID（可为空=默认模板）'; +COMMENT ON TABLE "public"."users" IS '用户基础表，用于隔离每个用户自己的知识库和文档'; + +-- ---------------------------- +-- Indexes structure for table agent_task_steps +-- ---------------------------- +CREATE INDEX "idx_agent_task_steps_started_at" ON "public"."agent_task_steps" USING btree ( + "started_at" "pg_catalog"."timestamptz_ops" DESC NULLS FIRST +); +CREATE INDEX "idx_agent_task_steps_task_id_step" ON "public"."agent_task_steps" USING btree ( + "task_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST, + "step_index" "pg_catalog"."int4_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table agent_task_steps +-- ---------------------------- +ALTER TABLE "public"."agent_task_steps" ADD CONSTRAINT "agent_task_steps_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table agent_tasks +-- ---------------------------- +CREATE INDEX "idx_agent_tasks_session_id" ON "public"."agent_tasks" USING btree ( + "session_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_agent_tasks_started_at" ON "public"."agent_tasks" USING btree ( + "started_at" "pg_catalog"."timestamptz_ops" DESC NULLS FIRST +); +CREATE INDEX "idx_agent_tasks_trace_id" ON "public"."agent_tasks" USING btree ( + "trace_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_agent_tasks_user_id" ON "public"."agent_tasks" USING btree ( + "user_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table agent_tasks +-- ---------------------------- +ALTER TABLE "public"."agent_tasks" ADD CONSTRAINT "agent_tasks_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table chat_messages +-- ---------------------------- +CREATE INDEX "idx_chat_messages_session_id_created_at" ON "public"."chat_messages" USING btree ( + "session_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "created_at" "pg_catalog"."timestamptz_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table chat_messages +-- ---------------------------- +ALTER TABLE "public"."chat_messages" ADD CONSTRAINT "chat_messages_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table chat_sessions +-- ---------------------------- +CREATE INDEX "idx_chat_sessions_user_id_updated_at" ON "public"."chat_sessions" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "updated_at" "pg_catalog"."timestamptz_ops" DESC NULLS FIRST +); + +-- ---------------------------- +-- Primary Key structure for table chat_sessions +-- ---------------------------- +ALTER TABLE "public"."chat_sessions" ADD CONSTRAINT "chat_sessions_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table chat_summaries +-- ---------------------------- +CREATE INDEX "idx_chat_summaries_session_id" ON "public"."chat_summaries" USING btree ( + "session_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Uniques structure for table chat_summaries +-- ---------------------------- +ALTER TABLE "public"."chat_summaries" ADD CONSTRAINT "chat_summaries_session_unique" UNIQUE ("session_id"); + +-- ---------------------------- +-- Primary Key structure for table chat_summaries +-- ---------------------------- +ALTER TABLE "public"."chat_summaries" ADD CONSTRAINT "chat_summaries_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table chat_traces +-- ---------------------------- +CREATE INDEX "idx_chat_traces_created_at" ON "public"."chat_traces" USING btree ( + "created_at" "pg_catalog"."timestamptz_ops" DESC NULLS FIRST +); +CREATE INDEX "idx_chat_traces_request_id" ON "public"."chat_traces" USING btree ( + "request_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_chat_traces_session_id" ON "public"."chat_traces" USING btree ( + "session_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_chat_traces_user_id" ON "public"."chat_traces" USING btree ( + "user_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table chat_traces +-- ---------------------------- +ALTER TABLE "public"."chat_traces" ADD CONSTRAINT "chat_traces_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table dingtalk_user_bindings +-- ---------------------------- +CREATE INDEX "idx_dingtalk_user_bindings_corp_id" ON "public"."dingtalk_user_bindings" USING btree ( + "corp_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_dingtalk_user_bindings_user_id" ON "public"."dingtalk_user_bindings" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Uniques structure for table dingtalk_user_bindings +-- ---------------------------- +ALTER TABLE "public"."dingtalk_user_bindings" ADD CONSTRAINT "dingtalk_user_bindings_user_unique" UNIQUE ("user_id"); +ALTER TABLE "public"."dingtalk_user_bindings" ADD CONSTRAINT "dingtalk_user_bindings_union_unique" UNIQUE ("ding_union_id"); + +-- ---------------------------- +-- Primary Key structure for table dingtalk_user_bindings +-- ---------------------------- +ALTER TABLE "public"."dingtalk_user_bindings" ADD CONSTRAINT "dingtalk_user_bindings_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table document_chunks +-- ---------------------------- +CREATE INDEX "idx_document_chunks_document_id" ON "public"."document_chunks" USING btree ( + "document_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); +CREATE INDEX "idx_document_chunks_embedding" ON "public"."document_chunks" USING ivfflat ( + "embedding" "public"."vector_cosine_ops" +) WITH (lists = 100) WHERE embedding IS NOT NULL; +CREATE INDEX "idx_document_chunks_keywords_gin" ON "public"."document_chunks" USING gin ( + "keywords" COLLATE "pg_catalog"."default" "pg_catalog"."array_ops" +) WHERE keywords IS NOT NULL; +CREATE INDEX "idx_document_chunks_user_kb" ON "public"."document_chunks" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "knowledge_base_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Uniques structure for table document_chunks +-- ---------------------------- +ALTER TABLE "public"."document_chunks" ADD CONSTRAINT "document_chunks_version_index_unique" UNIQUE ("version_id", "chunk_index"); + +-- ---------------------------- +-- Primary Key structure for table document_chunks +-- ---------------------------- +ALTER TABLE "public"."document_chunks" ADD CONSTRAINT "document_chunks_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table document_processing_jobs +-- ---------------------------- +CREATE INDEX "idx_document_processing_jobs_document_id" ON "public"."document_processing_jobs" USING btree ( + "document_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table document_processing_jobs +-- ---------------------------- +ALTER TABLE "public"."document_processing_jobs" ADD CONSTRAINT "document_processing_jobs_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table document_versions +-- ---------------------------- +CREATE INDEX "idx_document_versions_document_id" ON "public"."document_versions" USING btree ( + "document_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Uniques structure for table document_versions +-- ---------------------------- +ALTER TABLE "public"."document_versions" ADD CONSTRAINT "document_versions_document_version_unique" UNIQUE ("document_id", "version_no"); + +-- ---------------------------- +-- Primary Key structure for table document_versions +-- ---------------------------- +ALTER TABLE "public"."document_versions" ADD CONSTRAINT "document_versions_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table documents +-- ---------------------------- +CREATE INDEX "idx_documents_user_external" ON "public"."documents" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "source_type" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST, + "external_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +) WHERE external_id::text <> ''::text; +CREATE INDEX "idx_documents_user_kb" ON "public"."documents" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "knowledge_base_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table documents +-- ---------------------------- +ALTER TABLE "public"."documents" ADD CONSTRAINT "documents_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table knowledge_bases +-- ---------------------------- +CREATE INDEX "idx_knowledge_bases_user_id" ON "public"."knowledge_bases" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); +CREATE UNIQUE INDEX "knowledge_bases_user_name_normal_unique" ON "public"."knowledge_bases" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "name" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +) WHERE status = 1; + +-- ---------------------------- +-- Primary Key structure for table knowledge_bases +-- ---------------------------- +ALTER TABLE "public"."knowledge_bases" ADD CONSTRAINT "knowledge_bases_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table message_feedback +-- ---------------------------- +CREATE INDEX "idx_message_feedback_message_id" ON "public"."message_feedback" USING btree ( + "message_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_message_feedback_session_id" ON "public"."message_feedback" USING btree ( + "session_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_message_feedback_trace_id" ON "public"."message_feedback" USING btree ( + "trace_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_message_feedback_user_id" ON "public"."message_feedback" USING btree ( + "user_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table message_feedback +-- ---------------------------- +ALTER TABLE "public"."message_feedback" ADD CONSTRAINT "message_feedback_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table models +-- ---------------------------- +CREATE INDEX "idx_models_model_id" ON "public"."models" USING btree ( + "model_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE UNIQUE INDEX "idx_models_model_id_unique" ON "public"."models" USING btree ( + "model_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +) WHERE is_enabled = true; + +-- ---------------------------- +-- Primary Key structure for table models +-- ---------------------------- +ALTER TABLE "public"."models" ADD CONSTRAINT "models_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table role_templates +-- ---------------------------- +CREATE UNIQUE INDEX "idx_role_templates_builtin_key" ON "public"."role_templates" USING btree ( + "builtin_key" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +) WHERE builtin_key::text <> ''::text; +CREATE INDEX "idx_role_templates_is_enabled" ON "public"."role_templates" USING btree ( + "is_enabled" "pg_catalog"."bool_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table role_templates +-- ---------------------------- +ALTER TABLE "public"."role_templates" ADD CONSTRAINT "role_templates_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table storage_quotas +-- ---------------------------- +CREATE UNIQUE INDEX "storage_quotas_user_unique" ON "public"."storage_quotas" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table storage_quotas +-- ---------------------------- +ALTER TABLE "public"."storage_quotas" ADD CONSTRAINT "storage_quotas_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table sync_items +-- ---------------------------- +CREATE INDEX "idx_sync_items_source_parent" ON "public"."sync_items" USING btree ( + "sync_source_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "parent_external_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_sync_items_user_kb" ON "public"."sync_items" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "knowledge_base_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Uniques structure for table sync_items +-- ---------------------------- +ALTER TABLE "public"."sync_items" ADD CONSTRAINT "sync_items_user_source_external_unique" UNIQUE ("user_id", "sync_source_id", "external_id"); + +-- ---------------------------- +-- Primary Key structure for table sync_items +-- ---------------------------- +ALTER TABLE "public"."sync_items" ADD CONSTRAINT "sync_items_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table sync_jobs +-- ---------------------------- +CREATE INDEX "idx_sync_jobs_source_id" ON "public"."sync_jobs" USING btree ( + "sync_source_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); +CREATE INDEX "idx_sync_jobs_user_kb" ON "public"."sync_jobs" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "knowledge_base_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table sync_jobs +-- ---------------------------- +ALTER TABLE "public"."sync_jobs" ADD CONSTRAINT "sync_jobs_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table sync_sources +-- ---------------------------- +CREATE INDEX "idx_sync_sources_user_kb" ON "public"."sync_sources" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "knowledge_base_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table sync_sources +-- ---------------------------- +ALTER TABLE "public"."sync_sources" ADD CONSTRAINT "sync_sources_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table tool_providers +-- ---------------------------- +CREATE INDEX "idx_tool_providers_is_enabled" ON "public"."tool_providers" USING btree ( + "is_enabled" "pg_catalog"."bool_ops" ASC NULLS LAST +); +CREATE INDEX "idx_tool_providers_tool_type_id" ON "public"."tool_providers" USING btree ( + "tool_type_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table tool_providers +-- ---------------------------- +ALTER TABLE "public"."tool_providers" ADD CONSTRAINT "tool_providers_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table tool_types +-- ---------------------------- +CREATE INDEX "idx_tool_types_is_enabled" ON "public"."tool_types" USING btree ( + "is_enabled" "pg_catalog"."bool_ops" ASC NULLS LAST +); +CREATE UNIQUE INDEX "idx_tool_types_tool_key" ON "public"."tool_types" USING btree ( + "tool_key" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table tool_types +-- ---------------------------- +ALTER TABLE "public"."tool_types" ADD CONSTRAINT "tool_types_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table user_memories +-- ---------------------------- +CREATE INDEX "idx_user_memories_user_active" ON "public"."user_memories" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST, + "is_active" "pg_catalog"."bool_ops" ASC NULLS LAST, + "updated_at" "pg_catalog"."timestamptz_ops" DESC NULLS FIRST +); + +-- ---------------------------- +-- Uniques structure for table user_memories +-- ---------------------------- +ALTER TABLE "public"."user_memories" ADD CONSTRAINT "user_memories_user_content_unique" UNIQUE ("user_id", "content"); + +-- ---------------------------- +-- Primary Key structure for table user_memories +-- ---------------------------- +ALTER TABLE "public"."user_memories" ADD CONSTRAINT "user_memories_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table user_model_configs +-- ---------------------------- +CREATE INDEX "idx_user_model_configs_user_id" ON "public"."user_model_configs" USING btree ( + "user_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); +CREATE INDEX "idx_user_model_configs_user_id_model_id" ON "public"."user_model_configs" USING btree ( + "user_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST, + "model_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table user_model_configs +-- ---------------------------- +ALTER TABLE "public"."user_model_configs" ADD CONSTRAINT "user_model_configs_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table user_preferences +-- ---------------------------- +CREATE INDEX "idx_user_preferences_user_id" ON "public"."user_preferences" USING btree ( + "user_id" "pg_catalog"."uuid_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Uniques structure for table user_preferences +-- ---------------------------- +ALTER TABLE "public"."user_preferences" ADD CONSTRAINT "user_preferences_user_unique" UNIQUE ("user_id"); + +-- ---------------------------- +-- Primary Key structure for table user_preferences +-- ---------------------------- +ALTER TABLE "public"."user_preferences" ADD CONSTRAINT "user_preferences_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table user_tool_configs +-- ---------------------------- +CREATE INDEX "idx_user_tool_configs_user_id" ON "public"."user_tool_configs" USING btree ( + "user_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table user_tool_configs +-- ---------------------------- +ALTER TABLE "public"."user_tool_configs" ADD CONSTRAINT "user_tool_configs_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Indexes structure for table users +-- ---------------------------- +CREATE INDEX "idx_users_role_template_id" ON "public"."users" USING btree ( + "role_template_id" COLLATE "pg_catalog"."default" "pg_catalog"."text_ops" ASC NULLS LAST +); + +-- ---------------------------- +-- Primary Key structure for table users +-- ---------------------------- +ALTER TABLE "public"."users" ADD CONSTRAINT "users_pkey" PRIMARY KEY ("id"); + +-- ---------------------------- +-- Foreign Keys structure for table tool_providers +-- ---------------------------- +ALTER TABLE "public"."tool_providers" ADD CONSTRAINT "fk_tool_providers_tool_type" FOREIGN KEY ("tool_type_id") REFERENCES "public"."tool_types" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- ---------------------------- +-- Foreign Keys structure for table user_tool_configs +-- ---------------------------- +ALTER TABLE "public"."user_tool_configs" ADD CONSTRAINT "fk_user_tool_configs_tool_provider" FOREIGN KEY ("provider_id") REFERENCES "public"."tool_providers" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE "public"."user_tool_configs" ADD CONSTRAINT "fk_user_tool_configs_tool_type" FOREIGN KEY ("tool_type_id") REFERENCES "public"."tool_types" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +COMMIT;