From d2d09c92cbac4c3630a19c7105a9c7f4f3dba73a Mon Sep 17 00:00:00 2001 From: Chenyme <118253778+chenyme@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:32:26 +0800 Subject: [PATCH] feat: preserve historical images in multi turn conversations --- .../application/conversation/service.go | 3 + .../conversation/service_branch.go | 27 ++- .../conversation/service_branch_test.go | 13 ++ .../conversation/service_file_context.go | 63 +++++- .../conversation/service_file_context_test.go | 196 +++++++++++++++++ .../conversation/service_file_pipeline.go | 2 +- .../service_image_context_cache.go | 105 +++++++++ .../application/conversation/service_media.go | 21 +- .../conversation/service_message_context.go | 199 +++++++++++++++++- .../conversation/service_message_send.go | 11 +- .../service_prompt_fingerprint.go | 16 +- .../service_stateful_responses_test.go | 29 +++ .../http/conversation/handler_message_send.go | 4 + 13 files changed, 661 insertions(+), 28 deletions(-) create mode 100644 backend/internal/application/conversation/service_image_context_cache.go diff --git a/backend/internal/application/conversation/service.go b/backend/internal/application/conversation/service.go index 22583b85..c6c3c4f9 100644 --- a/backend/internal/application/conversation/service.go +++ b/backend/internal/application/conversation/service.go @@ -99,6 +99,7 @@ type Service struct { snapshotCache sync.Map // conversationID (uint) → *cachedSnapshot userMemCache sync.Map // userID (uint) → *cachedUserMemories userSettingCache sync.Map // "userID:key" (string) → *cachedUserSetting + imageContextCache *preparedConversationImageCache } func (s *Service) llmAttribution() (string, string) { @@ -133,6 +134,7 @@ type AttachmentInput struct { RagOptOut bool // 用户是否关闭该文件的 RAG；RAG 段直接复用，无需重查 DB ChunkCount int // 向量分块数；RAG 缓存 key 需要 Current bool // 是否为本轮用户显式上传的附件 + MessageRole string ContextMode string } @@ -253,6 +255,7 @@ func NewServiceWithRuntime( storeProvider: appstorage.NewRuntimeProvider(cfg, nil), logger: logger, generationStreams: newGenerationStreamRegistry(cache, defaultGenerationStreamOptions()), + imageContextCache: defaultPreparedConversationImageCache(), } if extractSvc == nil { extractSvc = extraction.NewServiceWithRuntime(cfg) diff --git a/backend/internal/application/conversation/service_branch.go b/backend/internal/application/conversation/service_branch.go index 9fe74ae3..05125e05 100644 --- a/backend/internal/application/conversation/service_branch.go +++ b/backend/internal/application/conversation/service_branch.go @@ -11,6 +11,8 @@ import ( "go.uber.org/zap" ) +const estimatedConversationImageTokens int64 = 1024 + type messageBranchState struct { ExistingMessages []model.Message ParentMessageID *uint @@ -311,8 +313,9 @@ func truncateContextByTokenBudget(messages []model.Message, budgetTokens int, in } total := 0 cutFrom := len(messages) + imageTokenReserve := conversationImageTokenReserveByMessage(messages) for i := len(messages) - 1; i >= 0; i-- { - msgTokens := int(estimateDomainMessageTokens(messages[i], includeReasoningContent)) + msgTokens := int(estimateDomainMessageTokens(messages[i], includeReasoningContent) + imageTokenReserve[i]) if total+msgTokens > budgetTokens && cutFrom < len(messages) { break } @@ -322,6 +325,28 @@ func truncateContextByTokenBudget(messages []model.Message, budgetTokens int, in return messages[cutFrom:] } +func conversationImageTokenReserveByMessage(messages []model.Message) map[int]int64 { + reserve := make(map[int]int64) + remaining := maxConversationImageContextCount + for messageIndex := len(messages) - 1; messageIndex >= 0 && remaining > 0; messageIndex-- { + message := messages[messageIndex] + if !strings.EqualFold(strings.TrimSpace(message.Role), "user") { + continue + } + refs := parseAttachmentSnapshotRefs(message.Attachments) + for attachmentIndex := len(refs) - 1; attachmentIndex >= 0 && remaining > 0; attachmentIndex-- { + ref := refs[attachmentIndex] + mimeType := firstNonEmptyString(ref.DetectedMIME, ref.MimeType) + if normalizeAttachmentKind(ref.Kind, mimeType) != "image" { + continue + } + reserve[messageIndex] += estimatedConversationImageTokens + remaining-- + } + } + return reserve +} + func estimateDomainMessageTokens(message model.Message, includeReasoningContent bool) int64 { tokens := estimateTokens(message.Content) if includeReasoningContent && message.Role == "assistant" { diff --git a/backend/internal/application/conversation/service_branch_test.go b/backend/internal/application/conversation/service_branch_test.go index 8ef4847c..660605a4 100644 --- a/backend/internal/application/conversation/service_branch_test.go +++ b/backend/internal/application/conversation/service_branch_test.go @@ -296,6 +296,19 @@ func TestTruncateContextByTokenBudgetCountsAssistantReasoningWhenEnabled(t *test } } +func TestTruncateContextByTokenBudgetReservesHistoricalImageTokens(t *testing.T) { + messages := []model.Message{ + {ID: 1, Role: "user", Content: "first", Attachments: `[{"file_id":"image_1","kind":"image","mime_type":"image/png"}]`}, + {ID: 2, Role: "assistant", Content: "ok"}, + {ID: 3, Role: "user", Content: "next"}, + } + + got := truncateContextByTokenBudget(messages, 100, false) + if len(got) != 2 || got[0].ID != 2 || got[1].ID != 3 { + t.Fatalf("expected image token reserve to trim the oldest image turn, got %#v", got) + } +} + func TestBuildBranchMessagePathReusesExistingUserForAssistantRetry(t *testing.T) { rootID := uint(1) userID := uint(2) diff --git a/backend/internal/application/conversation/service_file_context.go b/backend/internal/application/conversation/service_file_context.go index 878bea82..de6fccd2 100644 --- a/backend/internal/application/conversation/service_file_context.go +++ b/backend/internal/application/conversation/service_file_context.go @@ -19,7 +19,10 @@ const ( ) type attachmentSnapshotRef struct { - FileID string `json:"file_id"` + FileID string `json:"file_id"` + Kind string `json:"kind"` + MimeType string `json:"mime_type"` + DetectedMIME string `json:"detected_mime"` } type conversationFileContextPlan struct { @@ -59,6 +62,17 @@ func collectConversationFileIDs(messages []model.Message, currentFileIDs []strin } func parseAttachmentSnapshotFileIDs(raw string) []string { + items := parseAttachmentSnapshotRefs(raw) + result := make([]string, 0, len(items)) + for _, item := range items { + if fileID := strings.TrimSpace(item.FileID); fileID != "" { + result = append(result, fileID) + } + } + return result +} + +func parseAttachmentSnapshotRefs(raw string) []attachmentSnapshotRef { payload := strings.TrimSpace(raw) if payload == "" || payload == "[]" { return nil @@ -67,13 +81,7 @@ func parseAttachmentSnapshotFileIDs(raw string) []string { if err := json.Unmarshal([]byte(payload), &items); err != nil { return nil } - result := make([]string, 0, len(items)) - for _, item := range items { - if fileID := strings.TrimSpace(item.FileID); fileID != "" { - result = append(result, fileID) - } - } - return result + return items } func filterCurrentAttachments(items []AttachmentInput) []AttachmentInput { @@ -86,6 +94,29 @@ func filterCurrentAttachments(items []AttachmentInput) []AttachmentInput { return result } +func bindAttachmentMessageRoles(items []AttachmentInput, messages []model.Message) []AttachmentInput { + if len(items) == 0 || len(messages) == 0 { + return items + } + roles := make(map[string]string) + for _, message := range messages { + role := strings.ToLower(strings.TrimSpace(message.Role)) + if role != "user" && role != "assistant" { + continue + } + for _, fileID := range parseAttachmentSnapshotFileIDs(message.Attachments) { + if role == "user" || roles[fileID] == "" { + roles[fileID] = role + } + } + } + result := append([]AttachmentInput(nil), items...) + for index := range result { + result[index].MessageRole = roles[strings.TrimSpace(result[index].FileID)] + } + return result +} + func filterAttachmentsByContextMode(items []AttachmentInput, contextMode string) []AttachmentInput { result := make([]AttachmentInput, 0) for _, item := range items { @@ -105,6 +136,9 @@ func isStableTextAttachment(item AttachmentInput) bool { func shouldShowAttachmentProcessTrace(items []AttachmentInput) bool { for _, item := range items { + if strings.EqualFold(strings.TrimSpace(item.ContextMode), fileContextModeDirectImage) && !item.Current { + continue + } if item.Current { return true } @@ -115,6 +149,17 @@ func shouldShowAttachmentProcessTrace(items []AttachmentInput) bool { return false } +func attachmentProcessTraceItems(items []AttachmentInput) []AttachmentInput { + result := make([]AttachmentInput, 0, len(items)) + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(item.ContextMode), fileContextModeDirectImage) && !item.Current { + continue + } + result = append(result, item) + } + return result +} + func buildConversationFileContextPlan( attachments []AttachmentInput, fileMode string, @@ -128,7 +173,7 @@ func buildConversationFileContextPlan( } for _, item := range attachments { kind := normalizeAttachmentKind(item.Kind, item.DetectedMIME) - if kind == "image" && item.Current { + if kind == "image" && (item.Current || strings.EqualFold(strings.TrimSpace(item.MessageRole), "user")) { item.ContextMode = fileContextModeDirectImage plan.Attachments = append(plan.Attachments, item) plan.FullAttachments = append(plan.FullAttachments, item) diff --git a/backend/internal/application/conversation/service_file_context_test.go b/backend/internal/application/conversation/service_file_context_test.go index 39965b63..d1d5b80b 100644 --- a/backend/internal/application/conversation/service_file_context_test.go +++ b/backend/internal/application/conversation/service_file_context_test.go @@ -1,15 +1,36 @@ package conversation import ( + "bytes" + "context" + "errors" + "fmt" + "image" + "image/color" + "image/png" "strings" "testing" + appstorage "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/objectstorage" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" domainmemory "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/memory" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/llm" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/objectstore" ) +type conversationTestStoreProvider struct { + store objectstore.Store + opens int +} + +func (p *conversationTestStoreProvider) Open(context.Context) (objectstore.Store, error) { + p.opens++ + return p.store, nil +} + +var _ appstorage.Provider = (*conversationTestStoreProvider)(nil) + func TestCollectConversationFileIDsIgnoresFailedHistoricalMessages(t *testing.T) { messages := []model.Message{ { @@ -34,6 +55,130 @@ func TestCollectConversationFileIDsIgnoresFailedHistoricalMessages(t *testing.T) } } +func TestBindAttachmentMessageRolesPrefersUserOwnership(t *testing.T) { + items := []AttachmentInput{{FileID: "shared"}, {FileID: "assistant_only"}} + messages := []model.Message{ + {Role: "assistant", Attachments: `[{"file_id":"shared"},{"file_id":"assistant_only"}]`}, + {Role: "user", Attachments: `[{"file_id":"shared"}]`}, + } + + got := bindAttachmentMessageRoles(items, messages) + if got[0].MessageRole != "user" || got[1].MessageRole != "assistant" { + t.Fatalf("expected user role to win for shared attachment, got %#v", got) + } +} + +func TestConversationImageRefsPreferRecentImagesWithinBudget(t *testing.T) { + messages := make([]model.Message, 0, maxConversationImageContextCount+1) + attachments := make([]AttachmentInput, 0, maxConversationImageContextCount+1) + for index := 0; index <= maxConversationImageContextCount; index++ { + fileID := fmt.Sprintf("image_%d", index) + messages = append(messages, model.Message{Role: "user", Attachments: fmt.Sprintf(`[{"file_id":%q,"kind":"image","mime_type":"image/png"}]`, fileID)}) + attachments = append(attachments, AttachmentInput{FileID: fileID, Kind: "image", MimeType: "image/png", ContextMode: fileContextModeDirectImage}) + } + + refs := conversationImageRefs(messages, attachments, maxConversationImageContextCount) + if len(refs) != maxConversationImageContextCount { + t.Fatalf("expected %d recent images, got %#v", maxConversationImageContextCount, refs) + } + if refs[0].fileID != "image_1" || refs[len(refs)-1].fileID != "image_10" { + t.Fatalf("expected oldest image to be trimmed, got %#v", refs) + } +} + +func TestInjectConversationImageContextKeepsOwnershipAndUsesCache(t *testing.T) { + store := objectstore.NewLocal(t.TempDir()) + for key, data := range map[string][]byte{ + "images/one": []byte("image-one"), + "images/two": []byte("image-two"), + } { + if _, err := store.Put(t.Context(), key, bytes.NewReader(data), objectstore.PutOptions{ContentType: "image/png"}); err != nil { + t.Fatalf("put test image %s: %v", key, err) + } + } + + domainMessages := []model.Message{ + {Role: "user", Content: "描述第一张图片", Attachments: `[{"file_id":"image_1","kind":"image","mime_type":"image/png"}]`}, + {Role: "assistant", Content: "第一张是雪山"}, + {Role: "user", Content: "第二张和第一张有什么不同", Attachments: `[{"file_id":"image_2","kind":"image","mime_type":"image/png"}]`}, + {Role: "assistant", Content: "第二张是河谷"}, + {Role: "user", Content: "第一张图片里有没有云朵"}, + } + attachments := []AttachmentInput{ + {FileID: "image_1", Kind: "image", MimeType: "image/png", StoragePath: "images/one", ContextMode: fileContextModeDirectImage}, + {FileID: "image_2", Kind: "image", MimeType: "image/png", StoragePath: "images/two", ContextMode: fileContextModeDirectImage}, + } + provider := &conversationTestStoreProvider{store: store} + service := &Service{storeProvider: provider, imageContextCache: defaultPreparedConversationImageCache()} + history := historyMessagesFromDomain(domainMessages, historyMessageOptions{}) + + got, err := service.injectConversationImageContext(t.Context(), history, domainMessages, attachments, config.Config{ImageMaxDimension: 1024}) + if err != nil { + t.Fatalf("inject historical images: %v", err) + } + if len(got[0].Parts) != 2 || string(got[0].Parts[1].Data) != "image-one" { + t.Fatalf("expected first image on first user message, got %#v", got[0]) + } + if len(got[2].Parts) != 2 || string(got[2].Parts[1].Data) != "image-two" { + t.Fatalf("expected second image on second user message, got %#v", got[2]) + } + if _, err = service.injectConversationImageContext(t.Context(), history, domainMessages, attachments, config.Config{ImageMaxDimension: 1024}); err != nil { + t.Fatalf("inject cached historical images: %v", err) + } + if provider.opens != 1 { + t.Fatalf("expected prepared image cache to avoid repeated storage opens, got %d", provider.opens) + } +} + +func TestInjectConversationImageContextRejectsMissingAndOversizedContext(t *testing.T) { + domainMessages := []model.Message{{Role: "user", Attachments: `[{"file_id":"missing","kind":"image","mime_type":"image/png"}]`}} + service := &Service{imageContextCache: defaultPreparedConversationImageCache()} + _, err := service.injectConversationImageContext(t.Context(), historyMessagesFromDomain(domainMessages, historyMessageOptions{}), domainMessages, nil, config.Config{}) + if !errors.Is(err, ErrInvalidFileReference) { + t.Fatalf("expected missing historical image to fail explicitly, got %v", err) + } + + largeData := make([]byte, 11*1024*1024) + cache := defaultPreparedConversationImageCache() + attachments := []AttachmentInput{ + {FileID: "one", Kind: "image", MimeType: "image/png", StoragePath: "one", ContextMode: fileContextModeDirectImage}, + {FileID: "two", Kind: "image", MimeType: "image/png", StoragePath: "two", ContextMode: fileContextModeDirectImage}, + } + for _, att := range attachments { + cache.put(preparedConversationImageCacheKey(att, 1024, "image/png"), preparedConversationImage{data: largeData, mimeType: "image/png"}) + } + service = &Service{imageContextCache: cache} + domainMessages = []model.Message{ + {Role: "user", Attachments: `[{"file_id":"one","kind":"image","mime_type":"image/png"}]`}, + {Role: "assistant"}, + {Role: "user", Attachments: `[{"file_id":"two","kind":"image","mime_type":"image/png"}]`}, + } + _, err = service.injectConversationImageContext(t.Context(), historyMessagesFromDomain(domainMessages, historyMessageOptions{}), domainMessages, attachments, config.Config{ImageMaxDimension: 1024}) + if !errors.Is(err, ErrFileTooLarge) { + t.Fatalf("expected aggregate image context budget failure, got %v", err) + } +} + +func TestResizeImageIfNeededReturnsActualMIME(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 4, 4)) + for y := 0; y < 4; y++ { + for x := 0; x < 4; x++ { + img.Set(x, y, color.RGBA{R: 120, G: 80, B: 40, A: 255}) + } + } + var source bytes.Buffer + if err := png.Encode(&source, img); err != nil { + t.Fatalf("encode source image: %v", err) + } + resized, mimeType := resizeImageIfNeeded(source.Bytes(), "image/webp", 2) + if mimeType != "image/jpeg" { + t.Fatalf("expected resized non-PNG image to report JPEG, got %q", mimeType) + } + if _, format, err := image.Decode(bytes.NewReader(resized)); err != nil || format != "jpeg" { + t.Fatalf("expected JPEG bytes, format=%q err=%v", format, err) + } +} + func TestInjectUserContextUsesCompactXMLForRAG(t *testing.T) { messages := []llm.Message{{Role: "user", Content: "怎么发布？"}} chunks := []model.RAGChunk{{ @@ -53,6 +198,26 @@ func TestInjectUserContextUsesCompactXMLForRAG(t *testing.T) { } } +func TestInjectUserContextPreservesExistingImageParts(t *testing.T) { + messages := []llm.Message{{ + Role: "user", + Parts: []llm.ContentPart{ + {Kind: llm.ContentPartText, Text: "继续分析"}, + {Kind: llm.ContentPartImage, MimeType: "image/png", Data: []byte("image")}, + }, + }} + got := injectUserContext(t.Context(), messages, userContextInput{RAGChunks: []model.RAGChunk{{FileName: "note.md", Content: "偏好简洁回答"}}}, config.Config{}, nil) + if len(got) != 1 || len(got[0].Parts) != 2 { + t.Fatalf("expected text and existing image parts, got %#v", got) + } + if got[0].Parts[1].Kind != llm.ContentPartImage || string(got[0].Parts[1].Data) != "image" { + t.Fatalf("expected existing image part to be preserved, got %#v", got[0].Parts) + } + if !strings.Contains(got[0].Parts[0].Text, "继续分析") || !strings.Contains(got[0].Parts[0].Text, "偏好简洁回答") { + t.Fatalf("expected dynamic context and original text, got %q", got[0].Parts[0].Text) + } +} + func TestPrependStableFileContextKeepsFilesAtPromptTop(t *testing.T) { messages := []llm.Message{ {Role: "user", Content: "第一轮问题"}, @@ -206,6 +371,23 @@ func TestBuildConversationFileContextPlanOnlyDirectUploadsCurrentImages(t *testi } } +func TestBuildConversationFileContextPlanDirectUploadsHistoricalUserImages(t *testing.T) { + plan := buildConversationFileContextPlan([]AttachmentInput{{ + FileID: "file_history_user_image", + Kind: "image", + MimeType: "image/png", + DetectedMIME: "image/png", + MessageRole: "user", + }}, "rag", config.Config{RAGEnabled: true, EmbeddingEnabled: true}, "gpt-5.5", "", true) + + if len(plan.FullAttachments) != 1 || plan.FullAttachments[0].ContextMode != fileContextModeDirectImage { + t.Fatalf("expected historical user image to remain direct image context, got %#v", plan) + } + if len(plan.RAGAttachments) != 0 { + t.Fatalf("expected historical user image not to fall back to OCR/RAG, got %#v", plan.RAGAttachments) + } +} + func TestBuildConversationFileContextPlanUsesRAGForHistoricalImageOCRWhenRequested(t *testing.T) { cfg := config.Config{RAGEnabled: true, EmbeddingEnabled: true} plan := buildConversationFileContextPlan([]AttachmentInput{{ @@ -293,6 +475,20 @@ func TestShouldShowAttachmentProcessTraceSkipsHistoricalSkippedOnly(t *testing.T } } +func TestShouldShowAttachmentProcessTraceSkipsHistoricalDirectImages(t *testing.T) { + items := []AttachmentInput{{ + FileID: "file_history_image", + Kind: "image", + ContextMode: fileContextModeDirectImage, + }} + if shouldShowAttachmentProcessTrace(items) { + t.Fatal("expected historical direct images to stay out of repeated process traces") + } + if got := attachmentProcessTraceItems(items); len(got) != 0 { + t.Fatalf("expected historical direct images to be filtered from trace payload, got %#v", got) + } +} + func TestShouldShowAttachmentProcessTraceKeepsCurrentOrIncludedFiles(t *testing.T) { if !shouldShowAttachmentProcessTrace([]AttachmentInput{{ FileID: "file_current_image", diff --git a/backend/internal/application/conversation/service_file_pipeline.go b/backend/internal/application/conversation/service_file_pipeline.go index a4c26b26..df156e7b 100644 --- a/backend/internal/application/conversation/service_file_pipeline.go +++ b/backend/internal/application/conversation/service_file_pipeline.go @@ -278,7 +278,7 @@ func (s *Service) hydrateAttachmentsForSend( } for i, att := range attachments { if strings.TrimSpace(att.FileID) == "" || - (att.FileCategory == fileCategoryImage && (att.Current || !cfg.ExtractImageOCREnabled)) { + (att.FileCategory == fileCategoryImage && (att.Current || strings.EqualFold(strings.TrimSpace(att.MessageRole), "user") || !cfg.ExtractImageOCREnabled)) { continue } i, att := i, att // 闭包捕获 diff --git a/backend/internal/application/conversation/service_image_context_cache.go b/backend/internal/application/conversation/service_image_context_cache.go new file mode 100644 index 00000000..3d5ba87a --- /dev/null +++ b/backend/internal/application/conversation/service_image_context_cache.go @@ -0,0 +1,105 @@ +package conversation + +import ( + "container/list" + "fmt" + "strings" + "sync" +) + +const ( + maxPreparedConversationImageCacheEntries = 64 + maxPreparedConversationImageCacheBytes = 64 * 1024 * 1024 +) + +type preparedConversationImage struct { + data []byte + mimeType string +} + +type preparedConversationImageCacheEntry struct { + key string + image preparedConversationImage +} + +type preparedConversationImageCache struct { + mu sync.Mutex + entries map[string]*list.Element + recency *list.List + totalBytes int + maxEntries int + maxBytes int +} + +func newPreparedConversationImageCache(maxEntries int, maxBytes int) *preparedConversationImageCache { + return &preparedConversationImageCache{ + entries: make(map[string]*list.Element), + recency: list.New(), + maxEntries: maxEntries, + maxBytes: maxBytes, + } +} + +func defaultPreparedConversationImageCache() *preparedConversationImageCache { + return newPreparedConversationImageCache( + maxPreparedConversationImageCacheEntries, + maxPreparedConversationImageCacheBytes, + ) +} + +func preparedConversationImageCacheKey(att AttachmentInput, maxDim int, mimeType string) string { + identity := strings.TrimSpace(att.SHA256) + if identity == "" { + identity = strings.TrimSpace(att.FileID) + } + if identity == "" { + return "" + } + return fmt.Sprintf("%s:%d:%s", identity, maxDim, resolveImageMimeType(mimeType)) +} + +func (c *preparedConversationImageCache) get(key string) (preparedConversationImage, bool) { + if c == nil || strings.TrimSpace(key) == "" { + return preparedConversationImage{}, false + } + c.mu.Lock() + defer c.mu.Unlock() + element, ok := c.entries[key] + if !ok { + return preparedConversationImage{}, false + } + c.recency.MoveToFront(element) + return element.Value.(preparedConversationImageCacheEntry).image, true +} + +func (c *preparedConversationImageCache) put(key string, image preparedConversationImage) { + if c == nil || strings.TrimSpace(key) == "" || len(image.data) == 0 || c.maxEntries <= 0 || c.maxBytes <= 0 || len(image.data) > c.maxBytes { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + if existing, ok := c.entries[key]; ok { + entry := existing.Value.(preparedConversationImageCacheEntry) + c.totalBytes -= len(entry.image.data) + entry.image = image + existing.Value = entry + c.totalBytes += len(image.data) + c.recency.MoveToFront(existing) + } else { + element := c.recency.PushFront(preparedConversationImageCacheEntry{key: key, image: image}) + c.entries[key] = element + c.totalBytes += len(image.data) + } + + for len(c.entries) > c.maxEntries || c.totalBytes > c.maxBytes { + oldest := c.recency.Back() + if oldest == nil { + break + } + entry := oldest.Value.(preparedConversationImageCacheEntry) + delete(c.entries, entry.key) + c.totalBytes -= len(entry.image.data) + c.recency.Remove(oldest) + } +} diff --git a/backend/internal/application/conversation/service_media.go b/backend/internal/application/conversation/service_media.go index f78cb014..c7a3ea52 100644 --- a/backend/internal/application/conversation/service_media.go +++ b/backend/internal/application/conversation/service_media.go @@ -11,29 +11,30 @@ import ( "path/filepath" "strings" - _ "image/gif" // 注册 GIF 解码器。 _ "golang.org/x/image/webp" + _ "image/gif" // 注册 GIF 解码器。 ) const maxMediaImageEditInputPixels = 64 * 1024 * 1024 // resizeImageIfNeeded 在图片尺寸超过 maxDim 时进行缩放并重新编码。 -// 若解码/编码失败则返回原始字节，不报错，保证降级可用。 +// 返回的 MIME 始终与实际字节编码一致；失败时保留原始数据和 MIME。 // 使用最近邻插值以降低 CPU 开销，缩略图语义信息仍足够供 LLM 识别。 -func resizeImageIfNeeded(data []byte, mimeType string, maxDim int) []byte { +func resizeImageIfNeeded(data []byte, mimeType string, maxDim int) ([]byte, string) { + mime := resolveImageMimeType(mimeType) if maxDim <= 0 || len(data) == 0 { - return data + return data, mime } src, _, err := image.Decode(bytes.NewReader(data)) if err != nil { - return data // 无法解码时返回原始数据，由上游模型按原图处理。 + return data, mime // 无法解码时返回原始数据，由上游模型按原图处理。 } bounds := src.Bounds() w, h := bounds.Dx(), bounds.Dy() if w <= maxDim && h <= maxDim { - return data + return data, mime } var scale float64 @@ -68,18 +69,18 @@ func resizeImageIfNeeded(data []byte, mimeType string, maxDim int) []byte { } var buf bytes.Buffer - mime := strings.ToLower(strings.TrimSpace(mimeType)) switch { case strings.Contains(mime, "png"): if encErr := png.Encode(&buf, dst); encErr != nil { - return data + return data, mime } + return buf.Bytes(), "image/png" default: // jpeg 及其他格式统一使用 JPEG 输出 if encErr := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); encErr != nil { - return data + return data, mime } + return buf.Bytes(), "image/jpeg" } - return buf.Bytes() } // resolveImageMimeType 规范化图片 MIME 类型，未知时默认为 image/jpeg。 diff --git a/backend/internal/application/conversation/service_message_context.go b/backend/internal/application/conversation/service_message_context.go index 9b254da0..d1685597 100644 --- a/backend/internal/application/conversation/service_message_context.go +++ b/backend/internal/application/conversation/service_message_context.go @@ -15,11 +15,18 @@ import ( domainmemory "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/memory" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/llm" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/objectstore" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/conv" ) const MessageErrorCodeMediaImageStreamUnsupported = "media.image_stream_unsupported" +const ( + maxConversationImageContextCount = 10 + maxConversationImageContextBytes = 20 * 1024 * 1024 + maxConversationImageSourceBytes = 50 * 1024 * 1024 +) + func normalizePublicID(raw string) string { return conv.NormalizePublicID(raw) } @@ -736,6 +743,168 @@ func stableAttachmentSortKey(att AttachmentInput) string { return "3:" } +type conversationImageRef struct { + messageIndex int + fileID string +} + +func conversationImageRefs(messages []domainconversation.Message, attachments []AttachmentInput, limit int) []conversationImageRef { + if len(messages) == 0 || limit <= 0 { + return nil + } + available := make(map[string]AttachmentInput, len(attachments)) + current := make(map[string]struct{}) + for _, att := range attachments { + fileID := strings.TrimSpace(att.FileID) + if fileID == "" { + continue + } + if att.Current { + current[fileID] = struct{}{} + continue + } + if strings.EqualFold(strings.TrimSpace(att.ContextMode), fileContextModeDirectImage) && + normalizeAttachmentKind(att.Kind, firstNonEmptyString(att.DetectedMIME, att.MimeType)) == "image" { + available[fileID] = att + } + } + + refs := make([]conversationImageRef, 0) + historyIndex := 0 + for _, message := range messages { + if message.Role != "user" && message.Role != "assistant" && message.Role != "system" { + continue + } + if message.Role == "user" { + for _, snapshot := range parseAttachmentSnapshotRefs(message.Attachments) { + fileID := strings.TrimSpace(snapshot.FileID) + if fileID == "" { + continue + } + if _, isCurrent := current[fileID]; isCurrent { + continue + } + _, isAvailable := available[fileID] + snapshotMIME := firstNonEmptyString(snapshot.DetectedMIME, snapshot.MimeType) + if !isAvailable && normalizeAttachmentKind(snapshot.Kind, snapshotMIME) != "image" { + continue + } + refs = append(refs, conversationImageRef{messageIndex: historyIndex, fileID: fileID}) + } + } + historyIndex++ + } + if len(refs) > limit { + refs = refs[len(refs)-limit:] + } + return refs +} + +func (s *Service) injectConversationImageContext( + ctx context.Context, + messages []llm.Message, + domainMessages []domainconversation.Message, + attachments []AttachmentInput, + cfg config.Config, +) ([]llm.Message, error) { + refs := conversationImageRefs(domainMessages, attachments, maxConversationImageContextCount) + if len(refs) == 0 { + return messages, nil + } + + attachmentByFileID := make(map[string]AttachmentInput, len(attachments)) + for _, att := range attachments { + attachmentByFileID[strings.TrimSpace(att.FileID)] = att + } + maxDim := cfg.ImageMaxDimension + if maxDim <= 0 { + maxDim = 1024 + } + cache := s.imageContextCache + if cache == nil { + cache = defaultPreparedConversationImageCache() + } + storeProvider := s.storeProvider + if storeProvider == nil { + storeProvider = appstorage.NewRuntimeProvider(config.NewRuntime(cfg), nil) + } + + var store objectstore.Store + partsByRef := make(map[int]llm.ContentPart, len(refs)) + loadedByFileID := make(map[string]llm.ContentPart, len(refs)) + totalBytes := 0 + for index := len(refs) - 1; index >= 0; index-- { + ref := refs[index] + part, loaded := loadedByFileID[ref.fileID] + if !loaded { + att, ok := attachmentByFileID[ref.fileID] + if !ok || strings.TrimSpace(att.StoragePath) == "" { + return nil, fmt.Errorf("%w: historical image %s", ErrInvalidFileReference, ref.fileID) + } + mime := resolveImageMimeType(firstNonEmptyString(att.DetectedMIME, att.MimeType)) + cacheKey := preparedConversationImageCacheKey(att, maxDim, mime) + if cached, ok := cache.get(cacheKey); ok { + part = llm.ContentPart{Kind: llm.ContentPartImage, MimeType: cached.mimeType, Data: cached.data} + } else { + if store == nil { + openedStore, openErr := storeProvider.Open(ctx) + if openErr != nil { + return nil, fmt.Errorf("%w: open object storage: %v", ErrFileNotFound, openErr) + } + store = openedStore + } + reader, _, openErr := store.Open(ctx, strings.TrimSpace(att.StoragePath)) + if openErr != nil { + return nil, fmt.Errorf("%w: historical image %s: %v", ErrFileNotFound, ref.fileID, openErr) + } + data, readErr := io.ReadAll(io.LimitReader(reader, maxConversationImageSourceBytes+1)) + closeErr := reader.Close() + if readErr != nil { + return nil, fmt.Errorf("%w: read historical image %s: %v", ErrFileNotFound, ref.fileID, readErr) + } + if closeErr != nil { + return nil, fmt.Errorf("%w: close historical image %s: %v", ErrFileNotFound, ref.fileID, closeErr) + } + if len(data) == 0 { + return nil, fmt.Errorf("%w: historical image %s is empty", ErrInvalidFileReference, ref.fileID) + } + if len(data) > maxConversationImageSourceBytes { + return nil, fmt.Errorf("%w: historical image %s exceeds source limit", ErrFileTooLarge, ref.fileID) + } + resized, actualMIME := resizeImageIfNeeded(data, mime, maxDim) + part = llm.ContentPart{Kind: llm.ContentPartImage, MimeType: actualMIME, Data: resized} + cache.put(cacheKey, preparedConversationImage{data: resized, mimeType: actualMIME}) + } + loadedByFileID[ref.fileID] = part + } + if len(part.Data) == 0 { + return nil, fmt.Errorf("%w: historical image %s is empty", ErrInvalidFileReference, ref.fileID) + } + if totalBytes+len(part.Data) > maxConversationImageContextBytes { + return nil, fmt.Errorf("%w: historical image context exceeds %d bytes", ErrFileTooLarge, maxConversationImageContextBytes) + } + totalBytes += len(part.Data) + partsByRef[index] = part + } + + result := cloneLLMMessages(messages) + for index, ref := range refs { + part := partsByRef[index] + if ref.messageIndex < 0 || ref.messageIndex >= len(result) { + return nil, fmt.Errorf("%w: historical image message index", ErrInvalidFileReference) + } + message := result[ref.messageIndex] + message.Parts = append([]llm.ContentPart(nil), message.Parts...) + if len(message.Parts) == 0 && strings.TrimSpace(message.Content) != "" { + message.Parts = append(message.Parts, llm.ContentPart{Kind: llm.ContentPartText, Text: message.Content}) + message.Content = "" + } + message.Parts = append(message.Parts, part) + result[ref.messageIndex] = message + } + return result, nil +} + func imageAttachmentsForCurrentUser(attachments []AttachmentInput) []AttachmentInput { if len(attachments) == 0 { return nil @@ -783,7 +952,12 @@ func injectUserContext( } lastUserMsg := messages[lastUserIdx] - imageParts := make([]llm.ContentPart, 0, len(input.Attachments)) + imageParts := make([]llm.ContentPart, 0, len(lastUserMsg.Parts)+len(input.Attachments)) + for _, part := range lastUserMsg.Parts { + if part.Kind == llm.ContentPartImage && len(part.Data) > 0 { + imageParts = append(imageParts, part) + } + } contextXML := buildUserContextXML(input) for _, att := range input.Attachments { @@ -811,10 +985,10 @@ func injectUserContext( continue } mime := resolveImageMimeType(att.MimeType) - resized := resizeImageIfNeeded(imgData, mime, maxDim) + resized, actualMIME := resizeImageIfNeeded(imgData, mime, maxDim) imageParts = append(imageParts, llm.ContentPart{ Kind: llm.ContentPartImage, - MimeType: mime, + MimeType: actualMIME, Data: resized, }) } @@ -824,7 +998,7 @@ func injectUserContext( return messages } - content := strings.TrimSpace(lastUserMsg.Content) + content := strings.TrimSpace(userMessageText(lastUserMsg)) if !contextXML.empty() { content = buildUserContextPrompt(content, contextXML) } @@ -851,6 +1025,23 @@ func injectUserContext( return result } +func userMessageText(message llm.Message) string { + if strings.TrimSpace(message.Content) != "" || len(message.Parts) == 0 { + return message.Content + } + var builder strings.Builder + for _, part := range message.Parts { + if part.Kind != llm.ContentPartText && part.Kind != llm.ContentPartFile { + continue + } + if builder.Len() > 0 { + builder.WriteString("\n") + } + builder.WriteString(part.Text) + } + return builder.String() +} + func formatAttachmentFileContext(fileName string, text string) string { name := strings.TrimSpace(fileName) if name == "" { diff --git a/backend/internal/application/conversation/service_message_send.go b/backend/internal/application/conversation/service_message_send.go index 46668029..2cfd745c 100644 --- a/backend/internal/application/conversation/service_message_send.go +++ b/backend/internal/application/conversation/service_message_send.go @@ -488,6 +488,7 @@ func (s *Service) sendMessageInternal( retErr = err return nil, err } + conversationAttachments = bindAttachmentMessageRoles(conversationAttachments, promptMessages) conversationAttachments, err = s.hydrateAttachmentsForSend(ctx, input.UserID, conversationAttachments, input.OnEvent) if err != nil { retErr = err @@ -502,6 +503,11 @@ func (s *Service) sendMessageInternal( historyMsgs := historyMessagesFromDomain(promptMessages, historyMessageOptions{ ReasoningContentPassback: reasoningContentPassback, }) + historyMsgs, err = s.injectConversationImageContext(ctx, historyMsgs, promptMessages, fileContextPlan.FullAttachments, cfg) + if err != nil { + retErr = err + return nil, err + } if len(historyMsgs) == 0 { historyMsgs = append(historyMsgs, llm.Message{ Role: "user", @@ -545,8 +551,9 @@ func (s *Service) sendMessageInternal( } } llmMessages, _ := assembler.Assemble(historyMsgs) - if traceRecorder != nil && shouldShowAttachmentProcessTrace(fileContextPlan.Attachments) { - summary, markdown, payload := buildAttachmentProcessTrace(fileMode, fileContextPlan.Attachments) + processTraceAttachments := attachmentProcessTraceItems(fileContextPlan.Attachments) + if traceRecorder != nil && shouldShowAttachmentProcessTrace(processTraceAttachments) { + summary, markdown, payload := buildAttachmentProcessTrace(fileMode, processTraceAttachments) traceRecorder.appendProcessSection(summary, markdown, payload, messageTraceStatusStreaming) } diff --git a/backend/internal/application/conversation/service_prompt_fingerprint.go b/backend/internal/application/conversation/service_prompt_fingerprint.go index 4d451658..fc918f93 100644 --- a/backend/internal/application/conversation/service_prompt_fingerprint.go +++ b/backend/internal/application/conversation/service_prompt_fingerprint.go @@ -230,7 +230,21 @@ func buildNextStatefulPrefixMessages(messages []llm.Message, currentUserContent return appendAssistantStateMessage(messages, assistantText, reasoningContent) } result := cloneLLMMessages(messages[:lastUserIndex]) - result = append(result, llm.Message{Role: "user", Content: currentUserContent}) + currentUser := llm.Message{Role: "user", Content: currentUserContent} + imageParts := make([]llm.ContentPart, 0) + for _, part := range messages[lastUserIndex].Parts { + if part.Kind == llm.ContentPartImage && len(part.Data) > 0 { + imageParts = append(imageParts, part) + } + } + if len(imageParts) > 0 { + currentUser.Content = "" + if strings.TrimSpace(currentUserContent) != "" { + currentUser.Parts = append(currentUser.Parts, llm.ContentPart{Kind: llm.ContentPartText, Text: currentUserContent}) + } + currentUser.Parts = append(currentUser.Parts, imageParts...) + } + result = append(result, currentUser) result = append(result, llm.Message{Role: "assistant", Content: assistantText, ReasoningContent: reasoningContent}) return result } diff --git a/backend/internal/application/conversation/service_stateful_responses_test.go b/backend/internal/application/conversation/service_stateful_responses_test.go index 662659ce..02ceaed3 100644 --- a/backend/internal/application/conversation/service_stateful_responses_test.go +++ b/backend/internal/application/conversation/service_stateful_responses_test.go @@ -277,6 +277,35 @@ func TestBuildNextStatefulPrefixMessagesKeepsAssistantReasoning(t *testing.T) { } } +func TestBuildNextStatefulPrefixMessagesKeepsRebuildableUserImages(t *testing.T) { + imageData := []byte("image-data") + firstPrompt := []llm.Message{{ + Role: "user", + Parts: []llm.ContentPart{ + {Kind: llm.ContentPartText, Text: "第一轮"}, + {Kind: llm.ContentPartImage, MimeType: "image/png", Data: imageData}, + }, + }} + stored := buildPromptStateFingerprint(promptStateFingerprintInput{ + Messages: buildNextStatefulPrefixMessages(firstPrompt, "第一轮", "第一轮回答", ""), + }) + secondPrompt := []llm.Message{ + { + Role: "user", + Parts: []llm.ContentPart{ + {Kind: llm.ContentPartText, Text: "第一轮"}, + {Kind: llm.ContentPartImage, MimeType: "image/png", Data: imageData}, + }, + }, + {Role: "assistant", Content: "第一轮回答"}, + {Role: "user", Content: "继续"}, + } + prefix := buildPromptStateFingerprint(promptStateFingerprintInput{Messages: promptStatePrefixMessages(secondPrompt)}) + if stored != prefix { + t.Fatal("expected historical user image to preserve previous_response_id fingerprint") + } +} + func TestPromptStateFingerprintUsesRebuildableHistoryWhenCurrentUserHasDynamicContext(t *testing.T) { firstPrompt := []llm.Message{ {Role: "system", Content: "稳定文件"}, diff --git a/backend/internal/transport/http/conversation/handler_message_send.go b/backend/internal/transport/http/conversation/handler_message_send.go index f28ea13b..3f6f524f 100644 --- a/backend/internal/transport/http/conversation/handler_message_send.go +++ b/backend/internal/transport/http/conversation/handler_message_send.go @@ -334,6 +334,10 @@ func handleSendMessageError(c *gin.Context, err error) { response.Error(c, http.StatusNotFound, "conversation not found") case errors.Is(err, appconversation.ErrInvalidFileReference): response.Error(c, http.StatusBadRequest, "invalid file reference") + case errors.Is(err, appconversation.ErrFileNotFound): + response.Error(c, http.StatusNotFound, "file not found") + case errors.Is(err, appconversation.ErrFileTooLarge): + response.Error(c, http.StatusRequestEntityTooLarge, "file too large") case errors.Is(err, appconversation.ErrTooManyMessageFiles): response.Error(c, http.StatusBadRequest, "too many files in one message") case errors.Is(err, appconversation.ErrTooManySelectedTools):