From f420e47f952d8868e0964e348796bf239cb08e43 Mon Sep 17 00:00:00 2001 From: Loi Nguyen Date: Thu, 20 Aug 2026 09:15:15 +0700 Subject: [PATCH] fix(ollama): preserve multimodal images (#17663) ### Summary - Convert multimodal text blocks into Ollama message content. - Attach image references through Ollama native images arrays for synchronous and streaming chat requests. - Strip data-URI headers while preserving raw base64 and URL image values. - Add regression coverage for both request paths. Closes #17332 --------- Co-authored-by: Codex --- internal/entity/models/ollama.go | 90 ++++++++++++++++++++- internal/entity/models/ollama_test.go | 109 ++++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 2 deletions(-) diff --git a/internal/entity/models/ollama.go b/internal/entity/models/ollama.go index e82fa29bef..c01584221b 100644 --- a/internal/entity/models/ollama.go +++ b/internal/entity/models/ollama.go @@ -52,6 +52,92 @@ func (o *OllamaModel) Name() string { return "Ollama" } +func buildOllamaRequestBody(cfg *ChatConfig, modelName string, messages []Message, stream bool) map[string]any { + reqBody := buildRequestBody(cfg, modelName, messages, stream) + reqBody["messages"] = buildOllamaMessages(messages) + return reqBody +} + +func buildOllamaMessages(messages []Message) []map[string]any { + apiMessages := buildChatMessages(messages) + for i, message := range messages { + content, images, ok := ollamaMultimodalContent(message.Content) + if !ok { + continue + } + apiMessages[i]["content"] = content + if len(images) > 0 { + apiMessages[i]["images"] = images + } + } + return apiMessages +} + +func ollamaMultimodalContent(content interface{}) (string, []string, bool) { + var parts []interface{} + switch value := content.(type) { + case []interface{}: + parts = value + case []map[string]interface{}: + parts = make([]interface{}, len(value)) + for i := range value { + parts[i] = value[i] + } + default: + return "", nil, false + } + + var textParts []string + var images []string + for _, part := range parts { + partMap, ok := part.(map[string]interface{}) + if !ok { + if text, ok := part.(string); ok { + textParts = append(textParts, text) + } + continue + } + + partType, _ := partMap["type"].(string) + switch partType { + case "text", "input_text": + if text, ok := partMap["text"].(string); ok { + textParts = append(textParts, text) + } + case "image_url": + if imageURL := ollamaImageURL(partMap["image_url"]); imageURL != "" { + images = append(images, cleanOllamaImageData(imageURL)) + } + } + } + + return strings.Join(textParts, "\n"), images, true +} + +func ollamaImageURL(value interface{}) string { + switch image := value.(type) { + case string: + return image + case map[string]interface{}: + url, _ := image["url"].(string) + return url + case map[string]string: + return image["url"] + default: + return "" + } +} + +func cleanOllamaImageData(image string) string { + const base64Marker = ";base64," + if strings.HasPrefix(image, "data:") { + if marker := strings.Index(image, base64Marker); marker >= 0 { + return image[marker+len(base64Marker):] + } + } + return image +} + func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) { if len(messages) == 0 { return nil, fmt.Errorf("message is nil") @@ -70,7 +156,7 @@ func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, me } // Build request body - reqBody := buildRequestBody(chatModelConfig, modelName, messages, false) + reqBody := buildOllamaRequestBody(chatModelConfig, modelName, messages, false) if chatModelConfig != nil { if chatModelConfig.Effort != nil && *chatModelConfig.Effort != "" { @@ -108,7 +194,7 @@ func (o *OllamaModel) ChatStreamlyWithSender(ctx context.Context, modelName stri } // Build request body with streaming enabled - reqBody := buildRequestBody(modelConfig, modelName, messages, true) + reqBody := buildOllamaRequestBody(modelConfig, modelName, messages, true) if modelConfig.Effort != nil && *modelConfig.Effort != "" { if strings.HasPrefix(strings.ToLower(modelName), "gpt-oss") { diff --git a/internal/entity/models/ollama_test.go b/internal/entity/models/ollama_test.go index 786ad13f40..e625ed763a 100644 --- a/internal/entity/models/ollama_test.go +++ b/internal/entity/models/ollama_test.go @@ -17,9 +17,11 @@ package models import ( + "encoding/json" "io" "net/http" "net/http/httptest" + "ragflow/internal/common" "testing" ) @@ -27,6 +29,113 @@ func newOllamaForListModelsTest(baseURL string) *OllamaModel { return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Models: "api/tags"}) } +func newOllamaForChatTest(baseURL string) *OllamaModel { + return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "api/chat", AsyncChat: "api/chat"}) +} + +func ollamaMultimodalTestMessages() []Message { + return []Message{{Role: "user", Content: []interface{}{ + map[string]interface{}{"type": "text", "text": "describe"}, + map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/png;base64,aGVsbG8="}}, + map[string]interface{}{"type": "text", "text": "carefully"}, + map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "cmF3LWltYWdl"}}, + map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "https://example.com/cat.png"}}, + }}} +} + +func assertOllamaMultimodalRequest(t *testing.T, body map[string]interface{}, stream bool) { + t.Helper() + if body["stream"] != stream { + t.Errorf("stream=%v, want %v", body["stream"], stream) + } + messages, ok := body["messages"].([]interface{}) + if !ok || len(messages) != 1 { + t.Errorf("messages=%v, want one message", body["messages"]) + return + } + message, ok := messages[0].(map[string]interface{}) + if !ok { + t.Errorf("message=%v, want object", messages[0]) + return + } + if message["content"] != "describe\ncarefully" { + t.Errorf("content=%v, want joined text parts", message["content"]) + } + images, ok := message["images"].([]interface{}) + if !ok { + t.Errorf("images=%v, want array", message["images"]) + return + } + want := []string{"aGVsbG8=", "cmF3LWltYWdl", "https://example.com/cat.png"} + if len(images) != len(want) { + t.Errorf("images=%v, want %v", images, want) + return + } + for i, expected := range want { + if images[i] != expected { + t.Errorf("images[%d]=%v, want %q", i, images[i], expected) + } + } +} + +func TestOllamaChatMapsMultimodalImages(t *testing.T) { + withSSRFBypass(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode request: %v", err) + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + assertOllamaMultimodalRequest(t, body, false) + _, _ = io.WriteString(w, `{"message":{"content":"ok","thinking":""}}`) + })) + defer srv.Close() + + response, err := newOllamaForChatTest(srv.URL).ChatWithMessages( + t.Context(), + "llava", + ollamaMultimodalTestMessages(), + &APIConfig{}, + &ChatConfig{}, + &common.ModelUsage{}, + ) + if err != nil { + t.Fatalf("ChatWithMessages: %v", err) + } + if response.Answer == nil || *response.Answer != "ok" { + t.Fatalf("answer=%v, want ok", response.Answer) + } +} + +func TestOllamaStreamingChatMapsMultimodalImages(t *testing.T) { + withSSRFBypass(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode request: %v", err) + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + assertOllamaMultimodalRequest(t, body, true) + _, _ = io.WriteString(w, "{\"message\":{\"content\":\"ok\"},\"done\":false}\n{\"done\":true}\n") + })) + defer srv.Close() + + err := newOllamaForChatTest(srv.URL).ChatStreamlyWithSender( + t.Context(), + "llava", + ollamaMultimodalTestMessages(), + &APIConfig{}, + &ChatConfig{}, + &common.ModelUsage{}, + func(*string, *string) error { return nil }, + ) + if err != nil { + t.Fatalf("ChatStreamlyWithSender: %v", err) + } +} + func TestOllamaListModels(t *testing.T) { withSSRFBypass(t) ctx := t.Context()