fix(ollama): preserve multimodal images (#17663)

### Summary

- Convert multimodal text blocks into Ollama message content.
- Attach image references through Ollama native images arrays for
synchronous and streaming chat requests.
- Strip data-URI headers while preserving raw base64 and URL image
values.
- Add regression coverage for both request paths.

Closes #17332

---------

Co-authored-by: Codex <codex@openai.com>

internal/entity/models/ollama.go

@@ -52,6 +52,92 @@ func (o *OllamaModel) Name() string {
 	return "Ollama"
 }
 
+func buildOllamaRequestBody(cfg *ChatConfig, modelName string, messages []Message, stream bool) map[string]any {
+	reqBody := buildRequestBody(cfg, modelName, messages, stream)
+	reqBody["messages"] = buildOllamaMessages(messages)
+	return reqBody
+}
+
+func buildOllamaMessages(messages []Message) []map[string]any {
+	apiMessages := buildChatMessages(messages)
+	for i, message := range messages {
+		content, images, ok := ollamaMultimodalContent(message.Content)
+		if !ok {
+			continue
+		}
+		apiMessages[i]["content"] = content
+		if len(images) > 0 {
+			apiMessages[i]["images"] = images
+		}
+	}
+	return apiMessages
+}
+
+func ollamaMultimodalContent(content interface{}) (string, []string, bool) {
+	var parts []interface{}
+	switch value := content.(type) {
+	case []interface{}:
+		parts = value
+	case []map[string]interface{}:
+		parts = make([]interface{}, len(value))
+		for i := range value {
+			parts[i] = value[i]
+		}
+	default:
+		return "", nil, false
+	}
+
+	var textParts []string
+	var images []string
+	for _, part := range parts {
+		partMap, ok := part.(map[string]interface{})
+		if !ok {
+			if text, ok := part.(string); ok {
+				textParts = append(textParts, text)
+			}
+			continue
+		}
+
+		partType, _ := partMap["type"].(string)
+		switch partType {
+		case "text", "input_text":
+			if text, ok := partMap["text"].(string); ok {
+				textParts = append(textParts, text)
+			}
+		case "image_url":
+			if imageURL := ollamaImageURL(partMap["image_url"]); imageURL != "" {
+				images = append(images, cleanOllamaImageData(imageURL))
+			}
+		}
+	}
+
+	return strings.Join(textParts, "\n"), images, true
+}
+
+func ollamaImageURL(value interface{}) string {
+	switch image := value.(type) {
+	case string:
+		return image
+	case map[string]interface{}:
+		url, _ := image["url"].(string)
+		return url
+	case map[string]string:
+		return image["url"]
+	default:
+		return ""
+	}
+}
+
+func cleanOllamaImageData(image string) string {
+	const base64Marker = ";base64,"
+	if strings.HasPrefix(image, "data:") {
+		if marker := strings.Index(image, base64Marker); marker >= 0 {
+			return image[marker+len(base64Marker):]
+		}
+	}
+	return image
+}
+
 func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) {
 	if len(messages) == 0 {
 		return nil, fmt.Errorf("message is nil")
@@ -70,7 +156,7 @@ func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, me
 	}
 
 	// Build request body
-	reqBody := buildRequestBody(chatModelConfig, modelName, messages, false)
+	reqBody := buildOllamaRequestBody(chatModelConfig, modelName, messages, false)
 
 	if chatModelConfig != nil {
 		if chatModelConfig.Effort != nil && *chatModelConfig.Effort != "" {
@@ -108,7 +194,7 @@ func (o *OllamaModel) ChatStreamlyWithSender(ctx context.Context, modelName stri
 	}
 
 	// Build request body with streaming enabled
-	reqBody := buildRequestBody(modelConfig, modelName, messages, true)
+	reqBody := buildOllamaRequestBody(modelConfig, modelName, messages, true)
 
 	if modelConfig.Effort != nil && *modelConfig.Effort != "" {
 		if strings.HasPrefix(strings.ToLower(modelName), "gpt-oss") {

internal/entity/models/ollama_test.go

@@ -17,9 +17,11 @@
 package models
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"ragflow/internal/common"
 	"testing"
 )
 
@@ -27,6 +29,113 @@ func newOllamaForListModelsTest(baseURL string) *OllamaModel {
 	return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Models: "api/tags"})
 }
 
+func newOllamaForChatTest(baseURL string) *OllamaModel {
+	return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "api/chat", AsyncChat: "api/chat"})
+}
+
+func ollamaMultimodalTestMessages() []Message {
+	return []Message{{Role: "user", Content: []interface{}{
+		map[string]interface{}{"type": "text", "text": "describe"},
+		map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/png;base64,aGVsbG8="}},
+		map[string]interface{}{"type": "text", "text": "carefully"},
+		map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "cmF3LWltYWdl"}},
+		map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "https://example.com/cat.png"}},
+	}}}
+}
+
+func assertOllamaMultimodalRequest(t *testing.T, body map[string]interface{}, stream bool) {
+	t.Helper()
+	if body["stream"] != stream {
+		t.Errorf("stream=%v, want %v", body["stream"], stream)
+	}
+	messages, ok := body["messages"].([]interface{})
+	if !ok || len(messages) != 1 {
+		t.Errorf("messages=%v, want one message", body["messages"])
+		return
+	}
+	message, ok := messages[0].(map[string]interface{})
+	if !ok {
+		t.Errorf("message=%v, want object", messages[0])
+		return
+	}
+	if message["content"] != "describe\ncarefully" {
+		t.Errorf("content=%v, want joined text parts", message["content"])
+	}
+	images, ok := message["images"].([]interface{})
+	if !ok {
+		t.Errorf("images=%v, want array", message["images"])
+		return
+	}
+	want := []string{"aGVsbG8=", "cmF3LWltYWdl", "https://example.com/cat.png"}
+	if len(images) != len(want) {
+		t.Errorf("images=%v, want %v", images, want)
+		return
+	}
+	for i, expected := range want {
+		if images[i] != expected {
+			t.Errorf("images[%d]=%v, want %q", i, images[i], expected)
+		}
+	}
+}
+
+func TestOllamaChatMapsMultimodalImages(t *testing.T) {
+	withSSRFBypass(t)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]interface{}
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Errorf("decode request: %v", err)
+			http.Error(w, "invalid request", http.StatusBadRequest)
+			return
+		}
+		assertOllamaMultimodalRequest(t, body, false)
+		_, _ = io.WriteString(w, `{"message":{"content":"ok","thinking":""}}`)
+	}))
+	defer srv.Close()
+
+	response, err := newOllamaForChatTest(srv.URL).ChatWithMessages(
+		t.Context(),
+		"llava",
+		ollamaMultimodalTestMessages(),
+		&APIConfig{},
+		&ChatConfig{},
+		&common.ModelUsage{},
+	)
+	if err != nil {
+		t.Fatalf("ChatWithMessages: %v", err)
+	}
+	if response.Answer == nil || *response.Answer != "ok" {
+		t.Fatalf("answer=%v, want ok", response.Answer)
+	}
+}
+
+func TestOllamaStreamingChatMapsMultimodalImages(t *testing.T) {
+	withSSRFBypass(t)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]interface{}
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Errorf("decode request: %v", err)
+			http.Error(w, "invalid request", http.StatusBadRequest)
+			return
+		}
+		assertOllamaMultimodalRequest(t, body, true)
+		_, _ = io.WriteString(w, "{\"message\":{\"content\":\"ok\"},\"done\":false}\n{\"done\":true}\n")
+	}))
+	defer srv.Close()
+
+	err := newOllamaForChatTest(srv.URL).ChatStreamlyWithSender(
+		t.Context(),
+		"llava",
+		ollamaMultimodalTestMessages(),
+		&APIConfig{},
+		&ChatConfig{},
+		&common.ModelUsage{},
+		func(*string, *string) error { return nil },
+	)
+	if err != nil {
+		t.Fatalf("ChatStreamlyWithSender: %v", err)
+	}
+}
+
 func TestOllamaListModels(t *testing.T) {
 	withSSRFBypass(t)
 	ctx := t.Context()