feat: add chat-level TTS and ASR endpoints for Go API server (#17036)

internal/handler/chat_audio.go

@@ -0,0 +1,266 @@
+//
+//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package handler
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"go.uber.org/zap"
+
+	"ragflow/internal/common"
+	"ragflow/internal/entity"
+	modelModule "ragflow/internal/entity/models"
+)
+
+const (
+	// chatAudioSpeechMaxBodyBytes caps the JSON body of the TTS endpoint.
+	chatAudioSpeechMaxBodyBytes int64 = 1 << 20 // 1 MiB
+	// chatAudioUploadMaxBytes caps the multipart body of the transcription endpoint.
+	chatAudioUploadMaxBytes int64 = 64 << 20 // 64 MiB
+)
+
+// chatAudioSpeechRequest is the request body for POST /api/v1/chat/audio/speech.
+type chatAudioSpeechRequest struct {
+	Text string `json:"text" binding:"required"`
+}
+
+// ttsSegmentSplitRegex splits TTS input into synthesis segments on CJK
+// punctuation (，。/《》？；：！), ASCII ':' and ';', and newlines.
+var ttsSegmentSplitRegex = regexp.MustCompile("[，。/《》？；：！\\n\\r:;]+")
+
+// ChatAudioSpeech converts text to speech using the tenant's default TTS model
+// and streams the concatenated MP3 segments as an audio/mpeg response.
+func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		common.ErrorWithCode(c, errorCode, errorMessage)
+		return
+	}
+
+	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, chatAudioSpeechMaxBodyBytes)
+
+	var req chatAudioSpeechRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		common.ResponseWithCodeData(c, common.CodeArgumentError, nil, err.Error())
+		return
+	}
+
+	if h.llm == nil {
+		common.ErrorWithCode(c, common.CodeServerError, "TTS service not available")
+		return
+	}
+
+	driver, modelName, apiConfig, _, err := h.llm.GetTenantDefaultModelByType(user.ID, entity.ModelTypeTTS)
+	if err != nil {
+		common.ErrorWithCode(c, common.CodeDataError, err.Error())
+		return
+	}
+
+	segments := ttsSegmentSplitRegex.Split(req.Text, -1)
+	headerWritten := false
+	for i, seg := range segments {
+		seg = strings.TrimSpace(seg)
+		if seg == "" {
+			continue
+		}
+		resp, err := driver.AudioSpeech(&modelName, &seg, apiConfig, &modelModule.TTSConfig{Format: "mp3"})
+		if err != nil {
+			common.Warn("chat TTS synthesis failed",
+				zap.Int("segmentIndex", i),
+				zap.Int("segmentLen", len(seg)),
+				zap.Error(err))
+			continue
+		}
+		if resp == nil || len(resp.Audio) == 0 {
+			continue
+		}
+		if !headerWritten {
+			// Commit the audio headers only once the first chunk is available,
+			// so a fully failed synthesis can still return a JSON error status.
+			c.Header("Content-Type", "audio/mpeg")
+			c.Header("Cache-Control", "no-cache")
+			c.Header("Connection", "keep-alive")
+			c.Header("X-Accel-Buffering", "no")
+			c.Writer.WriteHeader(http.StatusOK)
+			headerWritten = true
+		}
+		if _, werr := c.Writer.Write(resp.Audio); werr != nil {
+			return
+		}
+		c.Writer.Flush()
+	}
+	if !headerWritten {
+		common.ErrorWithCode(c, common.CodeServerError, "TTS synthesis produced no audio")
+	}
+}
+
+// chatAudioAllowedExts is the set of audio file extensions accepted by the
+// transcription endpoint.
+var chatAudioAllowedExts = map[string]struct{}{
+	".wav":  {},
+	".mp3":  {},
+	".m4a":  {},
+	".aac":  {},
+	".flac": {},
+	".ogg":  {},
+	".webm": {},
+	".opus": {},
+	".wma":  {},
+}
+
+// chatAudioAllowedExtsList is the sorted, human-readable form of
+// chatAudioAllowedExts, computed once for error messages.
+var chatAudioAllowedExtsList = func() string {
+	exts := make([]string, 0, len(chatAudioAllowedExts))
+	for ext := range chatAudioAllowedExts {
+		exts = append(exts, ext)
+	}
+	sort.Strings(exts)
+	return strings.Join(exts, ", ")
+}()
+
+// ChatAudioTranscription transcribes an uploaded audio file using the tenant's
+// default ASR model. It supports both a single JSON response and SSE streaming.
+func (h *ChatHandler) ChatAudioTranscription(c *gin.Context) {
+	user, errorCode, errorMessage := GetUser(c)
+	if errorCode != common.CodeSuccess {
+		common.ErrorWithCode(c, errorCode, errorMessage)
+		return
+	}
+
+	if h.llm == nil {
+		common.ErrorWithCode(c, common.CodeServerError, "ASR service not available")
+		return
+	}
+
+	// Cap the body before any multipart parsing so an oversized upload is
+	// rejected instead of being drained into memory or spooled to disk.
+	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, chatAudioUploadMaxBytes)
+	if cl := c.Request.ContentLength; cl > chatAudioUploadMaxBytes {
+		common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "request body too large.")
+		return
+	}
+	if err := c.Request.ParseMultipartForm(chatAudioUploadMaxBytes); err != nil {
+		common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "invalid multipart form: "+err.Error())
+		return
+	}
+	defer func() {
+		if c.Request.MultipartForm != nil {
+			_ = c.Request.MultipartForm.RemoveAll()
+		}
+	}()
+
+	fileHeader, err := c.FormFile("file")
+	if err != nil {
+		common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Missing 'file' in multipart form-data")
+		return
+	}
+
+	suffix := strings.ToLower(filepath.Ext(fileHeader.Filename))
+	if _, ok := chatAudioAllowedExts[suffix]; suffix == "" || !ok {
+		common.ResponseWithCodeData(c, common.CodeArgumentError, nil,
+			fmt.Sprintf("Unsupported audio format: %s. Allowed: %s", suffix, chatAudioAllowedExtsList))
+		return
+	}
+
+	// Save the uploaded file to a temporary location so the model driver can read it.
+	tmpFile, err := os.CreateTemp("", "*"+suffix)
+	if err != nil {
+		common.ErrorWithCode(c, common.CodeServerError, "Failed to create temp audio file: "+err.Error())
+		return
+	}
+	tmpPath := tmpFile.Name()
+	_ = tmpFile.Close()
+	defer func() { _ = os.Remove(tmpPath) }()
+
+	if err := c.SaveUploadedFile(fileHeader, tmpPath); err != nil {
+		common.ErrorWithCode(c, common.CodeServerError, "Failed to save audio file: "+err.Error())
+		return
+	}
+
+	driver, modelName, apiConfig, _, err := h.llm.GetTenantDefaultModelByType(user.ID, entity.ModelTypeSpeech2Text)
+	if err != nil {
+		common.ErrorWithCode(c, common.CodeDataError, err.Error())
+		return
+	}
+
+	streamMode := strings.ToLower(c.PostForm("stream")) == "true"
+	if streamMode {
+		disableWriteDeadlineForSSE(c)
+		c.Header("Content-Type", "text/event-stream")
+		c.Header("Cache-Control", "no-cache")
+		c.Header("Connection", "keep-alive")
+		c.Header("X-Accel-Buffering", "no")
+		c.Writer.WriteHeader(http.StatusOK)
+
+		doneSent := false
+		sender := func(content, _ *string) error {
+			if content == nil {
+				return nil
+			}
+			event := map[string]interface{}{"text": *content}
+			if *content == "[DONE]" {
+				event["event"] = "done"
+				doneSent = true
+			} else {
+				event["event"] = "partial"
+			}
+			data, _ := json.Marshal(event)
+			if _, err := c.Writer.WriteString(fmt.Sprintf("data: %s\n\n", data)); err != nil {
+				return err
+			}
+			c.Writer.Flush()
+			return nil
+		}
+
+		if err := driver.TranscribeAudioWithSender(&modelName, &tmpPath, apiConfig, &modelModule.ASRConfig{}, sender); err != nil {
+			errEvent := map[string]interface{}{"event": "error", "text": err.Error()}
+			data, _ := json.Marshal(errEvent)
+			_, _ = c.Writer.WriteString(fmt.Sprintf("data: %s\n\n", data))
+			c.Writer.Flush()
+			return
+		}
+		if !doneSent {
+			// Drivers are expected to terminate the stream with "[DONE]"; send
+			// it here when they do not, so clients always see a completion event.
+			doneEvent := map[string]interface{}{"event": "done", "text": "[DONE]"}
+			data, _ := json.Marshal(doneEvent)
+			_, _ = c.Writer.WriteString(fmt.Sprintf("data: %s\n\n", data))
+			c.Writer.Flush()
+		}
+		return
+	}
+
+	resp, err := driver.TranscribeAudio(&modelName, &tmpPath, apiConfig, &modelModule.ASRConfig{})
+	if err != nil {
+		common.ErrorWithCode(c, common.CodeServerError, err.Error())
+		return
+	}
+	if resp == nil {
+		common.ErrorWithCode(c, common.CodeServerError, "empty transcription response")
+		return
+	}
+
+	common.SuccessWithData(c, map[string]string{"text": resp.Text}, "success")
+}

internal/router/router.go

@@ -352,6 +352,8 @@ func (r *Router) Setup(engine *gin.Engine) {
 				chat.POST("/completions", r.chatSessionHandler.ChatCompletions)
 				chat.POST("/mindmap", r.chatHandler.MindMap)
 				chat.POST("/recommendation", r.chatHandler.Recommendation)
+				chat.POST("/audio/speech", r.chatHandler.ChatAudioSpeech)
+				chat.POST("/audio/transcription", r.chatHandler.ChatAudioTranscription)
 			}
 			v1.POST("/openai/:chat_id/chat/completions", r.openaiChatHandler.OpenAIChatCompletions)