diff --git a/internal/handler/chat_audio.go b/internal/handler/chat_audio.go new file mode 100644 index 0000000000..40e86d7e86 --- /dev/null +++ b/internal/handler/chat_audio.go @@ -0,0 +1,266 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "ragflow/internal/common" + "ragflow/internal/entity" + modelModule "ragflow/internal/entity/models" +) + +const ( + // chatAudioSpeechMaxBodyBytes caps the JSON body of the TTS endpoint. + chatAudioSpeechMaxBodyBytes int64 = 1 << 20 // 1 MiB + // chatAudioUploadMaxBytes caps the multipart body of the transcription endpoint. + chatAudioUploadMaxBytes int64 = 64 << 20 // 64 MiB +) + +// chatAudioSpeechRequest is the request body for POST /api/v1/chat/audio/speech. +type chatAudioSpeechRequest struct { + Text string `json:"text" binding:"required"` +} + +// ttsSegmentSplitRegex splits TTS input into synthesis segments on CJK +// punctuation (,。/《》?;:!), ASCII ':' and ';', and newlines. +var ttsSegmentSplitRegex = regexp.MustCompile("[,。/《》?;:!\\n\\r:;]+") + +// ChatAudioSpeech converts text to speech using the tenant's default TTS model +// and streams the concatenated MP3 segments as an audio/mpeg response. +func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) { + user, errorCode, errorMessage := GetUser(c) + if errorCode != common.CodeSuccess { + common.ErrorWithCode(c, errorCode, errorMessage) + return + } + + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, chatAudioSpeechMaxBodyBytes) + + var req chatAudioSpeechRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ResponseWithCodeData(c, common.CodeArgumentError, nil, err.Error()) + return + } + + if h.llm == nil { + common.ErrorWithCode(c, common.CodeServerError, "TTS service not available") + return + } + + driver, modelName, apiConfig, _, err := h.llm.GetTenantDefaultModelByType(user.ID, entity.ModelTypeTTS) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + + segments := ttsSegmentSplitRegex.Split(req.Text, -1) + headerWritten := false + for i, seg := range segments { + seg = strings.TrimSpace(seg) + if seg == "" { + continue + } + resp, err := driver.AudioSpeech(&modelName, &seg, apiConfig, &modelModule.TTSConfig{Format: "mp3"}) + if err != nil { + common.Warn("chat TTS synthesis failed", + zap.Int("segmentIndex", i), + zap.Int("segmentLen", len(seg)), + zap.Error(err)) + continue + } + if resp == nil || len(resp.Audio) == 0 { + continue + } + if !headerWritten { + // Commit the audio headers only once the first chunk is available, + // so a fully failed synthesis can still return a JSON error status. + c.Header("Content-Type", "audio/mpeg") + c.Header("Cache-Control", "no-cache") + c.Header("Connection", "keep-alive") + c.Header("X-Accel-Buffering", "no") + c.Writer.WriteHeader(http.StatusOK) + headerWritten = true + } + if _, werr := c.Writer.Write(resp.Audio); werr != nil { + return + } + c.Writer.Flush() + } + if !headerWritten { + common.ErrorWithCode(c, common.CodeServerError, "TTS synthesis produced no audio") + } +} + +// chatAudioAllowedExts is the set of audio file extensions accepted by the +// transcription endpoint. +var chatAudioAllowedExts = map[string]struct{}{ + ".wav": {}, + ".mp3": {}, + ".m4a": {}, + ".aac": {}, + ".flac": {}, + ".ogg": {}, + ".webm": {}, + ".opus": {}, + ".wma": {}, +} + +// chatAudioAllowedExtsList is the sorted, human-readable form of +// chatAudioAllowedExts, computed once for error messages. +var chatAudioAllowedExtsList = func() string { + exts := make([]string, 0, len(chatAudioAllowedExts)) + for ext := range chatAudioAllowedExts { + exts = append(exts, ext) + } + sort.Strings(exts) + return strings.Join(exts, ", ") +}() + +// ChatAudioTranscription transcribes an uploaded audio file using the tenant's +// default ASR model. It supports both a single JSON response and SSE streaming. +func (h *ChatHandler) ChatAudioTranscription(c *gin.Context) { + user, errorCode, errorMessage := GetUser(c) + if errorCode != common.CodeSuccess { + common.ErrorWithCode(c, errorCode, errorMessage) + return + } + + if h.llm == nil { + common.ErrorWithCode(c, common.CodeServerError, "ASR service not available") + return + } + + // Cap the body before any multipart parsing so an oversized upload is + // rejected instead of being drained into memory or spooled to disk. + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, chatAudioUploadMaxBytes) + if cl := c.Request.ContentLength; cl > chatAudioUploadMaxBytes { + common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "request body too large.") + return + } + if err := c.Request.ParseMultipartForm(chatAudioUploadMaxBytes); err != nil { + common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "invalid multipart form: "+err.Error()) + return + } + defer func() { + if c.Request.MultipartForm != nil { + _ = c.Request.MultipartForm.RemoveAll() + } + }() + + fileHeader, err := c.FormFile("file") + if err != nil { + common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Missing 'file' in multipart form-data") + return + } + + suffix := strings.ToLower(filepath.Ext(fileHeader.Filename)) + if _, ok := chatAudioAllowedExts[suffix]; suffix == "" || !ok { + common.ResponseWithCodeData(c, common.CodeArgumentError, nil, + fmt.Sprintf("Unsupported audio format: %s. Allowed: %s", suffix, chatAudioAllowedExtsList)) + return + } + + // Save the uploaded file to a temporary location so the model driver can read it. + tmpFile, err := os.CreateTemp("", "*"+suffix) + if err != nil { + common.ErrorWithCode(c, common.CodeServerError, "Failed to create temp audio file: "+err.Error()) + return + } + tmpPath := tmpFile.Name() + _ = tmpFile.Close() + defer func() { _ = os.Remove(tmpPath) }() + + if err := c.SaveUploadedFile(fileHeader, tmpPath); err != nil { + common.ErrorWithCode(c, common.CodeServerError, "Failed to save audio file: "+err.Error()) + return + } + + driver, modelName, apiConfig, _, err := h.llm.GetTenantDefaultModelByType(user.ID, entity.ModelTypeSpeech2Text) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + + streamMode := strings.ToLower(c.PostForm("stream")) == "true" + if streamMode { + disableWriteDeadlineForSSE(c) + c.Header("Content-Type", "text/event-stream") + c.Header("Cache-Control", "no-cache") + c.Header("Connection", "keep-alive") + c.Header("X-Accel-Buffering", "no") + c.Writer.WriteHeader(http.StatusOK) + + doneSent := false + sender := func(content, _ *string) error { + if content == nil { + return nil + } + event := map[string]interface{}{"text": *content} + if *content == "[DONE]" { + event["event"] = "done" + doneSent = true + } else { + event["event"] = "partial" + } + data, _ := json.Marshal(event) + if _, err := c.Writer.WriteString(fmt.Sprintf("data: %s\n\n", data)); err != nil { + return err + } + c.Writer.Flush() + return nil + } + + if err := driver.TranscribeAudioWithSender(&modelName, &tmpPath, apiConfig, &modelModule.ASRConfig{}, sender); err != nil { + errEvent := map[string]interface{}{"event": "error", "text": err.Error()} + data, _ := json.Marshal(errEvent) + _, _ = c.Writer.WriteString(fmt.Sprintf("data: %s\n\n", data)) + c.Writer.Flush() + return + } + if !doneSent { + // Drivers are expected to terminate the stream with "[DONE]"; send + // it here when they do not, so clients always see a completion event. + doneEvent := map[string]interface{}{"event": "done", "text": "[DONE]"} + data, _ := json.Marshal(doneEvent) + _, _ = c.Writer.WriteString(fmt.Sprintf("data: %s\n\n", data)) + c.Writer.Flush() + } + return + } + + resp, err := driver.TranscribeAudio(&modelName, &tmpPath, apiConfig, &modelModule.ASRConfig{}) + if err != nil { + common.ErrorWithCode(c, common.CodeServerError, err.Error()) + return + } + if resp == nil { + common.ErrorWithCode(c, common.CodeServerError, "empty transcription response") + return + } + + common.SuccessWithData(c, map[string]string{"text": resp.Text}, "success") +} diff --git a/internal/router/router.go b/internal/router/router.go index 0d5353db6b..0f81a9d018 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -352,6 +352,8 @@ func (r *Router) Setup(engine *gin.Engine) { chat.POST("/completions", r.chatSessionHandler.ChatCompletions) chat.POST("/mindmap", r.chatHandler.MindMap) chat.POST("/recommendation", r.chatHandler.Recommendation) + chat.POST("/audio/speech", r.chatHandler.ChatAudioSpeech) + chat.POST("/audio/transcription", r.chatHandler.ChatAudioTranscription) } v1.POST("/openai/:chat_id/chat/completions", r.openaiChatHandler.OpenAIChatCompletions)