diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.md index 5088f30ef2..25b1592dfc 100644 --- a/docs/references/http_api_reference.md +++ b/docs/references/http_api_reference.md @@ -3032,10 +3032,11 @@ curl --request POST \ - `"use_kg"`: `boolean` - `"reasoning"`: `boolean` - `"cross_languages"`: `list[string]` - - `"web_search_provider"`: `string` The web search service to use. Supported values are `"tavily"`, `"querit"`, and `"serply"`. Defaults to `"tavily"` when omitted. + - `"web_search_provider"`: `string` The web search service to use. Supported values are `"tavily"`, `"querit"`, `"serply"`, and `"youcom"`. If omitted, Tavily is selected only when `"tavily_api_key"` is configured; otherwise web search is disabled. - `"tavily_api_key"`: `string` - `"querit_api_key"`: `string` The Querit API key. Set `web_search_provider` to `"querit"` when using this field. - `"serply_api_key"`: `string` The [Serply](https://serply.io) API key. Set `web_search_provider` to `"serply"` when using this field. See the [Serply documentation](https://serply.io/docs) for details. + - `"youcom_api_key"`: `string` The You.com API key. Set `web_search_provider` to `"youcom"` when using this field. Optional: You.com serves a rate-limited keyless endpoint, so `"youcom"` works with this field omitted, and a key lifts those limits. - `"toc_enhance"`: `boolean` - `"similarity_threshold"`: (*Body parameter*), `float` - `"vector_similarity_weight"`: (*Body parameter*), `float` diff --git a/internal/service/web_search_provider.go b/internal/service/web_search_provider.go index 9ee5b5e890..5cb7272b3d 100644 --- a/internal/service/web_search_provider.go +++ b/internal/service/web_search_provider.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "time" ) @@ -32,13 +33,25 @@ const ( webSearchProviderTavily = "tavily" webSearchProviderQuerit = "querit" webSearchProviderSerply = "serply" + webSearchProviderYouCom = "youcom" queritWebSearchEndpoint = "https://api.querit.ai/v1/search" serplyWebSearchEndpoint = "https://api.serply.io/v1/search/" + // You.com serves the same response shape from two endpoints. The keyless + // one is rate-limited but needs no credentials; the keyed one lifts those + // limits. The keyless endpoint rejects an X-API-Key header, so the endpoint + // and the headers are always chosen together. + youComWebSearchEndpoint = "https://api.you.com/v1/search" + youComKeylessWebSearchEndpoint = "https://api.you.com/v1/agents/search" + youComWebSearchResultCount = 6 + // Identifies RAGFlow to You.com. On the keyless endpoint there is no key to + // attribute traffic to, so this is the only signal available. + youComWebSearchUserAgent = "RAGFlow youdotcom-integration/infiniflow-ragflow" ) var ( queritWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second} serplyWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second} + youComWebSearchHTTPClient = &http.Client{Timeout: 30 * time.Second} ) type webSearchProviderConfig struct { @@ -61,6 +74,9 @@ func resolveWebSearchProvider(promptConfig map[string]interface{}) *webSearchPro } apiKeyField := "" + // You.com is usable with no credentials at all; every other provider here + // requires a key before it can be selected. + keyOptional := false switch provider { case webSearchProviderTavily: apiKeyField = "tavily_api_key" @@ -68,13 +84,16 @@ func resolveWebSearchProvider(promptConfig map[string]interface{}) *webSearchPro apiKeyField = "querit_api_key" case webSearchProviderSerply: apiKeyField = "serply_api_key" + case webSearchProviderYouCom: + apiKeyField = "youcom_api_key" + keyOptional = true default: return nil } apiKey, _ := promptConfig[apiKeyField].(string) apiKey = strings.TrimSpace(apiKey) - if apiKey == "" { + if apiKey == "" && !keyOptional { return nil } return &webSearchProviderConfig{ @@ -110,6 +129,14 @@ func (s *ChatPipelineService) retrieveWebSearch( provider.APIKey, question, ) + case webSearchProviderYouCom: + return retrieveYouComWebSearch( + ctx, + youComWebSearchHTTPClient, + youComEndpointFor(provider.APIKey), + provider.APIKey, + question, + ) default: return nil, fmt.Errorf("unsupported web search provider %q", provider.Provider) } @@ -142,6 +169,14 @@ func (dr *DeepResearcher) retrieveWebSearch( provider.APIKey, query, ) + case webSearchProviderYouCom: + return retrieveYouComWebSearch( + ctx, + youComWebSearchHTTPClient, + youComEndpointFor(provider.APIKey), + provider.APIKey, + query, + ) default: return nil, fmt.Errorf("unsupported web search provider %q", provider.Provider) } @@ -376,3 +411,131 @@ func decodeSerplyWebSearchResults(responseBody []byte) ([]serplyWebSearchResult, } return results, nil } + +type youComWebSearchResult struct { + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + Snippets []string `json:"snippets"` +} + +type youComWebSearchResponse struct { + Results struct { + Web []youComWebSearchResult `json:"web"` + News []youComWebSearchResult `json:"news"` + } `json:"results"` +} + +// youComEndpointFor picks the keyless endpoint when no key is configured. The +// keyless endpoint rejects an X-API-Key header, so callers must never send a +// key to it. +func youComEndpointFor(apiKey string) string { + if strings.TrimSpace(apiKey) == "" { + return youComKeylessWebSearchEndpoint + } + return youComWebSearchEndpoint +} + +// youComContent prefers the extracted page passages. News hits carry only a +// description. +func youComContent(result youComWebSearchResult) string { + passages := make([]string, 0, len(result.Snippets)) + for _, snippet := range result.Snippets { + if strings.TrimSpace(snippet) != "" { + passages = append(passages, snippet) + } + } + if len(passages) > 0 { + return strings.Join(passages, "\n") + } + return strings.TrimSpace(result.Description) +} + +func retrieveYouComWebSearch( + ctx context.Context, + client *http.Client, + endpoint string, + apiKey string, + query string, +) (map[string]interface{}, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("youcom: new request: %w", err) + } + queryParams := request.URL.Query() + queryParams.Set("query", query) + queryParams.Set("count", strconv.Itoa(youComWebSearchResultCount)) + request.URL.RawQuery = queryParams.Encode() + + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", youComWebSearchUserAgent) + if trimmedKey := strings.TrimSpace(apiKey); trimmedKey != "" { + request.Header.Set("X-API-Key", trimmedKey) + } + + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("youcom: do request: %w", err) + } + defer response.Body.Close() + + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("youcom: status %d", response.StatusCode) + } + + responseBody, err := io.ReadAll(response.Body) + if err != nil { + return nil, fmt.Errorf("youcom: read response: %w", err) + } + + var decoded youComWebSearchResponse + if err := json.Unmarshal(responseBody, &decoded); err != nil { + return nil, fmt.Errorf("youcom: decode response: %w", err) + } + + // `count` applies per response section, so web and news together can exceed + // it. Web results lead; the merged list is trimmed back afterwards. + merged := make([]youComWebSearchResult, 0, len(decoded.Results.Web)+len(decoded.Results.News)) + merged = append(merged, decoded.Results.Web...) + merged = append(merged, decoded.Results.News...) + + chunks := make([]map[string]interface{}, 0, len(merged)) + docAggs := make([]interface{}, 0, len(merged)) + for _, result := range merged { + if len(chunks) >= youComWebSearchResultCount { + break + } + content := youComContent(result) + if content == "" { + continue + } + chunkID := "youcom-" + result.URL + chunks = append(chunks, map[string]interface{}{ + "chunk_id": chunkID, + "content_ltks": tokenizeText(content), + "content_with_weight": content, + "doc_id": chunkID, + "docnm_kwd": result.Title, + "kb_id": []interface{}{}, + "important_kwd": []interface{}{}, + "image_id": "", + "similarity": float64(1), + "vector_similarity": float64(1), + "term_similarity": float64(0), + "vector": []float64{}, + "positions": []interface{}{}, + "url": result.URL, + }) + docAggs = append(docAggs, map[string]interface{}{ + "doc_name": result.Title, + "doc_id": chunkID, + "count": 1, + "url": result.URL, + }) + } + + return map[string]interface{}{ + "chunks": chunks, + "doc_aggs": docAggs, + }, nil +} diff --git a/internal/service/web_search_provider_test.go b/internal/service/web_search_provider_test.go index 3fd0870c90..7e9c6812ca 100644 --- a/internal/service/web_search_provider_test.go +++ b/internal/service/web_search_provider_test.go @@ -18,6 +18,7 @@ package service import ( "encoding/json" + "fmt" "net/http" "net/http/httptest" "net/url" @@ -371,3 +372,253 @@ func TestDecodeSerplyWebSearchResultsAcceptsMissingResults(t *testing.T) { t.Fatalf("results = %#v, want empty", results) } } + +func TestResolveWebSearchProviderSelectsYouComWithoutAKey(t *testing.T) { + // You.com is the only provider usable with no credentials at all. + provider := resolveWebSearchProvider(map[string]interface{}{ + "web_search_provider": "youcom", + }) + + if provider == nil { + t.Fatal("provider is nil") + } + if provider.Provider != webSearchProviderYouCom { + t.Fatalf("provider = %q, want %q", provider.Provider, webSearchProviderYouCom) + } + if provider.APIKey != "" { + t.Fatalf("api key = %q, want empty", provider.APIKey) + } +} + +func TestResolveWebSearchProviderTrimsOptionalYouComKey(t *testing.T) { + provider := resolveWebSearchProvider(map[string]interface{}{ + "web_search_provider": "youcom", + "youcom_api_key": " ydc-test ", + "tavily_api_key": "tvly-test", + }) + + if provider == nil { + t.Fatal("provider is nil") + } + if provider.APIKey != "ydc-test" { + t.Fatalf("api key = %q, want %q", provider.APIKey, "ydc-test") + } +} + +func TestResolveWebSearchProviderStillRequiresKeysForKeyedProviders(t *testing.T) { + // The You.com carve-out must not relax any other provider. + for _, provider := range []string{"tavily", "querit", "serply"} { + t.Run(provider, func(t *testing.T) { + if got := resolveWebSearchProvider(map[string]interface{}{ + "web_search_provider": provider, + }); got != nil { + t.Fatalf("provider = %+v, want nil", got) + } + }) + } +} + +func TestYouComEndpointForPicksKeylessWithoutAKey(t *testing.T) { + cases := []struct { + name string + apiKey string + want string + }{ + {name: "no key", apiKey: "", want: youComKeylessWebSearchEndpoint}, + {name: "whitespace key", apiKey: " ", want: youComKeylessWebSearchEndpoint}, + {name: "key set", apiKey: "ydc-test", want: youComWebSearchEndpoint}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := youComEndpointFor(tc.apiKey); got != tc.want { + t.Fatalf("endpoint = %q, want %q", got, tc.want) + } + }) + } +} + +func TestYouComContentDiscardsBlankSnippetsAndDescriptions(t *testing.T) { + cases := []struct { + name string + result youComWebSearchResult + want string + }{ + { + name: "joins non-blank passages", + result: youComWebSearchResult{Snippets: []string{"a", " ", "b"}, Description: "ignored"}, + want: "a\nb", + }, + { + name: "falls back to the description", + result: youComWebSearchResult{Snippets: []string{" "}, Description: " desc "}, + want: "desc", + }, + { + name: "whitespace-only description yields nothing", + result: youComWebSearchResult{Description: " "}, + want: "", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := youComContent(tc.result); got != tc.want { + t.Fatalf("content = %q, want %q", got, tc.want) + } + }) + } +} + +func TestRetrieveYouComWebSearchSendsNoAuthHeaderWhenKeyless(t *testing.T) { + var gotAuth string + var gotUserAgent string + var gotQuery string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("X-API-Key") + gotUserAgent = r.Header.Get("User-Agent") + gotQuery = r.URL.Query().Get("query") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"results":{"web":[]}}`)) + })) + defer server.Close() + + if _, err := retrieveYouComWebSearch( + t.Context(), + server.Client(), + server.URL, + "", + "What is RAGFlow?", + ); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // The keyless endpoint rejects an auth header, so none may be sent. + if gotAuth != "" { + t.Fatalf("X-API-Key = %q, want empty", gotAuth) + } + if gotUserAgent != youComWebSearchUserAgent { + t.Fatalf("User-Agent = %q, want %q", gotUserAgent, youComWebSearchUserAgent) + } + if gotQuery != "What is RAGFlow?" { + t.Fatalf("query = %q, want %q", gotQuery, "What is RAGFlow?") + } +} + +func TestRetrieveYouComWebSearchReturnsReferenceShape(t *testing.T) { + var gotAuth string + var gotCount string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("X-API-Key") + gotCount = r.URL.Query().Get("count") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"results":{"web":[{"url":"https://example.com/ragflow","title":"RAGFlow","description":"Meta description.","snippets":["First passage.","Second passage."]}],"news":[{"url":"https://news.example.com/ragflow","title":"RAGFlow ships","description":"News description only."}]}}`)) + })) + defer server.Close() + + result, err := retrieveYouComWebSearch( + t.Context(), + server.Client(), + server.URL, + "ydc-test", + "What is RAGFlow?", + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if gotAuth != "ydc-test" { + t.Fatalf("X-API-Key = %q, want %q", gotAuth, "ydc-test") + } + if gotCount != "6" { + t.Fatalf("count = %q, want %q", gotCount, "6") + } + + chunks, ok := result["chunks"].([]map[string]interface{}) + if !ok { + t.Fatalf("chunks type = %T, want []map[string]interface{}", result["chunks"]) + } + if len(chunks) != 2 { + t.Fatalf("chunks = %d, want 2", len(chunks)) + } + // Web hits carry extracted passages; news hits fall back to the description. + if got := chunks[0]["content_with_weight"]; got != "First passage.\nSecond passage." { + t.Fatalf("web content = %q", got) + } + if got := chunks[1]["content_with_weight"]; got != "News description only." { + t.Fatalf("news content = %q", got) + } + if got := chunks[0]["url"]; got != "https://example.com/ragflow" { + t.Fatalf("url = %q", got) + } + + docAggs, ok := result["doc_aggs"].([]interface{}) + if !ok { + t.Fatalf("doc_aggs type = %T, want []interface{}", result["doc_aggs"]) + } + if len(docAggs) != 2 { + t.Fatalf("doc_aggs = %d, want 2", len(docAggs)) + } +} + +func TestRetrieveYouComWebSearchSkipsBlankContent(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"results":{"web":[{"url":"https://example.com/blank","title":"Blank","description":" "}]}}`)) + })) + defer server.Close() + + result, err := retrieveYouComWebSearch(t.Context(), server.Client(), server.URL, "", "q") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if chunks := result["chunks"].([]map[string]interface{}); len(chunks) != 0 { + t.Fatalf("chunks = %d, want 0", len(chunks)) + } +} + +func TestRetrieveYouComWebSearchCapsMergedSections(t *testing.T) { + web := make([]map[string]string, 0, 6) + news := make([]map[string]string, 0, 6) + for i := 0; i < 6; i++ { + web = append(web, map[string]string{"url": fmt.Sprintf("https://example.com/w%d", i), "description": "d"}) + news = append(news, map[string]string{"url": fmt.Sprintf("https://example.com/n%d", i), "description": "d"}) + } + payload, err := json.Marshal(map[string]interface{}{ + "results": map[string]interface{}{"web": web, "news": news}, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(payload) + })) + defer server.Close() + + result, err := retrieveYouComWebSearch(t.Context(), server.Client(), server.URL, "", "q") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // `count` applies per section, so the merged list is trimmed back to 6. + chunks := result["chunks"].([]map[string]interface{}) + if len(chunks) != youComWebSearchResultCount { + t.Fatalf("chunks = %d, want %d", len(chunks), youComWebSearchResultCount) + } +} + +func TestRetrieveYouComWebSearchRejectsErrorStatuses(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusPaymentRequired) + })) + defer server.Close() + + if _, err := retrieveYouComWebSearch(t.Context(), server.Client(), server.URL, "", "q"); err == nil { + t.Fatal("expected an error for a non-2xx status") + } +} diff --git a/rag/utils/web_search_conn.py b/rag/utils/web_search_conn.py index 6e6e428766..08d9749441 100644 --- a/rag/utils/web_search_conn.py +++ b/rag/utils/web_search_conn.py @@ -20,10 +20,16 @@ from typing import Protocol from rag.utils.querit_conn import Querit from rag.utils.serply_conn import Serply from rag.utils.tavily_conn import Tavily +from rag.utils.youcom_conn import YouCom WEB_SEARCH_PROVIDER_TAVILY = "tavily" WEB_SEARCH_PROVIDER_QUERIT = "querit" WEB_SEARCH_PROVIDER_SERPLY = "serply" +WEB_SEARCH_PROVIDER_YOUCOM = "youcom" + +# You.com serves a keyless endpoint, so it is usable with no credentials at all. +# Every other provider here requires a key before it can be selected. +KEYLESS_WEB_SEARCH_PROVIDERS = frozenset({WEB_SEARCH_PROVIDER_YOUCOM}) logger = logging.getLogger(__name__) @@ -42,6 +48,8 @@ def has_web_search_provider(prompt_config: dict | None) -> bool: if not prompt_config: return False provider = prompt_config.get("web_search_provider", WEB_SEARCH_PROVIDER_TAVILY) + if provider in KEYLESS_WEB_SEARCH_PROVIDERS: + return True if provider == WEB_SEARCH_PROVIDER_TAVILY: return bool(_get_api_key(prompt_config, "tavily_api_key")) if provider == WEB_SEARCH_PROVIDER_QUERIT: @@ -57,7 +65,12 @@ def create_web_search_provider(prompt_config: dict | None) -> WebSearchProvider return None provider = prompt_config.get("web_search_provider", WEB_SEARCH_PROVIDER_TAVILY) - if provider not in (WEB_SEARCH_PROVIDER_TAVILY, WEB_SEARCH_PROVIDER_QUERIT, WEB_SEARCH_PROVIDER_SERPLY): + if provider not in ( + WEB_SEARCH_PROVIDER_TAVILY, + WEB_SEARCH_PROVIDER_QUERIT, + WEB_SEARCH_PROVIDER_SERPLY, + WEB_SEARCH_PROVIDER_YOUCOM, + ): logger.debug("Web search provider resolution: provider=%s status=invalid", provider) return None if not has_web_search_provider(prompt_config): @@ -69,4 +82,7 @@ def create_web_search_provider(prompt_config: dict | None) -> WebSearchProvider return Querit(_get_api_key(prompt_config, "querit_api_key")) if provider == WEB_SEARCH_PROVIDER_SERPLY: return Serply(_get_api_key(prompt_config, "serply_api_key")) + if provider == WEB_SEARCH_PROVIDER_YOUCOM: + # The key is optional: an empty one selects the keyless endpoint. + return YouCom(_get_api_key(prompt_config, "youcom_api_key")) return Tavily(_get_api_key(prompt_config, "tavily_api_key")) diff --git a/rag/utils/youcom_conn.py b/rag/utils/youcom_conn.py new file mode 100644 index 0000000000..ed7202eddd --- /dev/null +++ b/rag/utils/youcom_conn.py @@ -0,0 +1,152 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +from typing import Any + +import requests + +from common.http_client import DEFAULT_TIMEOUT +from common.misc_utils import get_uuid +from rag.nlp import rag_tokenizer + +logger = logging.getLogger(__name__) + +# You.com serves the same response shape from two endpoints. The keyless one is +# rate-limited but needs no credentials; the keyed one lifts those limits and +# exposes the full Search API. The keyless endpoint rejects an X-API-Key header, +# so the endpoint and the headers are always chosen together. +YOUCOM_SEARCH_URL = "https://api.you.com/v1/search" +YOUCOM_KEYLESS_SEARCH_URL = "https://api.you.com/v1/agents/search" +YOUCOM_RESULT_COUNT = 6 +# Identifies RAGFlow to You.com. On the keyless endpoint there is no key to +# attribute traffic to, so this is the only signal available. +YOUCOM_USER_AGENT = "RAGFlow youdotcom-integration/infiniflow-ragflow" + + +class YouCom: + def __init__(self, api_key: str = ""): + self.api_key = api_key.strip() if isinstance(api_key, str) else "" + + def search(self, query: str) -> list[dict[str, Any]]: + headers = { + "Accept": "application/json", + "User-Agent": YOUCOM_USER_AGENT, + } + url = YOUCOM_KEYLESS_SEARCH_URL + if self.api_key: + url = YOUCOM_SEARCH_URL + headers["X-API-Key"] = self.api_key + + try: + response = requests.get( + url, + headers=headers, + params={"query": query, "count": YOUCOM_RESULT_COUNT}, + timeout=DEFAULT_TIMEOUT, + ) + response.raise_for_status() + response_data = response.json() + if not isinstance(response_data, dict): + raise TypeError("You.com API response must be a JSON object.") + + results_container = response_data.get("results", {}) + if not isinstance(results_container, dict): + raise TypeError("You.com API response field results must be an object.") + + normalized_results = [] + # `count` applies per section, so web and news together can exceed + # it. Web results lead; the merged list is trimmed back afterwards. + for section in ("web", "news"): + section_results = results_container.get(section, []) + if section_results is None: + continue + if not isinstance(section_results, list): + raise TypeError(f"You.com API response field results.{section} must be an array.") + for result in section_results: + if not isinstance(result, dict): + continue + content = _youcom_content(result) + if not content: + continue + normalized_results.append( + { + "url": _youcom_text(result.get("url")), + "title": _youcom_text(result.get("title")), + "content": content, + "score": 1.0, + } + ) + return normalized_results[:YOUCOM_RESULT_COUNT] + except requests.HTTPError as error: + # Never log the exception message: requests builds it from the + # response URL, and the query is a URL parameter here. + status = error.response.status_code if error.response is not None else "unknown" + logger.error("You.com search failed: HTTP %s", status) + return [] + except (requests.RequestException, TypeError, ValueError) as error: + logger.error("You.com search failed: %s", type(error).__name__) + return [] + + def retrieve_chunks(self, question: str) -> dict[str, list]: + chunks = [] + doc_aggs = [] + for result in self.search(question): + chunk_id = get_uuid() + chunks.append( + { + "chunk_id": chunk_id, + "content_ltks": rag_tokenizer.tokenize(result["content"]), + "content_with_weight": result["content"], + "doc_id": chunk_id, + "docnm_kwd": result["title"], + "kb_id": [], + "important_kwd": [], + "image_id": "", + "similarity": result["score"], + "vector_similarity": 1.0, + "term_similarity": 0, + "vector": [], + "positions": [], + "url": result["url"], + } + ) + doc_aggs.append( + { + "doc_name": result["title"], + "doc_id": chunk_id, + "count": 1, + "url": result["url"], + } + ) + # Deliberately logs counts only: the query and the retrieved page text + # are user data and must not reach the logs. + logger.info("[YouCom] retrieved %s chunks (keyed=%s)", len(chunks), bool(self.api_key)) + return {"chunks": chunks, "doc_aggs": doc_aggs} + + +def _youcom_content(result: dict[str, Any]) -> str: + """Prefer the extracted page passages; news hits only carry a description.""" + snippets = result.get("snippets") + if isinstance(snippets, list): + joined = "\n".join(_youcom_text(snippet) for snippet in snippets if _youcom_text(snippet).strip()) + if joined: + return joined + return _youcom_text(result.get("description")) + + +def _youcom_text(value: Any) -> str: + return "" if value is None else str(value) diff --git a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py index 75e368de30..b8dd422dd4 100644 --- a/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py +++ b/test/testcases/test_http_api/test_session_management/test_session_sdk_routes_unit.py @@ -1706,6 +1706,23 @@ def test_chatbot_routes_auth_stream_nonstream_unit(monkeypatch): assert res["code"] == 0 assert res["data"]["has_web_search_provider"] is True + # You.com is keyless, so selecting it enables the flag with no key set. + youcom_dialog = SimpleNamespace( + name="My You.com Bot", + icon="avatar.png", + tenant_id="tenant-1", + status="1", + llm_id="", + prompt_config={ + "prologue": "Hello!", + "web_search_provider": "youcom", + }, + ) + monkeypatch.setattr(module.DialogService, "get_by_id", lambda _dialog_id: (True, youcom_dialog)) + res = _run(inspect.unwrap(module.chatbots_inputs)("dialog-youcom")) + assert res["code"] == 0 + assert res["data"]["has_web_search_provider"] is True + @pytest.mark.p2 def test_agentbot_routes_auth_stream_nonstream_unit(monkeypatch): diff --git a/test/unit_test/rag/utils/test_web_search_conn.py b/test/unit_test/rag/utils/test_web_search_conn.py index eb66f2d4d1..9c61bd4fcb 100644 --- a/test/unit_test/rag/utils/test_web_search_conn.py +++ b/test/unit_test/rag/utils/test_web_search_conn.py @@ -135,3 +135,47 @@ def test_has_web_search_provider_follows_selected_provider(): "tavily_api_key": "tvly-test", } ) + + +def test_create_web_search_provider_selects_youcom_without_a_key(monkeypatch): + """You.com is the only provider usable with no credentials at all.""" + created_with = [] + provider = object() + + monkeypatch.setattr(web_search_conn, "YouCom", lambda api_key: created_with.append(api_key) or provider) + + result = web_search_conn.create_web_search_provider({"web_search_provider": "youcom"}) + + assert result is provider + assert created_with == [""] + + +def test_create_web_search_provider_passes_the_optional_youcom_key(monkeypatch): + created_with = [] + provider = object() + + monkeypatch.setattr(web_search_conn, "YouCom", lambda api_key: created_with.append(api_key) or provider) + + result = web_search_conn.create_web_search_provider( + { + "web_search_provider": "youcom", + "youcom_api_key": " ydc-test ", + "tavily_api_key": "tvly-test", + } + ) + + assert result is provider + assert created_with == ["ydc-test"] + + +def test_has_web_search_provider_is_true_for_keyless_youcom(): + assert web_search_conn.has_web_search_provider({"web_search_provider": "youcom"}) + assert web_search_conn.has_web_search_provider({"web_search_provider": "youcom", "youcom_api_key": ""}) + assert web_search_conn.has_web_search_provider({"web_search_provider": "youcom", "youcom_api_key": "ydc-test"}) + + +def test_keyless_carve_out_does_not_relax_keyed_providers(): + """The You.com carve-out must not make any other provider key-optional.""" + for provider in ("tavily", "querit", "serply"): + assert not web_search_conn.has_web_search_provider({"web_search_provider": provider}) + assert web_search_conn.create_web_search_provider({"web_search_provider": provider}) is None diff --git a/test/unit_test/rag/utils/test_youcom_conn.py b/test/unit_test/rag/utils/test_youcom_conn.py new file mode 100644 index 0000000000..7dc6fddfb0 --- /dev/null +++ b/test/unit_test/rag/utils/test_youcom_conn.py @@ -0,0 +1,210 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from rag.utils import youcom_conn + + +class _Response: + status_code = 200 + + def __init__(self, payload=None): + self._payload = payload if payload is not None else _default_payload() + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def _default_payload(): + return { + "results": { + "web": [ + { + "title": "RAGFlow", + "url": "https://example.com/ragflow", + "description": "Meta description.", + "snippets": ["RAGFlow is an open-source RAG engine.", "It does deep document understanding."], + } + ], + "news": [ + { + "title": "RAGFlow ships", + "url": "https://news.example.com/ragflow", + "description": "News description only.", + } + ], + } + } + + +def _capture_get(monkeypatch, response=None): + request = {} + + def fake_get(url, *, headers, params, timeout): + request.update(url=url, headers=headers, params=params, timeout=timeout) + return response if response is not None else _Response() + + monkeypatch.setattr(youcom_conn.requests, "get", fake_get) + return request + + +def test_youcom_search_uses_keyless_endpoint_without_a_key(monkeypatch): + request = _capture_get(monkeypatch) + + results = youcom_conn.YouCom("").search("What is RAGFlow?") + + assert request["url"] == "https://api.you.com/v1/agents/search" + # The keyless endpoint rejects an auth header, so none may be sent. + assert "X-API-Key" not in request["headers"] + assert request["params"] == {"query": "What is RAGFlow?", "count": 6} + assert len(results) == 2 + + +def test_youcom_search_uses_keyed_endpoint_when_a_key_is_set(monkeypatch): + request = _capture_get(monkeypatch) + + youcom_conn.YouCom("ydc-test").search("What is RAGFlow?") + + assert request["url"] == "https://api.you.com/v1/search" + assert request["headers"]["X-API-Key"] == "ydc-test" + + +def test_youcom_search_trims_the_key_and_identifies_ragflow(monkeypatch): + request = _capture_get(monkeypatch) + + youcom_conn.YouCom(" ydc-test ").search("What is RAGFlow?") + + assert request["headers"]["X-API-Key"] == "ydc-test" + assert request["headers"]["User-Agent"] == "RAGFlow youdotcom-integration/infiniflow-ragflow" + + +def test_youcom_search_prefers_passages_and_falls_back_to_description(monkeypatch): + _capture_get(monkeypatch) + + results = youcom_conn.YouCom("").search("What is RAGFlow?") + + assert results == [ + { + "url": "https://example.com/ragflow", + "title": "RAGFlow", + "content": "RAGFlow is an open-source RAG engine.\nIt does deep document understanding.", + "score": 1.0, + }, + { + "url": "https://news.example.com/ragflow", + "title": "RAGFlow ships", + "content": "News description only.", + "score": 1.0, + }, + ] + + +def test_youcom_search_skips_results_without_content(monkeypatch): + payload = {"results": {"web": [{"title": "No content", "url": "https://example.com/empty", "snippets": [" "]}]}} + _capture_get(monkeypatch, _Response(payload)) + + assert youcom_conn.YouCom("").search("What is RAGFlow?") == [] + + +def test_youcom_search_caps_the_merged_sections(monkeypatch): + payload = { + "results": { + "web": [{"title": f"w{i}", "url": f"https://example.com/w{i}", "description": "d"} for i in range(6)], + "news": [{"title": f"n{i}", "url": f"https://example.com/n{i}", "description": "d"} for i in range(6)], + } + } + _capture_get(monkeypatch, _Response(payload)) + + results = youcom_conn.YouCom("").search("What is RAGFlow?") + + # `count` applies per section, so the merged list is trimmed back to 6. + assert len(results) == 6 + assert [result["title"] for result in results] == ["w0", "w1", "w2", "w3", "w4", "w5"] + + +def test_youcom_search_returns_empty_on_malformed_payloads(monkeypatch): + _capture_get(monkeypatch, _Response(["not", "an", "object"])) + assert youcom_conn.YouCom("").search("q") == [] + + _capture_get(monkeypatch, _Response({"results": "not-an-object"})) + assert youcom_conn.YouCom("").search("q") == [] + + _capture_get(monkeypatch, _Response({"results": {"web": "not-an-array"}})) + assert youcom_conn.YouCom("").search("q") == [] + + +class _ErrorResponse: + """A response whose raise_for_status() carries the full request URL, the way + requests builds it — including the query string.""" + + status_code = 402 + url = "https://api.you.com/v1/agents/search?query=my%20private%20query&count=6" + + def raise_for_status(self): + raise youcom_conn.requests.HTTPError( + f"402 Client Error: Payment Required for url: {self.url}", + response=self, + ) + + def json(self): # pragma: no cover - never reached + return {} + + +def test_youcom_search_never_logs_the_query_or_key_on_http_errors(monkeypatch, caplog): + """The query is a URL parameter, so the requests error message contains it.""" + monkeypatch.setattr(youcom_conn.requests, "get", lambda *a, **k: _ErrorResponse()) + + with caplog.at_level("ERROR"): + assert youcom_conn.YouCom("ydc-secret").search("my private query") == [] + + assert "my private query" not in caplog.text + assert "my%20private%20query" not in caplog.text + assert "ydc-secret" not in caplog.text + # Only the status code is useful and safe to record. + assert "402" in caplog.text + + +def test_youcom_search_logs_only_the_exception_type_on_transport_errors(monkeypatch, caplog): + def fake_get(url, *, headers, params, timeout): + raise youcom_conn.requests.ConnectionError(f"failed connecting to {url}?query=my%20private%20query") + + monkeypatch.setattr(youcom_conn.requests, "get", fake_get) + + with caplog.at_level("ERROR"): + assert youcom_conn.YouCom("ydc-secret").search("my private query") == [] + + assert "my%20private%20query" not in caplog.text + assert "ydc-secret" not in caplog.text + assert "ConnectionError" in caplog.text + + +def test_youcom_retrieve_chunks_returns_ragflow_chunk_shape(monkeypatch): + _capture_get(monkeypatch) + + retrieved = youcom_conn.YouCom("").retrieve_chunks("What is RAGFlow?") + + assert set(retrieved) == {"chunks", "doc_aggs"} + assert len(retrieved["chunks"]) == 2 + chunk = retrieved["chunks"][0] + assert chunk["url"] == "https://example.com/ragflow" + assert chunk["docnm_kwd"] == "RAGFlow" + assert chunk["content_with_weight"].startswith("RAGFlow is an open-source RAG engine.") + assert chunk["similarity"] == 1.0 + doc_agg = retrieved["doc_aggs"][0] + assert doc_agg["doc_name"] == "RAGFlow" + assert doc_agg["doc_id"] == chunk["chunk_id"] diff --git a/web/src/assets/svg/youcom.svg b/web/src/assets/svg/youcom.svg new file mode 100644 index 0000000000..9a5a676a84 --- /dev/null +++ b/web/src/assets/svg/youcom.svg @@ -0,0 +1,10 @@ + + You.com + + + + + + + + diff --git a/web/src/components/web-search-form-field.tsx b/web/src/components/web-search-form-field.tsx index dda7515572..903698ebdf 100644 --- a/web/src/components/web-search-form-field.tsx +++ b/web/src/components/web-search-form-field.tsx @@ -17,6 +17,7 @@ import queritLogo from '@/assets/querit.png'; import serplyLogo from '@/assets/serply.png'; import tavilyLogo from '@/assets/svg/tavily.svg'; +import youcomLogo from '@/assets/svg/youcom.svg'; import { RAGFlowSelect } from '@/components/ui/select'; import { WebSearchProvider } from '@/constants/chat'; import { useTranslate } from '@/hooks/common-hooks'; @@ -52,6 +53,11 @@ const providerOptions = [ logo: serplyLogo, value: WebSearchProvider.Serply, }, + { + name: 'You.com', + logo: youcomLogo, + value: WebSearchProvider.YouCom, + }, ] .sort((left, right) => left.name.localeCompare(right.name)) .map(({ name, logo, value }) => ({ @@ -91,6 +97,14 @@ const providerKeyConfig = { placeholder: 'serplyApiKeyMessage', helpUrl: 'https://serply.io', }, + [WebSearchProvider.YouCom]: { + name: 'prompt_config.youcom_api_key', + label: 'You.com API Key', + tip: 'youcomApiKeyTip', + placeholder: 'youcomApiKeyMessage', + helpUrl: + 'https://you.com/platform?utm_source=infiniflow-ragflow&utm_medium=oss_integration&utm_campaign=2026-08-oss-integrations&utm_content=app', + }, } as const; export function WebSearchFormField({ prefix = '' }: IProps) { diff --git a/web/src/constants/chat.ts b/web/src/constants/chat.ts index 021ce50cef..a898b983fc 100644 --- a/web/src/constants/chat.ts +++ b/web/src/constants/chat.ts @@ -60,4 +60,13 @@ export enum WebSearchProvider { Tavily = 'tavily', Querit = 'querit', Serply = 'serply', + YouCom = 'youcom', } + +/** + * Providers usable with no credentials at all. You.com serves a rate-limited + * keyless endpoint; every other provider requires a key before it can be used. + */ +export const KEYLESS_WEB_SEARCH_PROVIDERS: readonly WebSearchProvider[] = [ + WebSearchProvider.YouCom, +]; diff --git a/web/src/interfaces/database/chat.ts b/web/src/interfaces/database/chat.ts index f981963daf..fea56e8593 100644 --- a/web/src/interfaces/database/chat.ts +++ b/web/src/interfaces/database/chat.ts @@ -23,6 +23,7 @@ export interface PromptConfig { tavily_api_key?: string; querit_api_key?: string; serply_api_key?: string; + youcom_api_key?: string; web_search_provider?: WebSearchProvider; toc_enhance?: boolean; reference_metadata?: { diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index e338e356c8..95563f3759 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -1206,6 +1206,9 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s serplyApiKeyTip: 'When Serply is selected, its web search results supplement dataset retrieval.', serplyApiKeyMessage: 'Please enter your Serply API Key', + youcomApiKeyTip: + 'Optional. You.com works without a key on its rate-limited endpoint; add a key to lift those limits.', + youcomApiKeyMessage: 'Optional — leave blank to use the free tier', tavilyApiKeyHelp: 'How to get it?', crossLanguage: 'Cross-language search', crossLanguagePlaceholder: 'Select value', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 217b942591..308630b031 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -1094,6 +1094,9 @@ NER：使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R queritApiKeyMessage: '请输入你的 Querit API Key', serplyApiKeyTip: '选择 Serply 后，将使用 Serply 的网络搜索结果补充知识库检索。', serplyApiKeyMessage: '请输入你的 Serply API Key', + youcomApiKeyTip: + '可选。You.com 在限速端点上无需 API Key 即可使用；填写 Key 可解除限速。', + youcomApiKeyMessage: '可选 —— 留空则使用免费额度', tavilyApiKeyHelp: '如何获取？', crossLanguage: '跨语言搜索', crossLanguagePlaceholder: '请选择', diff --git a/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx b/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx index 6f0fba63b3..69d59187eb 100644 --- a/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx +++ b/web/src/pages/next-chats/chat/app-settings/use-chat-setting-schema.tsx @@ -36,11 +36,13 @@ export function useChatSettingSchema(staleDatasetIds: Set) { tavily_api_key: z.string().optional(), querit_api_key: z.string().optional(), serply_api_key: z.string().optional(), + youcom_api_key: z.string().optional(), web_search_provider: z .enum([ WebSearchProvider.Tavily, WebSearchProvider.Querit, WebSearchProvider.Serply, + WebSearchProvider.YouCom, ]) .optional(), reasoning: z.boolean().optional(), diff --git a/web/src/pages/next-chats/chat/use-show-internet.test.ts b/web/src/pages/next-chats/chat/use-show-internet.test.ts index 34bf3f653a..1d4e28580c 100644 --- a/web/src/pages/next-chats/chat/use-show-internet.test.ts +++ b/web/src/pages/next-chats/chat/use-show-internet.test.ts @@ -1,6 +1,10 @@ import { WebSearchProvider } from '@/constants/chat'; import type { PromptConfig } from '@/interfaces/database/chat'; -import { getWebSearchApiKey, getWebSearchProvider } from './web-search-api-key'; +import { + getWebSearchApiKey, + getWebSearchProvider, + hasWebSearchProvider, +} from './web-search-api-key'; describe('getWebSearchProvider', () => { it('does not select a provider for a new unconfigured dialog', () => { @@ -90,3 +94,52 @@ describe('getWebSearchApiKey', () => { expect(getWebSearchApiKey(promptConfig)).toBeUndefined(); }); }); + +describe('hasWebSearchProvider', () => { + it('is false for a new unconfigured dialog', () => { + expect(hasWebSearchProvider({} as PromptConfig)).toBe(false); + }); + + it('requires a key for providers that need one', () => { + expect( + hasWebSearchProvider({ + web_search_provider: WebSearchProvider.Querit, + } as PromptConfig), + ).toBe(false); + + expect( + hasWebSearchProvider({ + web_search_provider: WebSearchProvider.Querit, + querit_api_key: 'querit-test', + } as PromptConfig), + ).toBe(true); + }); + + it('is true for keyless You.com with no key configured', () => { + expect( + hasWebSearchProvider({ + web_search_provider: WebSearchProvider.YouCom, + } as PromptConfig), + ).toBe(true); + + expect( + hasWebSearchProvider({ + web_search_provider: WebSearchProvider.YouCom, + youcom_api_key: '', + } as PromptConfig), + ).toBe(true); + }); +}); + +describe('You.com key selection', () => { + it('uses only the selected You.com key', () => { + const promptConfig = { + web_search_provider: WebSearchProvider.YouCom, + youcom_api_key: 'ydc-test', + tavily_api_key: 'tvly-test', + } as PromptConfig; + + expect(getWebSearchProvider(promptConfig)).toBe(WebSearchProvider.YouCom); + expect(getWebSearchApiKey(promptConfig)).toBe('ydc-test'); + }); +}); diff --git a/web/src/pages/next-chats/chat/use-show-internet.ts b/web/src/pages/next-chats/chat/use-show-internet.ts index 3db50dd0a9..56c19c9496 100644 --- a/web/src/pages/next-chats/chat/use-show-internet.ts +++ b/web/src/pages/next-chats/chat/use-show-internet.ts @@ -1,9 +1,8 @@ import { useFetchChat } from '@/hooks/use-chat-request'; -import { isEmpty } from 'lodash'; -import { getWebSearchApiKey } from './web-search-api-key'; +import { hasWebSearchProvider } from './web-search-api-key'; export function useShowInternet() { const { data: currentDialog } = useFetchChat(); - return !isEmpty(getWebSearchApiKey(currentDialog?.prompt_config)); + return hasWebSearchProvider(currentDialog?.prompt_config); } diff --git a/web/src/pages/next-chats/chat/web-search-api-key.ts b/web/src/pages/next-chats/chat/web-search-api-key.ts index 6a7aabf298..b45752a0fb 100644 --- a/web/src/pages/next-chats/chat/web-search-api-key.ts +++ b/web/src/pages/next-chats/chat/web-search-api-key.ts @@ -1,4 +1,7 @@ -import { WebSearchProvider } from '@/constants/chat'; +import { + KEYLESS_WEB_SEARCH_PROVIDERS, + WebSearchProvider, +} from '@/constants/chat'; import type { PromptConfig } from '@/interfaces/database/chat'; export function getWebSearchProvider(promptConfig?: PromptConfig) { @@ -7,7 +10,8 @@ export function getWebSearchProvider(promptConfig?: PromptConfig) { if ( provider === WebSearchProvider.Tavily || provider === WebSearchProvider.Querit || - provider === WebSearchProvider.Serply + provider === WebSearchProvider.Serply || + provider === WebSearchProvider.YouCom ) { return provider; } @@ -37,9 +41,29 @@ export function getWebSearchApiKey(promptConfig?: PromptConfig) { case WebSearchProvider.Serply: apiKey = promptConfig?.serply_api_key; break; + case WebSearchProvider.YouCom: + apiKey = promptConfig?.youcom_api_key; + break; default: return undefined; } return typeof apiKey === 'string' ? apiKey.trim() : undefined; } + +/** + * Whether web search is usable as configured. Most providers need a key; a + * keyless provider is usable as soon as it is selected. + */ +export function hasWebSearchProvider(promptConfig?: PromptConfig) { + const provider = getWebSearchProvider(promptConfig); + + if (provider === undefined) { + return false; + } + if (KEYLESS_WEB_SEARCH_PROVIDERS.includes(provider)) { + return true; + } + + return Boolean(getWebSearchApiKey(promptConfig)); +}