From 2b4a80fcb24d0058c97f645c44151ffd09313676 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Tue, 2 Jun 2026 21:44:58 +0300 Subject: [PATCH] fix default search limits --- README.md | 4 +-- baidu/all_test.go | 19 +++++++++++- baidu/search.go | 4 +-- baidu/url.go | 8 +++-- bing/bing_test.go | 3 ++ bing/url.go | 2 +- cmd/root.go | 2 +- core/common.go | 15 ++++++++- core/common_test.go | 58 ++++++++++++++++++++++++++++++++++ core/server_test.go | 34 ++++++++++++++++++++ docs/openapi.yaml | 6 ++-- duckduckgo/search.go | 4 +-- duckduckgo/search_test.go | 65 +++++++++++++++++++++++++++++++++++++++ ecosia/search.go | 14 +++------ google/search.go | 4 +-- google/search_test.go | 8 +++++ google/url.go | 4 +-- yandex/search.go | 18 +++++------ 18 files changed, 231 insertions(+), 41 deletions(-) create mode 100644 core/common_test.go diff --git a/README.md b/README.md index 2810d14..30363de 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ Common parameters: | `date` | Date range | `20250101..20251231` | | `file` | File extension | `pdf`, `doc`, `xls` | | `site` | Site-specific search | `github.com` | -| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` | +| `limit` | Number of organic results, max 100. When omitted or `<=10`, only the first SERP page is parsed. | `25`, `50` | | `start` | Pagination offset | `0`, `10`, `20` | | `format` | Output format | `json`, `markdown`, `text`, `ndjson` | @@ -133,7 +133,7 @@ Engine-specific parameters: | Parameter | Supported engines | Notes | | ---------- | ----------------- | ---------------------------------------------------------------------- | | `filter` | `google` | Duplicate filter: `true` hides similar results, `false` includes them. | -| `features` | browser `Search` | Populate `serp_features[]` from the live page . | +| `features` | browser `Search` | Populate `serp_features[]` from the live page. Defaults to `true`. | ## Search Response Example diff --git a/baidu/all_test.go b/baidu/all_test.go index cc60ff2..11692a9 100644 --- a/baidu/all_test.go +++ b/baidu/all_test.go @@ -35,7 +35,7 @@ func TestBuildURL(t *testing.T) { if got := params.Get("gpc"); got != "stf=1388534400,1672531200|stftype=2" { t.Fatalf("unexpected gpc value: %q", got) } - if got := params.Get("rn"); got != "10" { + if got := params.Get("rn"); got != "" { t.Fatalf("unexpected rn value: %q", got) } if got := params.Get("pn"); got != "" { @@ -140,6 +140,23 @@ func TestBuildImageURL(t *testing.T) { } }, }, + { + name: "small limit omits result count param", + query: core.Query{ + Text: "golang", + Limit: 10, + }, + pageNum: 2, + check: func(t *testing.T, params url.Values, _ string) { + t.Helper() + if got := params.Get("rn"); got != "" { + t.Fatalf("rn should be omitted when Limit<=10, got %q", got) + } + if got := params.Get("pn"); got != "60" { + t.Fatalf("unexpected pn value: %q", got) + } + }, + }, { name: "empty fields return error", query: core.Query{}, diff --git a/baidu/search.go b/baidu/search.go index 13cd3d6..69f2b92 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -278,13 +278,13 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se return false, nil } - for len(searchResults) < query.Limit { + for core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) { done, err := fetchPage() if err != nil { return nil, err } searchPage++ - if done { + if done || !core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) { break } } diff --git a/baidu/url.go b/baidu/url.go index c8eee0f..2c6e449 100644 --- a/baidu/url.go +++ b/baidu/url.go @@ -65,7 +65,7 @@ func BuildURL(q core.Query) (string, error) { logrus.Warn("File search not supported") } - if q.Limit != 0 { + if q.Limit > 10 { params.Add("rn", strconv.Itoa(q.Limit)) } if q.Start < 0 { @@ -104,8 +104,10 @@ func BuildImageURL(q core.Query, pageNum int) (string, error) { return "", errors.New("Empty query built") } - if q.Limit != 0 { - params.Add("rn", "30") // Results per page + if q.Limit > 10 { + params.Add("rn", "30") // Results per page + } + if pageNum > 0 { params.Add("pn", strconv.Itoa(pageNum*30)) // Offset } diff --git a/bing/bing_test.go b/bing/bing_test.go index c5a9965..01e1f48 100644 --- a/bing/bing_test.go +++ b/bing/bing_test.go @@ -82,6 +82,9 @@ func TestBuildURL(t *testing.T) { if got := params.Get("filters"); got != `ex1:"ez5_20454_20570"` { t.Fatalf("unexpected filters value: %q", got) } + if got := params.Get("count"); got != "" { + t.Fatalf("count should be omitted when Limit<=10, got %q", got) + } // LangCode unset → no locale params; let Bing pick defaults // from the request rather than biasing toward en-US. for _, key := range []string{"mkt", "setlang", "cc"} { diff --git a/bing/url.go b/bing/url.go index c2f0811..92fdb8f 100644 --- a/bing/url.go +++ b/bing/url.go @@ -89,7 +89,7 @@ func BuildURL(q core.Query) (string, error) { if q.Start > 0 { // Bing uses 1-based first-result index for pagination. params.Add("first", strconv.Itoa(q.Start+1)) - } else if q.Limit > 0 { + } else if q.Limit > 10 { params.Add("count", strconv.Itoa(q.Limit)) } diff --git a/cmd/root.go b/cmd/root.go index 2d10f11..6420723 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( ) const ( - version = "0.7.15" + version = "0.7.16" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/common.go b/core/common.go index 3892110..2f71269 100644 --- a/core/common.go +++ b/core/common.go @@ -171,6 +171,19 @@ func OrganicLimitReached(results []SearchResult, limit int) bool { return limit > 0 && CountOrganicResults(results) >= limit } +// ShouldFetchResultPage reports whether a paginated engine should fetch another +// SERP page. Small/default limits should use the first SERP page as-is instead +// of chasing a target count across multiple page loads. +func ShouldFetchResultPage(collected, limit, pagesFetched int) bool { + if pagesFetched <= 0 { + return true + } + if limit > 0 && collected >= limit { + return false + } + return limit > defaultQueryLimit +} + // LimitOrganicResults keeps all ads and at most limit non-ad results. func LimitOrganicResults(results []SearchResult, limit int) []SearchResult { if limit <= 0 { @@ -356,7 +369,7 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { return errInvalidParam(fmt.Sprintf("filter: %v", err)) } - searchQuery.Features, err = strconv.ParseBool(reqCtx.Query("features", "0")) + searchQuery.Features, err = strconv.ParseBool(reqCtx.Query("features", "1")) if err != nil { return errInvalidParam(fmt.Sprintf("features: %v", err)) } diff --git a/core/common_test.go b/core/common_test.go new file mode 100644 index 0000000..e2293b0 --- /dev/null +++ b/core/common_test.go @@ -0,0 +1,58 @@ +package core + +import "testing" + +func TestShouldFetchResultPage(t *testing.T) { + tests := []struct { + name string + collected int + limit int + pagesFetched int + want bool + }{ + { + name: "always fetch first page", + limit: 10, + pagesFetched: 0, + want: true, + }, + { + name: "unset limit stops after first page", + collected: 8, + limit: 0, + pagesFetched: 1, + want: false, + }, + { + name: "default limit stops after first page even if short", + collected: 8, + limit: 10, + pagesFetched: 1, + want: false, + }, + { + name: "larger limit can fetch another short page", + collected: 8, + limit: 11, + pagesFetched: 1, + want: true, + }, + { + name: "larger limit stops when satisfied", + collected: 11, + limit: 11, + pagesFetched: 1, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ShouldFetchResultPage(tt.collected, tt.limit, tt.pagesFetched) + if got != tt.want { + t.Fatalf("ShouldFetchResultPage(%d, %d, %d) = %t, want %t", + tt.collected, tt.limit, tt.pagesFetched, got, tt.want) + } + }) + } +} diff --git a/core/server_test.go b/core/server_test.go index 7ea18df..697bb3b 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -341,6 +341,40 @@ func TestInvalidQueryParametersReturnJSONError(t *testing.T) { } } +func TestFeaturesDefaultEnabledAndCanBeDisabled(t *testing.T) { + var got []bool + engine := &engineMock{ + name: "google", + initialized: true, + searchFn: func(_ context.Context, q Query) ([]SearchResult, error) { + got = append(got, q.Features) + return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil + }, + } + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 0 + srv := NewServerWithOptions("127.0.0.1", 7240, opts, engine) + + resp := request(t, srv, "/google/search?text=golang") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected default features request to succeed, got %d", resp.StatusCode) + } + resp = request(t, srv, "/google/search?text=golang&features=false") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected features=false request to succeed, got %d", resp.StatusCode) + } + + if len(got) != 2 { + t.Fatalf("expected 2 engine calls, got %d", len(got)) + } + if !got[0] { + t.Fatal("expected features to default to true") + } + if got[1] { + t.Fatal("expected features=false to disable feature extraction") + } +} + func TestMegaEnginesEndpointResponseFormat(t *testing.T) { google := &engineMock{name: "google", initialized: true} yandex := &engineMock{name: "yandex", initialized: false} diff --git a/docs/openapi.yaml b/docs/openapi.yaml index b6b007e..958d64a 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -637,12 +637,12 @@ components: name: limit in: query required: false - description: Maximum organic results to return (1–100). Ads may be returned in addition. + description: Maximum organic results to return (1-100). Ads may be returned in addition. Omitted or small limits (<=10) parse only the first SERP page; larger limits may paginate when an engine supports it. schema: type: integer minimum: 1 maximum: 100 - default: 25 + default: 10 example: 10 StartQuery: name: start @@ -672,7 +672,7 @@ components: supported by the engine. schema: type: boolean - default: false + default: true EnginesQuery: name: engines in: query diff --git a/duckduckgo/search.go b/duckduckgo/search.go index a80832f..b0e8bfd 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -219,13 +219,13 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] return false, nil } - for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit { + for core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) { done, err := fetchPage() if err != nil { return nil, err } searchPage++ - if done || (query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit) { + if done || !core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) { break } if err := core.SleepContext(ctx, ddg.pageSleep); err != nil { diff --git a/duckduckgo/search_test.go b/duckduckgo/search_test.go index e26f22e..c82c588 100644 --- a/duckduckgo/search_test.go +++ b/duckduckgo/search_test.go @@ -256,3 +256,68 @@ func TestDuckDuckGoLanguageMapping(t *testing.T) { }) } } + +func TestShouldFetchDuckDuckGoPage(t *testing.T) { + results := func(organic, ads int) []core.SearchResult { + out := make([]core.SearchResult, 0, organic+ads) + for i := 0; i < ads; i++ { + out = append(out, core.SearchResult{URL: "https://ad.example/" + string(rune('a'+i)), Ad: true}) + } + for i := 0; i < organic; i++ { + out = append(out, core.SearchResult{URL: "https://example.com/" + string(rune('a'+i))}) + } + return out + } + + tests := []struct { + name string + results []core.SearchResult + limit int + pagesFetched int + want bool + }{ + { + name: "first page is always fetched", + limit: 10, + pagesFetched: 0, + want: true, + }, + { + name: "default limit does not chase a short first page", + results: results(8, 2), + limit: 10, + pagesFetched: 1, + want: false, + }, + { + name: "explicit larger limit can paginate", + results: results(8, 0), + limit: 11, + pagesFetched: 1, + want: true, + }, + { + name: "satisfied larger limit stops", + results: results(11, 0), + limit: 11, + pagesFetched: 1, + want: false, + }, + { + name: "unset internal query stops after first page", + results: results(8, 0), + limit: 0, + pagesFetched: 1, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := core.ShouldFetchResultPage(core.CountOrganicResults(tt.results), tt.limit, tt.pagesFetched) + if got != tt.want { + t.Fatalf("ShouldFetchResultPage() = %t, want %t", got, tt.want) + } + }) + } +} diff --git a/ecosia/search.go b/ecosia/search.go index 9885b4f..0cbb2d3 100644 --- a/ecosia/search.go +++ b/ecosia/search.go @@ -203,16 +203,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S return false, nil } - for query.Limit <= 0 || core.CountOrganicResults(all) < query.Limit { + for core.ShouldFetchResultPage(core.CountOrganicResults(all), query.Limit, pageNum-firstPage) { done, err := fetchPage() if err != nil { return nil, err } pageNum++ - if done { - break - } - if query.Limit > 0 && core.CountOrganicResults(all) >= query.Limit { + if done || !core.ShouldFetchResultPage(core.CountOrganicResults(all), query.Limit, pageNum-firstPage) { break } if err := core.SleepContext(ctx, e.pageSleep); err != nil { @@ -347,16 +344,13 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c return false, nil } - for query.Limit <= 0 || len(out) < query.Limit { + for core.ShouldFetchResultPage(len(out), query.Limit, pageNum) { done, err := fetchPage() if err != nil { return nil, err } pageNum++ - if done { - break - } - if query.Limit > 0 && len(out) >= query.Limit { + if done || !core.ShouldFetchResultPage(len(out), query.Limit, pageNum) { break } if err := core.SleepContext(ctx, e.pageSleep); err != nil { diff --git a/google/search.go b/google/search.go index ef4213f..82a65bc 100644 --- a/google/search.go +++ b/google/search.go @@ -493,7 +493,7 @@ func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.S // cells can hang on right-click. Cap iterations as a last-resort guard. const maxImagePasses = 20 stagnant := 0 - for pass := 0; pass < maxImagePasses && len(searchResultsMap) < query.Limit; pass++ { + for pass := 0; pass < maxImagePasses && core.ShouldFetchResultPage(len(searchResultsMap), query.Limit, pass); pass++ { if err := ctx.Err(); err != nil { return *core.ConvertSearchResultsMap(searchResultsMap), err } @@ -526,7 +526,7 @@ func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.S if err := r.Remove(); err != nil { gogl.logger.Debug("Failed to remove parsed image element: %s", err) } - if len(searchResultsMap) >= query.Limit { + if query.Limit > 0 && len(searchResultsMap) >= query.Limit { break } } diff --git a/google/search_test.go b/google/search_test.go index 867dff4..c7ea644 100644 --- a/google/search_test.go +++ b/google/search_test.go @@ -72,12 +72,16 @@ func TestBuildSearchURL(t *testing.T) { Text: "megadeth tickets", Filter: true, LangCode: "DE", + Limit: 10, }, check: func(t *testing.T, params url.Values, host string) { t.Helper() if host != "www.google.de" { t.Fatalf("unexpected host: %s", host) } + if got := params.Get("num"); got != "" { + t.Fatalf("num should be omitted when Limit<=10, got %q", got) + } if got := params.Get("hl"); got != "de" { t.Fatalf("unexpected hl value: %q", got) } @@ -316,12 +320,16 @@ func TestBuildImageSearchURL(t *testing.T) { Text: "mountains", LangCode: "en", Region: "GB", + Limit: 10, }, check: func(t *testing.T, params url.Values, host string) { t.Helper() if host != "www.google.co.uk" { t.Fatalf("unexpected host: %s", host) } + if got := params.Get("num"); got != "" { + t.Fatalf("num should be omitted when Limit<=10, got %q", got) + } if got := params.Get("gl"); got != "gb" { t.Fatalf("unexpected gl value: %q", got) } diff --git a/google/url.go b/google/url.go index 43a259a..6bbbc2a 100644 --- a/google/url.go +++ b/google/url.go @@ -270,7 +270,7 @@ func BuildURL(q core.Query) (string, error) { } // Limit number of results - if q.Limit != 0 { + if q.Limit > 10 { params.Add("num", strconv.Itoa(q.Limit)) } @@ -352,7 +352,7 @@ func BuildImageURL(q core.Query) (string, error) { } // Limit number of results - if q.Limit != 0 { + if q.Limit > 10 { params.Add("num", strconv.Itoa(q.Limit)) } diff --git a/yandex/search.go b/yandex/search.go index 368bcc8..f6b9c42 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -243,7 +243,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor return false, nil } - for !core.OrganicLimitReached(allResults, query.Limit) { + for core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage-startPage) { done, err := fetchPage() if err != nil { // Yandex commonly challenges rapid pagination, so a later page can @@ -257,7 +257,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor break } searchPage++ - if done || core.OrganicLimitReached(allResults, query.Limit) { + if done || !core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage-startPage) { break } if err := core.SleepContext(ctx, yand.pageSleep); err != nil { @@ -281,7 +281,6 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S yand.logger.Debug("Starting image search, query: %+v", query) searchResults := []core.SearchResult{} - allowPagination := query.Limit > 30 searchPage := 0 // fetchPage loads one image page and appends parsed results. @@ -343,22 +342,19 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S } searchResults = append(searchResults, res) - } - if len(searchResults) >= query.Limit { - return true, nil - } - if searchPage == 1 && !allowPagination { - return true, nil + if query.Limit > 0 && len(searchResults) >= query.Limit { + return true, nil + } } return false, nil } - for len(searchResults) < query.Limit { + for core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) { done, err := fetchPage() if err != nil { return searchResults, err } - if done { + if done || !core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) { break } }