From 7cdd358cad1945d9d0a1323619ac8d9b2d224dd9 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Tue, 17 Mar 2026 01:54:25 +0300 Subject: [PATCH] Add filter param to Google. Add start param for supporting engines --- README.md | 44 +++++++++++++++++++++++++++++++++++++++----- baidu/search.go | 2 +- baidu/search_raw.go | 5 +++++ baidu/url.go | 7 +++++++ bing/search.go | 2 +- bing/url.go | 11 +++++++++-- cmd/root.go | 2 +- cmd/search.go | 1 + core/common.go | 26 ++++++++++++++++++++++++++ google/search.go | 8 +++++--- google/search_raw.go | 6 ++++++ google/url.go | 13 +++++++++++++ yandex/search.go | 17 ++++++++++++++++- yandex/search_raw.go | 20 +++++++++++++++++++- 14 files changed, 149 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 742ab91..9c2083e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# OpenSERP (Search Engine Results Page) +# OpenSERP (Search Engine Results) ![OpenSERP](/logo.svg) @@ -8,7 +8,7 @@ -**OpenSERP** provides free API access to multiple search engines including **[Google, Yandex, Baidu, Bing, DuckDuckGo]**. Get comprehensive search results without expensive API subscriptions! +**OpenSERP** provides free API and CLI access to multiple search engines including **[Google, Yandex, Baidu, Bing, DuckDuckGo]**. Get comprehensive search results without expensive API subscriptions! ## Features @@ -75,8 +75,8 @@ curl "http://localhost:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bin { "rank": 2, "url": "https://www.bing.com/ck/a?!&&p=6f15ac4589858d0a104cd6f55cc8e91e8d8d6da91f905b626921f67f2323a467JmltdHM9MTc1OTE5MDQwMA&ptn=3&ver=2&hsh=4&fclid=2357c2f4-6131-68de-359f-d48c607c691d&u=a1aHR0cHM6Ly93d3cuZ29sZGVucmV0cmlldmVyZm9ydW0uY29tL3RocmVhZHMvdW5kZXJzdGFuZGluZy13aHktZ29sZGVuLXJldHJpZXZlciVFMiU4MCU5OXMtbGlmZXNwYW4taGFsdmVkLWluLXRoZS1sYXN0LTM1LXllYXJzLjM1NzMyMi8&ntb=1", - "title": "Golden Retriever Dog Forums\nhttps://www.goldenretrieverforum.com › threads › understanding-why-g…", - "description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now living until 9 or 10 years old. Golden Retrievers seem to be dying mostly of bone …", + "title": "Golden Retriever Dog Forums\nhttps://www.goldenretrieverforum.com › threads › understanding-why-g...", + "description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now living until 9 or 10 years old. Golden Retrievers seem to be dying mostly of bone ...", "ad": false, "engine": "bing" }, @@ -119,7 +119,14 @@ curl "http://localhost:7000/mega/engines" | `file` | File extension | `PDF`, `DOC`, `XLS` | | `site` | Site-specific search | `github.com`, `stackoverflow.com` | | `limit` | Number of results | `10`, `25`, `50` | -| `answers` | Include Q&A results | `true`, `false` | + +### Engine-Specific Parameters + +| Parameter | Supported engines | Notes | +| --------- | ----------------------------------- | ---------------------------------------------------------------------- | +| `start` | `google`, `bing`, `yandex`, `baidu` | Web search pagination offset. | +| `filter` | `google` | Duplicate filter (`true` => hide similar, `false` => include similar). | +| `answers` | `google` | Include Google answer boxes in output (negative ranks). | ### Individual Engine Examples @@ -129,6 +136,15 @@ curl "http://localhost:7000/duck/search?text=golang&limit=7" # Google search curl "http://localhost:7000/google/search?text=golang&lang=EN&limit=10" + +# Google search (pagination + include similar/hidden results) +curl "http://localhost:7000/google/search?text=golang&lang=EN&limit=10&start=10&filter=false" + +# Bing search (offset 20 => around page 3 for 10 results/page) +curl "http://localhost:7000/bing/search?text=golang&limit=10&start=20" + +# Yandex search (offset 10 => second page) +curl "http://localhost:7000/yandex/search?text=golang&limit=10&start=10" ``` ### Image Search @@ -153,6 +169,24 @@ OpenSERP supports HTTP and SOCKS5 proxies with authentication: ./openserp search bing "query" --proxy http://user:pass@127.0.0.1:8080 ``` +## Health Check + +```bash +curl -i "http://127.0.0.1:7000/health" +``` + +Response includes: + +- `status`: `healthy`, `degraded`, or `unhealthy` +- `uptime` +- per-engine readiness in `engines` +- basic runtime stats in `system` + +HTTP status behavior: + +- `200` for `healthy` and `degraded` +- `503` for `unhealthy` + ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/baidu/search.go b/baidu/search.go index f2ecabc..b547be1 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -131,7 +131,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { } desc = strings.ReplaceAll(desc, title, "") - gR := core.SearchResult{Rank: i + 1, URL: linkText.String(), Title: title, Description: desc} + gR := core.SearchResult{Rank: query.Start + i + 1, URL: linkText.String(), Title: title, Description: desc} searchResults = append(searchResults, gR) } diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 9da8e71..421768f 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -138,6 +138,11 @@ func Search(query core.Query) ([]core.SearchResult, error) { if err != nil { return nil, err } + if query.Start > 0 { + for i := range results { + results[i].Rank = query.Start + i + 1 + } + } logrus.Debugf("Baidu Raw results : %v", results) return core.DeduplicateResults(results), nil diff --git a/baidu/url.go b/baidu/url.go index 3e46952..2e310d5 100644 --- a/baidu/url.go +++ b/baidu/url.go @@ -66,6 +66,13 @@ func BuildURL(q core.Query) (string, error) { if q.Limit != 0 { params.Add("rn", strconv.Itoa(q.Limit)) } + if q.Start < 0 { + return "", errors.New("incorrect start provided") + } + if q.Start > 0 { + // Baidu uses "pn" as result offset for pagination. + params.Add("pn", strconv.Itoa(q.Start)) + } if len(params.Get("wd")) == 0 { return "", errors.New("Empty query built") diff --git a/bing/search.go b/bing/search.go index 40f7a00..e39f029 100644 --- a/bing/search.go +++ b/bing/search.go @@ -140,7 +140,7 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { } bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements)) - rank := 0 + rank := query.Start for _, result := range organicElements { srchRes := core.SearchResult{} diff --git a/bing/url.go b/bing/url.go index 5bce130..84f8661 100644 --- a/bing/url.go +++ b/bing/url.go @@ -43,8 +43,15 @@ func BuildURL(q core.Query) (string, error) { params.Add("setlang", strings.ToLower(q.LangCode)) } - // Set result offset (pagination) - Bing uses "first" parameter - if q.Limit > 0 { + // Set result offset (pagination) - Bing uses "first" parameter. + // When first is present, Bing may ignore custom count and return default page size. + if q.Start < 0 { + return "", errors.New("incorrect start provided") + } + if q.Start > 0 { + // Bing uses 1-based first-result index for pagination. + params.Add("first", strconv.Itoa(q.Start+1)) + } else if q.Limit > 0 { params.Add("count", strconv.Itoa(q.Limit)) } diff --git a/cmd/root.go b/cmd/root.go index 3b6fe79..509ce85 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,7 @@ import ( ) const ( - version = "0.5.4" + version = "0.5.5" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/cmd/search.go b/cmd/search.go index 9b243a2..a32720c 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -30,6 +30,7 @@ func search(cmd *cobra.Command, args []string) { query := core.Query{ Text: args[1], Limit: 10, + Filter: true, ProxyURL: config.App.ProxyURL, Insecure: config.App.Insecure, } diff --git a/core/common.go b/core/common.go index 6670c2c..82e5793 100644 --- a/core/common.go +++ b/core/common.go @@ -60,11 +60,23 @@ type Query struct { Filetype string // File extension to search. Site string // Search site Limit int // Limit the number of results + Start int // Search offset for pagination (Google uses 0, 10, 20...) + Filter bool // Filter duplicates (google) (false: include similar, true: hide similar) Answers bool // Include question and answers from SERP page to results with negative indexes ProxyURL string // Proxy URL for raw requests Insecure bool // Allow insecure TLS connections } +func ComputePagination(start int, pageSize int) (int, int, error) { + if pageSize <= 0 { + return 0, 0, errors.New("pageSize must be > 0") + } + if start < 0 { + return 0, 0, errors.New("start must be >= 0") + } + return start / pageSize, start % pageSize, nil +} + func (q Query) IsEmpty() bool { if q.Site == "" && q.Filetype == "" && q.Text == "" { return true @@ -85,6 +97,20 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { } searchQuery.Limit = limit + start, err := strconv.Atoi(reqCtx.Query("start", "0")) + if err != nil { + return err + } + if start < 0 { + return errors.New("start must be >= 0") + } + searchQuery.Start = start + + searchQuery.Filter, err = strconv.ParseBool(reqCtx.Query("filter", "1")) + if err != nil { + return err + } + searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0")) if err != nil { return err diff --git a/google/search.go b/google/search.go index 0f2a2c9..04b21ae 100644 --- a/google/search.go +++ b/google/search.go @@ -113,9 +113,11 @@ func (gogl *Google) checkCaptcha(page *rod.Page) bool { func (gogl *Google) preparePage(page *rod.Page) { // Remove "similar queries" lists - _, err := page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove()); })();") + _, err := page.Eval(`() => { + document.querySelectorAll("div[data-initq]").forEach((el) => el.remove()) + }`) if err != nil { - gogl.logger.Error("Page preparation failed: %s", err) + gogl.logger.Debug("Page preparation skipped: %s", err) } } @@ -185,7 +187,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { return nil, err } - rank := 0 + rank := query.Start for _, resEl := range searchResultElems { srchRes := core.SearchResult{} diff --git a/google/search_raw.go b/google/search_raw.go index 7fd403d..599bc5f 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -162,6 +162,12 @@ func Search(query core.Query) ([]core.SearchResult, error) { if err != nil { return nil, err } + + if query.Start > 0 { + for i := range results { + results[i].Rank = query.Start + i + 1 + } + } logrus.Debugf("Google Raw results : %v", results) return results, nil diff --git a/google/url.go b/google/url.go index 84bffb1..fb1d5d3 100644 --- a/google/url.go +++ b/google/url.go @@ -259,6 +259,19 @@ func BuildURL(q core.Query) (string, error) { params.Add("num", strconv.Itoa(q.Limit)) } + // Set result offset for pagination + if q.Start < 0 { + return "", errors.New("incorrect start param provided") + } + if q.Start > 0 { + params.Add("start", strconv.Itoa(q.Start)) + } + + // Google default is filter=1; send only when user asks to include similar results. + if !q.Filter { + params.Add("filter", "0") + } + if q.LangCode != "" { params.Add("hl", q.LangCode) params.Add("lr", "lang_"+strings.ToLower(q.LangCode)) diff --git a/yandex/search.go b/yandex/search.go index 8807125..dbc501a 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -115,9 +115,17 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { yand.logger.Debug("Starting search, query: %+v", query) + if query.Start < 0 { + return nil, fmt.Errorf("incorrect start provided") + } allResults := []core.SearchResult{} - searchPage := 0 + const pageSize = 10 + searchPage, skipOnFirstPage, err := core.ComputePagination(query.Start, pageSize) + if err != nil { + return nil, err + } + startPage := searchPage for len(allResults) < query.Limit { url, err := BuildURL(query, searchPage) @@ -158,6 +166,13 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { } r := yand.parseResults(elements, searchPage) + if searchPage == startPage && skipOnFirstPage > 0 { + if skipOnFirstPage >= len(r) { + r = []core.SearchResult{} + } else { + r = r[skipOnFirstPage:] + } + } allResults = append(allResults, r...) searchPage++ diff --git a/yandex/search_raw.go b/yandex/search_raw.go index 87d7e00..65b54f3 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -120,7 +120,12 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { } func Search(query core.Query) ([]core.SearchResult, error) { - googleURL, err := BuildURL(query, 1) + startPage, skipOnFirstPage, err := core.ComputePagination(query.Start, 10) + if err != nil { + return nil, err + } + + googleURL, err := BuildURL(query, startPage) if err != nil { return nil, err } @@ -136,6 +141,19 @@ func Search(query core.Query) ([]core.SearchResult, error) { if err != nil { return nil, err } + + if skipOnFirstPage > 0 { + if skipOnFirstPage >= len(results) { + results = []core.SearchResult{} + } else { + results = results[skipOnFirstPage:] + } + } + if query.Start > 0 { + for i := range results { + results[i].Rank = query.Start + i + 1 + } + } logrus.Debugf("Yandex Raw results : %v", results) return results, nil