Add filter param to Google. Add start param for supporting engines

README.md

@@ -1,4 +1,4 @@
-# OpenSERP (Search Engine Results Page)
+# OpenSERP (Search Engine Results)
 
 ![OpenSERP](/logo.svg)
 
@@ -8,7 +8,7 @@
 
 <!--[![Docker Pulls](https://img.shields.io/docker/pulls/karust/openserp)](https://hub.docker.com/repository/docker/karust/openserp)-->
 
-**OpenSERP** provides free API access to multiple search engines including **[Google, Yandex, Baidu, Bing, DuckDuckGo]**. Get comprehensive search results without expensive API subscriptions!
+**OpenSERP** provides free API and CLI access to multiple search engines including **[Google, Yandex, Baidu, Bing, DuckDuckGo]**. Get comprehensive search results without expensive API subscriptions!
 
 ## Features
 
@@ -75,8 +75,8 @@ curl "http://localhost:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bin
   {
     "rank": 2,
     "url": "https://www.bing.com/ck/a?!&&p=6f15ac4589858d0a104cd6f55cc8e91e8d8d6da91f905b626921f67f2323a467JmltdHM9MTc1OTE5MDQwMA&ptn=3&ver=2&hsh=4&fclid=2357c2f4-6131-68de-359f-d48c607c691d&u=a1aHR0cHM6Ly93d3cuZ29sZGVucmV0cmlldmVyZm9ydW0uY29tL3RocmVhZHMvdW5kZXJzdGFuZGluZy13aHktZ29sZGVuLXJldHJpZXZlciVFMiU4MCU5OXMtbGlmZXNwYW4taGFsdmVkLWluLXRoZS1sYXN0LTM1LXllYXJzLjM1NzMyMi8&ntb=1",
-    "title": "Golden Retriever Dog Forums\nhttps://www.goldenretrieverforum.com › threads › understanding-why-g…",
-    "description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now living until 9 or 10 years old. Golden Retrievers seem to be dying mostly of bone …",
+    "title": "Golden Retriever Dog Forums\nhttps://www.goldenretrieverforum.com › threads › understanding-why-g...",
+    "description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now living until 9 or 10 years old. Golden Retrievers seem to be dying mostly of bone ...",
     "ad": false,
     "engine": "bing"
   },
@@ -119,7 +119,14 @@ curl "http://localhost:7000/mega/engines"
 | `file`    | File extension       | `PDF`, `DOC`, `XLS`               |
 | `site`    | Site-specific search | `github.com`, `stackoverflow.com` |
 | `limit`   | Number of results    | `10`, `25`, `50`                  |
-| `answers` | Include Q&A results  | `true`, `false`                   |
+
+### Engine-Specific Parameters
+
+| Parameter | Supported engines                   | Notes                                                                  |
+| --------- | ----------------------------------- | ---------------------------------------------------------------------- |
+| `start`   | `google`, `bing`, `yandex`, `baidu` | Web search pagination offset.                                          |
+| `filter`  | `google`                            | Duplicate filter (`true` => hide similar, `false` => include similar). |
+| `answers` | `google`                            | Include Google answer boxes in output (negative ranks).                |
 
 ### Individual Engine Examples
 
@@ -129,6 +136,15 @@ curl "http://localhost:7000/duck/search?text=golang&limit=7"
 
 # Google search
 curl "http://localhost:7000/google/search?text=golang&lang=EN&limit=10"
+
+# Google search (pagination + include similar/hidden results)
+curl "http://localhost:7000/google/search?text=golang&lang=EN&limit=10&start=10&filter=false"
+
+# Bing search (offset 20 => around page 3 for 10 results/page)
+curl "http://localhost:7000/bing/search?text=golang&limit=10&start=20"
+
+# Yandex search (offset 10 => second page)
+curl "http://localhost:7000/yandex/search?text=golang&limit=10&start=10"
 ```
 
 ### Image Search
@@ -153,6 +169,24 @@ OpenSERP supports HTTP and SOCKS5 proxies with authentication:
 ./openserp search bing "query" --proxy http://user:pass@127.0.0.1:8080
 ```
 
+## Health Check
+
+```bash
+curl -i "http://127.0.0.1:7000/health"
+```
+
+Response includes:
+
+- `status`: `healthy`, `degraded`, or `unhealthy`
+- `uptime`
+- per-engine readiness in `engines`
+- basic runtime stats in `system`
+
+HTTP status behavior:
+
+- `200` for `healthy` and `degraded`
+- `503` for `unhealthy`
+
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

baidu/search.go

@@ -131,7 +131,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 		}
 		desc = strings.ReplaceAll(desc, title, "")
 
-		gR := core.SearchResult{Rank: i + 1, URL: linkText.String(), Title: title, Description: desc}
+		gR := core.SearchResult{Rank: query.Start + i + 1, URL: linkText.String(), Title: title, Description: desc}
 		searchResults = append(searchResults, gR)
 	}
 

baidu/search_raw.go

@@ -138,6 +138,11 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
+	if query.Start > 0 {
+		for i := range results {
+			results[i].Rank = query.Start + i + 1
+		}
+	}
 	logrus.Debugf("Baidu Raw results : %v", results)
 
 	return core.DeduplicateResults(results), nil

baidu/url.go

@@ -66,6 +66,13 @@ func BuildURL(q core.Query) (string, error) {
 	if q.Limit != 0 {
 		params.Add("rn", strconv.Itoa(q.Limit))
 	}
+	if q.Start < 0 {
+		return "", errors.New("incorrect start provided")
+	}
+	if q.Start > 0 {
+		// Baidu uses "pn" as result offset for pagination.
+		params.Add("pn", strconv.Itoa(q.Start))
+	}
 
 	if len(params.Get("wd")) == 0 {
 		return "", errors.New("Empty query built")

bing/search.go

@@ -140,7 +140,7 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
 
-	rank := 0
+	rank := query.Start
 	for _, result := range organicElements {
 		srchRes := core.SearchResult{}
 

bing/url.go

@@ -43,8 +43,15 @@ func BuildURL(q core.Query) (string, error) {
 		params.Add("setlang", strings.ToLower(q.LangCode))
 	}
 
-	// Set result offset (pagination) - Bing uses "first" parameter
-	if q.Limit > 0 {
+	// Set result offset (pagination) - Bing uses "first" parameter.
+	// When first is present, Bing may ignore custom count and return default page size.
+	if q.Start < 0 {
+		return "", errors.New("incorrect start provided")
+	}
+	if q.Start > 0 {
+		// Bing uses 1-based first-result index for pagination.
+		params.Add("first", strconv.Itoa(q.Start+1))
+	} else if q.Limit > 0 {
 		params.Add("count", strconv.Itoa(q.Limit))
 	}
 

cmd/root.go

@@ -13,7 +13,7 @@ import (
 )
 
 const (
-	version               = "0.5.4"
+	version               = "0.5.5"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )

cmd/search.go

@@ -30,6 +30,7 @@ func search(cmd *cobra.Command, args []string) {
 	query := core.Query{
 		Text:     args[1],
 		Limit:    10,
+		Filter:   true,
 		ProxyURL: config.App.ProxyURL,
 		Insecure: config.App.Insecure,
 	}

core/common.go

@@ -60,11 +60,23 @@ type Query struct {
 	Filetype     string // File extension to search.
 	Site         string // Search site
 	Limit        int    // Limit the number of results
+	Start        int    // Search offset for pagination (Google uses 0, 10, 20...)
+	Filter       bool   // Filter duplicates (google) (false: include similar, true: hide similar)
 	Answers      bool   // Include question and answers from SERP page to results with negative indexes
 	ProxyURL     string // Proxy URL for raw requests
 	Insecure     bool   // Allow insecure TLS connections
 }
 
+func ComputePagination(start int, pageSize int) (int, int, error) {
+	if pageSize <= 0 {
+		return 0, 0, errors.New("pageSize must be > 0")
+	}
+	if start < 0 {
+		return 0, 0, errors.New("start must be >= 0")
+	}
+	return start / pageSize, start % pageSize, nil
+}
+
 func (q Query) IsEmpty() bool {
 	if q.Site == "" && q.Filetype == "" && q.Text == "" {
 		return true
@@ -85,6 +97,20 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
 	}
 	searchQuery.Limit = limit
 
+	start, err := strconv.Atoi(reqCtx.Query("start", "0"))
+	if err != nil {
+		return err
+	}
+	if start < 0 {
+		return errors.New("start must be >= 0")
+	}
+	searchQuery.Start = start
+
+	searchQuery.Filter, err = strconv.ParseBool(reqCtx.Query("filter", "1"))
+	if err != nil {
+		return err
+	}
+
 	searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0"))
 	if err != nil {
 		return err

google/search.go

@@ -113,9 +113,11 @@ func (gogl *Google) checkCaptcha(page *rod.Page) bool {
 
 func (gogl *Google) preparePage(page *rod.Page) {
 	// Remove "similar queries" lists
-	_, err := page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
+	_, err := page.Eval(`() => {
+		document.querySelectorAll("div[data-initq]").forEach((el) => el.remove())
+	}`)
 	if err != nil {
-		gogl.logger.Error("Page preparation failed: %s", err)
+		gogl.logger.Debug("Page preparation skipped: %s", err)
 	}
 }
 
@@ -185,7 +187,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 		return nil, err
 	}
 
-	rank := 0
+	rank := query.Start
 	for _, resEl := range searchResultElems {
 		srchRes := core.SearchResult{}
 

google/search_raw.go

@@ -162,6 +162,12 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	if query.Start > 0 {
+		for i := range results {
+			results[i].Rank = query.Start + i + 1
+		}
+	}
 	logrus.Debugf("Google Raw results : %v", results)
 
 	return results, nil

google/url.go

@@ -259,6 +259,19 @@ func BuildURL(q core.Query) (string, error) {
 		params.Add("num", strconv.Itoa(q.Limit))
 	}
 
+	// Set result offset for pagination
+	if q.Start < 0 {
+		return "", errors.New("incorrect start param provided")
+	}
+	if q.Start > 0 {
+		params.Add("start", strconv.Itoa(q.Start))
+	}
+
+	// Google default is filter=1; send only when user asks to include similar results.
+	if !q.Filter {
+		params.Add("filter", "0")
+	}
+
 	if q.LangCode != "" {
 		params.Add("hl", q.LangCode)
 		params.Add("lr", "lang_"+strings.ToLower(q.LangCode))

yandex/search.go

@@ -115,9 +115,17 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
 
 func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 	yand.logger.Debug("Starting search, query: %+v", query)
+	if query.Start < 0 {
+		return nil, fmt.Errorf("incorrect start provided")
+	}
 
 	allResults := []core.SearchResult{}
-	searchPage := 0
+	const pageSize = 10
+	searchPage, skipOnFirstPage, err := core.ComputePagination(query.Start, pageSize)
+	if err != nil {
+		return nil, err
+	}
+	startPage := searchPage
 
 	for len(allResults) < query.Limit {
 		url, err := BuildURL(query, searchPage)
@@ -158,6 +166,13 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 		}
 
 		r := yand.parseResults(elements, searchPage)
+		if searchPage == startPage && skipOnFirstPage > 0 {
+			if skipOnFirstPage >= len(r) {
+				r = []core.SearchResult{}
+			} else {
+				r = r[skipOnFirstPage:]
+			}
+		}
 		allResults = append(allResults, r...)
 
 		searchPage++

yandex/search_raw.go

@@ -120,7 +120,12 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
 }
 
 func Search(query core.Query) ([]core.SearchResult, error) {
-	googleURL, err := BuildURL(query, 1)
+	startPage, skipOnFirstPage, err := core.ComputePagination(query.Start, 10)
+	if err != nil {
+		return nil, err
+	}
+
+	googleURL, err := BuildURL(query, startPage)
 	if err != nil {
 		return nil, err
 	}
@@ -136,6 +141,19 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	if skipOnFirstPage > 0 {
+		if skipOnFirstPage >= len(results) {
+			results = []core.SearchResult{}
+		} else {
+			results = results[skipOnFirstPage:]
+		}
+	}
+	if query.Start > 0 {
+		for i := range results {
+			results[i].Rank = query.Start + i + 1
+		}
+	}
 	logrus.Debugf("Yandex Raw results : %v", results)
 
 	return results, nil