fix: Remove duplicated results for all search engines and both modes

baidu/search.go

@@ -138,7 +138,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }
 
 func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
@@ -231,5 +231,5 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }

baidu/search_raw.go

@@ -113,5 +113,5 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Baidu Raw results : %v", results)
 
-	return results, nil
+	return core.DeduplicateResults(results), nil
 }

core/common.go

@@ -20,6 +20,26 @@ type SearchResult struct {
 	Ad          bool   `json:"ad"`
 }
 
+func DeduplicateResults(results []SearchResult) []SearchResult {
+	unique := make(map[string]bool)
+	var deduped []SearchResult
+
+	for _, result := range results {
+		if result.URL == "" {
+			continue
+		}
+		if !unique[result.URL] {
+			unique[result.URL] = true
+			deduped = append(deduped, result)
+		}
+	}
+
+	sort.Slice(deduped, func(i, j int) bool {
+		return deduped[i].Rank < deduped[j].Rank
+	})
+	return deduped
+}
+
 func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
 	searchResults := []SearchResult{}
 

google/search.go

@@ -132,6 +132,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
 	btnElms[3].Click(proto.InputMouseButtonLeft, 1)
 
 }
+
 func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Google search, query: %+v", query)
 
@@ -281,23 +282,11 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 				srchRes.URL = href.String()
 			}
 
-			// Skip if URL is empty or we've already seen this URL
+			// Skip if URL is empty
 			if srchRes.URL == "" {
 				continue
 			}
 
-			// Check for duplicates
-			isDuplicate := false
-			for _, existing := range searchResults {
-				if existing.URL == srchRes.URL {
-					isDuplicate = true
-					break
-				}
-			}
-			if isDuplicate {
-				continue
-			}
-
 			// Get description using multiple fallback strategies
 			desc := ""
 			if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil {
@@ -337,7 +326,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 		searchResults = append(searchResults, srchRes)
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }
 
 func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {

google/search_raw.go

@@ -115,7 +115,7 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
 	}
 
 	logrus.Tracef("Google search document size: %d", len(doc.Text()))
-	return results, err
+	return core.DeduplicateResults(results), err
 }
 
 func Search(query core.Query) ([]core.SearchResult, error) {

yandex/search.go

@@ -169,7 +169,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 		time.Sleep(yand.pageSleep)
 	}
 
-	return allResults, nil
+	return core.DeduplicateResults(allResults), nil
 }
 
 func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
@@ -242,5 +242,5 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		return searchResults[i].Rank < searchResults[j].Rank
 	})
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }

yandex/search_raw.go

@@ -89,7 +89,7 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
 	}
 
 	logrus.Tracef("Yandex search document size: %d", len(doc.Text()))
-	return results, err
+	return core.DeduplicateResults(results), err
 }
 
 func Search(query core.Query) ([]core.SearchResult, error) {