diff --git a/baidu/search.go b/baidu/search.go index e4edd1e..ef87845 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -138,7 +138,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { } } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { @@ -231,5 +231,5 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { } } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 188fd7b..998e3b0 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -113,5 +113,5 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Baidu Raw results : %v", results) - return results, nil + return core.DeduplicateResults(results), nil } diff --git a/core/common.go b/core/common.go index bf0cfad..6670c2c 100644 --- a/core/common.go +++ b/core/common.go @@ -20,6 +20,26 @@ type SearchResult struct { Ad bool `json:"ad"` } +func DeduplicateResults(results []SearchResult) []SearchResult { + unique := make(map[string]bool) + var deduped []SearchResult + + for _, result := range results { + if result.URL == "" { + continue + } + if !unique[result.URL] { + unique[result.URL] = true + deduped = append(deduped, result) + } + } + + sort.Slice(deduped, func(i, j int) bool { + return deduped[i].Rank < deduped[j].Rank + }) + return deduped +} + func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult { searchResults := []SearchResult{} diff --git a/google/search.go b/google/search.go index 491bdd6..9d79e84 100644 --- a/google/search.go +++ b/google/search.go @@ -132,6 +132,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) { btnElms[3].Click(proto.InputMouseButtonLeft, 1) } + func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { logrus.Tracef("Start Google search, query: %+v", query) @@ -281,23 +282,11 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { srchRes.URL = href.String() } - // Skip if URL is empty or we've already seen this URL + // Skip if URL is empty if srchRes.URL == "" { continue } - // Check for duplicates - isDuplicate := false - for _, existing := range searchResults { - if existing.URL == srchRes.URL { - isDuplicate = true - break - } - } - if isDuplicate { - continue - } - // Get description using multiple fallback strategies desc := "" if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil { @@ -337,7 +326,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, srchRes) } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { diff --git a/google/search_raw.go b/google/search_raw.go index 59c15ff..72543bc 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -115,7 +115,7 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { } logrus.Tracef("Google search document size: %d", len(doc.Text())) - return results, err + return core.DeduplicateResults(results), err } func Search(query core.Query) ([]core.SearchResult, error) { diff --git a/yandex/search.go b/yandex/search.go index d7ffb5e..853df64 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -169,7 +169,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { time.Sleep(yand.pageSleep) } - return allResults, nil + return core.DeduplicateResults(allResults), nil } func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { @@ -242,5 +242,5 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { return searchResults[i].Rank < searchResults[j].Rank }) - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } diff --git a/yandex/search_raw.go b/yandex/search_raw.go index 102a721..69d8884 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -89,7 +89,7 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { } logrus.Tracef("Yandex search document size: %d", len(doc.Text())) - return results, err + return core.DeduplicateResults(results), err } func Search(query core.Query) ([]core.SearchResult, error) {