From 29a7c2ca142966babd574a575b1e035964294fda Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Sun, 23 Jul 2023 04:35:48 +0300 Subject: [PATCH] Yandex images scroll parsing --- yandex/search.go | 127 +++++++++++++++++++++--------------------- yandex/search_test.go | 10 +--- yandex/url.go | 4 +- 3 files changed, 67 insertions(+), 74 deletions(-) diff --git a/yandex/search.go b/yandex/search.go index 063d533..5d8b36e 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -5,6 +5,7 @@ import ( "time" "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/input" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" "golang.org/x/time/rate" @@ -177,85 +178,81 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { return allResults, nil } -func (yand *Yandex) parseImageResults(results rod.Elements, pageNum int) []core.SearchResult { - searchResults := []core.SearchResult{} - - for i, r := range results { - - dataAttr, err := r.Attribute("data-bem") - if err != nil { - continue - } - - var data YandexImageData - - err = json.Unmarshal([]byte(*dataAttr), &data) - if err != nil { - logrus.Errorf("Cannot unmarshal yandex image: %v\nData: %v", err, *dataAttr) - continue - } - - linkText := data.SerpItem.ImgHref - title := data.SerpItem.Snippet.Title - description := data.SerpItem.Snippet.Text - - r := core.SearchResult{Rank: (pageNum * 10) + (i + 1), URL: linkText, Title: title, Description: description} - searchResults = append(searchResults, r) - } - - return searchResults -} - func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { logrus.Tracef("Start Yandex image search, query: %+v", query) - allResults := []core.SearchResult{} - searchPage := 0 - - url, err := BuildImageURL(query, searchPage) + searchResultsMap := map[string]core.SearchResult{} + url, err := BuildImageURL(query) if err != nil { return nil, err } page := yand.Navigate(url) - - // Get all search results in page - searchRes, err := page.Timeout(yand.Timeout).Search("div.serp-item") - if err != nil { + if !yand.LeavePageOpen { defer page.Close() - logrus.Errorf("Cannot parse search results: %s", err) - return nil, core.ErrSearchTimeout } - // Check why no results, maybe captcha? - if searchRes == nil { - defer page.Close() + for len(searchResultsMap) < query.Limit { + page.WaitLoad() + page.Keyboard.Press(input.End) + page.WaitLoad() + time.Sleep(time.Duration(time.Second * 2)) - if yand.isNoResults(page) { - logrus.Errorf("No results found") - } else if yand.isCaptcha(page) { - logrus.Errorf("Yandex captcha occurred during: %s", url) - return nil, core.ErrCaptcha - } - return nil, err - } - - elements, err := searchRes.All() - if err != nil { - logrus.Errorf("Cannot get all elements from search results: %s", err) - return nil, err - } - - r := yand.parseImageResults(elements, searchPage) - allResults = append(allResults, r...) - - if !yand.Browser.LeavePageOpen { - // Close tab before opening new one during the cycle - err = page.Close() + // Get all search results in page + results, err := page.Timeout(yand.Timeout).Search("div.serp-item") if err != nil { - logrus.Error(err) + logrus.Errorf("Cannot find search results: %s", err) + } + + // Check why no results + if results == nil { + if yand.isCaptcha(page) { + logrus.Errorf("Yandex captcha occurred during: %s", url) + return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha + } else if yand.isNoResults(page) { + logrus.Errorf("No results found") + } + return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout + } + + for i := 0; i < results.ResultCount; i++ { + r, err := results.Get(i, 1) + if err != nil { + logrus.Errorf("Cannot [%v] element from search result, [%v total]: %s", i, results.ResultCount, err) + return *core.ConvertSearchResultsMap(searchResultsMap), err + } + + dataAttr, err := r[0].Attribute("data-bem") + if err != nil { + continue + } + + var data YandexImageData + + err = json.Unmarshal([]byte(*dataAttr), &data) + if err != nil { + logrus.Errorf("Cannot unmarshal yandex image data: %v\nData: %v", err, *dataAttr) + continue + } + + linkText := data.SerpItem.ImgHref + title := data.SerpItem.Snippet.Title + description := data.SerpItem.Snippet.Text + + yR := core.SearchResult{ + Rank: (i + 1), + URL: linkText, + Title: title, + Description: description, + } + + searchResultsMap[linkText+title] = yR + } + + if !yand.LeavePageOpen { + page.Close() } } - return allResults, nil + return *core.ConvertSearchResultsMap(searchResultsMap), nil } diff --git a/yandex/search_test.go b/yandex/search_test.go index 1a37a6c..a5bdedb 100644 --- a/yandex/search_test.go +++ b/yandex/search_test.go @@ -1,7 +1,6 @@ package yandex import ( - "fmt" "testing" "time" @@ -11,7 +10,7 @@ import ( var browser *core.Browser func init() { - opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 5} + opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 15, LeavePageOpen: true} browser, _ = core.NewBrowser(opts) } @@ -31,18 +30,15 @@ func TestSearchYandex(t *testing.T) { } func TestImageYandex(t *testing.T) { - yand := New(*browser, core.SearchEngineOptions{}) - query := core.Query{Text: "furry tiger"} + query := core.Query{Text: "furry tiger", Limit: 90} results, err := yand.SearchImage(query) if err != nil { t.Fatalf("Cannot [ImageYandex]: %s", err) } - if len(results) == 0 { + if len(results) < 90 { t.Fatalf("[ImageYandex] returned empty result") } - - fmt.Println(results) } diff --git a/yandex/url.go b/yandex/url.go index 2afca01..536d4ef 100644 --- a/yandex/url.go +++ b/yandex/url.go @@ -42,7 +42,7 @@ func BuildURL(q core.Query, page int) (string, error) { return base.String(), nil } -func BuildImageURL(q core.Query, page int) (string, error) { +func BuildImageURL(q core.Query) (string, error) { // TODO: Add other parameters base, _ := url.Parse(baseURL) base.Path += "images/search/" @@ -56,7 +56,7 @@ func BuildImageURL(q core.Query, page int) (string, error) { } params.Add("text", text) - params.Add("p", fmt.Sprint(page)) + //params.Add("p", fmt.Sprint(page)) } if len(params.Get("text")) == 0 {