From 46758db4282ab9fbb73d179977b3561e0e4b43cd Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Sat, 24 Jun 2023 22:27:54 +0300 Subject: [PATCH] Check and log errors --- core/browser.go | 28 +++++++++++++++-------- google/search.go | 37 ++++++++++++++++++++++-------- yandex/search.go | 59 +++++++++++++++++++++++++++++------------------- 3 files changed, 82 insertions(+), 42 deletions(-) diff --git a/core/browser.go b/core/browser.go index 15b13e6..0844a90 100644 --- a/core/browser.go +++ b/core/browser.go @@ -12,14 +12,16 @@ import ( ) type BrowserOpts struct { - IsHeadless bool // Use browser interface - IsLeakless bool // Force to kill browser - Timeout time.Duration // Timeout - LanguageCode string - WaitRequests bool // Wait requests to complete after navigation + IsHeadless bool // Use browser interface + IsLeakless bool // Force to kill browser + Timeout time.Duration // Timeout + LanguageCode string + WaitRequests bool // Wait requests to complete after navigation + LeavePageOpen bool // Leave pages and browser open } -func (o *BrowserOpts) Check() { +// Initialize browser parameters with default values if they are not set +func (o *BrowserOpts) Init() { if o.Timeout == 0 { o.Timeout = time.Second * 30 } @@ -36,7 +38,7 @@ type Browser struct { } func NewBrowser(opts BrowserOpts) (*Browser, error) { - opts.Check() + opts.Init() logrus.Debugf("Browser options: %+v", opts) path, has := launcher.LookPath() @@ -64,17 +66,25 @@ func (b *Browser) Navigate(URL string) *rod.Page { b.browser = rod.New().ControlURL(b.browserAddr) b.browser.MustConnect() - b.browser.SetCookies(nil) + //b.browser.SetCookies(nil) page := stealth.MustPage(b.browser) wait := page.MustWaitRequestIdle() page.Navigate(URL) - wait() + + // causes bugs in google + if b.WaitRequests { + wait() + } page.MustEmulate(devices.Device{ UserAgent: uarand.GetRandom(), AcceptLanguage: b.LanguageCode, }) + + // Wait till page loads + time.Sleep(time.Second * 1) + return page } diff --git a/google/search.go b/google/search.go index 14fe6a8..dc6f5a6 100644 --- a/google/search.go +++ b/google/search.go @@ -20,7 +20,7 @@ type Google struct { func New(browser core.Browser) *Google { gogl := Google{Browser: browser} - gogl.checkTimeout = time.Second * 2 + gogl.checkTimeout = time.Second * 5 gogl.findNumRgxp = regexp.MustCompile("\\d") return &gogl } @@ -49,6 +49,11 @@ func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) { return total, nil } +func (gogl *Google) preparePage(page *rod.Page) { + // Remove "similar queries" lists + page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove()); })();") +} + func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { logrus.Tracef("Start Google search, query: %+v", query) @@ -59,17 +64,21 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { if err != nil { return nil, err } + page := gogl.Navigate(url) + gogl.preparePage(page) totalResults, err := gogl.FindTotalResults(page) if err != nil { return nil, err } + logrus.Tracef("%d total results found", totalResults) + if totalResults == 0 { return searchResults, nil } - results, err := page.Search("div>div.g") + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang]") if err != nil { return nil, err } @@ -86,34 +95,42 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { continue } linkText, err := link.Property("href") + if err != nil { + logrus.Error("No `href` tag found") + } // Get title titleTag, err := link.Element("h3") if err != nil { - logrus.Error(err) + logrus.Error("No `h3` tag found") continue } title, err := titleTag.Text() if err != nil { + logrus.Error("Cannot extract text from title") title = "No title" - logrus.Error(err) } // Get description + // doesn't catch all descTag, err := r.Element(`div[data-sncf~="1"]`) - desc := "No description found" - if err == nil { + desc := "" + if err != nil { + logrus.Trace(`No description 'div[data-sncf~="1"]' tag found`) + } else { desc = descTag.MustText() } - gR := core.SearchResult{Rank: i, URL: linkText.String(), Title: title, Description: desc} + gR := core.SearchResult{Rank: i + 1, URL: linkText.String(), Title: title, Description: desc} searchResults = append(searchResults, gR) } - err = page.Close() - if err != nil { - logrus.Error(err) + if !gogl.Browser.LeavePageOpen { + err = page.Close() + if err != nil { + logrus.Error(err) + } } return searchResults, nil diff --git a/yandex/search.go b/yandex/search.go index 3e16499..2fb954f 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -1,8 +1,6 @@ package yandex import ( - "errors" - "fmt" "time" "github.com/go-rod/rod" @@ -13,13 +11,13 @@ import ( type Yandex struct { core.Browser checkTimeout time.Duration // Timeout for secondary elements check - pagesSleep time.Duration // Sleep between pages + pageSleep time.Duration // Sleep between pages } func New(browser core.Browser) *Yandex { yand := Yandex{Browser: browser} yand.checkTimeout = time.Second * 2 - yand.pagesSleep = time.Second * 1 + yand.pageSleep = time.Second * 1 return &yand } @@ -35,17 +33,16 @@ func (yand *Yandex) isCaptcha(page *rod.Page) bool { return true } +// Check if nothig is found func (yand *Yandex) isNoResults(page *rod.Page) bool { noResFound := false _, err := page.Timeout(yand.checkTimeout).Search("div.EmptySearchResults-Title") - fmt.Println(err) if err == nil { noResFound = true } _, err = page.Timeout(yand.checkTimeout).Search("div>div.RequestMeta-Message") - fmt.Println(err) if err == nil { noResFound = true } @@ -63,23 +60,29 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc continue } linkText, err := link.Property("href") + if err != nil { + logrus.Error("No `href` tag found") + } // Get title titleTag, err := link.Element("h2") if err != nil { - logrus.Error("No title tag found") + logrus.Error("No title `h2` tag found") continue } title, err := titleTag.Text() if err != nil { + logrus.Error("Cannot extract text from title") title = "No title" } // Get description descTag, err := r.Element(`span.OrganicTextContentSpan`) - desc := "No description found" - if err == nil { + desc := "" + if err != nil { + logrus.Trace("No description `span.OrganicTextContentSpan` tag found") + } else { desc = descTag.MustText() } @@ -103,33 +106,43 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { } page := yand.Navigate(url) - defer page.Close() - searchRes, _ := page.Timeout(yand.Timeout).Search("li.serp-item") - if searchRes != nil { - elements, _ := searchRes.All() - r := yand.parseResults(elements, searchPage) - allResults = append(allResults, r...) + // Get all search results in page + searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item") + if err != nil { + logrus.Errorf("Cannot parse search results: %s", err) } + // Check why no results, maybe captcha? if searchRes == nil { if yand.isNoResults(page) { - return allResults, nil + logrus.Errorf("No results found") } else if yand.isCaptcha(page) { - logrus.Error(errors.New("Yandex captcha occured during: " + url)) - return allResults, nil + logrus.Errorf("Yandex captcha occurred during: %s", url) } break } - searchPage++ - - err = page.Close() + elements, err := searchRes.All() if err != nil { - logrus.Error(err) + logrus.Errorf("Cannot get all elements from search results: %s", err) + break } - time.Sleep(yand.pagesSleep) + r := yand.parseResults(elements, searchPage) + allResults = append(allResults, r...) + + searchPage++ + + if !yand.Browser.LeavePageOpen { + // Close tab before opening new one during the cycle + err = page.Close() + if err != nil { + logrus.Error(err) + } + } + + time.Sleep(yand.pageSleep) } return allResults, nil