Check and log errors

core/browser.go

@@ -12,14 +12,16 @@ import (
 )
 
 type BrowserOpts struct {
-	IsHeadless   bool          // Use browser interface
-	IsLeakless   bool          // Force to kill browser
-	Timeout      time.Duration // Timeout
-	LanguageCode string
-	WaitRequests bool // Wait requests to complete after navigation
+	IsHeadless    bool          // Use browser interface
+	IsLeakless    bool          // Force to kill browser
+	Timeout       time.Duration // Timeout
+	LanguageCode  string
+	WaitRequests  bool // Wait requests to complete after navigation
+	LeavePageOpen bool // Leave pages and browser open
 }
 
-func (o *BrowserOpts) Check() {
+// Initialize browser parameters with default values if they are not set
+func (o *BrowserOpts) Init() {
 	if o.Timeout == 0 {
 		o.Timeout = time.Second * 30
 	}
@@ -36,7 +38,7 @@ type Browser struct {
 }
 
 func NewBrowser(opts BrowserOpts) (*Browser, error) {
-	opts.Check()
+	opts.Init()
 	logrus.Debugf("Browser options: %+v", opts)
 
 	path, has := launcher.LookPath()
@@ -64,17 +66,25 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 
 	b.browser = rod.New().ControlURL(b.browserAddr)
 	b.browser.MustConnect()
-	b.browser.SetCookies(nil)
+	//b.browser.SetCookies(nil)
 
 	page := stealth.MustPage(b.browser)
 	wait := page.MustWaitRequestIdle()
 	page.Navigate(URL)
-	wait()
+
+	// causes bugs in google
+	if b.WaitRequests {
+		wait()
+	}
 
 	page.MustEmulate(devices.Device{
 		UserAgent:      uarand.GetRandom(),
 		AcceptLanguage: b.LanguageCode,
 	})
+
+	// Wait till page loads
+	time.Sleep(time.Second * 1)
+
 	return page
 }
 

google/search.go

@@ -20,7 +20,7 @@ type Google struct {
 
 func New(browser core.Browser) *Google {
 	gogl := Google{Browser: browser}
-	gogl.checkTimeout = time.Second * 2
+	gogl.checkTimeout = time.Second * 5
 	gogl.findNumRgxp = regexp.MustCompile("\\d")
 	return &gogl
 }
@@ -49,6 +49,11 @@ func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) {
 	return total, nil
 }
 
+func (gogl *Google) preparePage(page *rod.Page) {
+	// Remove "similar queries" lists
+	page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
+}
+
 func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Google search, query: %+v", query)
 
@@ -59,17 +64,21 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	page := gogl.Navigate(url)
+	gogl.preparePage(page)
 
 	totalResults, err := gogl.FindTotalResults(page)
 	if err != nil {
 		return nil, err
 	}
+	logrus.Tracef("%d total results found", totalResults)
+
 	if totalResults == 0 {
 		return searchResults, nil
 	}
 
-	results, err := page.Search("div>div.g")
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang]")
 	if err != nil {
 		return nil, err
 	}
@@ -86,34 +95,42 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 			continue
 		}
 		linkText, err := link.Property("href")
+		if err != nil {
+			logrus.Error("No `href` tag found")
+		}
 
 		// Get title
 		titleTag, err := link.Element("h3")
 		if err != nil {
-			logrus.Error(err)
+			logrus.Error("No `h3` tag found")
 			continue
 		}
 
 		title, err := titleTag.Text()
 		if err != nil {
+			logrus.Error("Cannot extract text from title")
 			title = "No title"
-			logrus.Error(err)
 		}
 
 		// Get description
+		// doesn't catch all
 		descTag, err := r.Element(`div[data-sncf~="1"]`)
-		desc := "No description found"
-		if err == nil {
+		desc := ""
+		if err != nil {
+			logrus.Trace(`No description 'div[data-sncf~="1"]' tag found`)
+		} else {
 			desc = descTag.MustText()
 		}
 
-		gR := core.SearchResult{Rank: i, URL: linkText.String(), Title: title, Description: desc}
+		gR := core.SearchResult{Rank: i + 1, URL: linkText.String(), Title: title, Description: desc}
 		searchResults = append(searchResults, gR)
 	}
 
-	err = page.Close()
-	if err != nil {
-		logrus.Error(err)
+	if !gogl.Browser.LeavePageOpen {
+		err = page.Close()
+		if err != nil {
+			logrus.Error(err)
+		}
 	}
 
 	return searchResults, nil

yandex/search.go

@@ -1,8 +1,6 @@
 package yandex
 
 import (
-	"errors"
-	"fmt"
 	"time"
 
 	"github.com/go-rod/rod"
@@ -13,13 +11,13 @@ import (
 type Yandex struct {
 	core.Browser
 	checkTimeout time.Duration // Timeout for secondary elements check
-	pagesSleep   time.Duration // Sleep between pages
+	pageSleep    time.Duration // Sleep between pages
 }
 
 func New(browser core.Browser) *Yandex {
 	yand := Yandex{Browser: browser}
 	yand.checkTimeout = time.Second * 2
-	yand.pagesSleep = time.Second * 1
+	yand.pageSleep = time.Second * 1
 	return &yand
 }
 
@@ -35,17 +33,16 @@ func (yand *Yandex) isCaptcha(page *rod.Page) bool {
 	return true
 }
 
+// Check if nothig is found
 func (yand *Yandex) isNoResults(page *rod.Page) bool {
 	noResFound := false
 
 	_, err := page.Timeout(yand.checkTimeout).Search("div.EmptySearchResults-Title")
-	fmt.Println(err)
 	if err == nil {
 		noResFound = true
 	}
 
 	_, err = page.Timeout(yand.checkTimeout).Search("div>div.RequestMeta-Message")
-	fmt.Println(err)
 	if err == nil {
 		noResFound = true
 	}
@@ -63,23 +60,29 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
 			continue
 		}
 		linkText, err := link.Property("href")
+		if err != nil {
+			logrus.Error("No `href` tag found")
+		}
 
 		// Get title
 		titleTag, err := link.Element("h2")
 		if err != nil {
-			logrus.Error("No title tag found")
+			logrus.Error("No title `h2` tag found")
 			continue
 		}
 
 		title, err := titleTag.Text()
 		if err != nil {
+			logrus.Error("Cannot extract text from title")
 			title = "No title"
 		}
 
 		// Get description
 		descTag, err := r.Element(`span.OrganicTextContentSpan`)
-		desc := "No description found"
-		if err == nil {
+		desc := ""
+		if err != nil {
+			logrus.Trace("No description `span.OrganicTextContentSpan` tag found")
+		} else {
 			desc = descTag.MustText()
 		}
 
@@ -103,33 +106,43 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 		}
 
 		page := yand.Navigate(url)
-		defer page.Close()
 
-		searchRes, _ := page.Timeout(yand.Timeout).Search("li.serp-item")
-		if searchRes != nil {
-			elements, _ := searchRes.All()
-			r := yand.parseResults(elements, searchPage)
-			allResults = append(allResults, r...)
+		// Get all search results in page
+		searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item")
+		if err != nil {
+			logrus.Errorf("Cannot parse search results: %s", err)
 		}
 
+		// Check why no results, maybe captcha?
 		if searchRes == nil {
 			if yand.isNoResults(page) {
-				return allResults, nil
+				logrus.Errorf("No results found")
 			} else if yand.isCaptcha(page) {
-				logrus.Error(errors.New("Yandex captcha occured during: " + url))
-				return allResults, nil
+				logrus.Errorf("Yandex captcha occurred during: %s", url)
 			}
 			break
 		}
 
-		searchPage++
-
-		err = page.Close()
+		elements, err := searchRes.All()
 		if err != nil {
-			logrus.Error(err)
+			logrus.Errorf("Cannot get all elements from search results: %s", err)
+			break
 		}
 
-		time.Sleep(yand.pagesSleep)
+		r := yand.parseResults(elements, searchPage)
+		allResults = append(allResults, r...)
+
+		searchPage++
+
+		if !yand.Browser.LeavePageOpen {
+			// Close tab before opening new one during the cycle
+			err = page.Close()
+			if err != nil {
+				logrus.Error(err)
+			}
+		}
+
+		time.Sleep(yand.pageSleep)
 	}
 
 	return allResults, nil