Add captcha check and error

baidu/search.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-rod/rod"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
 )
@@ -22,6 +23,22 @@ func (baid *Baidu) Name() string {
 	return "baidu"
 }
 
+func (baid *Baidu) isCaptcha(page *rod.Page) bool {
+	_, err := page.Timeout(baid.checkTimeout).Search("div.passMod_dialog-body")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func (baid *Baidu) isTimeout(page *rod.Page) bool {
+	_, err := page.Timeout(baid.checkTimeout).Search("button.timeout-button")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
 func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Baidu search, query: %+v", query)
 
@@ -37,7 +54,21 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 
 	results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
 	if err != nil {
-		return nil, err
+		logrus.Errorf("Cannot parse search results: %s", err)
+	}
+
+	// Check why no results, maybe captcha?
+	if results == nil {
+		defer page.Close()
+
+		if baid.isCaptcha(page) {
+			logrus.Errorf("Baidu captcha occurred during: %s", url)
+			return searchResults, core.ErrCaptcha
+		} else if baid.isTimeout(page) {
+			logrus.Errorf("Baidu timeout occurred during: %s", url)
+			return searchResults, core.ErrCaptcha
+		}
+		return searchResults, nil
 	}
 
 	resultElements, err := results.All()

cmd/root.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	version                    = "0.1.1"
+	version                    = "0.1.2"
 	defaultConfigFilename      = "config"
 	envPrefix                  = "OPENSERP"
 	replaceHyphenWithCamelCase = false

core/common.go

@@ -7,6 +7,8 @@ import (
 	"github.com/gofiber/fiber/v2"
 )
 
+var ErrCaptcha = errors.New("Captcha detected")
+
 type SearchResult struct {
 	Rank        int    `json:"rank"`
 	URL         string `json:"url"`

core/server.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
@@ -39,6 +40,14 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server {
 
 			res, err := locEngine.Search(q)
 			if err != nil {
+				switch err {
+				case ErrCaptcha:
+					err = errors.New(fmt.Sprintf("Captcha found, please stop sending requests for a while\n%s", err))
+					c.Status(503)
+				default:
+					c.Status(500)
+				}
+
 				logrus.Errorf("Error during %s search: %s", locEngine.Name(), err)
 				return err
 			}

google/search.go

@@ -49,6 +49,14 @@ func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) {
 	return total, nil
 }
 
+func (gogl *Google) isCaptcha(page *rod.Page) bool {
+	_, err := page.Timeout(gogl.checkTimeout).Search("form#captcha-form")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
 func (gogl *Google) preparePage(page *rod.Page) {
 	// Remove "similar queries" lists
 	page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
@@ -68,21 +76,28 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	page := gogl.Navigate(url)
 	gogl.preparePage(page)
 
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang], div[data-surl][jsaction]")
+	if err != nil {
+		logrus.Errorf("Cannot parse search results: %s", err)
+	}
+
+	// Check why no results, maybe captcha?
+	if results == nil {
+		defer page.Close()
+
+		if gogl.isCaptcha(page) {
+			logrus.Errorf("Google captcha occurred during: %s", url)
+			return searchResults, core.ErrCaptcha
+		}
+		return searchResults, nil
+	}
+
 	totalResults, err := gogl.FindTotalResults(page)
 	if err != nil {
 		return nil, err
 	}
 	logrus.Tracef("%d total results found", totalResults)
 
-	if totalResults == 0 {
-		return searchResults, nil
-	}
-
-	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang]")
-	if err != nil {
-		return nil, err
-	}
-
 	resultElements, err := results.All()
 	if err != nil {
 		return nil, err

yandex/search.go

@@ -115,10 +115,13 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 
 		// Check why no results, maybe captcha?
 		if searchRes == nil {
+			defer page.Close()
+
 			if yand.isNoResults(page) {
 				logrus.Errorf("No results found")
 			} else if yand.isCaptcha(page) {
 				logrus.Errorf("Yandex captcha occurred during: %s", url)
+				return allResults, core.ErrCaptcha
 			}
 			break
 		}