diff --git a/baidu/search.go b/baidu/search.go index cdb02e0..219d554 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -4,6 +4,7 @@ import ( "strings" "time" + "github.com/go-rod/rod" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" ) @@ -22,6 +23,22 @@ func (baid *Baidu) Name() string { return "baidu" } +func (baid *Baidu) isCaptcha(page *rod.Page) bool { + _, err := page.Timeout(baid.checkTimeout).Search("div.passMod_dialog-body") + if err != nil { + return false + } + return true +} + +func (baid *Baidu) isTimeout(page *rod.Page) bool { + _, err := page.Timeout(baid.checkTimeout).Search("button.timeout-button") + if err != nil { + return false + } + return true +} + func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { logrus.Tracef("Start Baidu search, query: %+v", query) @@ -37,7 +54,21 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd") if err != nil { - return nil, err + logrus.Errorf("Cannot parse search results: %s", err) + } + + // Check why no results, maybe captcha? + if results == nil { + defer page.Close() + + if baid.isCaptcha(page) { + logrus.Errorf("Baidu captcha occurred during: %s", url) + return searchResults, core.ErrCaptcha + } else if baid.isTimeout(page) { + logrus.Errorf("Baidu timeout occurred during: %s", url) + return searchResults, core.ErrCaptcha + } + return searchResults, nil } resultElements, err := results.All() diff --git a/cmd/root.go b/cmd/root.go index 11d5725..9b1272b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( ) const ( - version = "0.1.1" + version = "0.1.2" defaultConfigFilename = "config" envPrefix = "OPENSERP" replaceHyphenWithCamelCase = false diff --git a/core/common.go b/core/common.go index 6f0dd7b..cd1c5bd 100644 --- a/core/common.go +++ b/core/common.go @@ -7,6 +7,8 @@ import ( "github.com/gofiber/fiber/v2" ) +var ErrCaptcha = errors.New("Captcha detected") + type SearchResult struct { Rank int `json:"rank"` URL string `json:"url"` diff --git a/core/server.go b/core/server.go index a0efd8f..09222b7 100644 --- a/core/server.go +++ b/core/server.go @@ -1,6 +1,7 @@ package core import ( + "errors" "fmt" "strings" @@ -39,6 +40,14 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server { res, err := locEngine.Search(q) if err != nil { + switch err { + case ErrCaptcha: + err = errors.New(fmt.Sprintf("Captcha found, please stop sending requests for a while\n%s", err)) + c.Status(503) + default: + c.Status(500) + } + logrus.Errorf("Error during %s search: %s", locEngine.Name(), err) return err } diff --git a/google/search.go b/google/search.go index dc6f5a6..cd8ee6f 100644 --- a/google/search.go +++ b/google/search.go @@ -49,6 +49,14 @@ func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) { return total, nil } +func (gogl *Google) isCaptcha(page *rod.Page) bool { + _, err := page.Timeout(gogl.checkTimeout).Search("form#captcha-form") + if err != nil { + return false + } + return true +} + func (gogl *Google) preparePage(page *rod.Page) { // Remove "similar queries" lists page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove()); })();") @@ -68,21 +76,28 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { page := gogl.Navigate(url) gogl.preparePage(page) + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang], div[data-surl][jsaction]") + if err != nil { + logrus.Errorf("Cannot parse search results: %s", err) + } + + // Check why no results, maybe captcha? + if results == nil { + defer page.Close() + + if gogl.isCaptcha(page) { + logrus.Errorf("Google captcha occurred during: %s", url) + return searchResults, core.ErrCaptcha + } + return searchResults, nil + } + totalResults, err := gogl.FindTotalResults(page) if err != nil { return nil, err } logrus.Tracef("%d total results found", totalResults) - if totalResults == 0 { - return searchResults, nil - } - - results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang]") - if err != nil { - return nil, err - } - resultElements, err := results.All() if err != nil { return nil, err diff --git a/yandex/search.go b/yandex/search.go index 2fb954f..73af772 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -115,10 +115,13 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { // Check why no results, maybe captcha? if searchRes == nil { + defer page.Close() + if yand.isNoResults(page) { logrus.Errorf("No results found") } else if yand.isCaptcha(page) { logrus.Errorf("Yandex captcha occurred during: %s", url) + return allResults, core.ErrCaptcha } break }