diff --git a/baidu/search.go b/baidu/search.go index 219d554..2cc0244 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -54,7 +54,9 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd") if err != nil { + defer page.Close() logrus.Errorf("Cannot parse search results: %s", err) + return nil, core.ErrSearchTimeout } // Check why no results, maybe captcha? @@ -63,12 +65,12 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { if baid.isCaptcha(page) { logrus.Errorf("Baidu captcha occurred during: %s", url) - return searchResults, core.ErrCaptcha + return nil, core.ErrCaptcha } else if baid.isTimeout(page) { logrus.Errorf("Baidu timeout occurred during: %s", url) - return searchResults, core.ErrCaptcha + return nil, core.ErrCaptcha } - return searchResults, nil + return nil, nil } resultElements, err := results.All() diff --git a/core/common.go b/core/common.go index cd1c5bd..89688bb 100644 --- a/core/common.go +++ b/core/common.go @@ -8,6 +8,7 @@ import ( ) var ErrCaptcha = errors.New("Captcha detected") +var ErrSearchTimeout = errors.New("Timeout. Cannot find element on page") type SearchResult struct { Rank int `json:"rank"` diff --git a/core/server.go b/core/server.go index 09222b7..00636f9 100644 --- a/core/server.go +++ b/core/server.go @@ -43,13 +43,12 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server { switch err { case ErrCaptcha: err = errors.New(fmt.Sprintf("Captcha found, please stop sending requests for a while\n%s", err)) - c.Status(503) - default: - c.Status(500) + case ErrSearchTimeout: + err = errors.New(fmt.Sprintf("Error: %s\nProbably need to update CSS selector", err)) } logrus.Errorf("Error during %s search: %s", locEngine.Name(), err) - return err + return fiber.NewError(fiber.StatusServiceUnavailable, err.Error()) } return c.JSON(res) diff --git a/google/search.go b/google/search.go index cd8ee6f..619c022 100644 --- a/google/search.go +++ b/google/search.go @@ -78,7 +78,9 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang], div[data-surl][jsaction]") if err != nil { + defer page.Close() logrus.Errorf("Cannot parse search results: %s", err) + return nil, core.ErrSearchTimeout } // Check why no results, maybe captcha? @@ -87,9 +89,9 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { if gogl.isCaptcha(page) { logrus.Errorf("Google captcha occurred during: %s", url) - return searchResults, core.ErrCaptcha + return nil, core.ErrCaptcha } - return searchResults, nil + return nil, err } totalResults, err := gogl.FindTotalResults(page) diff --git a/yandex/search.go b/yandex/search.go index 73af772..6c95fba 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -110,7 +110,9 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { // Get all search results in page searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item") if err != nil { + defer page.Close() logrus.Errorf("Cannot parse search results: %s", err) + return nil, core.ErrSearchTimeout } // Check why no results, maybe captcha? @@ -121,7 +123,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { logrus.Errorf("No results found") } else if yand.isCaptcha(page) { logrus.Errorf("Yandex captcha occurred during: %s", url) - return allResults, core.ErrCaptcha + return nil, core.ErrCaptcha } break }