zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-05 16:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Handle failed navigation

Browse Source
This commit is contained in:
Rustem Kamalov
2025-07-03 02:29:12 +03:00
parent ba94c69dd0
commit 7f0f32ac4e
5 changed files with 38 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
baidu/search.go
View File

@@ -74,7 +74,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
return nil, err
}
page := baid.Navigate(url)
page, err := baid.Navigate(url)
if err != nil {
return nil, err
}
results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
if err != nil {
@@ -154,7 +157,11 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
}
// Get anti-crawler cookies first, then reload page
page := baid.Navigate(url)
page, err := baid.Navigate(url)
if err != nil {
return nil, err
}
if !baid.LeavePageOpen {
defer page.Close()
}

10
core/browser.go
View File

@@ -97,7 +97,7 @@ func (b *Browser) IsInitialized() bool {
}
// Open URL
func (b *Browser) Navigate(URL string) *rod.Page {
func (b *Browser) Navigate(URL string) (*rod.Page, error) {
logrus.Debug("Navigate to: ", URL)
b.browser = rod.New().ControlURL(b.browserAddr)
@@ -127,7 +127,11 @@ func (b *Browser) Navigate(URL string) *rod.Page {
page.MustEmulate(devices.Device{
AcceptLanguage: b.LanguageCode,
})
page.MustNavigate(URL)
err := page.Navigate(URL)
if err != nil {
return nil, err
}
wait := page.MustWaitRequestIdle()
// may cause bugs with google
@@ -135,7 +139,7 @@ func (b *Browser) Navigate(URL string) *rod.Page {
wait()
}
return page
return page, nil
}
func (b *Browser) Close() error {

7
core/browser_test.go
View File

@@ -38,13 +38,12 @@ func TestBot(t *testing.T) {
t.Fatalf("Error failed initializing browser: %s", err)
}
page := browser.Navigate("https://bot.sannysoft.com")
page, _ := browser.Navigate("https://bot.sannysoft.com")
page.MustScreenshotFullPage("./test/screenshot_bot.png")
page = browser.Navigate("https://www.whatismybrowser.com/")
page, _ = browser.Navigate("https://www.whatismybrowser.com/")
page.MustScreenshotFullPage("./test/screenshot_browser.png")
page = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
page, _ = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
page.MustScreenshotFullPage("./test/screenshot_creep.png")
}

14
google/search.go
View File

@@ -70,7 +70,7 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL)
resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL)
if err != nil {
logrus.Errorf("Error solving google captcha: %s", err)
return false
@@ -143,7 +143,11 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
if err != nil {
return nil, err
}
page := gogl.Navigate(url)
page, err := gogl.Navigate(url)
if err != nil {
return nil, err
}
defer gogl.close(page)
gogl.preparePage(page)
@@ -338,7 +342,11 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
return nil, err
}
page := gogl.Navigate(url)
page, err := gogl.Navigate(url)
if err != nil {
return nil, err
}
defer gogl.close(page)
for len(searchResultsMap) < query.Limit {

10
yandex/search.go
View File

@@ -124,7 +124,10 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
return nil, err
}
page := yand.Navigate(url)
page, err := yand.Navigate(url)
if err != nil {
return nil, err
}
// Get all search results in page
searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item")
@@ -185,7 +188,10 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
}
searchPage += 1
page := yand.Navigate(url)
page, err := yand.Navigate(url)
if err != nil {
return nil, err
}
if !yand.LeavePageOpen {
defer page.Close()