diff --git a/README.md b/README.md index 8a694a9..3dc098a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp) [![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp) -[![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/derailed/k9s/releases) +[![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases) API access for search engines results if available isn't free. @@ -60,7 +60,7 @@ openserp serve ``` * Or print results in CLI using `search` command: ```bash -openserp search google "how to get banned in google fast" # Change `google` to `yandex` or `baidu` +openserp search google "how to get banned from google fast" # Change `google` to `yandex` or `baidu` ``` As a result you should get JSON output containting search results: ```json diff --git a/core/browser.go b/core/browser.go index 0844a90..0bb3f38 100644 --- a/core/browser.go +++ b/core/browser.go @@ -16,18 +16,19 @@ type BrowserOpts struct { IsLeakless bool // Force to kill browser Timeout time.Duration // Timeout LanguageCode string - WaitRequests bool // Wait requests to complete after navigation - LeavePageOpen bool // Leave pages and browser open + WaitRequests bool // Wait requests to complete after navigation + LeavePageOpen bool // Leave pages and browser open + WaitLoadTime time.Duration // Time to wait till page loads } // Initialize browser parameters with default values if they are not set -func (o *BrowserOpts) Init() { +func (o *BrowserOpts) Check() { if o.Timeout == 0 { o.Timeout = time.Second * 30 } - if o.LanguageCode == "" { - o.LanguageCode = "en" + if o.WaitLoadTime == 0 { + o.WaitLoadTime = time.Second * 2 } } @@ -38,7 +39,7 @@ type Browser struct { } func NewBrowser(opts BrowserOpts) (*Browser, error) { - opts.Init() + opts.Check() logrus.Debugf("Browser options: %+v", opts) path, has := launcher.LookPath() @@ -66,11 +67,11 @@ func (b *Browser) Navigate(URL string) *rod.Page { b.browser = rod.New().ControlURL(b.browserAddr) b.browser.MustConnect() - //b.browser.SetCookies(nil) + b.browser.SetCookies(nil) page := stealth.MustPage(b.browser) wait := page.MustWaitRequestIdle() - page.Navigate(URL) + page.MustNavigate(URL) // causes bugs in google if b.WaitRequests { @@ -83,7 +84,7 @@ func (b *Browser) Navigate(URL string) *rod.Page { }) // Wait till page loads - time.Sleep(time.Second * 1) + time.Sleep(b.WaitLoadTime) return page } diff --git a/core/browser_test.go b/core/browser_test.go index 0453b52..efc921d 100644 --- a/core/browser_test.go +++ b/core/browser_test.go @@ -27,3 +27,24 @@ func TestCreateBrowser(t *testing.T) { // t.Fatalf("Error failed initializing leakless browser: %s", err) // } // } + +// Manually observe test results for now +func TestBot(t *testing.T) { + + var err error + opts := BrowserOpts{IsHeadless: false, IsLeakless: true, LeavePageOpen: true} + browser, err = NewBrowser(opts) + if err != nil { + t.Fatalf("Error failed initializing browser: %s", err) + } + + page := browser.Navigate("https://bot.sannysoft.com") + page.MustScreenshotFullPage("./test/screenshot_bot.png") + + page = browser.Navigate("https://www.whatismybrowser.com/") + page.MustScreenshotFullPage("./test/screenshot_browser.png") + + page = browser.Navigate("https://abrahamjuliot.github.io/creepjs/") + page.MustScreenshotFullPage("./test/screenshot_creep.png") + +}