diff --git a/cmd/root.go b/cmd/root.go index ac34a79..4c42138 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,7 @@ import ( ) const ( - version = "0.5.1" + version = "0.5.2" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/browser.go b/core/browser.go index 7debcfd..650dc2d 100644 --- a/core/browser.go +++ b/core/browser.go @@ -3,11 +3,13 @@ package core import ( "fmt" "net/url" + "strings" "time" "github.com/go-rod/rod" "github.com/go-rod/rod/lib/devices" "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/proto" "github.com/go-rod/stealth" "github.com/sirupsen/logrus" ) @@ -53,7 +55,8 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) { logrus.Debug("Browser found: ", has) // Create launcher - l := launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless) + l := launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Set("disable-blink-features", "AutomationControlled"). + Delete("enable-automation") // Configure proxy if specified if opts.ProxyURL != "" { @@ -124,28 +127,52 @@ func (b *Browser) Navigate(URL string) (*rod.Page, error) { b.browser.MustIgnoreCertErrors(true) } + ua := strings.ReplaceAll(b.browser.MustVersion().UserAgent, "HeadlessChrome/", "Chrome/") + var page *rod.Page + if b.UseStealth { page = stealth.MustPage(b.browser) - } else { - page = b.browser.MustPage(URL) - } + page.MustEmulate(devices.Device{ + AcceptLanguage: b.LanguageCode, + UserAgent: ua, + }) - page.MustEmulate(devices.Device{ - AcceptLanguage: b.LanguageCode, - }) + } else { + page = b.browser.MustPage("about:blank") + + page.MustEmulate(devices.Device{ + AcceptLanguage: b.LanguageCode, + UserAgent: ua, + }) + + proto.EmulationSetDeviceMetricsOverride{ + Width: 1920, + Height: 1080, + DeviceScaleFactor: 1, + Mobile: false, + ScreenWidth: &[]int{1920}[0], + ScreenHeight: &[]int{1080}[0], + }.Call(page) + } + //EnableCustomStealth(page) err := page.Navigate(URL) if err != nil { return nil, err } + // Avoid panics from MustWaitLoad when the target navigates/closes mid-wait + if werr := page.WaitLoad(); werr != nil { + logrus.Debugf("WaitLoad returned early: %v", werr) + } wait := page.MustWaitRequestIdle() // may cause bugs with google if b.WaitRequests { wait() } + time.Sleep(2 * time.Second) return page, nil } diff --git a/duckduckgo/search.go b/duckduckgo/search.go index aceb263..afeede4 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -2,6 +2,7 @@ package duckduckgo import ( "fmt" + "strings" "time" "github.com/go-rod/rod" @@ -36,9 +37,11 @@ func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter { } func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool { - // DuckDuckGo rarely shows captchas, but we can check for common patterns - _, err := page.Timeout(ddg.GetSelectorTimeout()).Search("div[class*='captcha']") - return err == nil + html, err := page.Timeout(ddg.GetSelectorTimeout()).HTML() + if err != nil { + return false + } + return strings.Contains(html, "bots user") } // Check if no results are found @@ -165,16 +168,22 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { allResults := []core.SearchResult{} searchPage := 0 - maxPages := 5 // Prevent infinite loops - consecutiveEmptyPages := 0 // Track consecutive pages with no valid results - for len(allResults) < query.Limit && searchPage < maxPages { + for len(allResults) < query.Limit { url, err := BuildURL(query) if err != nil { return nil, err } - page, err := ddg.Navigate(url) + // Navigate to a page that can detect headless browsers + page, err := ddg.Navigate("https://localhost:9001/?spjs_test_redirect=true") + page.WaitLoad() + if err != nil { + ddg.logger.Fatal("Error navigating to bot detection site: %s", err) + } + page.MustScreenshotFullPage("./ddg_headless1.png") + + page, err = ddg.Navigate(url) if err != nil { return nil, err } @@ -200,6 +209,8 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { break } } + + page.MustScreenshotFullPage("./duck.png") if searchErr != nil { defer page.Close() ddg.logger.Error("Cannot parse search results: %s", searchErr) @@ -227,22 +238,12 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { r := ddg.parseResults(elements, searchPage) - // Track consecutive empty pages if len(r) == 0 { - consecutiveEmptyPages++ - ddg.logger.Debug("No valid results found on page %d (consecutive empty: %d)", searchPage, consecutiveEmptyPages) - - // Break if we have too many consecutive empty pages - if consecutiveEmptyPages >= 2 { - ddg.logger.Warn("Too many consecutive empty pages, stopping search") - break - } - } else { - consecutiveEmptyPages = 0 // Reset counter when we find results + ddg.logger.Debug("No valid results found on page %d", searchPage) + return nil, core.ErrSearchTimeout } allResults = append(allResults, r...) - searchPage++ if !ddg.Browser.LeavePageOpen {