diff --git a/README.md b/README.md index 4f9bdf0..3b2d729 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,9 @@ A developer-friendly alternative to paid SERP API services! - 🌐 **Megasearch** - cross-engine aggregation with deduplication - 🖼 **Images** - image search is also available - 🎯 **Advanced filters** - language, date range, file type, and site queries -- 🌍 **Configurable** - proxy, cache, and resilient mode support +- 🌍 **Configurable** - proxy, cache, and resilient mode - 🐳 **Docker-ready** - local and container deployment -- 📝 **Data Formats** - JSON, Markdown, Text, NdJSON response format types are supported +- 📝 **Data Formats** - JSON, Markdown, Text, NdJSON response formats ## Quick Start⚡️ @@ -267,4 +267,4 @@ This project is licensed under the MIT License. See [LICENSE](LICENSE). Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md). -###### _"OpenSERP" is the name of this open-source project. Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._ +###### _"OpenSERP" is the name of this open-source project. The official [website](https://openserp.org) and [hosted](https://openserp.org/cloud) solution. Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._ diff --git a/baidu/search.go b/baidu/search.go index abaa229..763e9b7 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -10,7 +10,6 @@ import ( "github.com/go-rod/rod" "github.com/karust/openserp/core" - "golang.org/x/time/rate" ) type imageDataJson struct { @@ -54,12 +53,6 @@ func (baid *Baidu) Name() string { return "baidu" } -// GetRateLimiter returns a limiter configured from SearchEngineOptions. -func (baid *Baidu) GetRateLimiter() *rate.Limiter { - ratelimit := rate.Every(baid.GetRatelimit()) - return rate.NewLimiter(ratelimit, baid.RateBurst) -} - func (baid *Baidu) isCaptcha(page *rod.Page) bool { has, _, _ := page.Has(Selectors.Captcha) return has @@ -85,10 +78,7 @@ func (baid *Baidu) classifyBlockPage(page *rod.Page, url string) error { // Search executes a Baidu web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithMinimalBrowserProfile(ctx) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true) scoped := *baid scoped.logger = baid.logger.WithRequest(ctx) if scoped.Browser.WaitLoadTime == 0 || scoped.Browser.WaitLoadTime > 250*time.Millisecond { @@ -114,21 +104,12 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core if err != nil { return nil, err } - closePage := func() { - if baid.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - baid.logger.Debug("Page close error: %v", closeErr) - } - } + defer core.DeferClosePage(ctx, page, &baid.Browser)() searchResults, err := baid.waitForParsedSearchResults(ctx, page, url) if err != nil { - closePage() return nil, err } - closePage() for i := range searchResults { searchResults[i].Rank = query.Start + i + 1 @@ -185,10 +166,7 @@ func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Pag // SearchImage executes a Baidu image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithMinimalBrowserProfile(ctx) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true) scoped := *baid scoped.logger = baid.logger.WithRequest(ctx) baid = &scoped @@ -198,57 +176,46 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se searchResults := []core.SearchResult{} searchPage := 0 - for len(searchResults) < query.Limit { + // fetchPage loads one image-results page and appends parsed results. + // Returns (done, error): done=true ends the outer loop without error. + fetchPage := func() (bool, error) { url, err := BuildImageURL(query, searchPage) if err != nil { - return nil, err + return false, err } // First load often seeds anti-crawler cookies; results tend to become // available after one explicit reload. page, err := baid.Navigate(ctx, url) if err != nil { - return nil, err - } - closePage := func() { - if baid.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - baid.logger.Debug("Page close error: %v", closeErr) - } + return false, err } + defer core.DeferClosePage(ctx, page, &baid.Browser)() jsonWaitTimeout := baid.GetSelectorTimeout() if reloadErr := page.Reload(); reloadErr != nil { - closePage() - return nil, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } preElements, _, err := core.WaitForElements(ctx, page, Selectors.ImageJSONRoot, jsonWaitTimeout) if err != nil { if blockErr := baid.classifyBlockPage(page, url); blockErr != nil { - closePage() - return nil, blockErr + return false, blockErr } - closePage() baid.logger.Error("Cannot parse search results: %s", err) - return nil, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } if len(preElements) == 0 { if blockErr := baid.classifyBlockPage(page, url); blockErr != nil { - closePage() - return nil, blockErr + return false, blockErr } - closePage() - return nil, nil + return true, nil } jsonText, err := preElements[0].Text() if err != nil { - closePage() - return nil, err + return false, err } var data imageDataJson @@ -257,28 +224,23 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se jsonText = strings.ReplaceAll(jsonText, `\'`, "'") matchNewlines, err := regexp.Compile(`[\r\n\t]`) if err != nil { - closePage() - return nil, core.ErrParser + return false, core.ErrParser } escapeNewlines := func(s string) string { return matchNewlines.ReplaceAllString(s, "\\n") } re, err := regexp.Compile(`"[^"\\]*(?:\\[\s\S][^"\\]*)*"`) if err != nil { - closePage() - return nil, core.ErrParser + return false, core.ErrParser } fixedJson := re.ReplaceAllStringFunc(jsonText, escapeNewlines) - err = json.Unmarshal([]byte(fixedJson), &data) - if err != nil { - closePage() + if err := json.Unmarshal([]byte(fixedJson), &data); err != nil { baid.logger.Error("Failed to unmarshal JSON: %v", err) - return nil, core.ErrParser + return false, core.ErrParser } if len(data.Data) == 0 { - closePage() - break + return true, nil } for i, img := range data.Data { @@ -299,23 +261,26 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se img.Type, img.IsCopyright, ), - Ad: func() bool { - if img.AdType != "0" { - return true - } else { - return false - } - }(), + Ad: img.AdType != "0", } searchResults = append(searchResults, res) if query.Limit > 0 && len(searchResults) >= query.Limit { - break + return true, nil } } - searchPage += 1 + return false, nil + } - closePage() + for len(searchResults) < query.Limit { + done, err := fetchPage() + if err != nil { + return nil, err + } + searchPage++ + if done { + break + } } deduped := core.DeduplicateResults(searchResults) diff --git a/baidu/search_raw.go b/baidu/search_raw.go index ae4b9ac..2c020f7 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -25,9 +25,7 @@ func classifyBaiduRawHTML(body []byte) error { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.EnsureContext(ctx) - ctx = core.WithEngine(ctx, "baidu") - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, "baidu", false) defer func() { if recovered := recover(); recovered != nil { err = core.RecoverEnginePanicWithContext(ctx, "baidu", recovered, nil) diff --git a/bing/search.go b/bing/search.go index 297852b..06ec701 100644 --- a/bing/search.go +++ b/bing/search.go @@ -11,7 +11,6 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" "github.com/karust/openserp/core" - "golang.org/x/time/rate" ) // Bing implements core.SearchEngine for Bing SERP pages. @@ -35,12 +34,6 @@ func (bing *Bing) Name() string { return "bing" } -// GetRateLimiter returns a limiter configured from SearchEngineOptions. -func (bing *Bing) GetRateLimiter() *rate.Limiter { - ratelimit := rate.Every(bing.GetRatelimit()) - return rate.NewLimiter(ratelimit, bing.RateBurst) -} - func (bing *Bing) getTotalResults(page *rod.Page) (int, error) { results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Results) if err != nil { @@ -87,9 +80,7 @@ func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error { // Search executes a Bing web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false) scoped := *bing scoped.logger = bing.logger.WithRequest(ctx) bing = &scoped @@ -113,14 +104,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. if err != nil { return nil, err } - defer func() { - if bing.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - bing.logger.Debug("Page close error: %v", closeErr) - } - }() + defer core.DeferClosePage(ctx, page, &bing.Browser)() if bing.checkCaptcha(page) { bing.logger.Error("Captcha detected: %s", url) @@ -289,9 +273,7 @@ func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) { // SearchImage executes a Bing image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false) scoped := *bing scoped.logger = bing.logger.WithRequest(ctx) bing = &scoped @@ -310,14 +292,7 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea if err != nil { return nil, err } - defer func() { - if bing.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - bing.logger.Debug("Page close error: %v", closeErr) - } - }() + defer core.DeferClosePage(ctx, page, &bing.Browser)() // Check for captcha if bing.checkCaptcha(page) { diff --git a/config.yaml b/config.yaml index a4ad9fd..1ebfaa3 100644 --- a/config.yaml +++ b/config.yaml @@ -14,8 +14,9 @@ app: head: false # Headful mode leakless: false # Force browser process cleanup after request leave_head: false # Keep tabs open after request - max_processes: 5 # Concurrent Chrome processes + max_processes: 6 # Concurrent Chrome processes idle_ttl: 5m # close a Chrome that has not served traffic for this long + mega_timeout: 90s # max total wait for /mega/* requests; slow engines return partial results proxies: allow_request_proxy_url: false @@ -30,14 +31,14 @@ proxies: # - url: socks5h://127.0.0.1:1080 # tags: [eu] health: - failure_threshold: 3 # Disable proxy after this many consecutive failures + failure_threshold: 2 # Disable proxy after this many consecutive failures lanes: enabled: true # Reuse browser profile/cookies per engine + proxy session ID max_lanes: 100 # LRU cap for sticky lanes kept in worker memory drop_cookies_on_challenge: true # Clear lane cookies on captcha/challenge only cache: - ttl_seconds: 60 # Dedicated endpoint cache TTL in seconds (0 disables cache) + ttl_seconds: 120 # Dedicated endpoint cache TTL in seconds (0 disables cache) max_size: 1000 # Maximum cached dedicated responses before oldest-entry eviction resilience: @@ -63,29 +64,29 @@ captcha: solver_enabled: false # Global captcha solver gate (requires 2captcha.apikey) google: - rate_requests: 4 # Allowed average requests per minute - rate_burst: 2 # Burst requests before limiter applies - captcha: true # Engine-level solver flag (also requires captcha.solver_enabled=true) + rate_requests: 60 # Allowed average requests per minute + rate_burst: 3 # Burst requests before limiter applies + #captcha: true # Engine-level solver flag (also requires captcha.solver_enabled=true) yandex: - rate_requests: 4 - rate_burst: 2 + rate_requests: 60 + rate_burst: 3 baidu: - rate_requests: 4 - rate_burst: 2 + rate_requests: 60 + rate_burst: 3 # No proxy tag means direct traffic bing: - rate_requests: 4 - rate_burst: 2 + rate_requests: 60 + rate_burst: 3 # No proxy tag means direct traffic duckduckgo: - rate_requests: 4 - rate_burst: 2 + rate_requests: 60 + rate_burst: 3 ecosia: - rate_requests: 4 - rate_burst: 2 + rate_requests: 60 + rate_burst: 3 # No proxy tag means direct traffic diff --git a/core/common.go b/core/common.go index d9110d5..1b4cd24 100644 --- a/core/common.go +++ b/core/common.go @@ -11,6 +11,7 @@ import ( "time" "github.com/gofiber/fiber/v2" + "golang.org/x/time/rate" ) // ErrCaptcha is returned when the engine detects a captcha challenge page. @@ -322,10 +323,17 @@ func (o *SearchEngineOptions) Init() { } // GetRatelimit returns the interval between two allowed requests. +// Call Init() first so RateRequests / RateTime are non-zero. func (o *SearchEngineOptions) GetRatelimit() time.Duration { return (time.Duration(o.RateTime) * time.Second) / time.Duration(o.RateRequests) } +// GetRateLimiter returns a limiter configured from SearchEngineOptions. +// Call Init() first so RateBurst is non-zero. +func (o *SearchEngineOptions) GetRateLimiter() *rate.Limiter { + return rate.NewLimiter(rate.Every(o.GetRatelimit()), o.RateBurst) +} + // GetSelectorTimeout returns the selector wait timeout as time.Duration. func (o *SearchEngineOptions) GetSelectorTimeout() time.Duration { return time.Duration(o.SelectorTimeout) * time.Second diff --git a/core/context.go b/core/context.go index 62ce8bc..9ac5846 100644 --- a/core/context.go +++ b/core/context.go @@ -36,3 +36,14 @@ func SleepContext(ctx context.Context, d time.Duration) error { func IsContextDone(err error) bool { return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) } + +// PrepareEngineContext applies request-scoped metadata expected by all engine +// search implementations. +func PrepareEngineContext(ctx context.Context, query Query, engineName string, minimalBrowserProfile bool) context.Context { + ctx = WithEngine(EnsureContext(ctx), engineName) + ctx = WithProfileRegion(ctx, query.LangCode) + if minimalBrowserProfile { + ctx = WithMinimalBrowserProfile(ctx) + } + return WithQueryHash(ctx, QueryHashFromQuery(query)) +} diff --git a/core/page_helpers.go b/core/page_helpers.go index 3cfe6b1..8a9f1ab 100644 --- a/core/page_helpers.go +++ b/core/page_helpers.go @@ -79,6 +79,19 @@ func HasAnySelector(page *rod.Page, selectors []string) bool { return false } +// DeferClosePage returns a cleanup function that closes page unless the browser +// is configured to leave pages open for debugging. +func DeferClosePage(ctx context.Context, page *rod.Page, browser *Browser) func() { + return func() { + if browser != nil && browser.LeavePageOpen { + return + } + if err := ClosePageWithTimeout(ctx, page, time.Second); err != nil { + WithRequest(ctx).WithError(err).Debug("Page close error") + } + } +} + // HasAttribute reports whether el carries attr (regardless of value). func HasAttribute(el *rod.Element, attr string) bool { if el == nil { diff --git a/core/resilient.go b/core/resilient.go index 5465810..8de16ba 100644 --- a/core/resilient.go +++ b/core/resilient.go @@ -206,10 +206,11 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se results []SearchResult err error ) + requestCtx := proxyRequestContext(callCtx, engine.Name(), attemptQuery) if isImage { - results, err = engine.SearchImage(proxyRequestContext(callCtx, engine.Name(), attemptQuery), attemptQuery) + results, err = engine.SearchImage(requestCtx, attemptQuery) } else { - results, err = engine.Search(proxyRequestContext(callCtx, engine.Name(), attemptQuery), attemptQuery) + results, err = engine.Search(requestCtx, attemptQuery) } if reportToRegistry { diff --git a/duckduckgo/search.go b/duckduckgo/search.go index d90ce3a..cd138e8 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -8,7 +8,6 @@ import ( "github.com/go-rod/rod" "github.com/karust/openserp/core" - "golang.org/x/time/rate" ) // DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages. @@ -35,12 +34,6 @@ func (ddg *DuckDuckGo) Name() string { return "duckduckgo" } -// GetRateLimiter returns a limiter configured from SearchEngineOptions. -func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter { - ratelimit := rate.Every(ddg.GetRatelimit()) - return rate.NewLimiter(ratelimit, ddg.RateBurst) -} - func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool { if core.HasAnySelector(page, Selectors.CaptchaSelectors) { return true @@ -145,9 +138,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se // Search executes a DuckDuckGo web search and returns normalized search // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.WithEngine(core.EnsureContext(ctx), ddg.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, ddg.Name(), false) scoped := *ddg scoped.logger = ddg.logger.WithRequest(ctx) ddg = &scoped @@ -163,62 +154,54 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] allResults := []core.SearchResult{} searchPage := 0 - for len(allResults) < query.Limit { + // fetchPage loads one SERP page and appends parsed results. + // Returns (done, error): done=true ends the outer loop without error. + fetchPage := func() (bool, error) { url, err := BuildURL(query, searchPage) if err != nil { - return nil, err + return false, err } page, err := ddg.Navigate(ctx, url) if err != nil { - return nil, err - } - closePage := func() { - if ddg.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - ddg.logger.Debug("Page close error: %v", closeErr) - } + return false, err } + defer core.DeferClosePage(ctx, page, &ddg.Browser)() elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout()) if err != nil { if ddg.isNoResults(page) { ddg.logger.Warn("No results found") - closePage() - break + return true, nil } if ddg.isCaptcha(page) { ddg.logger.Error("Captcha detected: %s", url) - closePage() - return nil, core.ErrCaptcha + return false, core.ErrCaptcha } - closePage() ddg.logger.Error("Cannot parse search results: %s", err) - return nil, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } ddg.logger.Debug("Found results with selector: %s", selector) r := ddg.parseResults(elements, searchPage) - if len(r) == 0 { ddg.logger.Debug("No valid results found on page %d", searchPage) - closePage() - return nil, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } allResults = append(allResults, r...) + return false, nil + } + + for len(allResults) < query.Limit { + done, err := fetchPage() + if err != nil { + return nil, err + } searchPage++ - - // Close tab before opening new one during the cycle - closePage() - - // Break if we've reached or exceeded the limit - if len(allResults) >= query.Limit { + if done || len(allResults) >= query.Limit { break } - if err := core.SleepContext(ctx, ddg.pageSleep); err != nil { return nil, err } @@ -239,9 +222,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] // SearchImage executes a DuckDuckGo image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.WithEngine(core.EnsureContext(ctx), ddg.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, ddg.Name(), false) scoped := *ddg scoped.logger = ddg.logger.WithRequest(ctx) ddg = &scoped @@ -260,13 +241,7 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor return nil, err } - if !ddg.Browser.LeavePageOpen { - defer func() { - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - ddg.logger.Debug("Page close error: %v", closeErr) - } - }() - } + defer core.DeferClosePage(ctx, page, &ddg.Browser)() elements, selector, err := core.WaitForElements(ctx, page, Selectors.ImageResult, ddg.GetSelectorTimeout()) if err != nil { diff --git a/ecosia/search.go b/ecosia/search.go index 041dde4..8961e7e 100644 --- a/ecosia/search.go +++ b/ecosia/search.go @@ -15,7 +15,6 @@ import ( "github.com/go-rod/rod" "github.com/karust/openserp/core" - "golang.org/x/time/rate" ) // ecosiaPageSize is the organic-results-per-page count on Ecosia's web SERP. @@ -43,7 +42,6 @@ const ( cfBodyMarker = "not a bot" ) - // Ecosia implements core.SearchEngine for Ecosia SERP pages. Additional // documentation at https://support.ecosia.org/article/447-search-features. type Ecosia struct { @@ -66,11 +64,6 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Ecosia { // Name returns the stable engine identifier. func (e *Ecosia) Name() string { return "ecosia" } -// GetRateLimiter returns a limiter configured from SearchEngineOptions. -func (e *Ecosia) GetRateLimiter() *rate.Limiter { - return rate.NewLimiter(rate.Every(e.GetRatelimit()), e.RateBurst) -} - // isCaptcha reports whether the current page is a Cloudflare interstitial. // Checks the URL, title, then body text (cheapest first). func (e *Ecosia) isCaptcha(page *rod.Page) bool { @@ -132,9 +125,7 @@ func (e *Ecosia) parseResult(elem *rod.Element, rank int, ad bool) (core.SearchR // Search executes an Ecosia web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.WithEngine(core.EnsureContext(ctx), e.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, e.Name(), false) scoped := *e scoped.logger = e.logger.WithRequest(ctx) e = &scoped @@ -155,40 +146,32 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S return nil, err } nextAdRank := -1 - for query.Limit <= 0 || len(all) < query.Limit { + // fetchPage loads one SERP page and appends parsed results. + // Returns (done, error): done=true ends the outer loop without error. + fetchPage := func() (bool, error) { u, err := BuildURL(query, pageNum) if err != nil { - return nil, err + return false, err } page, err := e.Navigate(ctx, u) if err != nil { - return nil, err - } - closePage := func() { - if e.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - e.logger.Debug("Page close error: %v", closeErr) - } + return false, err } + defer core.DeferClosePage(ctx, page, &e.Browser)() if err := page.WaitLoad(); err != nil { - closePage() e.logger.Error("Page load wait failed: %s", err) - return nil, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } if _, err := page.Timeout(e.GetSelectorTimeout()).Element(Selectors.Mainline); err != nil { if e.isCaptcha(page) { - closePage() e.logger.Error("Captcha detected: %s", u) - return nil, core.ErrCaptcha + return false, core.ErrCaptcha } - closePage() e.logger.Warn("Mainline not found on page %d", pageNum) - break + return true, nil } organic, _ := page.Elements(Selectors.Result) @@ -196,9 +179,8 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S if len(organic) == 0 && len(ads) == 0 { // Empty mainline = zero-result query or end of pagination, not // a parser failure; don't trip the retry path. - closePage() e.logger.Debug("No results on page %d", pageNum) - break + return true, nil } for _, r := range organic { @@ -213,10 +195,18 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S nextAdRank-- } } + return false, nil + } - closePage() + for query.Limit <= 0 || len(all) < query.Limit { + done, err := fetchPage() + if err != nil { + return nil, err + } pageNum++ - + if done { + break + } if query.Limit > 0 && len(all) >= query.Limit { break } @@ -293,9 +283,7 @@ func (e *Ecosia) parseImageResult(el *rod.Element, rank int) (core.SearchResult, // query.Start is ignored: per-page card count varies, so callers should // drive depth through query.Limit alone. func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.WithEngine(core.EnsureContext(ctx), e.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, e.Name(), false) scoped := *e scoped.logger = e.logger.WithRequest(ctx) e = &scoped @@ -311,47 +299,38 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c out := []core.SearchResult{} pageNum := 0 nextRank := 1 - for query.Limit <= 0 || len(out) < query.Limit { + // fetchPage loads one image page and appends parsed results. + // Returns (done, error): done=true ends the outer loop without error. + fetchPage := func() (bool, error) { u, err := BuildImageURL(query, pageNum) if err != nil { - return nil, err + return false, err } page, err := e.Navigate(ctx, u) if err != nil { - return nil, err - } - closePage := func() { - if e.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - e.logger.Debug("Page close error: %v", closeErr) - } + return false, err } + defer core.DeferClosePage(ctx, page, &e.Browser)() if err := page.WaitLoad(); err != nil { - closePage() e.logger.Error("Page load wait failed: %s", err) - return nil, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } if _, err := page.Timeout(e.GetSelectorTimeout()).Element(Selectors.ImageResult); err != nil { if e.isCaptcha(page) { - closePage() e.logger.Error("Captcha detected: %s", u) - return nil, core.ErrCaptcha + return false, core.ErrCaptcha } - closePage() e.logger.Debug("No image results on page %d", pageNum) - break + return true, nil } elements, err := page.Elements(Selectors.ImageResult) if err != nil { - closePage() e.logger.Error("Cannot collect image results: %s", err) - return nil, core.ErrParser + return false, core.ErrParser } for _, el := range elements { @@ -363,10 +342,18 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c break } } + return false, nil + } - closePage() + for query.Limit <= 0 || len(out) < query.Limit { + done, err := fetchPage() + if err != nil { + return nil, err + } pageNum++ - + if done { + break + } if query.Limit > 0 && len(out) >= query.Limit { break } diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go index aede483..780607e 100644 --- a/ecosia/search_raw.go +++ b/ecosia/search_raw.go @@ -71,9 +71,7 @@ func imageResultParser(response *http.Response) ([]core.SearchResult, error) { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.EnsureContext(ctx) - ctx = core.WithEngine(ctx, "ecosia") - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, "ecosia", false) defer func() { if recovered := recover(); recovered != nil { err = core.RecoverEnginePanicWithContext(ctx, "ecosia", recovered, nil) diff --git a/google/search.go b/google/search.go index 7c24b8f..d315942 100644 --- a/google/search.go +++ b/google/search.go @@ -12,7 +12,6 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" "github.com/karust/openserp/core" - "golang.org/x/time/rate" ) // Google implements core.SearchEngine for Google SERP pages. @@ -38,12 +37,6 @@ func (gogl *Google) Name() string { return "google" } -// GetRateLimiter returns a limiter configured from SearchEngineOptions. -func (gogl *Google) GetRateLimiter() *rate.Limiter { - ratelimit := rate.Every(gogl.GetRatelimit()) - return rate.NewLimiter(ratelimit, gogl.RateBurst) -} - func (gogl *Google) getTotalResults(page *rod.Page) (int, error) { if gogl.rgxpGetDigits == nil { return 0, core.ErrParser @@ -178,10 +171,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) { // Search executes a Google web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.WithEngine(core.EnsureContext(ctx), gogl.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithMinimalBrowserProfile(ctx) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true) scoped := *gogl scoped.logger = gogl.logger.WithRequest(ctx) gogl = &scoped @@ -446,10 +436,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor // SearchImage executes a Google image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.WithEngine(core.EnsureContext(ctx), gogl.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithMinimalBrowserProfile(ctx) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true) scoped := *gogl scoped.logger = gogl.logger.WithRequest(ctx) gogl = &scoped @@ -591,10 +578,5 @@ func (gogl *Google) parseImageCell(r *rod.Element, dst map[string]core.SearchRes } func (gogl *Google) close(ctx context.Context, page *rod.Page) { - if !gogl.Browser.LeavePageOpen { - err := core.ClosePageWithTimeout(ctx, page, time.Second) - if err != nil { - gogl.logger.Debug("Page close error: %v", err) - } - } + core.DeferClosePage(ctx, page, &gogl.Browser)() } diff --git a/google/search_raw.go b/google/search_raw.go index afa2a2b..39c33e4 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -109,9 +109,7 @@ func classifyGoogleRawHTML(body []byte) error { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.EnsureContext(ctx) - ctx = core.WithEngine(ctx, "google") - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, "google", false) defer func() { if recovered := recover(); recovered != nil { err = core.RecoverEnginePanicWithContext(ctx, "google", recovered, nil) diff --git a/logo.svg b/logo.svg index 5438785..104911d 100644 --- a/logo.svg +++ b/logo.svg @@ -1,94 +1,107 @@ - - - OpenSERP – Multi‑Engine API Logo - Professional banner-like logo with a unified search bar and chips for Google, Bing, DuckDuckGo, Yandex and Baidu, plus an API caption. - + - - - - - - + + + - - - - - - - - OpenSERP - - + + OpenSERP - - - - - - - + + One API · Every search engine · Free & open-source + + + + + + + Google - - Search across Google, Bing, DuckDuckGo, Yandex, Baidu… - - - - - - - - - + + + + Bing + + + + + DuckDuckGo + + + + + Yandex + + + + + Baidu + + + + + Ecosia + + + + + more - - - - - - - Google - + + + + + + + + + + + + - - - - Bing - - - - - - DuckDuckGo - - - - - - Yandex - - - - - - Baidu + + + + + + localhost:7000/megasearch?q=… + + + Search - - - API + + + + + + + + - - + + + + + JSON + + + + NDJSON + + + + MD + + + + TXT + + + \ No newline at end of file diff --git a/yandex/search.go b/yandex/search.go index 7dbb94a..14f1e2f 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -9,7 +9,6 @@ import ( "github.com/go-rod/rod" "github.com/karust/openserp/core" - "golang.org/x/time/rate" ) // ImageEntity contains one image record from Yandex image search state JSON. @@ -60,12 +59,6 @@ func (yand *Yandex) Name() string { return "yandex" } -// GetRateLimiter returns a limiter configured from SearchEngineOptions. -func (yand *Yandex) GetRateLimiter() *rate.Limiter { - ratelimit := rate.Every(yand.GetRatelimit()) - return rate.NewLimiter(ratelimit, yand.RateBurst) -} - func (yand *Yandex) isCaptcha(page *rod.Page) bool { has, _, _ := page.Has(Selectors.Captcha) return has @@ -146,9 +139,7 @@ func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntit // Search executes a Yandex web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.WithEngine(core.EnsureContext(ctx), yand.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, yand.Name(), false) scoped := *yand scoped.logger = yand.logger.WithRequest(ctx) yand = &scoped @@ -172,40 +163,32 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor } startPage := searchPage - for len(allResults) < query.Limit { + // fetchPage loads one SERP page and appends parsed results. + // Returns (done, error): done=true ends the outer loop without error. + fetchPage := func() (bool, error) { url, err := BuildURL(query, searchPage) if err != nil { - return nil, err + return false, err } page, err := yand.Navigate(ctx, url) if err != nil { - return nil, err - } - closePage := func() { - if yand.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - yand.logger.Debug("Page close error: %v", closeErr) - } + return false, err } + defer core.DeferClosePage(ctx, page, &yand.Browser)() elements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, yand.GetSelectorTimeout()) if err != nil { if yand.isCaptcha(page) { yand.logger.Error("Captcha detected: %s", url) - closePage() - return nil, core.ErrCaptcha + return false, core.ErrCaptcha } if yand.isNoResults(page) { yand.logger.Warn("No results found") - closePage() - break + return true, nil } - closePage() yand.logger.Error("Cannot parse search results: %s", err) - return nil, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } r := yand.parseResults(elements, searchPage) @@ -217,12 +200,18 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor } } allResults = append(allResults, r...) + return false, nil + } + for len(allResults) < query.Limit { + done, err := fetchPage() + if err != nil { + return nil, err + } searchPage++ - - // Close tab before opening new one during the cycle - closePage() - + if done { + break + } if err := core.SleepContext(ctx, yand.pageSleep); err != nil { return nil, err } @@ -235,9 +224,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor // SearchImage executes a Yandex image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.WithEngine(core.EnsureContext(ctx), yand.Name()) - ctx = core.WithProfileRegion(ctx, query.LangCode) - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, yand.Name(), false) scoped := *yand scoped.logger = yand.logger.WithRequest(ctx) yand = &scoped @@ -248,25 +235,20 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S allowPagination := query.Limit > 30 searchPage := 0 - for len(searchResults) < query.Limit { + // fetchPage loads one image page and appends parsed results. + // Returns (done, error): done=true ends the outer loop. + fetchPage := func() (bool, error) { url, err := BuildImageURL(query, searchPage) if err != nil { - return nil, err + return false, err } searchPage += 1 page, err := yand.Navigate(ctx, url) if err != nil { - return nil, err - } - closePage := func() { - if yand.Browser.LeavePageOpen { - return - } - if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { - yand.logger.Debug("Page close error: %v", closeErr) - } + return false, err } + defer core.DeferClosePage(ctx, page, &yand.Browser)() results, _, err := core.WaitForElements( ctx, @@ -283,14 +265,12 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S if err != nil { if yand.isCaptcha(page) { yand.logger.Error("Captcha detected: %s", url) - closePage() - return searchResults, core.ErrCaptcha + return false, core.ErrCaptcha } if yand.isNoResults(page) { yand.logger.Warn("No results found") } - closePage() - return searchResults, core.ErrSearchTimeout + return false, core.ErrSearchTimeout } pageEntities := yand.parseImageEntities(results) @@ -301,8 +281,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S } } if len(pageEntities) == 0 { - closePage() - break + return true, nil } for id := range pageEntities { @@ -317,15 +296,22 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S searchResults = append(searchResults, res) } if len(searchResults) >= query.Limit { - closePage() - break + return true, nil } if searchPage == 1 && !allowPagination { - closePage() + return true, nil + } + return false, nil + } + + for len(searchResults) < query.Limit { + done, err := fetchPage() + if err != nil { + return searchResults, err + } + if done { break } - - closePage() } sort.Slice(searchResults, func(i, j int) bool { diff --git a/yandex/search_raw.go b/yandex/search_raw.go index a65be83..d613e3e 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -25,9 +25,7 @@ func classifyYandexRawHTML(body []byte) error { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.EnsureContext(ctx) - ctx = core.WithEngine(ctx, "yandex") - ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + ctx = core.PrepareEngineContext(ctx, query, "yandex", false) defer func() { if recovered := recover(); recovered != nil { err = core.RecoverEnginePanicWithContext(ctx, "yandex", recovered, nil)