From 5d5d135fbf511a3581efb9e1d0c6c335f0fcc2e3 Mon Sep 17 00:00:00 2001 From: Parash Subedi Date: Wed, 22 Jul 2026 02:56:20 +0545 Subject: [PATCH] feat: update DuckDuckGo search logic, use "more results" button for next page results (#41) * feat: implement windowOrganicResults function and update DuckDuckGo search logic * Add fallback ddg "more results" selector. Update test HTML + add test --------- Co-authored-by: Rustem Kamalov --- cmd/root.go | 2 +- duckduckgo/search.go | 148 ++++++++++++++---------- duckduckgo/search_test.go | 108 ++++++++--------- duckduckgo/selectors.go | 5 + duckduckgo/testdata/search_results.html | 2 +- 5 files changed, 142 insertions(+), 123 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 7104116..8c80bdc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( ) const ( - version = "0.8.11" + version = "0.8.12" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/duckduckgo/search.go b/duckduckgo/search.go index 205a79b..1980948 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -7,6 +7,7 @@ import ( "time" "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" "github.com/karust/openserp/core" ) @@ -14,8 +15,7 @@ import ( type DuckDuckGo struct { core.Browser core.SearchEngineOptions - pageSleep time.Duration // Sleep between pages - logger *core.EngineLogger + logger *core.EngineLogger } // New creates a DuckDuckGo engine instance with browser/runtime options applied. @@ -25,7 +25,6 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo { ddg.SearchEngineOptions = opts ddg.logger = core.NewEngineLogger("DuckDuckGo") - ddg.pageSleep = time.Second * 1 return &ddg } @@ -146,6 +145,35 @@ func ddgElementHasAdMarker(el *rod.Element) bool { return false } +func windowOrganicResults(results []core.SearchResult, start, limit int) []core.SearchResult { + out := make([]core.SearchResult, 0, len(results)) + skipped, kept := 0, 0 + for _, result := range results { + if !result.Ad && skipped < start { + skipped++ + continue + } + if !result.Ad && limit > 0 && kept >= limit { + continue + } + out = append(out, result) + if !result.Ad { + kept++ + } + } + return out +} + +func findMoreResultsButton(page *rod.Page) *rod.Element { + for _, selector := range Selectors.MoreResults { + has, button, err := page.Has(selector) + if err == nil && has && button != nil { + return button + } + } + return nil +} + // Search executes a DuckDuckGo web search and returns normalized search // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { @@ -155,72 +183,70 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] ddg = &scoped ddg.logger.Debug("Starting search, query: %+v", query) - - allResults := []core.SearchResult{} - var pageFeatures []core.SerpFeature - searchPage := 0 - - // fetchPage loads one SERP page and appends parsed results. - // Returns (done, error): done=true ends the outer loop without error. - fetchPage := func() (bool, error) { - url, err := BuildURL(query, searchPage) - if err != nil { - return false, err - } - - page, err := ddg.Navigate(ctx, url) - if err != nil { - return false, err - } - defer core.DeferClosePage(ctx, page, &ddg.Browser)() - - elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout()) - if err != nil { - if ddg.isNoResults(page) { - ddg.logger.Warn("No results found") - return true, nil - } - if ddg.isCaptcha(page) { - ddg.logger.Error("Captcha detected: %s", url) - return false, core.ErrCaptcha - } - ddg.logger.Error("Cannot parse search results: %s", err) - return false, core.ErrSearchTimeout - } - ddg.logger.Debug("Found results with selector: %s", selector) - - r := ddg.parseResults(elements, searchPage) - if len(r) == 0 { - ddg.logger.Debug("No valid results found on page %d", searchPage) - return false, core.ErrSearchTimeout - } - - if query.Features && searchPage == 0 { - pageFeatures = extractDDGFeaturesFromPage(page) - } - allResults = append(allResults, r...) - return false, nil + if query.Start < 0 { + return nil, fmt.Errorf("incorrect start provided") } - for core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) { - done, err := fetchPage() - if err != nil { - return nil, err + url, err := BuildURL(query, 0) + if err != nil { + return nil, err + } + page, err := ddg.Navigate(ctx, url) + if err != nil { + return nil, err + } + defer core.DeferClosePage(ctx, page, &ddg.Browser)() + + elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout()) + if err != nil { + if ddg.isNoResults(page) { + ddg.logger.Warn("No results found") + return []core.SearchResult{}, nil } - searchPage++ - if done || !core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) { + if ddg.isCaptcha(page) { + ddg.logger.Error("Captcha detected: %s", url) + return nil, core.ErrCaptcha + } + ddg.logger.Error("Cannot parse search results: %s", err) + return nil, core.ErrSearchTimeout + } + ddg.logger.Debug("Found results with selector: %s", selector) + + allResults := ddg.parseResults(elements, 0) + if len(allResults) == 0 { + return nil, core.ErrSearchTimeout + } + var pageFeatures []core.SerpFeature + if query.Features { + pageFeatures = extractDDGFeaturesFromPage(page) + } + + wantOrganic := query.Start + query.Limit + for core.CountOrganicResults(allResults) < wantOrganic { + button := findMoreResultsButton(page) + if button == nil { break } - if err := core.SleepContext(ctx, ddg.pageSleep); err != nil { - return nil, err + before := len(elements) + if err := button.Click(proto.InputMouseButtonLeft, 1); err != nil { + ddg.logger.Debug("More results click failed: %s", err) + break } + if err := page.Timeout(ddg.GetSelectorTimeout()).WaitElementsMoreThan(selector, before); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + ddg.logger.Debug("No additional results loaded") + break + } + elements, err = page.Elements(selector) + if err != nil || len(elements) <= before { + break + } + allResults = core.DeduplicateResults(append(allResults, ddg.parseResults(elements, 0)...)) } - // Deduplicate results - deduped := core.DeduplicateResults(allResults) - - // Trim to exact limit if necessary - deduped = core.LimitOrganicResults(deduped, query.Limit) + deduped := windowOrganicResults(core.DeduplicateResults(allResults), query.Start, query.Limit) ddg.logger.Info("Search completed: %d results", len(deduped)) return core.AttachFeaturesToFirstResult(deduped, pageFeatures), nil diff --git a/duckduckgo/search_test.go b/duckduckgo/search_test.go index c82c588..effdea7 100644 --- a/duckduckgo/search_test.go +++ b/duckduckgo/search_test.go @@ -1,10 +1,14 @@ package duckduckgo import ( + "context" "net/url" + "os" "testing" "github.com/karust/openserp/core" + "github.com/karust/openserp/testutil" + "github.com/karust/openserp/testutil/ithelper" ) func TestBuildURL(t *testing.T) { @@ -257,67 +261,51 @@ func TestDuckDuckGoLanguageMapping(t *testing.T) { } } -func TestShouldFetchDuckDuckGoPage(t *testing.T) { - results := func(organic, ads int) []core.SearchResult { - out := make([]core.SearchResult, 0, organic+ads) - for i := 0; i < ads; i++ { - out = append(out, core.SearchResult{URL: "https://ad.example/" + string(rune('a'+i)), Ad: true}) - } - for i := 0; i < organic; i++ { - out = append(out, core.SearchResult{URL: "https://example.com/" + string(rune('a'+i))}) - } - return out +func TestWindowOrganicResults(t *testing.T) { + results := []core.SearchResult{ + {URL: "https://ad.example", Ad: true}, + {URL: "https://example.com/1", Rank: 1}, + {URL: "https://example.com/2", Rank: 2}, + {URL: "https://example.com/3", Rank: 3}, + {URL: "https://example.com/4", Rank: 4}, } - tests := []struct { - name string - results []core.SearchResult - limit int - pagesFetched int - want bool - }{ - { - name: "first page is always fetched", - limit: 10, - pagesFetched: 0, - want: true, - }, - { - name: "default limit does not chase a short first page", - results: results(8, 2), - limit: 10, - pagesFetched: 1, - want: false, - }, - { - name: "explicit larger limit can paginate", - results: results(8, 0), - limit: 11, - pagesFetched: 1, - want: true, - }, - { - name: "satisfied larger limit stops", - results: results(11, 0), - limit: 11, - pagesFetched: 1, - want: false, - }, - { - name: "unset internal query stops after first page", - results: results(8, 0), - limit: 0, - pagesFetched: 1, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := core.ShouldFetchResultPage(core.CountOrganicResults(tt.results), tt.limit, tt.pagesFetched) - if got != tt.want { - t.Fatalf("ShouldFetchResultPage() = %t, want %t", got, tt.want) - } - }) + got := windowOrganicResults(results, 1, 2) + if len(got) != 3 || !got[0].Ad || got[1].Rank != 2 || got[2].Rank != 3 { + t.Fatalf("unexpected result window: %#v", got) + } +} + +func TestFindMoreResultsButton(t *testing.T) { + testutil.RequireIntegration(t) + + fixture, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + browser := ithelper.CreateBrowser(t) + page, err := browser.Navigate(context.Background(), "about:blank") + if err != nil { + t.Fatalf("navigate: %v", err) + } + defer core.DeferClosePage(context.Background(), page, browser)() + + cases := []struct { + html string + wantHit bool + }{ + {string(fixture), true}, // real DDG markup + {``, true}, // renamed id, substring fallback + {`
no button
`, false}, + } + + for _, tc := range cases { + if err := page.SetDocumentContent(tc.html); err != nil { + t.Fatalf("set content: %v", err) + } + if hit := findMoreResultsButton(page) != nil; hit != tc.wantHit { + t.Fatalf("hit=%v want=%v", hit, tc.wantHit) + } } } diff --git a/duckduckgo/selectors.go b/duckduckgo/selectors.go index 50b7af4..8c80e69 100644 --- a/duckduckgo/selectors.go +++ b/duckduckgo/selectors.go @@ -14,6 +14,7 @@ var Selectors = struct { Desc []string Link []string AdBadge []string + MoreResults []string ImageResult []string ImageImg []string ImageTitle []string @@ -83,6 +84,10 @@ var Selectors = struct { ".ad-badge", ".result--ad", }, + MoreResults: []string{ + "#more-results", + "[id*='more-result']", + }, ImageResult: []string{ "figure[data-testid='image-result']", "figure", diff --git a/duckduckgo/testdata/search_results.html b/duckduckgo/testdata/search_results.html index b63ca4d..17ee327 100644 --- a/duckduckgo/testdata/search_results.html +++ b/duckduckgo/testdata/search_results.html @@ -1 +1 @@ -how to fetch in javascript at DuckDuckGo

  2. To fetch data in JavaScript, use thefetch()method, which takes a URL as an argument and returns a Promise that resolves to a Response object. You can then handle the response using methods like.json()to extract the data you need.

    MozillaMedium

    Fetching Data in JavaScript

    To fetch data in JavaScript, you can use thefetch()method. This method is designed to make HTTP requests to a specified URL and handle the responses.

    Basic Usage of fetch()

    Thefetch()method takes a URL as its first argument and returns a Promise that resolves to a Response object. Here’s a simple example:

    javascript
    fetch('http://test.test').then(response=>response.json()).then(data=>console.log(data)).catch(error=>console.error('Error:', error));

    Key Steps in Using fetch()

    1. Make a Request: Call thefetch()method with the desired URL.
    2. Handle the Response: Use the.then()method to process the Response object.
      • Use.json()to convert the response body to JSON format.
    3. Error Handling: Use.catch()to handle any errors that may occur during the fetch operation.

    Auto-generated based on listed sources. May contain inaccuracies.

    Was this helpful?

  3. Mozilla Developer

    http://test.test› en-US › docs › Web › API › Fetch_API › Using_Fetch

    Using the Fetch API - Web APIs | MDN - MDN Web Docs

    Aug 20, 2025Using theFetchAPI TheFetchAPI provides aJavaScriptinterface for making HTTP requests and processing the responses.Fetchis the modern replacement for XMLHttpRequest: unlike XMLHttpRequest, which uses callbacks,Fetchis promise-based and is integrated with features of the modern web such as service workers and Cross-Origin Resource Sharing (CORS). With theFetchAPI, you make a request ...

  6. w3reference.com

    http://test.test› blog › how-to-use-javascript-fetch-api-step-by-step-guide-with-examples

    How to Use JavaScript Fetch API: Step-by-Step Guide with Examples

    1 day agoTheFetchAPI has emerged as the standard for handling HTTP requests inJavaScript, replacing the older XMLHttpRequest with a more modern, promise-based interface. Unlike XMLHttpRequest,Fetchis designed to work seamlessly with promises and async/await, making asynchronous code easier to read and maintain.

  7. FreeCodecamp

    http://test.test› news › how-to-fetch-data-from-an-api-using-the-fetch-api-in-javascript

    How to Fetch Data from an API Using the Fetch API in JavaScript

    TheFetchAPI is a modernJavaScriptinterface for making network requests, primarily designed to replace the older XMLHttpRequest. It provides a more straightforward and flexible way to handle HTTP requests, making it easier for developers to work with APIs andfetchdata from servers.

  8. DEV Community

    http://test.test› wisdomudo › how-to-use-the-fetch-api-in-javascript-for-http-requests-110n

    How to Use the Fetch API in JavaScript for HTTP Requests

    Oct 15, 2025Practical examples of usingFetchinreal-world projects Understanding theFetchAPI Before usingFetch, it's important to understand what it does. TheFetchAPI is a built-inJavaScriptfeature that lets you make asynchronous HTTP requests, meaning your code canfetchdata from a server in the background without freezing the rest of the page.

  10. DigitalOcean

    http://test.test› community › tutorials › how-to-use-the-javascript-fetch-api-to-get-data

    How to Use JavaScript Fetch API: Step-by-Step Guide with Examples

    Feb 6, 2025With an understanding of the syntax for using theFetchAPI, you can now move on to usingfetch() on a real API. Step 2 — UsingFetchtoget Data from an API The following code samples will be based on the JSONPlaceholder API. Using the API, you will get ten users and display them on the page usingJavaScript.

  12. The Modern JavaScript Tutorial

    http://test.test› fetch

    Fetch - The Modern JavaScript Tutorial

    Otherwise, if afetchfails, or the response has non-200 status, we just return null in the resulting array. Please note: .then call is attached directly tofetch, so that when we have the response, it doesn't wait for other fetches, but starts to read .json () immediately.
  1. JavaScript

    This kind of functionality was previously achieved using XMLHttpRequest. Fetch provides a better alternative that can be easily used by other technologies such as Service Workers. Fetch also provides a single logical place to define other HTTP-related concepts such as CORS and extensions to HTTP.

    More at MDN Web Docs
    Source:MDN Web Docs
    Was this helpful?

Get Our Windows Browser

Protect your data as you search, browse, and use AI.

Download
Custom date rangeX
+openai at DuckDuckGo

  1. OpenAI

    http://test.test

    OpenAI | Research & Deployment

    We believe our research will eventually lead to artificial general intelligence, a system that can solve human-level problems. Building safe and beneficial AGI is our mission.

  7. Wikipedia

    http://test.test› wiki › OpenAI

    OpenAI - Wikipedia

    OpenAIis an American artificial intelligence (AI) research organization headquartered in San Francisco, consisting ofOpenAIGroup PBC, a for-profit public benefit corporation (PBC), partially controlled byOpenAIFoundation, a nonprofit.OpenAIdevelops generative AI models, particularly the GPT series of large language models.

  12. OpenAI is an American artificial intelligence research organization headquartered in San Francisco that develops large language models like the GPT family, text-to-image models like DALL·E, and text-to-video models like Sora, and its ChatGPT release in 2022 helped spark the generative AI boom.

    Wikipedia

    Auto-generated based on listed sources. May contain inaccuracies.

    Was this helpful?
  1. More Images

    OpenAI

    American artificial intelligence research organization
    openai.com
    OpenAI is an American artificial intelligence research organization headquartered in San Francisco, consisting of OpenAI Group PBC, a for-profit public benefit corporation, partially controlled by OpenAI Foundation, a nonprofit.OpenAI develops generative AI models, particularly the GPT series of large language models.Its release of ChatGPT in November 2022 has been credited with catalyzing the AI boom, and widespread interest in generative AI.OpenAI was founded in 2015 in Delaware as a nonprofitA for-profit subsidiary of the nonprofit was created in 2019, and a 2025 restructuring converted the subsidiary into a PBC that is 26% owned by the nonprofit.Microsoft previously invested over $13 billion into OpenAI, and provides Azure cloud computing resources.In October 2025, OpenAI conducted a $6.6 billion share sale that valued the company at $500 billion.Continued in Wikipedia
    TypePrivate
    IndustryArtificial intelligence
    FoundedDecember 08, 2015
    WebsiteWikipediaXInstagramYouTubeSoundCloud
    Source:Wikipedia
    Was this helpful?

Get Our Windows Browser

Protect your data as you search, browse, and use AI.

Download
Custom date rangeX