feat: update DuckDuckGo search logic, use "more results" button for next page results (#41)

* feat: implement windowOrganicResults function and update DuckDuckGo search logic
* Add fallback ddg "more results" selector. Update test HTML + add test

---------

Co-authored-by: Rustem Kamalov <karust13@gmail.com>

cmd/root.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	version               = "0.8.11"
+	version               = "0.8.12"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )

duckduckgo/search.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
 	"github.com/karust/openserp/core"
 )
 
@@ -14,8 +15,7 @@ import (
 type DuckDuckGo struct {
 	core.Browser
 	core.SearchEngineOptions
-	pageSleep time.Duration // Sleep between pages
-	logger    *core.EngineLogger
+	logger *core.EngineLogger
 }
 
 // New creates a DuckDuckGo engine instance with browser/runtime options applied.
@@ -25,7 +25,6 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo {
 	ddg.SearchEngineOptions = opts
 	ddg.logger = core.NewEngineLogger("DuckDuckGo")
 
-	ddg.pageSleep = time.Second * 1
 	return &ddg
 }
 
@@ -146,6 +145,35 @@ func ddgElementHasAdMarker(el *rod.Element) bool {
 	return false
 }
 
+func windowOrganicResults(results []core.SearchResult, start, limit int) []core.SearchResult {
+	out := make([]core.SearchResult, 0, len(results))
+	skipped, kept := 0, 0
+	for _, result := range results {
+		if !result.Ad && skipped < start {
+			skipped++
+			continue
+		}
+		if !result.Ad && limit > 0 && kept >= limit {
+			continue
+		}
+		out = append(out, result)
+		if !result.Ad {
+			kept++
+		}
+	}
+	return out
+}
+
+func findMoreResultsButton(page *rod.Page) *rod.Element {
+	for _, selector := range Selectors.MoreResults {
+		has, button, err := page.Has(selector)
+		if err == nil && has && button != nil {
+			return button
+		}
+	}
+	return nil
+}
+
 // Search executes a DuckDuckGo web search and returns normalized search
 // results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -155,72 +183,70 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
 	ddg = &scoped
 
 	ddg.logger.Debug("Starting search, query: %+v", query)
-
-	allResults := []core.SearchResult{}
-	var pageFeatures []core.SerpFeature
-	searchPage := 0
-
-	// fetchPage loads one SERP page and appends parsed results.
-	// Returns (done, error): done=true ends the outer loop without error.
-	fetchPage := func() (bool, error) {
-		url, err := BuildURL(query, searchPage)
-		if err != nil {
-			return false, err
-		}
-
-		page, err := ddg.Navigate(ctx, url)
-		if err != nil {
-			return false, err
-		}
-		defer core.DeferClosePage(ctx, page, &ddg.Browser)()
-
-		elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout())
-		if err != nil {
-			if ddg.isNoResults(page) {
-				ddg.logger.Warn("No results found")
-				return true, nil
-			}
-			if ddg.isCaptcha(page) {
-				ddg.logger.Error("Captcha detected: %s", url)
-				return false, core.ErrCaptcha
-			}
-			ddg.logger.Error("Cannot parse search results: %s", err)
-			return false, core.ErrSearchTimeout
-		}
-		ddg.logger.Debug("Found results with selector: %s", selector)
-
-		r := ddg.parseResults(elements, searchPage)
-		if len(r) == 0 {
-			ddg.logger.Debug("No valid results found on page %d", searchPage)
-			return false, core.ErrSearchTimeout
-		}
-
-		if query.Features && searchPage == 0 {
-			pageFeatures = extractDDGFeaturesFromPage(page)
-		}
-		allResults = append(allResults, r...)
-		return false, nil
+	if query.Start < 0 {
+		return nil, fmt.Errorf("incorrect start provided")
 	}
 
-	for core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) {
-		done, err := fetchPage()
-		if err != nil {
-			return nil, err
+	url, err := BuildURL(query, 0)
+	if err != nil {
+		return nil, err
+	}
+	page, err := ddg.Navigate(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer core.DeferClosePage(ctx, page, &ddg.Browser)()
+
+	elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout())
+	if err != nil {
+		if ddg.isNoResults(page) {
+			ddg.logger.Warn("No results found")
+			return []core.SearchResult{}, nil
 		}
-		searchPage++
-		if done || !core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) {
+		if ddg.isCaptcha(page) {
+			ddg.logger.Error("Captcha detected: %s", url)
+			return nil, core.ErrCaptcha
+		}
+		ddg.logger.Error("Cannot parse search results: %s", err)
+		return nil, core.ErrSearchTimeout
+	}
+	ddg.logger.Debug("Found results with selector: %s", selector)
+
+	allResults := ddg.parseResults(elements, 0)
+	if len(allResults) == 0 {
+		return nil, core.ErrSearchTimeout
+	}
+	var pageFeatures []core.SerpFeature
+	if query.Features {
+		pageFeatures = extractDDGFeaturesFromPage(page)
+	}
+
+	wantOrganic := query.Start + query.Limit
+	for core.CountOrganicResults(allResults) < wantOrganic {
+		button := findMoreResultsButton(page)
+		if button == nil {
 			break
 		}
-		if err := core.SleepContext(ctx, ddg.pageSleep); err != nil {
-			return nil, err
+		before := len(elements)
+		if err := button.Click(proto.InputMouseButtonLeft, 1); err != nil {
+			ddg.logger.Debug("More results click failed: %s", err)
+			break
 		}
+		if err := page.Timeout(ddg.GetSelectorTimeout()).WaitElementsMoreThan(selector, before); err != nil {
+			if ctx.Err() != nil {
+				return nil, ctx.Err()
+			}
+			ddg.logger.Debug("No additional results loaded")
+			break
+		}
+		elements, err = page.Elements(selector)
+		if err != nil || len(elements) <= before {
+			break
+		}
+		allResults = core.DeduplicateResults(append(allResults, ddg.parseResults(elements, 0)...))
 	}
 
-	// Deduplicate results
-	deduped := core.DeduplicateResults(allResults)
-
-	// Trim to exact limit if necessary
-	deduped = core.LimitOrganicResults(deduped, query.Limit)
+	deduped := windowOrganicResults(core.DeduplicateResults(allResults), query.Start, query.Limit)
 
 	ddg.logger.Info("Search completed: %d results", len(deduped))
 	return core.AttachFeaturesToFirstResult(deduped, pageFeatures), nil

duckduckgo/search_test.go

@@ -1,10 +1,14 @@
 package duckduckgo
 
 import (
+	"context"
 	"net/url"
+	"os"
 	"testing"
 
 	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/testutil"
+	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestBuildURL(t *testing.T) {
@@ -257,67 +261,51 @@ func TestDuckDuckGoLanguageMapping(t *testing.T) {
 	}
 }
 
-func TestShouldFetchDuckDuckGoPage(t *testing.T) {
-	results := func(organic, ads int) []core.SearchResult {
-		out := make([]core.SearchResult, 0, organic+ads)
-		for i := 0; i < ads; i++ {
-			out = append(out, core.SearchResult{URL: "https://ad.example/" + string(rune('a'+i)), Ad: true})
-		}
-		for i := 0; i < organic; i++ {
-			out = append(out, core.SearchResult{URL: "https://example.com/" + string(rune('a'+i))})
-		}
-		return out
+func TestWindowOrganicResults(t *testing.T) {
+	results := []core.SearchResult{
+		{URL: "https://ad.example", Ad: true},
+		{URL: "https://example.com/1", Rank: 1},
+		{URL: "https://example.com/2", Rank: 2},
+		{URL: "https://example.com/3", Rank: 3},
+		{URL: "https://example.com/4", Rank: 4},
 	}
 
-	tests := []struct {
-		name         string
-		results      []core.SearchResult
-		limit        int
-		pagesFetched int
-		want         bool
-	}{
-		{
-			name:         "first page is always fetched",
-			limit:        10,
-			pagesFetched: 0,
-			want:         true,
-		},
-		{
-			name:         "default limit does not chase a short first page",
-			results:      results(8, 2),
-			limit:        10,
-			pagesFetched: 1,
-			want:         false,
-		},
-		{
-			name:         "explicit larger limit can paginate",
-			results:      results(8, 0),
-			limit:        11,
-			pagesFetched: 1,
-			want:         true,
-		},
-		{
-			name:         "satisfied larger limit stops",
-			results:      results(11, 0),
-			limit:        11,
-			pagesFetched: 1,
-			want:         false,
-		},
-		{
-			name:         "unset internal query stops after first page",
-			results:      results(8, 0),
-			limit:        0,
-			pagesFetched: 1,
-			want:         false,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := core.ShouldFetchResultPage(core.CountOrganicResults(tt.results), tt.limit, tt.pagesFetched)
-			if got != tt.want {
-				t.Fatalf("ShouldFetchResultPage() = %t, want %t", got, tt.want)
-			}
-		})
+	got := windowOrganicResults(results, 1, 2)
+	if len(got) != 3 || !got[0].Ad || got[1].Rank != 2 || got[2].Rank != 3 {
+		t.Fatalf("unexpected result window: %#v", got)
+	}
+}
+
+func TestFindMoreResultsButton(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	fixture, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+
+	browser := ithelper.CreateBrowser(t)
+	page, err := browser.Navigate(context.Background(), "about:blank")
+	if err != nil {
+		t.Fatalf("navigate: %v", err)
+	}
+	defer core.DeferClosePage(context.Background(), page, browser)()
+
+	cases := []struct {
+		html    string
+		wantHit bool
+	}{
+		{string(fixture), true},                               // real DDG markup
+		{`<button id="js-more-results-btn">x</button>`, true}, // renamed id, substring fallback
+		{`<div>no button</div>`, false},
+	}
+
+	for _, tc := range cases {
+		if err := page.SetDocumentContent(tc.html); err != nil {
+			t.Fatalf("set content: %v", err)
+		}
+		if hit := findMoreResultsButton(page) != nil; hit != tc.wantHit {
+			t.Fatalf("hit=%v want=%v", hit, tc.wantHit)
+		}
 	}
 }

duckduckgo/selectors.go

@@ -14,6 +14,7 @@ var Selectors = struct {
 	Desc             []string
 	Link             []string
 	AdBadge          []string
+	MoreResults      []string
 	ImageResult      []string
 	ImageImg         []string
 	ImageTitle       []string
@@ -83,6 +84,10 @@ var Selectors = struct {
 		".ad-badge",
 		".result--ad",
 	},
+	MoreResults: []string{
+		"#more-results",
+		"[id*='more-result']",
+	},
 	ImageResult: []string{
 		"figure[data-testid='image-result']",
 		"figure",