diff --git a/cmd/root.go b/cmd/root.go index 7104116..8c80bdc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( ) const ( - version = "0.8.11" + version = "0.8.12" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/duckduckgo/search.go b/duckduckgo/search.go index 205a79b..1980948 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -7,6 +7,7 @@ import ( "time" "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" "github.com/karust/openserp/core" ) @@ -14,8 +15,7 @@ import ( type DuckDuckGo struct { core.Browser core.SearchEngineOptions - pageSleep time.Duration // Sleep between pages - logger *core.EngineLogger + logger *core.EngineLogger } // New creates a DuckDuckGo engine instance with browser/runtime options applied. @@ -25,7 +25,6 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo { ddg.SearchEngineOptions = opts ddg.logger = core.NewEngineLogger("DuckDuckGo") - ddg.pageSleep = time.Second * 1 return &ddg } @@ -146,6 +145,35 @@ func ddgElementHasAdMarker(el *rod.Element) bool { return false } +func windowOrganicResults(results []core.SearchResult, start, limit int) []core.SearchResult { + out := make([]core.SearchResult, 0, len(results)) + skipped, kept := 0, 0 + for _, result := range results { + if !result.Ad && skipped < start { + skipped++ + continue + } + if !result.Ad && limit > 0 && kept >= limit { + continue + } + out = append(out, result) + if !result.Ad { + kept++ + } + } + return out +} + +func findMoreResultsButton(page *rod.Page) *rod.Element { + for _, selector := range Selectors.MoreResults { + has, button, err := page.Has(selector) + if err == nil && has && button != nil { + return button + } + } + return nil +} + // Search executes a DuckDuckGo web search and returns normalized search // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { @@ -155,72 +183,70 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] ddg = &scoped ddg.logger.Debug("Starting search, query: %+v", query) - - allResults := []core.SearchResult{} - var pageFeatures []core.SerpFeature - searchPage := 0 - - // fetchPage loads one SERP page and appends parsed results. - // Returns (done, error): done=true ends the outer loop without error. - fetchPage := func() (bool, error) { - url, err := BuildURL(query, searchPage) - if err != nil { - return false, err - } - - page, err := ddg.Navigate(ctx, url) - if err != nil { - return false, err - } - defer core.DeferClosePage(ctx, page, &ddg.Browser)() - - elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout()) - if err != nil { - if ddg.isNoResults(page) { - ddg.logger.Warn("No results found") - return true, nil - } - if ddg.isCaptcha(page) { - ddg.logger.Error("Captcha detected: %s", url) - return false, core.ErrCaptcha - } - ddg.logger.Error("Cannot parse search results: %s", err) - return false, core.ErrSearchTimeout - } - ddg.logger.Debug("Found results with selector: %s", selector) - - r := ddg.parseResults(elements, searchPage) - if len(r) == 0 { - ddg.logger.Debug("No valid results found on page %d", searchPage) - return false, core.ErrSearchTimeout - } - - if query.Features && searchPage == 0 { - pageFeatures = extractDDGFeaturesFromPage(page) - } - allResults = append(allResults, r...) - return false, nil + if query.Start < 0 { + return nil, fmt.Errorf("incorrect start provided") } - for core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) { - done, err := fetchPage() - if err != nil { - return nil, err + url, err := BuildURL(query, 0) + if err != nil { + return nil, err + } + page, err := ddg.Navigate(ctx, url) + if err != nil { + return nil, err + } + defer core.DeferClosePage(ctx, page, &ddg.Browser)() + + elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout()) + if err != nil { + if ddg.isNoResults(page) { + ddg.logger.Warn("No results found") + return []core.SearchResult{}, nil } - searchPage++ - if done || !core.ShouldFetchResultPage(core.CountOrganicResults(allResults), query.Limit, searchPage) { + if ddg.isCaptcha(page) { + ddg.logger.Error("Captcha detected: %s", url) + return nil, core.ErrCaptcha + } + ddg.logger.Error("Cannot parse search results: %s", err) + return nil, core.ErrSearchTimeout + } + ddg.logger.Debug("Found results with selector: %s", selector) + + allResults := ddg.parseResults(elements, 0) + if len(allResults) == 0 { + return nil, core.ErrSearchTimeout + } + var pageFeatures []core.SerpFeature + if query.Features { + pageFeatures = extractDDGFeaturesFromPage(page) + } + + wantOrganic := query.Start + query.Limit + for core.CountOrganicResults(allResults) < wantOrganic { + button := findMoreResultsButton(page) + if button == nil { break } - if err := core.SleepContext(ctx, ddg.pageSleep); err != nil { - return nil, err + before := len(elements) + if err := button.Click(proto.InputMouseButtonLeft, 1); err != nil { + ddg.logger.Debug("More results click failed: %s", err) + break } + if err := page.Timeout(ddg.GetSelectorTimeout()).WaitElementsMoreThan(selector, before); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + ddg.logger.Debug("No additional results loaded") + break + } + elements, err = page.Elements(selector) + if err != nil || len(elements) <= before { + break + } + allResults = core.DeduplicateResults(append(allResults, ddg.parseResults(elements, 0)...)) } - // Deduplicate results - deduped := core.DeduplicateResults(allResults) - - // Trim to exact limit if necessary - deduped = core.LimitOrganicResults(deduped, query.Limit) + deduped := windowOrganicResults(core.DeduplicateResults(allResults), query.Start, query.Limit) ddg.logger.Info("Search completed: %d results", len(deduped)) return core.AttachFeaturesToFirstResult(deduped, pageFeatures), nil diff --git a/duckduckgo/search_test.go b/duckduckgo/search_test.go index c82c588..effdea7 100644 --- a/duckduckgo/search_test.go +++ b/duckduckgo/search_test.go @@ -1,10 +1,14 @@ package duckduckgo import ( + "context" "net/url" + "os" "testing" "github.com/karust/openserp/core" + "github.com/karust/openserp/testutil" + "github.com/karust/openserp/testutil/ithelper" ) func TestBuildURL(t *testing.T) { @@ -257,67 +261,51 @@ func TestDuckDuckGoLanguageMapping(t *testing.T) { } } -func TestShouldFetchDuckDuckGoPage(t *testing.T) { - results := func(organic, ads int) []core.SearchResult { - out := make([]core.SearchResult, 0, organic+ads) - for i := 0; i < ads; i++ { - out = append(out, core.SearchResult{URL: "https://ad.example/" + string(rune('a'+i)), Ad: true}) - } - for i := 0; i < organic; i++ { - out = append(out, core.SearchResult{URL: "https://example.com/" + string(rune('a'+i))}) - } - return out +func TestWindowOrganicResults(t *testing.T) { + results := []core.SearchResult{ + {URL: "https://ad.example", Ad: true}, + {URL: "https://example.com/1", Rank: 1}, + {URL: "https://example.com/2", Rank: 2}, + {URL: "https://example.com/3", Rank: 3}, + {URL: "https://example.com/4", Rank: 4}, } - tests := []struct { - name string - results []core.SearchResult - limit int - pagesFetched int - want bool - }{ - { - name: "first page is always fetched", - limit: 10, - pagesFetched: 0, - want: true, - }, - { - name: "default limit does not chase a short first page", - results: results(8, 2), - limit: 10, - pagesFetched: 1, - want: false, - }, - { - name: "explicit larger limit can paginate", - results: results(8, 0), - limit: 11, - pagesFetched: 1, - want: true, - }, - { - name: "satisfied larger limit stops", - results: results(11, 0), - limit: 11, - pagesFetched: 1, - want: false, - }, - { - name: "unset internal query stops after first page", - results: results(8, 0), - limit: 0, - pagesFetched: 1, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := core.ShouldFetchResultPage(core.CountOrganicResults(tt.results), tt.limit, tt.pagesFetched) - if got != tt.want { - t.Fatalf("ShouldFetchResultPage() = %t, want %t", got, tt.want) - } - }) + got := windowOrganicResults(results, 1, 2) + if len(got) != 3 || !got[0].Ad || got[1].Rank != 2 || got[2].Rank != 3 { + t.Fatalf("unexpected result window: %#v", got) + } +} + +func TestFindMoreResultsButton(t *testing.T) { + testutil.RequireIntegration(t) + + fixture, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + browser := ithelper.CreateBrowser(t) + page, err := browser.Navigate(context.Background(), "about:blank") + if err != nil { + t.Fatalf("navigate: %v", err) + } + defer core.DeferClosePage(context.Background(), page, browser)() + + cases := []struct { + html string + wantHit bool + }{ + {string(fixture), true}, // real DDG markup + {``, true}, // renamed id, substring fallback + {`
To fetch data in JavaScript, you can use thefetch()method. This method is designed to make HTTP requests to a specified URL and handle the responses.
Thefetch()method takes a URL as its first argument and returns a Promise that resolves to a Response object. Here’s a simple example:
Mozilla Developer
http://test.test› en-US › docs › Web › API › Fetch_API › Using_Fetch
JavaScript Tutorial
http://test.test› web-apis › javascript-fetch-api
The Modern JavaScript Tutorial
http://test.test› fetch
OpenAI
http://test.test
GPT-4 is more creative and collaborative than ever before. It can generate, edit, and iterate with users on creative and technical writing tasks, such as composing songs, writing screenplays, or learning a user's writing style.
Build on the OpenAI API Platform Sign up or login with an OpenAI account to build with the OpenAI API.
Codex in ChatGPT helps engineering teams build faster with AI coding agents. Complete pull requests, refactors, code reviews, automations, and more across parallel workflows.
Our API platform offers our latest models and guides for safety best practices.
Improve the quality and efficiency of work for every employee with OpenAI technology.
Explore how our network of customers is using our technology to advance their goals.
ChatGPT
http://test.test
Wikipedia
http://test.test› wiki › OpenAI
ChatGPT
http://test.test› overview
OpenAI is an American artificial intelligence research organization headquartered in San Francisco that develops large language models like the GPT family, text-to-image models like DALL·E, and text-to-video models like Sora, and its ChatGPT release in 2022 helped spark the generative AI boom.