diff --git a/cmd/root.go b/cmd/root.go index 8751984..403097d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -36,6 +36,7 @@ type Config struct { BaiduConfig EngineConfig `mapstructure:"baidu"` BingConfig EngineConfig `mapstructure:"bing"` DuckDuckGoConfig EngineConfig `mapstructure:"duckduckgo"` + EcosiaConfig EngineConfig `mapstructure:"ecosia"` } type Config2Captcha struct { @@ -195,6 +196,7 @@ func sanitizedConfigForLog(cfg Config) map[string]interface{} { "baidu": cfg.BaiduConfig, "bing": cfg.BingConfig, "duckduckgo": cfg.DuckDuckGoConfig, + "ecosia": cfg.EcosiaConfig, } } diff --git a/cmd/search.go b/cmd/search.go index 66483f5..d3ebeae 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -12,6 +12,7 @@ import ( "github.com/karust/openserp/bing" "github.com/karust/openserp/core" "github.com/karust/openserp/duckduckgo" + "github.com/karust/openserp/ecosia" "github.com/karust/openserp/google" "github.com/karust/openserp/yandex" "github.com/sirupsen/logrus" @@ -21,7 +22,7 @@ import ( var searchCMD = &cobra.Command{ Use: "search", Aliases: []string{"find"}, - Short: "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo)", + Short: "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)", Args: cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)), Run: search, } @@ -142,6 +143,8 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string, engine = bing.New(*browser, config.BingConfig.SearchEngineOptions) case "duckduckgo": engine = duckduckgo.New(*browser, config.DuckDuckGoConfig.SearchEngineOptions) + case "ecosia": + engine = ecosia.New(*browser, config.EcosiaConfig.SearchEngineOptions) default: return nil, fmt.Errorf("no %q search engine found", engineType) } @@ -160,6 +163,8 @@ func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error) return google.Search(ctx, query) case "baidu": return baidu.Search(ctx, query) + case "ecosia": + return ecosia.Search(ctx, query) case "bing": logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.") return nil, fmt.Errorf("bing does not support raw requests mode") diff --git a/cmd/serve.go b/cmd/serve.go index f46cfda..8453c39 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -16,6 +16,7 @@ import ( "github.com/karust/openserp/bing" "github.com/karust/openserp/core" "github.com/karust/openserp/duckduckgo" + "github.com/karust/openserp/ecosia" "github.com/karust/openserp/google" "github.com/karust/openserp/yandex" "github.com/sirupsen/logrus" @@ -633,6 +634,13 @@ func browserEngineSpecs() []browserEngineSpec { return duckduckgo.New(browser, opts) }, }, + { + name: "ecosia", + opts: config.EcosiaConfig.SearchEngineOptions, + factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { + return ecosia.New(browser, opts) + }, + }, } } diff --git a/config.yaml b/config.yaml index ba5dae1..6a8906b 100644 --- a/config.yaml +++ b/config.yaml @@ -84,3 +84,8 @@ bing: duckduckgo: rate_requests: 4 rate_burst: 2 + +ecosia: + rate_requests: 4 + rate_burst: 2 + # No proxy tag means direct traffic diff --git a/ecosia/search.go b/ecosia/search.go new file mode 100644 index 0000000..f958c7d --- /dev/null +++ b/ecosia/search.go @@ -0,0 +1,421 @@ +// Package ecosia implements an Ecosia SERP scraper (web + image search). +// +// Ecosia (https://www.ecosia.org/) is a Berlin-based search engine that +// proxies results from Bing, Google, and its own EUSP/Staan index +// (https://staan.ai/) which underlying provider serves a query depends on +// market and device. +package ecosia + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/go-rod/rod" + "github.com/karust/openserp/core" + "golang.org/x/time/rate" +) + +// ecosiaPageSize is the organic-results-per-page count on Ecosia's web SERP. +// Image search uses the same ?p=N param shape but its per-page count varies, +// so SearchImage paginates by Limit alone. +const ecosiaPageSize = 10 + +// startPage translates q.Start (a 0-based result offset, Google/Bing +// convention) into Ecosia's 0-based page index and the rank of its first +// result. Off-grid offsets round down to a page boundary since Ecosia +// exposes no per-result offset param. +func startPage(start int) (pageNum, startRank int, err error) { + if start < 0 { + return 0, 0, errors.New("incorrect start provided") + } + pageNum = start / ecosiaPageSize + startRank = pageNum*ecosiaPageSize + 1 + return pageNum, startRank, nil +} + +// Cloudflare interstitial markers. URL/title are CF defaults. +const ( + cfURLPath = "cdn-cgi" + cfPageTitle = "just a moment" + cfBodyMarker = "not a bot" +) + +// Ecosia's SERP DOM shape varies by the underlying provider chosen per market +// (Bing, Google, or EUSP per Ecosia's search-features doc). The data-test-id +// attributes are the most stable surface across providers; class names drift. +var sel = struct { + Mainline string + Result string + Ad string + ResultLink string + Title string + Desc string + ImageResult string + ImageLink string + ImageSource string + ImageDims string +}{ + Mainline: "[data-test-id='mainline']", + Result: "[data-test-id='mainline-result-web']", + Ad: "[data-test-id='mainline-result-ad']", + ResultLink: "[data-test-id='result-link']", + Title: "[data-test-id='result-title']", + Desc: "[data-test-id='result-description']", + ImageResult: "[data-test-id='images-result']", + ImageLink: "[data-test-id='image-result-link']", + ImageSource: "[data-test-id='image-result-source']", + ImageDims: "[data-test-id='image-result-dimensions']", +} + +// Ecosia implements core.SearchEngine for Ecosia SERP pages. Additional +// documentation at https://support.ecosia.org/article/447-search-features. +type Ecosia struct { + core.Browser + core.SearchEngineOptions + pageSleep time.Duration // Sleep between pages + logger *core.EngineLogger +} + +// New creates an Ecosia engine instance with browser/runtime options applied. +func New(browser core.Browser, opts core.SearchEngineOptions) *Ecosia { + e := Ecosia{Browser: browser} + opts.Init() + e.SearchEngineOptions = opts + e.logger = core.NewEngineLogger("Ecosia") + e.pageSleep = time.Second + return &e +} + +// Name returns the stable engine identifier. +func (e *Ecosia) Name() string { return "ecosia" } + +// GetRateLimiter returns a limiter configured from SearchEngineOptions. +func (e *Ecosia) GetRateLimiter() *rate.Limiter { + return rate.NewLimiter(rate.Every(e.GetRatelimit()), e.RateBurst) +} + +// isCaptcha reports whether the current page is a Cloudflare interstitial. +// Checks the URL, title, then body text (cheapest first). +func (e *Ecosia) isCaptcha(page *rod.Page) bool { + info, err := page.Info() + if err == nil { + if strings.Contains(strings.ToLower(info.URL), cfURLPath) { + return true + } + if strings.Contains(strings.ToLower(info.Title), cfPageTitle) { + return true + } + } + html, err := page.Timeout(e.GetSelectorTimeout()).HTML() + if err != nil { + return false + } + return strings.Contains(strings.ToLower(html), cfBodyMarker) +} + +func (e *Ecosia) parseResult(elem *rod.Element, rank int, ad bool) (core.SearchResult, bool) { + link, err := elem.Element(sel.ResultLink) + if err != nil { + // Fall back to the first anchor when the test-id selector is absent. + link, err = elem.Element("a[href]") + if err != nil { + return core.SearchResult{}, false + } + } + href, err := link.Property("href") + if err != nil { + return core.SearchResult{}, false + } + hrefStr := strings.TrimSpace(href.String()) + if hrefStr == "" || strings.HasPrefix(hrefStr, "javascript:") { + return core.SearchResult{}, false + } + + title := "" + if t, err := elem.Element(sel.Title); err == nil { + title, _ = t.Text() + } else if t, err := elem.Element("h2, h3"); err == nil { + title, _ = t.Text() + } + + desc := "" + if d, err := elem.Element(sel.Desc); err == nil { + desc, _ = d.Text() + } + + return core.SearchResult{ + Rank: rank, + URL: hrefStr, + Title: strings.TrimSpace(title), + Description: strings.TrimSpace(desc), + Ad: ad, + }, true +} + +// Search executes an Ecosia web search and returns normalized search results. +// It may return core.ErrCaptcha or core.ErrSearchTimeout. +func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { + ctx = core.WithEngine(core.EnsureContext(ctx), e.Name()) + ctx = core.WithProfileRegion(ctx, query.LangCode) + ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + scoped := *e + scoped.logger = e.logger.WithRequest(ctx) + e = &scoped + + e.logger.Debug("Starting search, query: %+v", query) + defer func() { + if recovered := recover(); recovered != nil { + err = core.RecoverEnginePanicWithContext(ctx, e.Name(), recovered, e.logger) + results = nil + } + }() + + // nextRank counts up across pages for organic results; nextAdRank counts + // down from -1 so ads keep unique, order-preserving negative ranks. + all := []core.SearchResult{} + pageNum, nextRank, err := startPage(query.Start) + if err != nil { + return nil, err + } + nextAdRank := -1 + for query.Limit <= 0 || len(all) < query.Limit { + u, err := BuildURL(query, pageNum) + if err != nil { + return nil, err + } + + page, err := e.Navigate(ctx, u) + if err != nil { + return nil, err + } + closePage := func() { + if e.Browser.LeavePageOpen { + return + } + if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { + e.logger.Debug("Page close error: %v", closeErr) + } + } + + if err := page.WaitLoad(); err != nil { + closePage() + e.logger.Error("Page load wait failed: %s", err) + return nil, core.ErrSearchTimeout + } + + if _, err := page.Timeout(e.GetSelectorTimeout()).Element(sel.Mainline); err != nil { + if e.isCaptcha(page) { + closePage() + e.logger.Error("Captcha detected: %s", u) + return nil, core.ErrCaptcha + } + closePage() + e.logger.Warn("Mainline not found on page %d", pageNum) + break + } + + organic, _ := page.Elements(sel.Result) + ads, _ := page.Elements(sel.Ad) + if len(organic) == 0 && len(ads) == 0 { + // Empty mainline = zero-result query or end of pagination, not + // a parser failure; don't trip the retry path. + closePage() + e.logger.Debug("No results on page %d", pageNum) + break + } + + for _, r := range organic { + if res, ok := e.parseResult(r, nextRank, false); ok { + all = append(all, res) + nextRank++ + } + } + for _, r := range ads { + if res, ok := e.parseResult(r, nextAdRank, true); ok { + all = append(all, res) + nextAdRank-- + } + } + + closePage() + pageNum++ + + if query.Limit > 0 && len(all) >= query.Limit { + break + } + if err := core.SleepContext(ctx, e.pageSleep); err != nil { + return nil, err + } + } + + deduped := core.DeduplicateResults(all) + if query.Limit > 0 { + organic, ads := splitAds(deduped) + if len(organic) > query.Limit { + organic = organic[:query.Limit] + } + deduped = append(organic, ads...) + } + e.logger.Info("Search completed: %d results", len(deduped)) + return deduped, nil +} + +// parseImageResult extracts a single image card into a SearchResult, +// returning (_, false) if the card lacks a usable image URL. +func (e *Ecosia) parseImageResult(el *rod.Element, rank int) (core.SearchResult, bool) { + link, err := el.Element(sel.ImageLink) + if err != nil { + return core.SearchResult{}, false + } + href, err := link.Property("href") + if err != nil { + return core.SearchResult{}, false + } + imgURL := strings.TrimSpace(href.String()) + if imgURL == "" { + return core.SearchResult{}, false + } + + title := "" + if img, err := link.Element("img"); err == nil { + if alt, err := img.Attribute("alt"); err == nil && alt != nil { + title = strings.TrimSpace(*alt) + } + } + + source := "" + if s, err := el.Element(sel.ImageSource); err == nil { + source, _ = s.Text() + source = strings.TrimSpace(source) + } + dims := "" + if d, err := el.Element(sel.ImageDims); err == nil { + dims, _ = d.Text() + dims = strings.TrimSpace(dims) + } + + desc := source + if dims != "" { + if source != "" { + desc = fmt.Sprintf("%s (%s)", source, dims) + } else { + desc = dims + } + } + + return core.SearchResult{ + Rank: rank, + URL: imgURL, + Title: title, + Description: desc, + }, true +} + +// SearchImage executes an Ecosia image search and returns normalized image +// results. It may return core.ErrCaptcha or core.ErrSearchTimeout. +// query.Start is ignored: per-page card count varies, so callers should +// drive depth through query.Limit alone. +func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { + ctx = core.WithEngine(core.EnsureContext(ctx), e.Name()) + ctx = core.WithProfileRegion(ctx, query.LangCode) + ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + scoped := *e + scoped.logger = e.logger.WithRequest(ctx) + e = &scoped + + e.logger.Debug("Starting image search, query: %+v", query) + defer func() { + if recovered := recover(); recovered != nil { + err = core.RecoverEnginePanicWithContext(ctx, e.Name(), recovered, e.logger) + results = nil + } + }() + + out := []core.SearchResult{} + pageNum := 0 + nextRank := 1 + for query.Limit <= 0 || len(out) < query.Limit { + u, err := BuildImageURL(query, pageNum) + if err != nil { + return nil, err + } + + page, err := e.Navigate(ctx, u) + if err != nil { + return nil, err + } + closePage := func() { + if e.Browser.LeavePageOpen { + return + } + if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil { + e.logger.Debug("Page close error: %v", closeErr) + } + } + + if err := page.WaitLoad(); err != nil { + closePage() + e.logger.Error("Page load wait failed: %s", err) + return nil, core.ErrSearchTimeout + } + + if _, err := page.Timeout(e.GetSelectorTimeout()).Element(sel.ImageResult); err != nil { + if e.isCaptcha(page) { + closePage() + e.logger.Error("Captcha detected: %s", u) + return nil, core.ErrCaptcha + } + closePage() + e.logger.Debug("No image results on page %d", pageNum) + break + } + + elements, err := page.Elements(sel.ImageResult) + if err != nil { + closePage() + e.logger.Error("Cannot collect image results: %s", err) + return nil, core.ErrParser + } + + for _, el := range elements { + if res, ok := e.parseImageResult(el, nextRank); ok { + out = append(out, res) + nextRank++ + } + if query.Limit > 0 && len(out) >= query.Limit { + break + } + } + + closePage() + pageNum++ + + if query.Limit > 0 && len(out) >= query.Limit { + break + } + if err := core.SleepContext(ctx, e.pageSleep); err != nil { + return nil, err + } + } + + deduped := core.DeduplicateResults(out) + if query.Limit > 0 && len(deduped) > query.Limit { + deduped = deduped[:query.Limit] + } + e.logger.Info("Image search completed: %d results", len(deduped)) + return deduped, nil +} + +func splitAds(in []core.SearchResult) (organic, ads []core.SearchResult) { + for _, r := range in { + if r.Ad { + ads = append(ads, r) + } else { + organic = append(organic, r) + } + } + return +} diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go new file mode 100644 index 0000000..8c3383b --- /dev/null +++ b/ecosia/search_raw.go @@ -0,0 +1,172 @@ +package ecosia + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/corpix/uarand" + "github.com/karust/openserp/core" +) + +func ecosiaRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) { + baseClient, err := core.NewRawHTTPClient(query) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", uarand.GetRandom()) + core.SetAcceptLanguageHeader(req, query.LangCode) + + res, err := baseClient.Do(req) + if err != nil { + return nil, err + } + return res, nil +} + +// resultParser parses an Ecosia SERP HTML response into search results +// using goquery (no browser required). +func resultParser(response *http.Response) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + return nil, err + } + var ( + results []core.SearchResult + rank = 1 + ) + doc.Find(sel.Result).Each(func(_ int, s *goquery.Selection) { + href, ok := s.Find(sel.ResultLink).Attr("href") + if !ok || strings.TrimSpace(href) == "" { + return + } + var ( + title = strings.TrimSpace(s.Find(sel.Title).Text()) + desc = strings.TrimSpace(s.Find(sel.Desc).Text()) + ) + results = append(results, core.SearchResult{ + Rank: rank, + URL: href, + Title: title, + Description: desc, + }) + rank++ + }) + adRank := -1 + doc.Find(sel.Ad).Each(func(_ int, s *goquery.Selection) { + href, ok := s.Find(sel.ResultLink).Attr("href") + if !ok || strings.TrimSpace(href) == "" { + return + } + var ( + title = strings.TrimSpace(s.Find(sel.Title).Text()) + desc = strings.TrimSpace(s.Find(sel.Desc).Text()) + ) + results = append(results, core.SearchResult{ + Rank: adRank, + URL: href, + Title: title, + Description: desc, + Ad: true, + }) + adRank-- + }) + return results, nil +} + +// imageResultParser parses an Ecosia image SERP HTML response into search +// results using goquery (no browser required). +func imageResultParser(response *http.Response) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + return nil, err + } + var ( + results []core.SearchResult + rank = 1 + ) + doc.Find(sel.ImageResult).Each(func(_ int, s *goquery.Selection) { + href, ok := s.Find(sel.ImageLink).Attr("href") + if !ok || strings.TrimSpace(href) == "" { + return + } + title, _ := s.Find(sel.ImageLink).Find("img").Attr("alt") + title = strings.TrimSpace(title) + var ( + source = strings.TrimSpace(s.Find(sel.ImageSource).Text()) + dims = strings.TrimSpace(s.Find(sel.ImageDims).Text()) + desc = source + ) + if dims != "" { + if source != "" { + desc = source + " (" + dims + ")" + } else { + desc = dims + } + } + results = append(results, core.SearchResult{ + Rank: rank, + URL: href, + Title: title, + Description: desc, + }) + rank++ + }) + return results, nil +} + +func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { + ctx = core.EnsureContext(ctx) + ctx = core.WithEngine(ctx, "ecosia") + ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) + defer func() { + if recovered := recover(); recovered != nil { + err = core.RecoverEnginePanicWithContext(ctx, "ecosia", recovered, nil) + results = nil + } + }() + + pageNum, startRank, err := startPage(query.Start) + if err != nil { + return nil, err + } + + ecosiaURL, err := BuildURL(query, pageNum) + if err != nil { + return nil, err + } + core.WithRequest(ctx).WithField("url", ecosiaURL).Debug(fmt.Sprintf("Ecosia URL built: %s", ecosiaURL)) + + res, err := ecosiaRequest(ctx, ecosiaURL, query) + if err != nil { + return nil, err + } + defer core.DrainAndCloseResponse(res) + core.WithRequest(ctx).WithField("status_code", res.StatusCode).Debug( + fmt.Sprintf("Ecosia Raw response: code=%d", res.StatusCode), + ) + + parsedResults, err := resultParser(res) + if err != nil { + return nil, err + } + + // Ecosia paginates by page index, not result offset, so re-rank from + // the page boundary rather than query.Start (off-grid offsets round down). + for i := range parsedResults { + parsedResults[i].Rank = startRank + i + } + + core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug( + fmt.Sprintf("Ecosia Raw results : %v", parsedResults), + ) + + return core.DeduplicateResults(parsedResults), nil +} diff --git a/ecosia/search_raw_test.go b/ecosia/search_raw_test.go new file mode 100644 index 0000000..9154d1c --- /dev/null +++ b/ecosia/search_raw_test.go @@ -0,0 +1,84 @@ +package ecosia + +import ( + "testing" + + "github.com/karust/openserp/testutil" +) + +func TestEcosiaResultParser(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fixture string + wantCount int + }{ + {name: "search results", fixture: "search_results.html", wantCount: 10}, + {name: "no results", fixture: "search_no_results.html", wantCount: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + results, err := resultParser(testutil.ResponseFromFixture(t, tt.fixture)) + if err != nil { + t.Fatalf("resultParser() error = %v", err) + } + if len(results) != tt.wantCount { + t.Fatalf("expected %d results for %s, got %d", tt.wantCount, tt.fixture, len(results)) + } + if tt.wantCount == 0 { + return + } + + testutil.AssertSequentialRanks(t, results) + testutil.AssertFirstResultFilled(t, results) + }) + } +} + +func TestEcosiaImageResultParser(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fixture string + wantCount int + }{ + {name: "image results", fixture: "images_results.html", wantCount: 24}, + {name: "no results", fixture: "images_no_results.html", wantCount: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + results, err := imageResultParser(testutil.ResponseFromFixture(t, tt.fixture)) + if err != nil { + t.Fatalf("imageResultParser() error = %v", err) + } + if len(results) != tt.wantCount { + t.Fatalf("expected %d results for %s, got %d", tt.wantCount, tt.fixture, len(results)) + } + if tt.wantCount == 0 { + return + } + testutil.AssertSequentialRanks(t, results) + testutil.AssertFirstResultFilled(t, results) + }) + } +} + +func TestEcosiaResultParserEmptyHTML(t *testing.T) { + t.Parallel() + + results, err := resultParser(testutil.ResponseFromString("")) + if err != nil { + t.Fatalf("resultParser() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + } +} diff --git a/ecosia/search_test.go b/ecosia/search_test.go new file mode 100644 index 0000000..3203111 --- /dev/null +++ b/ecosia/search_test.go @@ -0,0 +1,280 @@ +package ecosia + +import ( + "net/url" + "testing" + + "github.com/karust/openserp/core" +) + +func TestBuildURL(t *testing.T) { + tests := []struct { + name string + query core.Query + page int + wantErr bool + check func(*testing.T, url.Values, string) + }{ + { + name: "basic search omits page param on page 0", + query: core.Query{Text: "llm tool use"}, + page: 0, + check: func(t *testing.T, params url.Values, host string) { + t.Helper() + if host != "www.ecosia.org" { + t.Fatalf("unexpected host: %s", host) + } + if got := params.Get("q"); got != "llm tool use" { + t.Fatalf("unexpected q: %q", got) + } + if got := params.Get("method"); got != "index" { + t.Fatalf("unexpected method: %q", got) + } + if got := params.Get("p"); got != "" { + t.Fatalf("expected no p param on page 0, got %q", got) + } + }, + }, + { + name: "site, filetype and pagination", + query: core.Query{ + Text: "поиск", + Site: "github.com", + Filetype: "pdf", + LangCode: "de-AT", + }, + page: 2, + check: func(t *testing.T, params url.Values, _ string) { + t.Helper() + if got := params.Get("q"); got != "поиск site:github.com filetype:pdf" { + t.Fatalf("unexpected q: %q", got) + } + if got := params.Get("p"); got != "2" { + t.Fatalf("unexpected p: %q", got) + } + if got := params.Get("mkt"); got != "" { + t.Fatalf("unexpected mkt: %q", got) + } + }, + }, + { + name: "5-day span buckets to week", + query: core.Query{Text: "golang", DateInterval: "20240101..20240106"}, + page: 0, + check: func(t *testing.T, params url.Values, _ string) { + t.Helper() + if got := params.Get("freshness"); got != "week" { + t.Fatalf("unexpected freshness: %q", got) + } + }, + }, + { + name: "long span is silently dropped", + query: core.Query{Text: "golang", DateInterval: "20240101..20240601"}, + page: 0, + check: func(t *testing.T, params url.Values, _ string) { + t.Helper() + if got := params.Get("freshness"); got != "" { + t.Fatalf("freshness should not be set for spans > 31d, got %q", got) + } + }, + }, + { + name: "non-spec keyword input errors", + query: core.Query{Text: "golang", DateInterval: "week"}, + wantErr: true, + }, + { + name: "malformed date errors", + query: core.Query{Text: "golang", DateInterval: "2024-01-01..2024-01-31"}, + wantErr: true, + }, + { + name: "end before start errors", + query: core.Query{Text: "golang", DateInterval: "20240131..20240101"}, + wantErr: true, + }, + { + name: "empty query returns error", + query: core.Query{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildURL(tt.query, tt.page) + if (err != nil) != tt.wantErr { + t.Fatalf("BuildURL() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + parsed, err := url.Parse(got) + if err != nil { + t.Fatalf("BuildURL() returned invalid URL: %v", err) + } + if tt.check != nil { + tt.check(t, parsed.Query(), parsed.Host) + } + }) + } +} + +func TestBuildImageURL(t *testing.T) { + got, err := BuildImageURL(core.Query{Text: "trees"}, 0) + if err != nil { + t.Fatalf("BuildImageURL() error = %v", err) + } + parsed, err := url.Parse(got) + if err != nil { + t.Fatalf("invalid URL: %v", err) + } + if parsed.Host != "www.ecosia.org" || parsed.Path != "/images" { + t.Fatalf("expected /images endpoint, got %s%s", parsed.Host, parsed.Path) + } + if parsed.Query().Get("q") != "trees" { + t.Fatalf("unexpected q: %q", parsed.Query().Get("q")) + } + if got := parsed.Query().Get("imageType"); got != "" { + t.Fatalf("expected no imageType when Filetype unset, got %q", got) + } + if got := parsed.Query().Get("p"); got != "" { + t.Fatalf("expected no p param on page 0, got %q", got) + } + + if _, err := BuildImageURL(core.Query{}, 0); err == nil { + t.Fatalf("expected error for empty query") + } +} + +func TestBuildImageURLPagination(t *testing.T) { + got, err := BuildImageURL(core.Query{Text: "trees"}, 2) + if err != nil { + t.Fatalf("BuildImageURL() error = %v", err) + } + parsed, err := url.Parse(got) + if err != nil { + t.Fatalf("invalid URL: %v", err) + } + if g := parsed.Query().Get("p"); g != "2" { + t.Fatalf("expected p=2, got %q", g) + } +} + +func TestBuildImageURLFreshness(t *testing.T) { + got, err := BuildImageURL(core.Query{Text: "trees", DateInterval: "20240101..20240101"}, 0) + if err != nil { + t.Fatalf("BuildImageURL() err = %v", err) + } + parsed, err := url.Parse(got) + if err != nil { + t.Fatalf("invalid URL: %v", err) + } + if g := parsed.Query().Get("freshness"); g != "day" { + t.Fatalf("expected freshness=day, got %q", g) + } +} + +func TestEcosiaFreshness(t *testing.T) { + tests := []struct { + name string + input string + want string + wantErr bool + }{ + {name: "empty input is no-op", input: "", want: ""}, + {name: "same day", input: "20240101..20240101", want: "day"}, + {name: "1-day span hits day boundary", input: "20240101..20240102", want: "day"}, + {name: "2-day span buckets to week", input: "20240101..20240103", want: "week"}, + {name: "7-day span hits week boundary", input: "20240101..20240108", want: "week"}, + {name: "8-day span buckets to month", input: "20240101..20240109", want: "month"}, + {name: "31-day span hits month boundary", input: "20240101..20240201", want: "month"}, + {name: "32-day span is dropped", input: "20240101..20240202", want: ""}, + {name: "very long span is dropped", input: "20200101..20240101", want: ""}, + {name: "missing separator errors", input: "20240101", wantErr: true}, + {name: "wrong format errors", input: "2024-01-01..2024-01-31", wantErr: true}, + {name: "non-numeric errors", input: "abcdefgh..ijklmnop", wantErr: true}, + {name: "end before start errors", input: "20240131..20240101", wantErr: true}, + {name: "keyword input errors", input: "week", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ecosiaFreshness(tt.input) + if (err != nil) != tt.wantErr { + t.Fatalf("ecosiaFreshness(%q) err = %v, wantErr %v", tt.input, err, tt.wantErr) + } + if tt.wantErr { + return + } + if got != tt.want { + t.Fatalf("ecosiaFreshness(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestBuildImageURLImageType(t *testing.T) { + tests := []struct { + name string + filetype string + want string + wantErr bool + }{ + {name: "clipart", filetype: "clipart", want: "clipart"}, + {name: "uppercase normalized", filetype: "PHOTO", want: "photo"}, + {name: "animatedgif", filetype: "animatedgif", want: "animatedgif"}, + {name: "rejects extension", filetype: "gif", wantErr: true}, + {name: "rejects unknown", filetype: "vector", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildImageURL(core.Query{Text: "trees", Filetype: tt.filetype}, 0) + if (err != nil) != tt.wantErr { + t.Fatalf("BuildImageURL() err = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + parsed, err := url.Parse(got) + if err != nil { + t.Fatalf("invalid URL: %v", err) + } + if g := parsed.Query().Get("imageType"); g != tt.want { + t.Fatalf("unexpected imageType: %q want %q", g, tt.want) + } + }) + } +} + +func TestStartPage(t *testing.T) { + tests := []struct { + name string + start int + wantPage int + wantRank int + wantErr bool + }{ + {name: "zero offset starts at page 0", start: 0, wantPage: 0, wantRank: 1}, + {name: "page-aligned offset 10", start: 10, wantPage: 1, wantRank: 11}, + {name: "page-aligned offset 50", start: 50, wantPage: 5, wantRank: 51}, + {name: "off-grid offset rounds down", start: 15, wantPage: 1, wantRank: 11}, + {name: "off-grid offset 24 rounds to page 2", start: 24, wantPage: 2, wantRank: 21}, + {name: "negative offset errors", start: -1, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + page, rank, err := startPage(tt.start) + if (err != nil) != tt.wantErr { + t.Fatalf("startPage(%d) err = %v, wantErr %v", tt.start, err, tt.wantErr) + } + if tt.wantErr { + return + } + if page != tt.wantPage || rank != tt.wantRank { + t.Fatalf("startPage(%d) = (%d, %d), want (%d, %d)", + tt.start, page, rank, tt.wantPage, tt.wantRank) + } + }) + } +} diff --git a/ecosia/testdata/images_no_results.html b/ecosia/testdata/images_no_results.html new file mode 100644 index 0000000..809ec8e --- /dev/null +++ b/ecosia/testdata/images_no_results.html @@ -0,0 +1,61 @@ + + + + + + + + + + + +"49a4f536d6a76c3c61a2a9d4b3e26d37dc25a4cf" - Ecosia - Images + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Skip to main content

Images

Unfortunately we didn’t find any results for “"49a4f536d6a76c3c61a2a9d4b3e26d37dc25a4cf"”

Try these tips:
  • Check that all words are spelled correctly
  • Try different or fewer keywords
+ + + + + + diff --git a/ecosia/testdata/images_results.html b/ecosia/testdata/images_results.html new file mode 100644 index 0000000..5f726dc --- /dev/null +++ b/ecosia/testdata/images_results.html @@ -0,0 +1,61 @@ + + + + + + + + + + + +oak - Ecosia - Images + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Skip to main content

Images

Shop for oak

+ + + + + + diff --git a/ecosia/testdata/search_no_results.html b/ecosia/testdata/search_no_results.html new file mode 100644 index 0000000..1880b90 --- /dev/null +++ b/ecosia/testdata/search_no_results.html @@ -0,0 +1,54 @@ + + + + + + + + + +"1234zzzzzzzzzzzz1234" - Ecosia + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Skip to main content

Search

Unfortunately we didn’t find any results for “"1234zzzzzzzzzzzz1234"”

Try these tips:
  • Check that all words are spelled correctly
  • Try different or fewer keywords
+ + + + + + diff --git a/ecosia/testdata/search_results.html b/ecosia/testdata/search_results.html new file mode 100644 index 0000000..1ed9a0f --- /dev/null +++ b/ecosia/testdata/search_results.html @@ -0,0 +1,56 @@ + + + + + + + + + +llm tool use - Ecosia + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Skip to main content

Search

Large language model

Type of machine learning model

A large language model (LLM) is a neural network trained on a vast amount of text for natural language processing tasks, especially language generation. LLMs can generate, summarize, translate and parse text in many contexts, and are a foundational technology behind modern chatbots. Biased or inaccurate training data c… Wikipedia

Searches related to llm tool use

llm tools list
llm tools examples
llm tool calling
llm tool use paper
local llm tool use
llm tools openai
llm with tools langchain
llm tools ai
Next
+ + + + + + diff --git a/ecosia/url.go b/ecosia/url.go new file mode 100644 index 0000000..22c0595 --- /dev/null +++ b/ecosia/url.go @@ -0,0 +1,152 @@ +package ecosia + +import ( + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/karust/openserp/core" +) + +const ( + baseURL = "https://www.ecosia.org/search" + imagesURL = "https://www.ecosia.org/images" +) + +// ecosiaFreshness maps a YYYYMMDD..YYYYMMDD DateInterval to Ecosia's freshness +// bucket: <= 1d -> day, <= 7d -> week, <= 31d -> month. Longer spans are +// dropped (returns "", nil) since Ecosia exposes no finer control. Malformed +// input is rejected so non-spec values do not silently lose the filter. +func ecosiaFreshness(dateInterval string) (string, error) { + s := strings.TrimSpace(dateInterval) + if s == "" { + return "", nil + } + parts := strings.Split(s, "..") + if len(parts) != 2 { + return "", errors.New("incorrect date interval provided, expected YYYYMMDD..YYYYMMDD") + } + start, err := time.Parse("20060102", parts[0]) + if err != nil { + return "", errors.New("invalid start date format, expected YYYYMMDD") + } + end, err := time.Parse("20060102", parts[1]) + if err != nil { + return "", errors.New("invalid end date format, expected YYYYMMDD") + } + span := end.Sub(start) + if span < 0 { + return "", errors.New("date interval end is before start") + } + switch { + case span <= 24*time.Hour: + return "day", nil + case span <= 7*24*time.Hour: + return "week", nil + case span <= 31*24*time.Hour: + return "month", nil + default: + return "", nil + } +} + +// BuildURL builds an Ecosia web search URL for the supplied query and +// 0-based page index. q.LangCode is not encoded — Ecosia takes region from +// the ECFG cookie / Accept-Language, set via the browser profile. +func BuildURL(q core.Query, page int) (string, error) { + base, err := url.Parse(baseURL) + if err != nil { + return "", err + } + + params := url.Values{} + params.Set("method", "index") + + text := q.Text + if q.Site != "" { + text += " site:" + q.Site + } + if q.Filetype != "" { + text += " filetype:" + q.Filetype + } + if strings.TrimSpace(text) == "" { + return "", errors.New("empty query built") + } + params.Set("q", text) + + f, err := ecosiaFreshness(q.DateInterval) + if err != nil { + return "", err + } + if f != "" { + params.Set("freshness", f) + } + + if page > 0 { + params.Set("p", fmt.Sprintf("%d", page)) + } + + base.RawQuery = params.Encode() + return base.String(), nil +} + +// validImageTypes are the categories Ecosia exposes on /images via the +// imageType URL param; q.Filetype is matched against this set. +var validImageTypes = map[string]struct{}{ + "clipart": {}, + "photo": {}, + "line": {}, + "animatedgif": {}, + "transparent": {}, +} + +// BuildImageURL builds an Ecosia image search URL for the supplied query +// and 0-based page index. Image search lives on /images, not /search. +// q.Filetype maps to imageType (clipart, photo, line, animatedgif, +// transparent), not a generic file extension. +// +// Of the six image filters Ecosia exposes (Size, Colour, Type, Time, Layouts, +// License), only Type (q.Filetype -> imageType) and Time (q.DateInterval -> +// freshness) are mapped; size/colour/layouts/license are not mapped currently. +func BuildImageURL(q core.Query, page int) (string, error) { + base, err := url.Parse(imagesURL) + if err != nil { + return "", err + } + + text := q.Text + if q.Site != "" { + text += " site:" + q.Site + } + if strings.TrimSpace(text) == "" { + return "", errors.New("empty query built") + } + + params := url.Values{} + params.Set("q", text) + + f, err := ecosiaFreshness(q.DateInterval) + if err != nil { + return "", err + } + if f != "" { + params.Set("freshness", f) + } + + if q.Filetype != "" { + v := strings.ToLower(strings.TrimSpace(q.Filetype)) + if _, ok := validImageTypes[v]; !ok { + return "", fmt.Errorf("unsupported imageType %q (want one of: clipart, photo, line, animatedgif, transparent)", q.Filetype) + } + params.Set("imageType", v) + } + + if page > 0 { + params.Set("p", fmt.Sprintf("%d", page)) + } + + base.RawQuery = params.Encode() + return base.String(), nil +}