diff --git a/README.md b/README.md index 35908bb..742ab91 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,10 @@ ![OpenSERP](/logo.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp) -[![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp) -[![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases) +[![Go Reference](https://pkg.go.dev/badge/github/karust/openserp?style=for-the-badge)](https://pkg.go.dev/github.com/karust/openserp) +[![release](https://img.shields.io/github/release/karust/openserp)](https://github.com/karust/openserp/releases) + + **OpenSERP** provides free API access to multiple search engines including **[Google, Yandex, Baidu, Bing, DuckDuckGo]**. Get comprehensive search results without expensive API subscriptions! @@ -12,6 +14,7 @@ - 🔍 **Multi-Engine Support**: Google, Yandex, Baidu, Bing, DuckDuckGo... - 🌐 **Megasearch**: Aggregate results from multiple engines simultaneously +- 🖼 **Images**: Image search is also available! - 🎯 **Advanced Filtering**: Language, date range, file type, site-specific searches - 🌍 **Proxy Support**: HTTP/SOCKS5 proxy support - 🐳 **Docker Ready**: Easy deployment with Docker @@ -51,10 +54,10 @@ go build -o openserp . curl "http://localhost:7000/mega/search?text=golang&limit=10" # Pick specific engines -curl "http://localhost:7000/mega/search?text=golang&engines=google,bing&limit=5" +curl "http://localhost:7000/mega/search?text=golang&engines=duckduckgo,bing&limit=15" # Advanced filtering -curl "http://localhost:7000/mega/search?text=golang&lang=EN&site=github.com&limit=8" +curl "http://localhost:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bing&limit=20&date=20251005..20251005&lang=EN" ``` - API response example: @@ -93,12 +96,6 @@ curl "http://localhost:7000/mega/search?text=golang&lang=EN&site=github.com&limi ```bash # Search images across ALL engines curl "http://localhost:7000/mega/image?text=golang logo&limit=20" - -# Pick specific engines for images -curl "http://localhost:7000/mega/image?text=golang&engines=google,bing&limit=15" - -# Language-specific image search -curl "http://localhost:7000/mega/image?text=golang&lang=EN&limit=10" ``` ### Available Engines diff --git a/bing/search.go b/bing/search.go index bc79d0a..40f7a00 100644 --- a/bing/search.go +++ b/bing/search.go @@ -204,7 +204,32 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, srchRes) } - return core.DeduplicateResults(searchResults), nil + // Deduplicate results + deduped := core.DeduplicateResults(searchResults) + + // Trim to exact limit if necessary (only organic results, not ads) + if query.Limit > 0 { + organicResults := []core.SearchResult{} + adResults := []core.SearchResult{} + + for _, result := range deduped { + if result.Ad { + adResults = append(adResults, result) + } else { + organicResults = append(organicResults, result) + } + } + + // Trim organic results to limit + if len(organicResults) > query.Limit { + organicResults = organicResults[:query.Limit] + } + + // Combine back: organic results + ads + deduped = append(organicResults, adResults...) + } + + return deduped, nil } // BingImageData represents the JSON structure in the m attribute of image elements diff --git a/cmd/root.go b/cmd/root.go index af8fba0..3b6fe79 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,7 @@ import ( ) const ( - version = "0.5.3" + version = "0.5.4" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/duckduckgo/search.go b/duckduckgo/search.go index 9f8ca48..1670ed4 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -170,7 +170,7 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { searchPage := 0 for len(allResults) < query.Limit { - url, err := BuildURL(query) + url, err := BuildURL(query, searchPage) if err != nil { return nil, err } @@ -202,7 +202,6 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { } } - page.MustScreenshotFullPage("./duck.png") if searchErr != nil { defer page.Close() ddg.logger.Error("Cannot parse search results: %s", searchErr) @@ -246,11 +245,24 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { } } + // Break if we've reached or exceeded the limit + if len(allResults) >= query.Limit { + break + } + time.Sleep(ddg.pageSleep) } - ddg.logger.Info("Search completed: %d results", len(allResults)) - return core.DeduplicateResults(allResults), nil + // Deduplicate results + deduped := core.DeduplicateResults(allResults) + + // Trim to exact limit if necessary + if len(deduped) > query.Limit { + deduped = deduped[:query.Limit] + } + + ddg.logger.Info("Search completed: %d results", len(deduped)) + return deduped, nil } func (ddg *DuckDuckGo) SearchImage(query core.Query) ([]core.SearchResult, error) { diff --git a/duckduckgo/url.go b/duckduckgo/url.go index e581111..d224620 100644 --- a/duckduckgo/url.go +++ b/duckduckgo/url.go @@ -5,13 +5,14 @@ import ( "fmt" "net/url" "strings" + "time" "github.com/karust/openserp/core" ) const baseURL = "https://duckduckgo.com" -func BuildURL(q core.Query) (string, error) { +func BuildURL(q core.Query, page int) (string, error) { base, err := url.Parse(baseURL) if err != nil { return "", err @@ -41,10 +42,25 @@ func BuildURL(q core.Query) (string, error) { if q.DateInterval != "" { intervals := strings.Split(q.DateInterval, "..") if len(intervals) != 2 { - return "", errors.New("incorrect data interval provided") + return "", errors.New("incorrect date interval provided") } - // DuckDuckGo uses different date format - params.Add("df", fmt.Sprintf("%s..%s", intervals[0], intervals[1])) + + // Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement) + startDate, err := time.Parse("20060102", intervals[0]) + if err != nil { + return "", errors.New("invalid start date format, expected YYYYMMDD") + } + + endDate, err := time.Parse("20060102", intervals[1]) + if err != nil { + return "", errors.New("invalid end date format, expected YYYYMMDD") + } + + // DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format + dateRange := fmt.Sprintf("%s..%s", + startDate.Format("2006-01-02"), + endDate.Format("2006-01-02")) + params.Add("df", dateRange) } // Set language @@ -56,6 +72,14 @@ func BuildURL(q core.Query) (string, error) { params.Add("t", "h") // HTML format params.Add("ia", "web") // Web search + // Add pagination parameter if not on first page + if page > 0 { + // DuckDuckGo uses 's' parameter for pagination (start offset) + // Each page has approximately 25-30 results, but we'll use conservative estimate + offset := page * 25 + params.Add("s", fmt.Sprintf("%d", offset)) + } + base.RawQuery = params.Encode() return base.String(), nil } @@ -89,6 +113,31 @@ func BuildImageURL(q core.Query) (string, error) { return "", errors.New("empty query built") } + // Set search date range + if q.DateInterval != "" { + intervals := strings.Split(q.DateInterval, "..") + if len(intervals) != 2 { + return "", errors.New("incorrect date interval provided") + } + + // Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement) + startDate, err := time.Parse("20060102", intervals[0]) + if err != nil { + return "", errors.New("invalid start date format, expected YYYYMMDD") + } + + endDate, err := time.Parse("20060102", intervals[1]) + if err != nil { + return "", errors.New("invalid end date format, expected YYYYMMDD") + } + + // DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format + dateRange := fmt.Sprintf("%s..%s", + startDate.Format("2006-01-02"), + endDate.Format("2006-01-02")) + params.Add("df", dateRange) + } + // Set language if q.LangCode != "" { params.Add("kl", strings.ToLower(q.LangCode))