Fix duckduck date and limit filtering. Deduplicate results

README.md

@@ -3,8 +3,10 @@
 ![OpenSERP](/logo.svg)
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp)
-[![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp)
-[![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases)
+[![Go Reference](https://pkg.go.dev/badge/github/karust/openserp?style=for-the-badge)](https://pkg.go.dev/github.com/karust/openserp)
+[![release](https://img.shields.io/github/release/karust/openserp)](https://github.com/karust/openserp/releases)
+
+<!--[![Docker Pulls](https://img.shields.io/docker/pulls/karust/openserp)](https://hub.docker.com/repository/docker/karust/openserp)-->
 
 **OpenSERP** provides free API access to multiple search engines including **[Google, Yandex, Baidu, Bing, DuckDuckGo]**. Get comprehensive search results without expensive API subscriptions!
 
@@ -12,6 +14,7 @@
 
 - 🔍 **Multi-Engine Support**: Google, Yandex, Baidu, Bing, DuckDuckGo...
 - 🌐 **Megasearch**: Aggregate results from multiple engines simultaneously
+- 🖼 **Images**: Image search is also available!
 - 🎯 **Advanced Filtering**: Language, date range, file type, site-specific searches
 - 🌍 **Proxy Support**: HTTP/SOCKS5 proxy support
 - 🐳 **Docker Ready**: Easy deployment with Docker
@@ -51,10 +54,10 @@ go build -o openserp .
 curl "http://localhost:7000/mega/search?text=golang&limit=10"
 
 # Pick specific engines
-curl "http://localhost:7000/mega/search?text=golang&engines=google,bing&limit=5"
+curl "http://localhost:7000/mega/search?text=golang&engines=duckduckgo,bing&limit=15"
 
 # Advanced filtering
-curl "http://localhost:7000/mega/search?text=golang&lang=EN&site=github.com&limit=8"
+curl "http://localhost:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bing&limit=20&date=20251005..20251005&lang=EN"
 ```
 
 - API response example:
@@ -93,12 +96,6 @@ curl "http://localhost:7000/mega/search?text=golang&lang=EN&site=github.com&limi
 ```bash
 # Search images across ALL engines
 curl "http://localhost:7000/mega/image?text=golang logo&limit=20"
-
-# Pick specific engines for images
-curl "http://localhost:7000/mega/image?text=golang&engines=google,bing&limit=15"
-
-# Language-specific image search
-curl "http://localhost:7000/mega/image?text=golang&lang=EN&limit=10"
 ```
 
 ### Available Engines

bing/search.go

@@ -204,7 +204,32 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
 		searchResults = append(searchResults, srchRes)
 	}
 
-	return core.DeduplicateResults(searchResults), nil
+	// Deduplicate results
+	deduped := core.DeduplicateResults(searchResults)
+
+	// Trim to exact limit if necessary (only organic results, not ads)
+	if query.Limit > 0 {
+		organicResults := []core.SearchResult{}
+		adResults := []core.SearchResult{}
+
+		for _, result := range deduped {
+			if result.Ad {
+				adResults = append(adResults, result)
+			} else {
+				organicResults = append(organicResults, result)
+			}
+		}
+
+		// Trim organic results to limit
+		if len(organicResults) > query.Limit {
+			organicResults = organicResults[:query.Limit]
+		}
+
+		// Combine back: organic results + ads
+		deduped = append(organicResults, adResults...)
+	}
+
+	return deduped, nil
 }
 
 // BingImageData represents the JSON structure in the m attribute of image elements

cmd/root.go

@@ -13,7 +13,7 @@ import (
 )
 
 const (
-	version               = "0.5.3"
+	version               = "0.5.4"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )

duckduckgo/search.go

@@ -170,7 +170,7 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) {
 	searchPage := 0
 
 	for len(allResults) < query.Limit {
-		url, err := BuildURL(query)
+		url, err := BuildURL(query, searchPage)
 		if err != nil {
 			return nil, err
 		}
@@ -202,7 +202,6 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) {
 			}
 		}
 
-		page.MustScreenshotFullPage("./duck.png")
 		if searchErr != nil {
 			defer page.Close()
 			ddg.logger.Error("Cannot parse search results: %s", searchErr)
@@ -246,11 +245,24 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) {
 			}
 		}
 
+		// Break if we've reached or exceeded the limit
+		if len(allResults) >= query.Limit {
+			break
+		}
+
 		time.Sleep(ddg.pageSleep)
 	}
 
-	ddg.logger.Info("Search completed: %d results", len(allResults))
-	return core.DeduplicateResults(allResults), nil
+	// Deduplicate results
+	deduped := core.DeduplicateResults(allResults)
+
+	// Trim to exact limit if necessary
+	if len(deduped) > query.Limit {
+		deduped = deduped[:query.Limit]
+	}
+
+	ddg.logger.Info("Search completed: %d results", len(deduped))
+	return deduped, nil
 }
 
 func (ddg *DuckDuckGo) SearchImage(query core.Query) ([]core.SearchResult, error) {

duckduckgo/url.go

@@ -5,13 +5,14 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/karust/openserp/core"
 )
 
 const baseURL = "https://duckduckgo.com"
 
-func BuildURL(q core.Query) (string, error) {
+func BuildURL(q core.Query, page int) (string, error) {
 	base, err := url.Parse(baseURL)
 	if err != nil {
 		return "", err
@@ -41,10 +42,25 @@ func BuildURL(q core.Query) (string, error) {
 	if q.DateInterval != "" {
 		intervals := strings.Split(q.DateInterval, "..")
 		if len(intervals) != 2 {
-			return "", errors.New("incorrect data interval provided")
+			return "", errors.New("incorrect date interval provided")
 		}
-		// DuckDuckGo uses different date format
-		params.Add("df", fmt.Sprintf("%s..%s", intervals[0], intervals[1]))
+
+		// Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement)
+		startDate, err := time.Parse("20060102", intervals[0])
+		if err != nil {
+			return "", errors.New("invalid start date format, expected YYYYMMDD")
+		}
+
+		endDate, err := time.Parse("20060102", intervals[1])
+		if err != nil {
+			return "", errors.New("invalid end date format, expected YYYYMMDD")
+		}
+
+		// DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format
+		dateRange := fmt.Sprintf("%s..%s",
+			startDate.Format("2006-01-02"),
+			endDate.Format("2006-01-02"))
+		params.Add("df", dateRange)
 	}
 
 	// Set language
@@ -56,6 +72,14 @@ func BuildURL(q core.Query) (string, error) {
 	params.Add("t", "h")    // HTML format
 	params.Add("ia", "web") // Web search
 
+	// Add pagination parameter if not on first page
+	if page > 0 {
+		// DuckDuckGo uses 's' parameter for pagination (start offset)
+		// Each page has approximately 25-30 results, but we'll use conservative estimate
+		offset := page * 25
+		params.Add("s", fmt.Sprintf("%d", offset))
+	}
+
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
@@ -89,6 +113,31 @@ func BuildImageURL(q core.Query) (string, error) {
 		return "", errors.New("empty query built")
 	}
 
+	// Set search date range
+	if q.DateInterval != "" {
+		intervals := strings.Split(q.DateInterval, "..")
+		if len(intervals) != 2 {
+			return "", errors.New("incorrect date interval provided")
+		}
+
+		// Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement)
+		startDate, err := time.Parse("20060102", intervals[0])
+		if err != nil {
+			return "", errors.New("invalid start date format, expected YYYYMMDD")
+		}
+
+		endDate, err := time.Parse("20060102", intervals[1])
+		if err != nil {
+			return "", errors.New("invalid end date format, expected YYYYMMDD")
+		}
+
+		// DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format
+		dateRange := fmt.Sprintf("%s..%s",
+			startDate.Format("2006-01-02"),
+			endDate.Format("2006-01-02"))
+		params.Add("df", dateRange)
+	}
+
 	// Set language
 	if q.LangCode != "" {
 		params.Add("kl", strings.ToLower(q.LangCode))