From 29c7b0fbe09640160efcfc1f1e04e60e0fbe60e9 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Wed, 22 Jul 2026 15:17:16 +0300 Subject: [PATCH] expose proxy-attempts + use-profile CORS headers; simplify DDG url builders --- config.yaml | 2 +- core/middleware.go | 2 +- core/middleware_test.go | 4 +- duckduckgo/search.go | 2 +- duckduckgo/url.go | 165 ++++++++++++++++------------------------ 5 files changed, 69 insertions(+), 106 deletions(-) diff --git a/config.yaml b/config.yaml index 19e4838..39dc90f 100644 --- a/config.yaml +++ b/config.yaml @@ -64,7 +64,7 @@ cors: enabled: true allow_origins: "*" allow_methods: "GET, POST, OPTIONS" - allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant" + allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant, X-Use-Profile" max_age: 86400 # 2captcha: diff --git a/core/middleware.go b/core/middleware.go index 0019ef1..f8a2940 100644 --- a/core/middleware.go +++ b/core/middleware.go @@ -30,7 +30,7 @@ type CORSConfig struct { const browserProfileIDHeader = "X-Browser-Profile-Id" const useProfileHeader = "X-Use-Profile" -const exposedResponseHeaders = "X-Request-ID, X-Cache, X-Fallback-Engine, X-Proxy-Mode, X-Proxy-Tag, X-Proxy-Used, X-Network-Bytes, " + browserProfileIDHeader +const exposedResponseHeaders = "X-Request-ID, X-Cache, X-Fallback-Engine, X-Proxy-Mode, X-Proxy-Tag, X-Proxy-Used, X-Proxy-Attempts, X-Network-Bytes, " + browserProfileIDHeader func DefaultCORSConfig() CORSConfig { return CORSConfig{ diff --git a/core/middleware_test.go b/core/middleware_test.go index 5198e3d..748aa98 100644 --- a/core/middleware_test.go +++ b/core/middleware_test.go @@ -63,8 +63,8 @@ func TestCORSMiddleware_UsesConfiguredHeaders(t *testing.T) { if got := resp.Header.Get("Access-Control-Max-Age"); got != "1200" { t.Fatalf("unexpected max-age: %q", got) } - if got := resp.Header.Get("Access-Control-Expose-Headers"); !strings.Contains(got, "X-Network-Bytes") || !strings.Contains(got, "X-Browser-Profile-Id") { - t.Fatalf("expected exposed headers to include network and browser profile headers, got %q", got) + if got := resp.Header.Get("Access-Control-Expose-Headers"); !strings.Contains(got, "X-Network-Bytes") || !strings.Contains(got, "X-Browser-Profile-Id") || !strings.Contains(got, "X-Proxy-Attempts") { + t.Fatalf("expected exposed headers to include network, browser profile, and proxy-attempts headers, got %q", got) } } diff --git a/duckduckgo/search.go b/duckduckgo/search.go index 1980948..dc9184b 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -99,7 +99,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se } // Validate that we have a proper URL - hrefStr := linkText.String() + hrefStr := strings.TrimSpace(linkText.String()) if hrefStr == "" || hrefStr == "#" || hrefStr == "javascript:void(0)" { ddg.logger.Debug("Invalid href: %s", hrefStr) continue diff --git a/duckduckgo/url.go b/duckduckgo/url.go index 116d689..6877374 100644 --- a/duckduckgo/url.go +++ b/duckduckgo/url.go @@ -79,6 +79,61 @@ func duckDuckGoKL(langCode, region string) string { return ddgKLByLocale[locale.Language] } +// addQueryText appends the "q" param, combining text with site/filetype +// operators. Returns an error when the resulting query is empty. +func addQueryText(params url.Values, q core.Query) error { + if q.Text == "" && q.Site == "" && q.Filetype == "" { + return errors.New("empty query built") + } + text := q.Text + if q.Site != "" { + text += " site:" + q.Site + } + if q.Filetype != "" { + text += " filetype:" + q.Filetype + } + params.Add("q", text) + return nil +} + +// addDateRange appends the "df" param, converting the YYYYMMDD..YYYYMMDD +// interval to DuckDuckGo's YYYY-MM-DD..YYYY-MM-DD form. No-op when unset. +func addDateRange(params url.Values, interval string) error { + if interval == "" { + return nil + } + intervals := strings.Split(interval, "..") + if len(intervals) != 2 { + return errors.New("incorrect date interval provided") + } + start, err := time.Parse("20060102", intervals[0]) + if err != nil { + return errors.New("invalid start date format, expected YYYYMMDD") + } + end, err := time.Parse("20060102", intervals[1]) + if err != nil { + return errors.New("invalid end date format, expected YYYYMMDD") + } + params.Add("df", start.Format("2006-01-02")+".."+end.Format("2006-01-02")) + return nil +} + +// buildParams assembles the query, date range, and locale params shared by web +// and image search. +func buildParams(q core.Query) (url.Values, error) { + params := url.Values{} + if err := addQueryText(params, q); err != nil { + return nil, err + } + if err := addDateRange(params, q.DateInterval); err != nil { + return nil, err + } + if kl := duckDuckGoKL(q.LangCode, q.Region); kl != "" { + params.Add("kl", kl) + } + return params, nil +} + // BuildURL builds a DuckDuckGo web search URL for the provided query and page // index. It returns an error when query text or date parameters are invalid. func BuildURL(q core.Query, page int) (string, error) { @@ -86,66 +141,17 @@ func BuildURL(q core.Query, page int) (string, error) { if err != nil { return "", err } - - base.Path += "" - params := url.Values{} - - // Set request text - if q.Text != "" || q.Site != "" || q.Filetype != "" { - text := q.Text - if q.Site != "" { - text += " site:" + q.Site - } - if q.Filetype != "" { - text += " filetype:" + q.Filetype - } - - params.Add("q", text) + params, err := buildParams(q) + if err != nil { + return "", err } - if len(params.Get("q")) == 0 { - return "", errors.New("empty query built") - } - - // Set search date range - if q.DateInterval != "" { - intervals := strings.Split(q.DateInterval, "..") - if len(intervals) != 2 { - return "", errors.New("incorrect date interval provided") - } - - // Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement) - startDate, err := time.Parse("20060102", intervals[0]) - if err != nil { - return "", errors.New("invalid start date format, expected YYYYMMDD") - } - - endDate, err := time.Parse("20060102", intervals[1]) - if err != nil { - return "", errors.New("invalid end date format, expected YYYYMMDD") - } - - // DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format - dateRange := fmt.Sprintf("%s..%s", - startDate.Format("2006-01-02"), - endDate.Format("2006-01-02")) - params.Add("df", dateRange) - } - - if kl := duckDuckGoKL(q.LangCode, q.Region); kl != "" { - params.Add("kl", kl) - } - - // DuckDuckGo specific parameters params.Add("t", "h") // HTML format params.Add("ia", "web") // Web search - // Add pagination parameter if not on first page + // Pagination uses 's' (start offset); ~25 results per page. if page > 0 { - // DuckDuckGo uses 's' parameter for pagination (start offset) - // Each page has approximately 25-30 results, but we'll use conservative estimate - offset := page * 25 - params.Add("s", fmt.Sprintf("%d", offset)) + params.Add("s", fmt.Sprintf("%d", page*25)) } base.RawQuery = params.Encode() @@ -159,58 +165,15 @@ func BuildImageURL(q core.Query) (string, error) { if err != nil { return "", err } + params, err := buildParams(q) + if err != nil { + return "", err + } - base.Path += "" - params := url.Values{} params.Add("t", "h") // HTML format params.Add("iax", "images") // Image search params.Add("ia", "images") - // Set request text - if q.Text != "" || q.Site != "" || q.Filetype != "" { - text := q.Text - if q.Site != "" { - text += " site:" + q.Site - } - if q.Filetype != "" { - text += " filetype:" + q.Filetype - } - - params.Add("q", text) - } - - if len(params.Get("q")) == 0 { - return "", errors.New("empty query built") - } - - // Set search date range - if q.DateInterval != "" { - intervals := strings.Split(q.DateInterval, "..") - if len(intervals) != 2 { - return "", errors.New("incorrect date interval provided") - } - - // Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement) - startDate, err := time.Parse("20060102", intervals[0]) - if err != nil { - return "", errors.New("invalid start date format, expected YYYYMMDD") - } - - endDate, err := time.Parse("20060102", intervals[1]) - if err != nil { - return "", errors.New("invalid end date format, expected YYYYMMDD") - } - - // DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format - dateRange := fmt.Sprintf("%s..%s", - startDate.Format("2006-01-02"), - endDate.Format("2006-01-02")) - params.Add("df", dateRange) - } - - if kl := duckDuckGoKL(q.LangCode, q.Region); kl != "" { - params.Add("kl", kl) - } base.RawQuery = params.Encode() return base.String(), nil }