zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-05 16:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

expose proxy-attempts + use-profile CORS headers; simplify DDG url builders

Browse Source
This commit is contained in:
Rustem Kamalov
2026-07-22 15:17:16 +03:00
parent 5d5d135fbf
commit 29c7b0fbe0
5 changed files with 69 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config.yaml
View File

@@ -64,7 +64,7 @@ cors:
enabled: true
allow_origins: "*"
allow_methods: "GET, POST, OPTIONS"
allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant"
allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant, X-Use-Profile"
max_age: 86400
# 2captcha:

2
core/middleware.go
View File

@@ -30,7 +30,7 @@ type CORSConfig struct {
const browserProfileIDHeader = "X-Browser-Profile-Id"
const useProfileHeader = "X-Use-Profile"
const exposedResponseHeaders = "X-Request-ID, X-Cache, X-Fallback-Engine, X-Proxy-Mode, X-Proxy-Tag, X-Proxy-Used, X-Network-Bytes, " + browserProfileIDHeader
const exposedResponseHeaders = "X-Request-ID, X-Cache, X-Fallback-Engine, X-Proxy-Mode, X-Proxy-Tag, X-Proxy-Used, X-Proxy-Attempts, X-Network-Bytes, " + browserProfileIDHeader
func DefaultCORSConfig() CORSConfig {
return CORSConfig{

4
core/middleware_test.go
View File

@@ -63,8 +63,8 @@ func TestCORSMiddleware_UsesConfiguredHeaders(t *testing.T) {
if got := resp.Header.Get("Access-Control-Max-Age"); got != "1200" {
t.Fatalf("unexpected max-age: %q", got)
}
if got := resp.Header.Get("Access-Control-Expose-Headers"); !strings.Contains(got, "X-Network-Bytes") || !strings.Contains(got, "X-Browser-Profile-Id") {
t.Fatalf("expected exposed headers to include network and browser profile headers, got %q", got)
if got := resp.Header.Get("Access-Control-Expose-Headers"); !strings.Contains(got, "X-Network-Bytes") || !strings.Contains(got, "X-Browser-Profile-Id") || !strings.Contains(got, "X-Proxy-Attempts") {
t.Fatalf("expected exposed headers to include network, browser profile, and proxy-attempts headers, got %q", got)
}
}

2
duckduckgo/search.go
View File

@@ -99,7 +99,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
}
// Validate that we have a proper URL
hrefStr := linkText.String()
hrefStr := strings.TrimSpace(linkText.String())
if hrefStr == "" || hrefStr == "#" || hrefStr == "javascript:void(0)" {
ddg.logger.Debug("Invalid href: %s", hrefStr)
continue

165
duckduckgo/url.go
View File

@@ -79,6 +79,61 @@ func duckDuckGoKL(langCode, region string) string {
return ddgKLByLocale[locale.Language]
}
// addQueryText appends the "q" param, combining text with site/filetype
// operators. Returns an error when the resulting query is empty.
func addQueryText(params url.Values, q core.Query) error {
if q.Text == "" && q.Site == "" && q.Filetype == "" {
return errors.New("empty query built")
}
text := q.Text
if q.Site != "" {
text += " site:" + q.Site
}
if q.Filetype != "" {
text += " filetype:" + q.Filetype
}
params.Add("q", text)
return nil
}
// addDateRange appends the "df" param, converting the YYYYMMDD..YYYYMMDD
// interval to DuckDuckGo's YYYY-MM-DD..YYYY-MM-DD form. No-op when unset.
func addDateRange(params url.Values, interval string) error {
if interval == "" {
return nil
}
intervals := strings.Split(interval, "..")
if len(intervals) != 2 {
return errors.New("incorrect date interval provided")
}
start, err := time.Parse("20060102", intervals[0])
if err != nil {
return errors.New("invalid start date format, expected YYYYMMDD")
}
end, err := time.Parse("20060102", intervals[1])
if err != nil {
return errors.New("invalid end date format, expected YYYYMMDD")
}
params.Add("df", start.Format("2006-01-02")+".."+end.Format("2006-01-02"))
return nil
}
// buildParams assembles the query, date range, and locale params shared by web
// and image search.
func buildParams(q core.Query) (url.Values, error) {
params := url.Values{}
if err := addQueryText(params, q); err != nil {
return nil, err
}
if err := addDateRange(params, q.DateInterval); err != nil {
return nil, err
}
if kl := duckDuckGoKL(q.LangCode, q.Region); kl != "" {
params.Add("kl", kl)
}
return params, nil
}
// BuildURL builds a DuckDuckGo web search URL for the provided query and page
// index. It returns an error when query text or date parameters are invalid.
func BuildURL(q core.Query, page int) (string, error) {
@@ -86,66 +141,17 @@ func BuildURL(q core.Query, page int) (string, error) {
if err != nil {
return "", err
}
base.Path += ""
params := url.Values{}
// Set request text
if q.Text != "" || q.Site != "" || q.Filetype != "" {
text := q.Text
if q.Site != "" {
text += " site:" + q.Site
}
if q.Filetype != "" {
text += " filetype:" + q.Filetype
}
params.Add("q", text)
params, err := buildParams(q)
if err != nil {
return "", err
}
if len(params.Get("q")) == 0 {
return "", errors.New("empty query built")
}
// Set search date range
if q.DateInterval != "" {
intervals := strings.Split(q.DateInterval, "..")
if len(intervals) != 2 {
return "", errors.New("incorrect date interval provided")
}
// Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement)
startDate, err := time.Parse("20060102", intervals[0])
if err != nil {
return "", errors.New("invalid start date format, expected YYYYMMDD")
}
endDate, err := time.Parse("20060102", intervals[1])
if err != nil {
return "", errors.New("invalid end date format, expected YYYYMMDD")
}
// DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format
dateRange := fmt.Sprintf("%s..%s",
startDate.Format("2006-01-02"),
endDate.Format("2006-01-02"))
params.Add("df", dateRange)
}
if kl := duckDuckGoKL(q.LangCode, q.Region); kl != "" {
params.Add("kl", kl)
}
// DuckDuckGo specific parameters
params.Add("t", "h") // HTML format
params.Add("ia", "web") // Web search
// Add pagination parameter if not on first page
// Pagination uses 's' (start offset); ~25 results per page.
if page > 0 {
// DuckDuckGo uses 's' parameter for pagination (start offset)
// Each page has approximately 25-30 results, but we'll use conservative estimate
offset := page * 25
params.Add("s", fmt.Sprintf("%d", offset))
params.Add("s", fmt.Sprintf("%d", page*25))
}
base.RawQuery = params.Encode()
@@ -159,58 +165,15 @@ func BuildImageURL(q core.Query) (string, error) {
if err != nil {
return "", err
}
params, err := buildParams(q)
if err != nil {
return "", err
}
base.Path += ""
params := url.Values{}
params.Add("t", "h") // HTML format
params.Add("iax", "images") // Image search
params.Add("ia", "images")
// Set request text
if q.Text != "" || q.Site != "" || q.Filetype != "" {
text := q.Text
if q.Site != "" {
text += " site:" + q.Site
}
if q.Filetype != "" {
text += " filetype:" + q.Filetype
}
params.Add("q", text)
}
if len(params.Get("q")) == 0 {
return "", errors.New("empty query built")
}
// Set search date range
if q.DateInterval != "" {
intervals := strings.Split(q.DateInterval, "..")
if len(intervals) != 2 {
return "", errors.New("incorrect date interval provided")
}
// Convert from YYYYMMDD to YYYY-MM-DD format (DuckDuckGo requirement)
startDate, err := time.Parse("20060102", intervals[0])
if err != nil {
return "", errors.New("invalid start date format, expected YYYYMMDD")
}
endDate, err := time.Parse("20060102", intervals[1])
if err != nil {
return "", errors.New("invalid end date format, expected YYYYMMDD")
}
// DuckDuckGo uses YYYY-MM-DD..YYYY-MM-DD format
dateRange := fmt.Sprintf("%s..%s",
startDate.Format("2006-01-02"),
endDate.Format("2006-01-02"))
params.Add("df", dateRange)
}
if kl := duckDuckGoKL(q.LangCode, q.Region); kl != "" {
params.Add("kl", kl)
}
base.RawQuery = params.Encode()
return base.String(), nil
}