diff --git a/README.md b/README.md index dc144c3..98d6152 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ API response example: "rank": 2, "url": "https://www.bing.com/ck/a?!&&p=6f15ac4589858d0a104cd6f55cc8", "title": "Golden Retriever Dog Forums", - "description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now...", + "description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now...", "ad": false, "engine": "bing" }, @@ -142,6 +142,87 @@ curl "http://127.0.0.1:7000/yandex/search?text=golang&limit=10&start=10" curl "http://127.0.0.1:7000/bing/image?text=golang&limit=20" ``` +## Response Examples + +Interactive docs (OpenAPI + Swagger UI) are available at: + +- `http://127.0.0.1:7000/docs` +- `http://127.0.0.1:7000/openapi.yaml` + +### Web Search Response (`//search`) + +```json +[ + { + "rank": 1, + "url": "https://go.dev/doc/", + "title": "Documentation - The Go Programming Language", + "description": "Official Go documentation, tutorials, references, and release notes.", + "ad": false + }, + { + "rank": 2, + "url": "https://pkg.go.dev/", + "title": "pkg.go.dev", + "description": "Go package discovery and API documentation.", + "ad": false + } +] +``` + +### Image Search Response (`//image`) + +```json +[ + { + "rank": 1, + "url": "https://golang.org/lib/godoc/images/go-logo-blue.svg", + "title": "Go Gopher Logo", + "description": "Source: https://go.dev/brand/", + "ad": false + }, + { + "rank": 2, + "url": "https://example.com/images/go-mascot.png", + "title": "Go mascot", + "description": "Height:800, Width:1200, Source Page: https://example.com/post", + "ad": false + } +] +``` + +### Error Responses + +`400 Bad Request` (invalid/missing query): + +```json +{ + "error": "bad_request", + "code": 400, + "message": "Query cannot be empty" +} +``` + +`503 Service Unavailable` (engine unavailable, captcha, timeout, or proxy path failure): + +```json +{ + "error": "service_unavailable", + "code": 503, + "message": "captcha found, please stop sending requests for a while: captcha detected" +} +``` + +### Response Headers + +| Header | Values/Examples | Meaning | +| ------------------- | ------------------------------- | ------------------------------------------------------------- | +| `X-Cache` | `HIT`, `MISS`, `BYPASS` | Cache result for this response. | +| `X-Fallback-Engine` | `google`, `bing`, `duckduckgo` | Present when dedicated endpoint used fallback engine. | +| `X-Proxy-Mode` | `off`, `single`, `pool` | Proxy policy mode applied by resilient search. | +| `X-Proxy-Tag` | `residential`, `datacenter`, `` | Selected proxy pool tag. Empty when proxy mode is off/direct. | +| `X-Proxy-Used` | `direct`, `socks5://host:port` | Actual upstream route used to execute request. | + ## 🌍 Proxy Support OpenSERP supports HTTP and SOCKS5 proxies. diff --git a/baidu/search.go b/baidu/search.go index 774b50f..545c0ef 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -31,12 +31,14 @@ type imageDataJson struct { } } +// Baidu implements core.SearchEngine for Baidu SERP pages. type Baidu struct { core.Browser core.SearchEngineOptions logger *core.EngineLogger } +// New creates a Baidu engine instance with browser/runtime options applied. func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu { baid := Baidu{Browser: browser} opts.Init() @@ -45,10 +47,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu { return &baid } +// Name returns the stable engine identifier. func (baid *Baidu) Name() string { return "baidu" } +// GetRateLimiter returns a limiter configured from SearchEngineOptions. func (baid *Baidu) GetRateLimiter() *rate.Limiter { ratelimit := rate.Every(baid.GetRatelimit()) return rate.NewLimiter(ratelimit, baid.RateBurst) @@ -64,6 +68,8 @@ func (baid *Baidu) isTimeout(page *rod.Page) bool { return err == nil } +// Search executes a Baidu web search and returns normalized search results. +// It may return core.ErrCaptcha or core.ErrSearchTimeout. func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { baid.logger.Debug("Starting search, query: %+v", query) @@ -145,6 +151,8 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { return core.DeduplicateResults(searchResults), nil } +// SearchImage executes a Baidu image search and returns normalized image +// results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { baid.logger.Debug("Starting image search, query: %+v", query) diff --git a/baidu/url.go b/baidu/url.go index 2e310d5..c8eee0f 100644 --- a/baidu/url.go +++ b/baidu/url.go @@ -22,6 +22,8 @@ func dateToTimestamp(date string) (int64, error) { return t.Unix(), nil } +// BuildURL builds a Baidu web search URL from Query fields. +// It returns an error when query text, date, or pagination parameters are invalid. func BuildURL(q core.Query) (string, error) { base, _ := url.Parse("https://www.baidu.com/") base.Path += "s" @@ -84,6 +86,8 @@ func BuildURL(q core.Query) (string, error) { return base.String(), nil } +// BuildImageURL builds a Baidu image search URL from Query fields and page +// index. It returns an error when the query text is empty. func BuildImageURL(q core.Query, pageNum int) (string, error) { base, _ := url.Parse("https://image.baidu.com/") base.Path += "search/acjson" diff --git a/bing/search.go b/bing/search.go index 5726adf..b71ff41 100644 --- a/bing/search.go +++ b/bing/search.go @@ -13,12 +13,14 @@ import ( "golang.org/x/time/rate" ) +// Bing implements core.SearchEngine for Bing SERP pages. type Bing struct { core.Browser core.SearchEngineOptions logger *core.EngineLogger } +// New creates a Bing engine instance with browser/runtime options applied. func New(browser core.Browser, opts core.SearchEngineOptions) *Bing { bing := Bing{Browser: browser} opts.Init() @@ -27,10 +29,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Bing { return &bing } +// Name returns the stable engine identifier. func (bing *Bing) Name() string { return "bing" } +// GetRateLimiter returns a limiter configured from SearchEngineOptions. func (bing *Bing) GetRateLimiter() *rate.Limiter { ratelimit := rate.Every(bing.GetRatelimit()) return rate.NewLimiter(ratelimit, bing.RateBurst) @@ -99,6 +103,8 @@ func (bing *Bing) close(page *rod.Page) { } } +// Search executes a Bing web search and returns normalized search results. +// It may return core.ErrCaptcha or core.ErrSearchTimeout. func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { bing.logger.Debug("Starting search, query: %+v", query) @@ -240,7 +246,7 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { return deduped, nil } -// BingImageData represents the JSON structure in the m attribute of image elements +// BingImageData represents metadata encoded in the image result `m` attribute. type BingImageData struct { T string `json:"t"` // Title Desc string `json:"desc"` // Description @@ -252,7 +258,8 @@ type BingImageData struct { MURL string `json:"murl"` // Image URL } -// SearchImage performs Bing image search and returns results +// SearchImage executes a Bing image search and returns normalized image +// results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) { bing.logger.Debug("Starting image search, query: %+v", query) diff --git a/bing/url.go b/bing/url.go index 84f8661..67d1d9d 100644 --- a/bing/url.go +++ b/bing/url.go @@ -12,6 +12,8 @@ import ( "github.com/sirupsen/logrus" ) +// BuildURL builds a Bing web search URL from Query fields. +// It returns an error when query text or date parameters are invalid. func BuildURL(q core.Query) (string, error) { base, err := url.Parse("https://www.bing.com") if err != nil { @@ -93,6 +95,8 @@ func BuildURL(q core.Query) (string, error) { return base.String(), nil } +// BuildImageURL builds a Bing image search URL from Query fields. +// It returns an error when the resulting query text is empty. func BuildImageURL(q core.Query) (string, error) { base, err := url.Parse("https://www.bing.com") if err != nil { diff --git a/core/browser.go b/core/browser.go index 70fb5e3..5845a36 100644 --- a/core/browser.go +++ b/core/browser.go @@ -17,23 +17,35 @@ import ( "github.com/sirupsen/logrus" ) +// BrowserOpts configures Chromium launch and navigation behavior. type BrowserOpts struct { - IsHeadless bool // Use browser interface - IsLeakless bool // Force to kill browser - Timeout time.Duration // Timeout - LanguageCode string - WaitRequests bool // Wait requests to complete after navigation - LeavePageOpen bool // Leave pages and browser open - WaitLoadTime time.Duration // Time to wait till page loads - CaptchaSolverApiKey string // 2Captcha api key - BrowserPath string // Explicit browser executable path - ProxyURL string // Proxy URL - Insecure bool // Allow insecure TLS connections - UseStealth bool // Use go-rod stealth plugin - + // IsHeadless runs Chromium without visible UI. + IsHeadless bool + // IsLeakless forces child browser process cleanup when the parent exits. + IsLeakless bool + // Timeout is applied to browser connect and page navigation operations. + Timeout time.Duration + // LanguageCode sets Accept-Language for emulated requests. + LanguageCode string + // WaitRequests waits for request-idle state after navigation. + WaitRequests bool + // LeavePageOpen keeps pages open after search operations. + LeavePageOpen bool + // WaitLoadTime is an additional fixed wait after load/idle checks. + WaitLoadTime time.Duration + // CaptchaSolverApiKey enables 2Captcha integration for supported engines. + CaptchaSolverApiKey string + // BrowserPath optionally points to a specific browser executable. + BrowserPath string + // ProxyURL defines the upstream proxy for browser traffic. + ProxyURL string + // Insecure allows invalid TLS certificates for browser requests. + Insecure bool + // UseStealth enables go-rod stealth page creation. + UseStealth bool } -// Initialize browser parameters with default values if they are not set +// Check applies default option values when optional fields are unset. func (o *BrowserOpts) Check() { if o.Timeout == 0 { o.Timeout = time.Second * 30 @@ -44,6 +56,7 @@ func (o *BrowserOpts) Check() { } } +// Browser wraps a launched Chromium instance used by engine implementations. type Browser struct { BrowserOpts browserAddr string @@ -51,6 +64,8 @@ type Browser struct { CaptchaSolver *CaptchaSolver } +// NewBrowser launches a new Chromium process via Rod launcher and returns a +// Browser wrapper configured with proxy and captcha solver settings. func NewBrowser(opts BrowserOpts) (*Browser, error) { opts.Check() logrus.Debugf("Browser options: %+v", opts) @@ -151,7 +166,7 @@ func resolveBrowserBinaryPath(browserPath string, lookPath func() (string, bool) return "", nil } -// Check whether browser instance is already created +// IsInitialized reports whether the browser launcher has been created. func (b *Browser) IsInitialized() bool { if b.browserAddr != "" { return true @@ -160,7 +175,9 @@ func (b *Browser) IsInitialized() bool { } } -// Open URL +// Navigate connects to Chromium, creates a page, applies stealth/emulation and +// proxy auth, then navigates to URL. It returns an initialized page ready for +// selector queries, or an error when browser setup/navigation fails. func (b *Browser) Navigate(URL string) (*rod.Page, error) { logrus.Debug("Navigate to: ", URL) @@ -280,6 +297,7 @@ func (b *Browser) Navigate(URL string) (*rod.Page, error) { return page, nil } +// Close closes the active browser connection. func (b *Browser) Close() error { return b.browser.Close() } diff --git a/core/common.go b/core/common.go index ff9689b..87731b2 100644 --- a/core/common.go +++ b/core/common.go @@ -9,17 +9,31 @@ import ( "github.com/gofiber/fiber/v2" ) +// ErrCaptcha is returned when the engine detects a captcha challenge page. +// This error is treated as non-retryable by resilient search policies. var ErrCaptcha = errors.New("captcha detected") + +// ErrSearchTimeout is returned when required SERP elements are not found before +// selector or page timeouts expire. var ErrSearchTimeout = errors.New("timeout. Cannot find element on page") +// SearchResult represents one normalized result item returned by any engine. type SearchResult struct { - Rank int `json:"rank"` - URL string `json:"url"` - Title string `json:"title"` + // Rank is a 1-based position in engine output. Some engines use negative + // ranks for non-organic blocks such as ads or instant answers. + Rank int `json:"rank"` + // URL is the canonical result URL. + URL string `json:"url"` + // Title is the result headline shown on the SERP. + Title string `json:"title"` + // Description is the snippet text associated with the result. Description string `json:"description"` - Ad bool `json:"ad"` + // Ad reports whether the result is sponsored. + Ad bool `json:"ad"` } +// DeduplicateResults removes items with duplicate URLs and returns a result set +// sorted by rank in ascending order. func DeduplicateResults(results []SearchResult) []SearchResult { unique := make(map[string]bool) var deduped []SearchResult @@ -40,6 +54,8 @@ func DeduplicateResults(results []SearchResult) []SearchResult { return deduped } +// ConvertSearchResultsMap converts a map-based collection to a rank-sorted +// slice and returns it by pointer. func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult { searchResults := []SearchResult{} @@ -53,21 +69,42 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search return &searchResults } +// Query holds request parameters used by HTTP handlers and search engines. +// Example minimal query: Query{Text: "golang", Limit: 10}. type Query struct { - Text string - LangCode string // eg. EN, ES, RU... - DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010 - Filetype string // File extension to search. - Site string // Search site - Limit int // Limit the number of results - Start int // Search offset for pagination (Google uses 0, 10, 20...) - Filter bool // Filter duplicates (google) (false: include similar, true: hide similar) - Answers bool // Include question and answers from SERP page to results with negative indexes - ProxyURL string // Proxy URL for raw requests - ProxyOverride string // Request-scoped proxy override: tag or direct - Insecure bool // Allow insecure TLS connections + // Text is the search phrase, for example "golang fiber tutorial". + Text string + // LangCode is an engine language hint such as "EN", "DE", or "RU". + LangCode string + // DateInterval filters by date range in YYYYMMDD..YYYYMMDD format. + // Example: "20250101..20250331". + DateInterval string + // Filetype is a file extension filter, for example "pdf" or "docx". + Filetype string + // Site restricts results to a specific domain, for example "github.com". + Site string + // Limit is the maximum number of results requested by the client. + Limit int + // Start is an engine pagination offset. Values are engine-specific: + // Google commonly uses 0,10,20 while some engines use page indexes. + Start int + // Filter controls duplicate filtering when supported by the engine. + // For Google, false includes similar results and true hides them. + Filter bool + // Answers enables parsing answer modules when supported by the engine. + // Such entries may be returned with negative rank values. + Answers bool + // ProxyURL is a direct proxy URL used by raw HTTP search paths. + ProxyURL string + // ProxyOverride is a request-scoped proxy policy override (tag or "direct"), + // typically parsed from the X-Use-Proxy header. + ProxyOverride string + // Insecure enables insecure TLS for request/browser execution. + Insecure bool } +// ComputePagination translates an absolute start offset into page index and +// in-page offset for a fixed page size. func ComputePagination(start int, pageSize int) (int, int, error) { if pageSize <= 0 { return 0, 0, errors.New("pageSize must be > 0") @@ -78,6 +115,7 @@ func ComputePagination(start int, pageSize int) (int, int, error) { return start / pageSize, start % pageSize, nil } +// IsEmpty reports whether query text operators are all absent. func (q Query) IsEmpty() bool { if q.Site == "" && q.Filetype == "" && q.Text == "" { return true @@ -85,6 +123,9 @@ func (q Query) IsEmpty() bool { return false } +// InitFromContext populates Query from HTTP query parameters and request +// headers. It validates numeric/boolean inputs and returns an error for empty +// search expressions. func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { searchQuery.Text = reqCtx.Query("text") searchQuery.LangCode = reqCtx.Query("lang") @@ -128,14 +169,23 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { return nil } +// SearchEngineOptions controls engine pacing, selector waits, and captcha +// handling behavior shared by browser and raw implementations. type SearchEngineOptions struct { - RateRequests int `mapstructure:"rate_requests"` - RateTime int64 `mapstructure:"rate_seconds"` - RateBurst int `mapstructure:"rate_burst"` - SelectorTimeout int64 `mapstructure:"selector_timeout"` // CSS selector timeout in seconds - IsSolveCaptcha bool `mapstructure:"captcha"` + // RateRequests is the allowed number of requests within RateTime seconds. + RateRequests int `mapstructure:"rate_requests"` + // RateTime defines the rate-limiting window size in seconds. + RateTime int64 `mapstructure:"rate_seconds"` + // RateBurst is the token bucket burst size for short spikes. + RateBurst int `mapstructure:"rate_burst"` + // SelectorTimeout is the per-selector wait timeout in seconds. + SelectorTimeout int64 `mapstructure:"selector_timeout"` + // IsSolveCaptcha enables automatic captcha solving when engine support and + // solver credentials are configured. + IsSolveCaptcha bool `mapstructure:"captcha"` } +// Init sets default option values when fields are zero. func (o *SearchEngineOptions) Init() { if o.RateRequests == 0 { o.RateRequests = 6 @@ -151,10 +201,12 @@ func (o *SearchEngineOptions) Init() { } } +// GetRatelimit returns the interval between two allowed requests. func (o *SearchEngineOptions) GetRatelimit() time.Duration { return (time.Duration(o.RateTime) * time.Second) / time.Duration(o.RateRequests) } +// GetSelectorTimeout returns the selector wait timeout as time.Duration. func (o *SearchEngineOptions) GetSelectorTimeout() time.Duration { return time.Duration(o.SelectorTimeout) * time.Second } diff --git a/core/server.go b/core/server.go index 73dc354..2eb9c8d 100644 --- a/core/server.go +++ b/core/server.go @@ -16,14 +16,24 @@ import ( "golang.org/x/time/rate" ) +// SearchEngine defines the contract required by the HTTP server and resilient +// search pipeline. type SearchEngine interface { + // Search runs a web search request and returns normalized results. + // Implementations should return sentinel errors such as ErrCaptcha and + // ErrSearchTimeout for policy-aware handling. Search(Query) ([]SearchResult, error) + // SearchImage runs an image search request and returns normalized results. SearchImage(Query) ([]SearchResult, error) + // IsInitialized reports whether the engine is ready to serve requests. IsInitialized() bool + // Name returns a stable engine identifier used in routes and telemetry. Name() string + // GetRateLimiter returns an engine-specific limiter used by resilient search. GetRateLimiter() *rate.Limiter } +// Server exposes OpenSERP HTTP endpoints backed by one or more search engines. type Server struct { app *fiber.App addr string @@ -34,15 +44,25 @@ type Server struct { opts ServerOptions } +// ServerOptions configures HTTP server middleware and resilience behavior. type ServerOptions struct { - CacheTTL time.Duration - CacheMaxSize int - EnableCORS bool - CORS CORSConfig + // CacheTTL controls response cache entry lifetime. Zero disables caching. + CacheTTL time.Duration + // CacheMaxSize is the maximum number of cached entries. + CacheMaxSize int + // EnableCORS enables cross-origin headers with the CORS config below. + EnableCORS bool + // CORS contains allowed origins, methods, and headers when CORS is enabled. + CORS CORSConfig + // AllowEndpointFallback allows dedicated engine routes to fall back to other + // healthy engines when the primary engine fails. AllowEndpointFallback bool - Resilience ResilientConfig + // Resilience defines retry/circuit-breaker/proxy strategy settings. + Resilience ResilientConfig } +// DefaultServerOptions returns production-oriented defaults for cache, CORS, +// and resilient search policies. func DefaultServerOptions() ServerOptions { return ServerOptions{ CacheTTL: 5 * time.Minute, @@ -54,10 +74,15 @@ func DefaultServerOptions() ServerOptions { } } +// NewServer creates a Server with DefaultServerOptions and registers all +// routes for the provided engines. func NewServer(host string, port int, searchEngines ...SearchEngine) *Server { return NewServerWithOptions(host, port, DefaultServerOptions(), searchEngines...) } +// NewServerWithOptions builds a Server, installs middleware, and registers API +// routes. The returned server is ready to Listen; call Shutdown for graceful +// stop. func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngines ...SearchEngine) *Server { addr := fmt.Sprintf("%s:%d", host, port) app := fiber.New(fiber.Config{ @@ -207,6 +232,7 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm return c.JSON(res) } +// HealthStatus is returned by /health and summarizes service state. type HealthStatus struct { Status string `json:"status"` Uptime string `json:"uptime"` @@ -214,6 +240,7 @@ type HealthStatus struct { System map[string]interface{} `json:"system"` } +// EngineHealth describes availability of one configured engine. type EngineHealth struct { Name string `json:"name"` Initialized bool `json:"initialized"` @@ -309,6 +336,7 @@ func (s *Server) handleCircuitBreakerStats(c *fiber.Ctx) error { }) } +// MegaSearchResult extends SearchResult with the engine source name. type MegaSearchResult struct { SearchResult Engine string `json:"engine"` @@ -607,10 +635,12 @@ func (s *Server) handleSwaggerUI(c *fiber.Ctx) error { return c.SendString(page) } +// Listen starts the Fiber HTTP server on the configured address. func (s *Server) Listen() error { return s.app.Listen(s.addr) } +// Shutdown gracefully stops the Fiber HTTP server. func (s *Server) Shutdown() error { return s.app.Shutdown() } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 37807f1..316691f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -175,33 +175,33 @@ Defaults below are the shipped defaults in `config.yaml` (if present). If the co ### `server` -| Key | Default | Description | -| --- | --- | --- | -| `server.host` | `0.0.0.0` | API bind host | -| `server.port` | `7000` | API bind port | -| `server.debug` | `false` | Debug mode, forces headful browser | -| `server.verbose` | `true` | Info-level request logs | -| `server.raw_requests` | `false` | `true` = raw HTTP mode | -| `server.insecure` | `true` | Allow insecure TLS connections | +| Key | Default | Description | +| --------------------- | --------- | ---------------------------------- | +| `server.host` | `0.0.0.0` | API bind host | +| `server.port` | `7000` | API bind port | +| `server.debug` | `false` | Debug mode, forces headful browser | +| `server.verbose` | `true` | Info-level request logs | +| `server.raw_requests` | `false` | `true` = raw HTTP mode | +| `server.insecure` | `true` | Allow insecure TLS connections | ### `app` -| Key | Default | Description | -| --- | --- | --- | -| `app.timeout` | `15` | Request timeout in seconds | -| `app.browser_path` | `""` | Custom browser binary path | -| `app.head` | `false` | Headful browser UI | -| `app.leakless` | `false` | Force browser process cleanup | -| `app.leave_head` | `false` | Keep browser tabs open | -| `app.stealth` | `false` | Enable stealth plugin | +| Key | Default | Description | +| ------------------ | ------- | ----------------------------- | +| `app.timeout` | `15` | Request timeout in seconds | +| `app.browser_path` | `""` | Custom browser binary path | +| `app.head` | `false` | Headful browser UI | +| `app.leakless` | `false` | Force browser process cleanup | +| `app.leave_head` | `false` | Keep browser tabs open | +| `app.stealth` | `false` | Enable stealth plugin | ### `proxies` -| Key | Default | Description | -| --- | --- | --- | -| `proxies.global` | unset | Force single proxy for all engines | -| `proxies.entries[]` | empty | Tagged proxy pool entries (`url`, `tags`) | -| `proxies.health.failure_threshold` | `3` | Disable proxy after N failures | +| Key | Default | Description | +| ---------------------------------- | ------- | ----------------------------------------- | +| `proxies.global` | unset | Force single proxy for all engines | +| `proxies.entries[]` | empty | Tagged proxy pool entries (`url`, `tags`) | +| `proxies.health.failure_threshold` | `3` | Disable proxy after N failures | Per-engine optional proxy tag: @@ -213,52 +213,52 @@ Per-engine optional proxy tag: ### `cache` -| Key | Default | Description | -| --- | --- | --- | -| `cache.ttl_seconds` | `60` | Response cache TTL (0 disables cache) | -| `cache.max_size` | `1000` | Max cached entries | +| Key | Default | Description | +| ------------------- | ------- | ------------------------------------- | +| `cache.ttl_seconds` | `60` | Response cache TTL (0 disables cache) | +| `cache.max_size` | `1000` | Max cached entries | ### `resilience` -| Key | Default | Description | -| --- | --- | --- | -| `resilience.max_retries` | `2` | Retry attempts per request | +| Key | Default | Description | +| ------------------------------------ | ------- | ------------------------------------------------------ | +| `resilience.max_retries` | `2` | Retry attempts per request | | `resilience.allow_endpoint_fallback` | `false` | Allow dedicated endpoints to fallback to other engines | ### `circuit_breaker` -| Key | Default | Description | -| --- | --- | --- | -| `circuit_breaker.failures` | `5` | Failures before opening circuit | -| `circuit_breaker.recovery_seconds` | `60` | Open -> half-open wait time | -| `circuit_breaker.successes` | `2` | Half-open successes to close circuit | +| Key | Default | Description | +| ---------------------------------- | ------- | ------------------------------------ | +| `circuit_breaker.failures` | `5` | Failures before opening circuit | +| `circuit_breaker.recovery_seconds` | `60` | Open -> half-open wait time | +| `circuit_breaker.successes` | `2` | Half-open successes to close circuit | ### `cors` -| Key | Default | Description | -| --- | --- | --- | -| `cors.enabled` | `true` | Enable CORS middleware | -| `cors.allow_origins` | `"*"` | Allowed origins | -| `cors.allow_methods` | `"GET, POST, OPTIONS"` | Allowed methods | -| `cors.allow_headers` | `"Origin, Content-Type, Accept, Authorization, X-Use-Proxy"` | Allowed headers | -| `cors.max_age` | `86400` | Preflight cache max age (seconds) | +| Key | Default | Description | +| -------------------- | ------------------------------------------------------------ | --------------------------------- | +| `cors.enabled` | `true` | Enable CORS middleware | +| `cors.allow_origins` | `"*"` | Allowed origins | +| `cors.allow_methods` | `"GET, POST, OPTIONS"` | Allowed methods | +| `cors.allow_headers` | `"Origin, Content-Type, Accept, Authorization, X-Use-Proxy"` | Allowed headers | +| `cors.max_age` | `86400` | Preflight cache max age (seconds) | ### `2captcha` -| Key | Default | Description | -| --- | --- | --- | -| `2captcha.apikey` | unset | Optional captcha solver key | +| Key | Default | Description | +| ----------------- | ------- | --------------------------- | +| `2captcha.apikey` | unset | Optional captcha solver key | ### Engine rate-limit defaults For each engine (`google`, `yandex`, `baidu`, `bing`, `duckduckgo`): -| Key | Default | Description | -| --- | --- | --- | -| `.rate_requests` | `4` | Average requests per minute | -| `.rate_burst` | `2` | Burst capacity | -| `.rate_seconds` | `60` (implicit) | Rate window seconds | -| `.selector_timeout` | `5` (implicit) | Selector wait timeout seconds | +| Key | Default | Description | +| --------------------------- | --------------- | ----------------------------- | +| `.rate_requests` | `4` | Average requests per minute | +| `.rate_burst` | `2` | Burst capacity | +| `.rate_seconds` | `60` (implicit) | Rate window seconds | +| `.selector_timeout` | `5` (implicit) | Selector wait timeout seconds | Google-only additional toggle: diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/duckduckgo/search.go b/duckduckgo/search.go index dfc0209..eb66b78 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -10,6 +10,7 @@ import ( "golang.org/x/time/rate" ) +// DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages. type DuckDuckGo struct { core.Browser core.SearchEngineOptions @@ -17,6 +18,7 @@ type DuckDuckGo struct { logger *core.EngineLogger } +// New creates a DuckDuckGo engine instance with browser/runtime options applied. func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo { ddg := DuckDuckGo{Browser: browser} opts.Init() @@ -27,10 +29,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo { return &ddg } +// Name returns the stable engine identifier. func (ddg *DuckDuckGo) Name() string { return "duckduckgo" } +// GetRateLimiter returns a limiter configured from SearchEngineOptions. func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter { ratelimit := rate.Every(ddg.GetRatelimit()) return rate.NewLimiter(ratelimit, ddg.RateBurst) @@ -163,6 +167,8 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se return searchResults } +// Search executes a DuckDuckGo web search and returns normalized search +// results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { ddg.logger.Debug("Starting search, query: %+v", query) @@ -265,6 +271,8 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { return deduped, nil } +// SearchImage executes a DuckDuckGo image search and returns normalized image +// results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) SearchImage(query core.Query) ([]core.SearchResult, error) { ddg.logger.Debug("Starting image search, query: %+v", query) diff --git a/duckduckgo/url.go b/duckduckgo/url.go index d224620..e78d6e3 100644 --- a/duckduckgo/url.go +++ b/duckduckgo/url.go @@ -12,6 +12,8 @@ import ( const baseURL = "https://duckduckgo.com" +// BuildURL builds a DuckDuckGo web search URL for the provided query and page +// index. It returns an error when query text or date parameters are invalid. func BuildURL(q core.Query, page int) (string, error) { base, err := url.Parse(baseURL) if err != nil { @@ -84,6 +86,8 @@ func BuildURL(q core.Query, page int) (string, error) { return base.String(), nil } +// BuildImageURL builds a DuckDuckGo image search URL from Query fields. +// It returns an error when query text or date parameters are invalid. func BuildImageURL(q core.Query) (string, error) { base, err := url.Parse(baseURL) if err != nil { diff --git a/google/search.go b/google/search.go index e38094a..97861b9 100644 --- a/google/search.go +++ b/google/search.go @@ -14,6 +14,7 @@ import ( "golang.org/x/time/rate" ) +// Google implements core.SearchEngine for Google SERP pages. type Google struct { core.Browser core.SearchEngineOptions @@ -21,6 +22,7 @@ type Google struct { logger *core.EngineLogger } +// New creates a Google engine instance with browser/runtime options applied. func New(browser core.Browser, opts core.SearchEngineOptions) *Google { gogl := Google{Browser: browser} opts.Init() @@ -30,10 +32,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Google { return &gogl } +// Name returns the stable engine identifier. func (gogl *Google) Name() string { return "google" } +// GetRateLimiter returns a limiter configured from SearchEngineOptions. func (gogl *Google) GetRateLimiter() *rate.Limiter { ratelimit := rate.Every(gogl.GetRatelimit()) return rate.NewLimiter(ratelimit, gogl.RateBurst) @@ -157,6 +161,8 @@ func (gogl *Google) acceptCookies(page *rod.Page) { } +// Search executes a Google web search and returns normalized search results. +// It may return core.ErrCaptcha or core.ErrSearchTimeout. func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { gogl.logger.Debug("Starting search, query: %+v", query) @@ -366,6 +372,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { return core.DeduplicateResults(searchResults), nil } +// SearchImage executes a Google image search and returns normalized image +// results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { gogl.logger.Debug("Starting image search, query: %+v", query) diff --git a/google/url.go b/google/url.go index fb1d5d3..fb7a617 100644 --- a/google/url.go +++ b/google/url.go @@ -11,6 +11,8 @@ import ( "github.com/sirupsen/logrus" ) +// GoogleDomains maps language/country hints to Google TLD suffixes used for +// URL construction. var GoogleDomains = map[string]string{ "": "com", "en": "com", @@ -213,7 +215,8 @@ var GoogleDomains = map[string]string{ "zw": "co.zw", } -// Build Google query URL from Query struct +// BuildURL builds a Google web search URL from Query fields. +// It returns an error when the resulting query text is empty or invalid. func BuildURL(q core.Query) (string, error) { googleBase := GoogleDomains[strings.ToLower(q.LangCode)] base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase)) @@ -286,6 +289,8 @@ func BuildURL(q core.Query) (string, error) { return base.String(), nil } +// BuildImageURL builds a Google image search URL from Query fields. +// It returns an error when the resulting query text is empty or invalid. func BuildImageURL(q core.Query) (string, error) { // TODO: Add new params googleBase := GoogleDomains[strings.ToLower(q.LangCode)] @@ -344,6 +349,7 @@ func BuildImageURL(q core.Query) (string, error) { return base.String(), nil } +// SourceImage contains parsed Google image metadata extracted from result links. type SourceImage struct { PageURL string OriginalURL string diff --git a/yandex/search.go b/yandex/search.go index dbc501a..ac2eac6 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -11,6 +11,7 @@ import ( "golang.org/x/time/rate" ) +// ImageEntity contains one image record from Yandex image search state JSON. type ImageEntity struct { ID string `json:"id"` Rank int `json:"pos"` @@ -23,6 +24,7 @@ type ImageEntity struct { IsGIF bool `json:"gifLabel"` } +// ImageData maps the subset of Yandex JSON state used by parser code. type ImageData struct { InitalState struct { SerpList struct { @@ -33,6 +35,7 @@ type ImageData struct { } `json:"initialState"` } +// Yandex implements core.SearchEngine for Yandex SERP pages. type Yandex struct { core.Browser core.SearchEngineOptions @@ -40,6 +43,7 @@ type Yandex struct { logger *core.EngineLogger } +// New creates a Yandex engine instance with browser/runtime options applied. func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex { yand := Yandex{Browser: browser} opts.Init() @@ -50,10 +54,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex { return &yand } +// Name returns the stable engine identifier. func (yand *Yandex) Name() string { return "yandex" } +// GetRateLimiter returns a limiter configured from SearchEngineOptions. func (yand *Yandex) GetRateLimiter() *rate.Limiter { ratelimit := rate.Every(yand.GetRatelimit()) return rate.NewLimiter(ratelimit, yand.RateBurst) @@ -113,6 +119,8 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc return searchResults } +// Search executes a Yandex web search and returns normalized search results. +// It may return core.ErrCaptcha or core.ErrSearchTimeout. func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { yand.logger.Debug("Starting search, query: %+v", query) if query.Start < 0 { @@ -192,6 +200,8 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { return core.DeduplicateResults(allResults), nil } +// SearchImage executes a Yandex image search and returns normalized image +// results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { yand.logger.Debug("Starting image search, query: %+v", query) diff --git a/yandex/url.go b/yandex/url.go index 646ac73..dea0028 100644 --- a/yandex/url.go +++ b/yandex/url.go @@ -10,6 +10,8 @@ import ( const baseURL = "https://www.yandex.com" +// BuildURL builds a Yandex web search URL for the provided query and page +// index. It returns an error when the resulting query text is empty. func BuildURL(q core.Query, page int) (string, error) { base, _ := url.Parse(baseURL) base.Path += "search/" @@ -42,6 +44,8 @@ func BuildURL(q core.Query, page int) (string, error) { return base.String(), nil } +// BuildImageURL builds a Yandex image search URL for the provided query and +// page index. It returns an error when the resulting query text is empty. func BuildImageURL(q core.Query, page int) (string, error) { // TODO: Add other parameters base, _ := url.Parse(baseURL)