zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-05 16:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

docs: add godoc for public api and readme response examples

Browse Source
This commit is contained in:
Rustem Kamalov
2026-04-20 01:20:52 +03:00
parent a9986fd8fe
commit 0b7fbd3a45
16 changed files with 339 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
README.md
View File

@@ -82,7 +82,7 @@ API response example:
"rank": 2,
"url": "https://www.bing.com/ck/a?!&&p=6f15ac4589858d0a104cd6f55cc8",
"title": "Golden Retriever Dog Forums",
"description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now...",
"description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now...",
"ad": false,
"engine": "bing"
},
@@ -142,6 +142,87 @@ curl "http://127.0.0.1:7000/yandex/search?text=golang&limit=10&start=10"
curl "http://127.0.0.1:7000/bing/image?text=golang&limit=20"
```
## Response Examples
Interactive docs (OpenAPI + Swagger UI) are available at:
- `http://127.0.0.1:7000/docs`
- `http://127.0.0.1:7000/openapi.yaml`
### Web Search Response (`/<engine>/search`)
```json
[
{
"rank": 1,
"url": "https://go.dev/doc/",
"title": "Documentation - The Go Programming Language",
"description": "Official Go documentation, tutorials, references, and release notes.",
"ad": false
},
{
"rank": 2,
"url": "https://pkg.go.dev/",
"title": "pkg.go.dev",
"description": "Go package discovery and API documentation.",
"ad": false
}
]
```
### Image Search Response (`/<engine>/image`)
```json
[
{
"rank": 1,
"url": "https://golang.org/lib/godoc/images/go-logo-blue.svg",
"title": "Go Gopher Logo",
"description": "Source: https://go.dev/brand/",
"ad": false
},
{
"rank": 2,
"url": "https://example.com/images/go-mascot.png",
"title": "Go mascot",
"description": "Height:800, Width:1200, Source Page: https://example.com/post",
"ad": false
}
]
```
### Error Responses
`400 Bad Request` (invalid/missing query):
```json
{
"error": "bad_request",
"code": 400,
"message": "Query cannot be empty"
}
```
`503 Service Unavailable` (engine unavailable, captcha, timeout, or proxy path failure):
```json
{
"error": "service_unavailable",
"code": 503,
"message": "captcha found, please stop sending requests for a while: captcha detected"
}
```
### Response Headers
| Header | Values/Examples | Meaning |
| ------------------- | ------------------------------- | ------------------------------------------------------------- |
| `X-Cache` | `HIT`, `MISS`, `BYPASS` | Cache result for this response. |
| `X-Fallback-Engine` | `google`, `bing`, `duckduckgo` | Present when dedicated endpoint used fallback engine. |
| `X-Proxy-Mode` | `off`, `single`, `pool` | Proxy policy mode applied by resilient search. |
| `X-Proxy-Tag` | `residential`, `datacenter`, `` | Selected proxy pool tag. Empty when proxy mode is off/direct. |
| `X-Proxy-Used` | `direct`, `socks5://host:port` | Actual upstream route used to execute request. |
## 🌍 Proxy Support
OpenSERP supports HTTP and SOCKS5 proxies.

8
baidu/search.go
View File

@@ -31,12 +31,14 @@ type imageDataJson struct {
}
}
// Baidu implements core.SearchEngine for Baidu SERP pages.
type Baidu struct {
core.Browser
core.SearchEngineOptions
logger *core.EngineLogger
}
// New creates a Baidu engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu {
baid := Baidu{Browser: browser}
opts.Init()
@@ -45,10 +47,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu {
return &baid
}
// Name returns the stable engine identifier.
func (baid *Baidu) Name() string {
return "baidu"
}
// GetRateLimiter returns a limiter configured from SearchEngineOptions.
func (baid *Baidu) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(baid.GetRatelimit())
return rate.NewLimiter(ratelimit, baid.RateBurst)
@@ -64,6 +68,8 @@ func (baid *Baidu) isTimeout(page *rod.Page) bool {
return err == nil
}
// Search executes a Baidu web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
baid.logger.Debug("Starting search, query: %+v", query)
@@ -145,6 +151,8 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
return core.DeduplicateResults(searchResults), nil
}
// SearchImage executes a Baidu image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
baid.logger.Debug("Starting image search, query: %+v", query)

4
baidu/url.go
View File

@@ -22,6 +22,8 @@ func dateToTimestamp(date string) (int64, error) {
return t.Unix(), nil
}
// BuildURL builds a Baidu web search URL from Query fields.
// It returns an error when query text, date, or pagination parameters are invalid.
func BuildURL(q core.Query) (string, error) {
base, _ := url.Parse("https://www.baidu.com/")
base.Path += "s"
@@ -84,6 +86,8 @@ func BuildURL(q core.Query) (string, error) {
return base.String(), nil
}
// BuildImageURL builds a Baidu image search URL from Query fields and page
// index. It returns an error when the query text is empty.
func BuildImageURL(q core.Query, pageNum int) (string, error) {
base, _ := url.Parse("https://image.baidu.com/")
base.Path += "search/acjson"

11
bing/search.go
View File

@@ -13,12 +13,14 @@ import (
"golang.org/x/time/rate"
)
// Bing implements core.SearchEngine for Bing SERP pages.
type Bing struct {
core.Browser
core.SearchEngineOptions
logger *core.EngineLogger
}
// New creates a Bing engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *Bing {
bing := Bing{Browser: browser}
opts.Init()
@@ -27,10 +29,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Bing {
return &bing
}
// Name returns the stable engine identifier.
func (bing *Bing) Name() string {
return "bing"
}
// GetRateLimiter returns a limiter configured from SearchEngineOptions.
func (bing *Bing) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(bing.GetRatelimit())
return rate.NewLimiter(ratelimit, bing.RateBurst)
@@ -99,6 +103,8 @@ func (bing *Bing) close(page *rod.Page) {
}
}
// Search executes a Bing web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
bing.logger.Debug("Starting search, query: %+v", query)
@@ -240,7 +246,7 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
return deduped, nil
}
// BingImageData represents the JSON structure in the m attribute of image elements
// BingImageData represents metadata encoded in the image result `m` attribute.
type BingImageData struct {
T string `json:"t"` // Title
Desc string `json:"desc"` // Description
@@ -252,7 +258,8 @@ type BingImageData struct {
MURL string `json:"murl"` // Image URL
}
// SearchImage performs Bing image search and returns results
// SearchImage executes a Bing image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
bing.logger.Debug("Starting image search, query: %+v", query)

4
bing/url.go
View File

@@ -12,6 +12,8 @@ import (
"github.com/sirupsen/logrus"
)
// BuildURL builds a Bing web search URL from Query fields.
// It returns an error when query text or date parameters are invalid.
func BuildURL(q core.Query) (string, error) {
base, err := url.Parse("https://www.bing.com")
if err != nil {
@@ -93,6 +95,8 @@ func BuildURL(q core.Query) (string, error) {
return base.String(), nil
}
// BuildImageURL builds a Bing image search URL from Query fields.
// It returns an error when the resulting query text is empty.
func BuildImageURL(q core.Query) (string, error) {
base, err := url.Parse("https://www.bing.com")
if err != nil {

50
core/browser.go
View File

@@ -17,23 +17,35 @@ import (
"github.com/sirupsen/logrus"
)
// BrowserOpts configures Chromium launch and navigation behavior.
type BrowserOpts struct {
IsHeadless bool // Use browser interface
IsLeakless bool // Force to kill browser
Timeout time.Duration // Timeout
LanguageCode string
WaitRequests bool // Wait requests to complete after navigation
LeavePageOpen bool // Leave pages and browser open
WaitLoadTime time.Duration // Time to wait till page loads
CaptchaSolverApiKey string // 2Captcha api key
BrowserPath string // Explicit browser executable path
ProxyURL string // Proxy URL
Insecure bool // Allow insecure TLS connections
UseStealth bool // Use go-rod stealth plugin
// IsHeadless runs Chromium without visible UI.
IsHeadless bool
// IsLeakless forces child browser process cleanup when the parent exits.
IsLeakless bool
// Timeout is applied to browser connect and page navigation operations.
Timeout time.Duration
// LanguageCode sets Accept-Language for emulated requests.
LanguageCode string
// WaitRequests waits for request-idle state after navigation.
WaitRequests bool
// LeavePageOpen keeps pages open after search operations.
LeavePageOpen bool
// WaitLoadTime is an additional fixed wait after load/idle checks.
WaitLoadTime time.Duration
// CaptchaSolverApiKey enables 2Captcha integration for supported engines.
CaptchaSolverApiKey string
// BrowserPath optionally points to a specific browser executable.
BrowserPath string
// ProxyURL defines the upstream proxy for browser traffic.
ProxyURL string
// Insecure allows invalid TLS certificates for browser requests.
Insecure bool
// UseStealth enables go-rod stealth page creation.
UseStealth bool
}
// Initialize browser parameters with default values if they are not set
// Check applies default option values when optional fields are unset.
func (o *BrowserOpts) Check() {
if o.Timeout == 0 {
o.Timeout = time.Second * 30
@@ -44,6 +56,7 @@ func (o *BrowserOpts) Check() {
}
}
// Browser wraps a launched Chromium instance used by engine implementations.
type Browser struct {
BrowserOpts
browserAddr string
@@ -51,6 +64,8 @@ type Browser struct {
CaptchaSolver *CaptchaSolver
}
// NewBrowser launches a new Chromium process via Rod launcher and returns a
// Browser wrapper configured with proxy and captcha solver settings.
func NewBrowser(opts BrowserOpts) (*Browser, error) {
opts.Check()
logrus.Debugf("Browser options: %+v", opts)
@@ -151,7 +166,7 @@ func resolveBrowserBinaryPath(browserPath string, lookPath func() (string, bool)
return "", nil
}
// Check whether browser instance is already created
// IsInitialized reports whether the browser launcher has been created.
func (b *Browser) IsInitialized() bool {
if b.browserAddr != "" {
return true
@@ -160,7 +175,9 @@ func (b *Browser) IsInitialized() bool {
}
}
// Open URL
// Navigate connects to Chromium, creates a page, applies stealth/emulation and
// proxy auth, then navigates to URL. It returns an initialized page ready for
// selector queries, or an error when browser setup/navigation fails.
func (b *Browser) Navigate(URL string) (*rod.Page, error) {
logrus.Debug("Navigate to: ", URL)
@@ -280,6 +297,7 @@ func (b *Browser) Navigate(URL string) (*rod.Page, error) {
return page, nil
}
// Close closes the active browser connection.
func (b *Browser) Close() error {
return b.browser.Close()
}

94
core/common.go
View File

@@ -9,17 +9,31 @@ import (
"github.com/gofiber/fiber/v2"
)
// ErrCaptcha is returned when the engine detects a captcha challenge page.
// This error is treated as non-retryable by resilient search policies.
var ErrCaptcha = errors.New("captcha detected")
// ErrSearchTimeout is returned when required SERP elements are not found before
// selector or page timeouts expire.
var ErrSearchTimeout = errors.New("timeout. Cannot find element on page")
// SearchResult represents one normalized result item returned by any engine.
type SearchResult struct {
Rank int `json:"rank"`
URL string `json:"url"`
Title string `json:"title"`
// Rank is a 1-based position in engine output. Some engines use negative
// ranks for non-organic blocks such as ads or instant answers.
Rank int `json:"rank"`
// URL is the canonical result URL.
URL string `json:"url"`
// Title is the result headline shown on the SERP.
Title string `json:"title"`
// Description is the snippet text associated with the result.
Description string `json:"description"`
Ad bool `json:"ad"`
// Ad reports whether the result is sponsored.
Ad bool `json:"ad"`
}
// DeduplicateResults removes items with duplicate URLs and returns a result set
// sorted by rank in ascending order.
func DeduplicateResults(results []SearchResult) []SearchResult {
unique := make(map[string]bool)
var deduped []SearchResult
@@ -40,6 +54,8 @@ func DeduplicateResults(results []SearchResult) []SearchResult {
return deduped
}
// ConvertSearchResultsMap converts a map-based collection to a rank-sorted
// slice and returns it by pointer.
func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
searchResults := []SearchResult{}
@@ -53,21 +69,42 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
return &searchResults
}
// Query holds request parameters used by HTTP handlers and search engines.
// Example minimal query: Query{Text: "golang", Limit: 10}.
type Query struct {
Text string
LangCode string // eg. EN, ES, RU...
DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010
Filetype string // File extension to search.
Site string // Search site
Limit int // Limit the number of results
Start int // Search offset for pagination (Google uses 0, 10, 20...)
Filter bool // Filter duplicates (google) (false: include similar, true: hide similar)
Answers bool // Include question and answers from SERP page to results with negative indexes
ProxyURL string // Proxy URL for raw requests
ProxyOverride string // Request-scoped proxy override: tag or direct
Insecure bool // Allow insecure TLS connections
// Text is the search phrase, for example "golang fiber tutorial".
Text string
// LangCode is an engine language hint such as "EN", "DE", or "RU".
LangCode string
// DateInterval filters by date range in YYYYMMDD..YYYYMMDD format.
// Example: "20250101..20250331".
DateInterval string
// Filetype is a file extension filter, for example "pdf" or "docx".
Filetype string
// Site restricts results to a specific domain, for example "github.com".
Site string
// Limit is the maximum number of results requested by the client.
Limit int
// Start is an engine pagination offset. Values are engine-specific:
// Google commonly uses 0,10,20 while some engines use page indexes.
Start int
// Filter controls duplicate filtering when supported by the engine.
// For Google, false includes similar results and true hides them.
Filter bool
// Answers enables parsing answer modules when supported by the engine.
// Such entries may be returned with negative rank values.
Answers bool
// ProxyURL is a direct proxy URL used by raw HTTP search paths.
ProxyURL string
// ProxyOverride is a request-scoped proxy policy override (tag or "direct"),
// typically parsed from the X-Use-Proxy header.
ProxyOverride string
// Insecure enables insecure TLS for request/browser execution.
Insecure bool
}
// ComputePagination translates an absolute start offset into page index and
// in-page offset for a fixed page size.
func ComputePagination(start int, pageSize int) (int, int, error) {
if pageSize <= 0 {
return 0, 0, errors.New("pageSize must be > 0")
@@ -78,6 +115,7 @@ func ComputePagination(start int, pageSize int) (int, int, error) {
return start / pageSize, start % pageSize, nil
}
// IsEmpty reports whether query text operators are all absent.
func (q Query) IsEmpty() bool {
if q.Site == "" && q.Filetype == "" && q.Text == "" {
return true
@@ -85,6 +123,9 @@ func (q Query) IsEmpty() bool {
return false
}
// InitFromContext populates Query from HTTP query parameters and request
// headers. It validates numeric/boolean inputs and returns an error for empty
// search expressions.
func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
searchQuery.Text = reqCtx.Query("text")
searchQuery.LangCode = reqCtx.Query("lang")
@@ -128,14 +169,23 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
return nil
}
// SearchEngineOptions controls engine pacing, selector waits, and captcha
// handling behavior shared by browser and raw implementations.
type SearchEngineOptions struct {
RateRequests int `mapstructure:"rate_requests"`
RateTime int64 `mapstructure:"rate_seconds"`
RateBurst int `mapstructure:"rate_burst"`
SelectorTimeout int64 `mapstructure:"selector_timeout"` // CSS selector timeout in seconds
IsSolveCaptcha bool `mapstructure:"captcha"`
// RateRequests is the allowed number of requests within RateTime seconds.
RateRequests int `mapstructure:"rate_requests"`
// RateTime defines the rate-limiting window size in seconds.
RateTime int64 `mapstructure:"rate_seconds"`
// RateBurst is the token bucket burst size for short spikes.
RateBurst int `mapstructure:"rate_burst"`
// SelectorTimeout is the per-selector wait timeout in seconds.
SelectorTimeout int64 `mapstructure:"selector_timeout"`
// IsSolveCaptcha enables automatic captcha solving when engine support and
// solver credentials are configured.
IsSolveCaptcha bool `mapstructure:"captcha"`
}
// Init sets default option values when fields are zero.
func (o *SearchEngineOptions) Init() {
if o.RateRequests == 0 {
o.RateRequests = 6
@@ -151,10 +201,12 @@ func (o *SearchEngineOptions) Init() {
}
}
// GetRatelimit returns the interval between two allowed requests.
func (o *SearchEngineOptions) GetRatelimit() time.Duration {
return (time.Duration(o.RateTime) * time.Second) / time.Duration(o.RateRequests)
}
// GetSelectorTimeout returns the selector wait timeout as time.Duration.
func (o *SearchEngineOptions) GetSelectorTimeout() time.Duration {
return time.Duration(o.SelectorTimeout) * time.Second
}

40
core/server.go
View File

@@ -16,14 +16,24 @@ import (
"golang.org/x/time/rate"
)
// SearchEngine defines the contract required by the HTTP server and resilient
// search pipeline.
type SearchEngine interface {
// Search runs a web search request and returns normalized results.
// Implementations should return sentinel errors such as ErrCaptcha and
// ErrSearchTimeout for policy-aware handling.
Search(Query) ([]SearchResult, error)
// SearchImage runs an image search request and returns normalized results.
SearchImage(Query) ([]SearchResult, error)
// IsInitialized reports whether the engine is ready to serve requests.
IsInitialized() bool
// Name returns a stable engine identifier used in routes and telemetry.
Name() string
// GetRateLimiter returns an engine-specific limiter used by resilient search.
GetRateLimiter() *rate.Limiter
}
// Server exposes OpenSERP HTTP endpoints backed by one or more search engines.
type Server struct {
app *fiber.App
addr string
@@ -34,15 +44,25 @@ type Server struct {
opts ServerOptions
}
// ServerOptions configures HTTP server middleware and resilience behavior.
type ServerOptions struct {
CacheTTL time.Duration
CacheMaxSize int
EnableCORS bool
CORS CORSConfig
// CacheTTL controls response cache entry lifetime. Zero disables caching.
CacheTTL time.Duration
// CacheMaxSize is the maximum number of cached entries.
CacheMaxSize int
// EnableCORS enables cross-origin headers with the CORS config below.
EnableCORS bool
// CORS contains allowed origins, methods, and headers when CORS is enabled.
CORS CORSConfig
// AllowEndpointFallback allows dedicated engine routes to fall back to other
// healthy engines when the primary engine fails.
AllowEndpointFallback bool
Resilience ResilientConfig
// Resilience defines retry/circuit-breaker/proxy strategy settings.
Resilience ResilientConfig
}
// DefaultServerOptions returns production-oriented defaults for cache, CORS,
// and resilient search policies.
func DefaultServerOptions() ServerOptions {
return ServerOptions{
CacheTTL: 5 * time.Minute,
@@ -54,10 +74,15 @@ func DefaultServerOptions() ServerOptions {
}
}
// NewServer creates a Server with DefaultServerOptions and registers all
// routes for the provided engines.
func NewServer(host string, port int, searchEngines ...SearchEngine) *Server {
return NewServerWithOptions(host, port, DefaultServerOptions(), searchEngines...)
}
// NewServerWithOptions builds a Server, installs middleware, and registers API
// routes. The returned server is ready to Listen; call Shutdown for graceful
// stop.
func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngines ...SearchEngine) *Server {
addr := fmt.Sprintf("%s:%d", host, port)
app := fiber.New(fiber.Config{
@@ -207,6 +232,7 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm
return c.JSON(res)
}
// HealthStatus is returned by /health and summarizes service state.
type HealthStatus struct {
Status string `json:"status"`
Uptime string `json:"uptime"`
@@ -214,6 +240,7 @@ type HealthStatus struct {
System map[string]interface{} `json:"system"`
}
// EngineHealth describes availability of one configured engine.
type EngineHealth struct {
Name string `json:"name"`
Initialized bool `json:"initialized"`
@@ -309,6 +336,7 @@ func (s *Server) handleCircuitBreakerStats(c *fiber.Ctx) error {
})
}
// MegaSearchResult extends SearchResult with the engine source name.
type MegaSearchResult struct {
SearchResult
Engine string `json:"engine"`
@@ -607,10 +635,12 @@ func (s *Server) handleSwaggerUI(c *fiber.Ctx) error {
return c.SendString(page)
}
// Listen starts the Fiber HTTP server on the configured address.
func (s *Server) Listen() error {
return s.app.Listen(s.addr)
}
// Shutdown gracefully stops the Fiber HTTP server.
func (s *Server) Shutdown() error {
return s.app.Shutdown()
}

98
docs/ARCHITECTURE.md
View File

@@ -175,33 +175,33 @@ Defaults below are the shipped defaults in `config.yaml` (if present). If the co
### `server`
| Key | Default | Description |
| --- | --- | --- |
| `server.host` | `0.0.0.0` | API bind host |
| `server.port` | `7000` | API bind port |
| `server.debug` | `false` | Debug mode, forces headful browser |
| `server.verbose` | `true` | Info-level request logs |
| `server.raw_requests` | `false` | `true` = raw HTTP mode |
| `server.insecure` | `true` | Allow insecure TLS connections |
| Key | Default | Description |
| --------------------- | --------- | ---------------------------------- |
| `server.host` | `0.0.0.0` | API bind host |
| `server.port` | `7000` | API bind port |
| `server.debug` | `false` | Debug mode, forces headful browser |
| `server.verbose` | `true` | Info-level request logs |
| `server.raw_requests` | `false` | `true` = raw HTTP mode |
| `server.insecure` | `true` | Allow insecure TLS connections |
### `app`
| Key | Default | Description |
| --- | --- | --- |
| `app.timeout` | `15` | Request timeout in seconds |
| `app.browser_path` | `""` | Custom browser binary path |
| `app.head` | `false` | Headful browser UI |
| `app.leakless` | `false` | Force browser process cleanup |
| `app.leave_head` | `false` | Keep browser tabs open |
| `app.stealth` | `false` | Enable stealth plugin |
| Key | Default | Description |
| ------------------ | ------- | ----------------------------- |
| `app.timeout` | `15` | Request timeout in seconds |
| `app.browser_path` | `""` | Custom browser binary path |
| `app.head` | `false` | Headful browser UI |
| `app.leakless` | `false` | Force browser process cleanup |
| `app.leave_head` | `false` | Keep browser tabs open |
| `app.stealth` | `false` | Enable stealth plugin |
### `proxies`
| Key | Default | Description |
| --- | --- | --- |
| `proxies.global` | unset | Force single proxy for all engines |
| `proxies.entries[]` | empty | Tagged proxy pool entries (`url`, `tags`) |
| `proxies.health.failure_threshold` | `3` | Disable proxy after N failures |
| Key | Default | Description |
| ---------------------------------- | ------- | ----------------------------------------- |
| `proxies.global` | unset | Force single proxy for all engines |
| `proxies.entries[]` | empty | Tagged proxy pool entries (`url`, `tags`) |
| `proxies.health.failure_threshold` | `3` | Disable proxy after N failures |
Per-engine optional proxy tag:
@@ -213,52 +213,52 @@ Per-engine optional proxy tag:
### `cache`
| Key | Default | Description |
| --- | --- | --- |
| `cache.ttl_seconds` | `60` | Response cache TTL (0 disables cache) |
| `cache.max_size` | `1000` | Max cached entries |
| Key | Default | Description |
| ------------------- | ------- | ------------------------------------- |
| `cache.ttl_seconds` | `60` | Response cache TTL (0 disables cache) |
| `cache.max_size` | `1000` | Max cached entries |
### `resilience`
| Key | Default | Description |
| --- | --- | --- |
| `resilience.max_retries` | `2` | Retry attempts per request |
| Key | Default | Description |
| ------------------------------------ | ------- | ------------------------------------------------------ |
| `resilience.max_retries` | `2` | Retry attempts per request |
| `resilience.allow_endpoint_fallback` | `false` | Allow dedicated endpoints to fallback to other engines |
### `circuit_breaker`
| Key | Default | Description |
| --- | --- | --- |
| `circuit_breaker.failures` | `5` | Failures before opening circuit |
| `circuit_breaker.recovery_seconds` | `60` | Open -> half-open wait time |
| `circuit_breaker.successes` | `2` | Half-open successes to close circuit |
| Key | Default | Description |
| ---------------------------------- | ------- | ------------------------------------ |
| `circuit_breaker.failures` | `5` | Failures before opening circuit |
| `circuit_breaker.recovery_seconds` | `60` | Open -> half-open wait time |
| `circuit_breaker.successes` | `2` | Half-open successes to close circuit |
### `cors`
| Key | Default | Description |
| --- | --- | --- |
| `cors.enabled` | `true` | Enable CORS middleware |
| `cors.allow_origins` | `"*"` | Allowed origins |
| `cors.allow_methods` | `"GET, POST, OPTIONS"` | Allowed methods |
| `cors.allow_headers` | `"Origin, Content-Type, Accept, Authorization, X-Use-Proxy"` | Allowed headers |
| `cors.max_age` | `86400` | Preflight cache max age (seconds) |
| Key | Default | Description |
| -------------------- | ------------------------------------------------------------ | --------------------------------- |
| `cors.enabled` | `true` | Enable CORS middleware |
| `cors.allow_origins` | `"*"` | Allowed origins |
| `cors.allow_methods` | `"GET, POST, OPTIONS"` | Allowed methods |
| `cors.allow_headers` | `"Origin, Content-Type, Accept, Authorization, X-Use-Proxy"` | Allowed headers |
| `cors.max_age` | `86400` | Preflight cache max age (seconds) |
### `2captcha`
| Key | Default | Description |
| --- | --- | --- |
| `2captcha.apikey` | unset | Optional captcha solver key |
| Key | Default | Description |
| ----------------- | ------- | --------------------------- |
| `2captcha.apikey` | unset | Optional captcha solver key |
### Engine rate-limit defaults
For each engine (`google`, `yandex`, `baidu`, `bing`, `duckduckgo`):
| Key | Default | Description |
| --- | --- | --- |
| `<engine>.rate_requests` | `4` | Average requests per minute |
| `<engine>.rate_burst` | `2` | Burst capacity |
| `<engine>.rate_seconds` | `60` (implicit) | Rate window seconds |
| `<engine>.selector_timeout` | `5` (implicit) | Selector wait timeout seconds |
| Key | Default | Description |
| --------------------------- | --------------- | ----------------------------- |
| `<engine>.rate_requests` | `4` | Average requests per minute |
| `<engine>.rate_burst` | `2` | Burst capacity |
| `<engine>.rate_seconds` | `60` (implicit) | Rate window seconds |
| `<engine>.selector_timeout` | `5` (implicit) | Selector wait timeout seconds |
Google-only additional toggle:

0
CONTRIBUTING.md → docs/CONTRIBUTING.md
View File

8
duckduckgo/search.go
View File

@@ -10,6 +10,7 @@ import (
"golang.org/x/time/rate"
)
// DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages.
type DuckDuckGo struct {
core.Browser
core.SearchEngineOptions
@@ -17,6 +18,7 @@ type DuckDuckGo struct {
logger *core.EngineLogger
}
// New creates a DuckDuckGo engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo {
ddg := DuckDuckGo{Browser: browser}
opts.Init()
@@ -27,10 +29,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo {
return &ddg
}
// Name returns the stable engine identifier.
func (ddg *DuckDuckGo) Name() string {
return "duckduckgo"
}
// GetRateLimiter returns a limiter configured from SearchEngineOptions.
func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(ddg.GetRatelimit())
return rate.NewLimiter(ratelimit, ddg.RateBurst)
@@ -163,6 +167,8 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
return searchResults
}
// Search executes a DuckDuckGo web search and returns normalized search
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) {
ddg.logger.Debug("Starting search, query: %+v", query)
@@ -265,6 +271,8 @@ func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) {
return deduped, nil
}
// SearchImage executes a DuckDuckGo image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) SearchImage(query core.Query) ([]core.SearchResult, error) {
ddg.logger.Debug("Starting image search, query: %+v", query)

4
duckduckgo/url.go
View File

@@ -12,6 +12,8 @@ import (
const baseURL = "https://duckduckgo.com"
// BuildURL builds a DuckDuckGo web search URL for the provided query and page
// index. It returns an error when query text or date parameters are invalid.
func BuildURL(q core.Query, page int) (string, error) {
base, err := url.Parse(baseURL)
if err != nil {
@@ -84,6 +86,8 @@ func BuildURL(q core.Query, page int) (string, error) {
return base.String(), nil
}
// BuildImageURL builds a DuckDuckGo image search URL from Query fields.
// It returns an error when query text or date parameters are invalid.
func BuildImageURL(q core.Query) (string, error) {
base, err := url.Parse(baseURL)
if err != nil {

8
google/search.go
View File

@@ -14,6 +14,7 @@ import (
"golang.org/x/time/rate"
)
// Google implements core.SearchEngine for Google SERP pages.
type Google struct {
core.Browser
core.SearchEngineOptions
@@ -21,6 +22,7 @@ type Google struct {
logger *core.EngineLogger
}
// New creates a Google engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
gogl := Google{Browser: browser}
opts.Init()
@@ -30,10 +32,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
return &gogl
}
// Name returns the stable engine identifier.
func (gogl *Google) Name() string {
return "google"
}
// GetRateLimiter returns a limiter configured from SearchEngineOptions.
func (gogl *Google) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(gogl.GetRatelimit())
return rate.NewLimiter(ratelimit, gogl.RateBurst)
@@ -157,6 +161,8 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
}
// Search executes a Google web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
gogl.logger.Debug("Starting search, query: %+v", query)
@@ -366,6 +372,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
return core.DeduplicateResults(searchResults), nil
}
// SearchImage executes a Google image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
gogl.logger.Debug("Starting image search, query: %+v", query)

8
google/url.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/sirupsen/logrus"
)
// GoogleDomains maps language/country hints to Google TLD suffixes used for
// URL construction.
var GoogleDomains = map[string]string{
"": "com",
"en": "com",
@@ -213,7 +215,8 @@ var GoogleDomains = map[string]string{
"zw": "co.zw",
}
// Build Google query URL from Query struct
// BuildURL builds a Google web search URL from Query fields.
// It returns an error when the resulting query text is empty or invalid.
func BuildURL(q core.Query) (string, error) {
googleBase := GoogleDomains[strings.ToLower(q.LangCode)]
base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase))
@@ -286,6 +289,8 @@ func BuildURL(q core.Query) (string, error) {
return base.String(), nil
}
// BuildImageURL builds a Google image search URL from Query fields.
// It returns an error when the resulting query text is empty or invalid.
func BuildImageURL(q core.Query) (string, error) {
// TODO: Add new params
googleBase := GoogleDomains[strings.ToLower(q.LangCode)]
@@ -344,6 +349,7 @@ func BuildImageURL(q core.Query) (string, error) {
return base.String(), nil
}
// SourceImage contains parsed Google image metadata extracted from result links.
type SourceImage struct {
PageURL string
OriginalURL string

10
yandex/search.go
View File

@@ -11,6 +11,7 @@ import (
"golang.org/x/time/rate"
)
// ImageEntity contains one image record from Yandex image search state JSON.
type ImageEntity struct {
ID string `json:"id"`
Rank int `json:"pos"`
@@ -23,6 +24,7 @@ type ImageEntity struct {
IsGIF bool `json:"gifLabel"`
}
// ImageData maps the subset of Yandex JSON state used by parser code.
type ImageData struct {
InitalState struct {
SerpList struct {
@@ -33,6 +35,7 @@ type ImageData struct {
} `json:"initialState"`
}
// Yandex implements core.SearchEngine for Yandex SERP pages.
type Yandex struct {
core.Browser
core.SearchEngineOptions
@@ -40,6 +43,7 @@ type Yandex struct {
logger *core.EngineLogger
}
// New creates a Yandex engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex {
yand := Yandex{Browser: browser}
opts.Init()
@@ -50,10 +54,12 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex {
return &yand
}
// Name returns the stable engine identifier.
func (yand *Yandex) Name() string {
return "yandex"
}
// GetRateLimiter returns a limiter configured from SearchEngineOptions.
func (yand *Yandex) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(yand.GetRatelimit())
return rate.NewLimiter(ratelimit, yand.RateBurst)
@@ -113,6 +119,8 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
return searchResults
}
// Search executes a Yandex web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
yand.logger.Debug("Starting search, query: %+v", query)
if query.Start < 0 {
@@ -192,6 +200,8 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
return core.DeduplicateResults(allResults), nil
}
// SearchImage executes a Yandex image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
yand.logger.Debug("Starting image search, query: %+v", query)

4
yandex/url.go
View File

@@ -10,6 +10,8 @@ import (
const baseURL = "https://www.yandex.com"
// BuildURL builds a Yandex web search URL for the provided query and page
// index. It returns an error when the resulting query text is empty.
func BuildURL(q core.Query, page int) (string, error) {
base, _ := url.Parse(baseURL)
base.Path += "search/"
@@ -42,6 +44,8 @@ func BuildURL(q core.Query, page int) (string, error) {
return base.String(), nil
}
// BuildImageURL builds a Yandex image search URL for the provided query and
// page index. It returns an error when the resulting query text is empty.
func BuildImageURL(q core.Query, page int) (string, error) {
// TODO: Add other parameters
base, _ := url.Parse(baseURL)