package core import ( "errors" "sort" "strconv" "time" "github.com/gofiber/fiber/v2" ) var ErrCaptcha = errors.New("captcha detected") var ErrSearchTimeout = errors.New("timeout. Cannot find element on page") type SearchResult struct { Rank int `json:"rank"` URL string `json:"url"` Title string `json:"title"` Description string `json:"description"` Ad bool `json:"ad"` } func DeduplicateResults(results []SearchResult) []SearchResult { unique := make(map[string]bool) var deduped []SearchResult for _, result := range results { if result.URL == "" { continue } if !unique[result.URL] { unique[result.URL] = true deduped = append(deduped, result) } } sort.Slice(deduped, func(i, j int) bool { return deduped[i].Rank < deduped[j].Rank }) return deduped } func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult { searchResults := []SearchResult{} for _, v := range searchResultsMap { searchResults = append(searchResults, v) } sort.Slice(searchResults, func(i, j int) bool { return searchResults[i].Rank < searchResults[j].Rank }) return &searchResults } type Query struct { Text string LangCode string // eg. EN, ES, RU... DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010 Filetype string // File extension to search. Site string // Search site Limit int // Limit the number of results Start int // Search offset for pagination (Google uses 0, 10, 20...) Filter bool // Filter duplicates (google) (false: include similar, true: hide similar) Answers bool // Include question and answers from SERP page to results with negative indexes ProxyURL string // Proxy URL for raw requests ProxyOverride string // Request-scoped proxy override: tag or direct Insecure bool // Allow insecure TLS connections } func ComputePagination(start int, pageSize int) (int, int, error) { if pageSize <= 0 { return 0, 0, errors.New("pageSize must be > 0") } if start < 0 { return 0, 0, errors.New("start must be >= 0") } return start / pageSize, start % pageSize, nil } func (q Query) IsEmpty() bool { if q.Site == "" && q.Filetype == "" && q.Text == "" { return true } return false } func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { searchQuery.Text = reqCtx.Query("text") searchQuery.LangCode = reqCtx.Query("lang") searchQuery.DateInterval = reqCtx.Query("date") searchQuery.Filetype = reqCtx.Query("file") searchQuery.Site = reqCtx.Query("site") limit, err := strconv.Atoi(reqCtx.Query("limit", "25")) if err != nil { return err } searchQuery.Limit = limit start, err := strconv.Atoi(reqCtx.Query("start", "0")) if err != nil { return err } if start < 0 { return errors.New("start must be >= 0") } searchQuery.Start = start searchQuery.Filter, err = strconv.ParseBool(reqCtx.Query("filter", "1")) if err != nil { return err } searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0")) if err != nil { return err } searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy")) if err != nil { return err } if searchQuery.IsEmpty() { return errors.New("Query cannot be empty") } return nil } type SearchEngineOptions struct { RateRequests int `mapstructure:"rate_requests"` RateTime int64 `mapstructure:"rate_seconds"` RateBurst int `mapstructure:"rate_burst"` SelectorTimeout int64 `mapstructure:"selector_timeout"` // CSS selector timeout in seconds IsSolveCaptcha bool `mapstructure:"captcha"` } func (o *SearchEngineOptions) Init() { if o.RateRequests == 0 { o.RateRequests = 6 } if o.RateTime == 0 { o.RateTime = 60 } if o.RateBurst == 0 { o.RateBurst = 1 } if o.SelectorTimeout == 0 { o.SelectorTimeout = 5 } } func (o *SearchEngineOptions) GetRatelimit() time.Duration { return (time.Duration(o.RateTime) * time.Second) / time.Duration(o.RateRequests) } func (o *SearchEngineOptions) GetSelectorTimeout() time.Duration { return time.Duration(o.SelectorTimeout) * time.Second }