From 74fc15a6e23c8f8b6ed546b1c099c061ae25750c Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Tue, 30 Sep 2025 05:27:54 +0300 Subject: [PATCH] Add DuckDuckGo. Add custom logger. Fix arg priority and other issues --- baidu/search.go | 33 +-- baidu/url.go | 6 +- bing/search.go | 83 +++++--- bing/url.go | 14 +- cmd/root.go | 59 ++++-- cmd/search.go | 24 ++- cmd/serve.go | 4 +- config.yaml | 10 +- core/logger.go | 99 +++++++-- duckduckgo/search.go | 427 ++++++++++++++++++++++++++++++++++++++ duckduckgo/search_test.go | 103 +++++++++ duckduckgo/url.go | 99 +++++++++ google/search.go | 69 +++--- yandex/search.go | 36 ++-- 14 files changed, 937 insertions(+), 129 deletions(-) create mode 100644 duckduckgo/search.go create mode 100644 duckduckgo/search_test.go create mode 100644 duckduckgo/url.go diff --git a/baidu/search.go b/baidu/search.go index fd7fffd..f2ecabc 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -8,7 +8,6 @@ import ( "github.com/go-rod/rod" "github.com/karust/openserp/core" - "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -35,12 +34,14 @@ type imageDataJson struct { type Baidu struct { core.Browser core.SearchEngineOptions + logger *core.EngineLogger } func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu { baid := Baidu{Browser: browser} opts.Init() baid.SearchEngineOptions = opts + baid.logger = core.NewEngineLogger("Baidu") return &baid } @@ -64,7 +65,7 @@ func (baid *Baidu) isTimeout(page *rod.Page) bool { } func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Baidu search, query: %+v", query) + baid.logger.Debug("Starting search, query: %+v", query) searchResults := []core.SearchResult{} @@ -82,7 +83,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd") if err != nil { defer page.Close() - logrus.Errorf("Cannot parse search results: %s", err) + baid.logger.Error("Cannot parse search results: %s", err) return nil, core.ErrSearchTimeout } @@ -91,10 +92,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { defer page.Close() if baid.isCaptcha(page) { - logrus.Errorf("Baidu captcha occurred during: %s", url) + baid.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } else if baid.isTimeout(page) { - logrus.Errorf("Baidu timeout occurred during: %s", url) + baid.logger.Error("Timeout occurred: %s", url) return nil, core.ErrCaptcha } return nil, nil @@ -113,13 +114,13 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { } linkText, err := link.Property("href") if err != nil { - logrus.Error("No `href` tag found") + baid.logger.Error("Missing href tag") } // Get title title, err := link.Text() if err != nil { - logrus.Error("Cannot extract text from title") + baid.logger.Error("Failed to extract title") title = "No title" } @@ -134,10 +135,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, gR) } - if !baid.LeavePageOpen { + if !baid.Browser.LeavePageOpen { err = page.Close() if err != nil { - logrus.Error(err) + baid.logger.Error("Page close error: %v", err) } } @@ -145,7 +146,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { } func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Baidu Image search, query: %+v", query) + baid.logger.Debug("Starting image search, query: %+v", query) searchResults := []core.SearchResult{} searchPage := 0 @@ -162,7 +163,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { return nil, err } - if !baid.LeavePageOpen { + if !baid.Browser.LeavePageOpen { defer page.Close() } page.Reload() @@ -171,7 +172,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { result, err := page.Timeout(baid.Timeout).Search("body > pre") if err != nil { defer page.Close() - logrus.Errorf("Cannot parse search results: %s", err) + baid.logger.Error("Cannot parse search results: %s", err) return nil, core.ErrSearchTimeout } @@ -180,10 +181,10 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { defer page.Close() if baid.isCaptcha(page) { - logrus.Errorf("Baidu captcha occurred during: %s", url) + baid.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } else if baid.isTimeout(page) { - logrus.Errorf("Baidu timeout occurred during: %s", url) + baid.logger.Error("Timeout occurred: %s", url) return nil, core.ErrCaptcha } return nil, nil @@ -207,7 +208,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { err = json.Unmarshal([]byte(fixedJson), &data) if err != nil { - logrus.Errorf("Cannot unmarshal JSON: %v\nData: %v", err, jsonText) + baid.logger.Error("Failed to unmarshal JSON: %v", err) return nil, err } @@ -233,7 +234,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { searchPage += 1 - if !baid.LeavePageOpen { + if !baid.Browser.LeavePageOpen { page.Close() } } diff --git a/baidu/url.go b/baidu/url.go index 666534b..3e46952 100644 --- a/baidu/url.go +++ b/baidu/url.go @@ -13,7 +13,7 @@ import ( ) func dateToTimestamp(date string) (int64, error) { - layout := "20060201" + layout := "20060102" t, err := time.Parse(layout, date) if err != nil { return 0, err @@ -55,12 +55,12 @@ func BuildURL(q core.Query) (string, error) { if q.LangCode != "" { //params.Add("rqlang", q.LangCode) - logrus.Warn("Baidu's Language specific search not supported yet") + logrus.Warn("Language search not supported") } if q.Filetype != "" { //params.Add("ft", q.Filetype) - logrus.Warn("Baidu's File search not supported yet") + logrus.Warn("File search not supported") } if q.Limit != 0 { diff --git a/bing/search.go b/bing/search.go index e075a4b..bc79d0a 100644 --- a/bing/search.go +++ b/bing/search.go @@ -10,19 +10,20 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" "github.com/karust/openserp/core" - "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) type Bing struct { core.Browser core.SearchEngineOptions + logger *core.EngineLogger } func New(browser core.Browser, opts core.SearchEngineOptions) *Bing { bing := Bing{Browser: browser} opts.Init() bing.SearchEngineOptions = opts + bing.logger = core.NewEngineLogger("Bing") return &bing } @@ -44,11 +45,36 @@ func (bing *Bing) getTotalResults(page *rod.Page) (int, error) { } func (bing *Bing) checkCaptcha(page *rod.Page) bool { - captcha, err := page.Timeout(bing.GetSelectorTimeout() / 2).Element("div#bxc") - if err != nil { + if page == nil { return false } - return captcha != nil + + if info, err := page.Info(); err == nil { + url := strings.ToLower(info.URL) + if strings.Contains(url, "turing") || strings.Contains(url, "captcha") { + return true + } + } + + timeout := bing.GetSelectorTimeout() / 2 + if timeout <= 0 { + timeout = time.Second * 2 + } + + selectors := []string{ + "div.captcha", + "div.captcha_header", + } + + for _, selector := range selectors { + has, err, _ := page.Timeout(timeout).Has(selector) + if err == nil && has { + bing.logger.Debug("Captcha detected: %s", selector) + return true + } + } + + return false } func (bing *Bing) acceptCookies(page *rod.Page) { @@ -61,13 +87,18 @@ func (bing *Bing) acceptCookies(page *rod.Page) { } func (bing *Bing) close(page *rod.Page) { - if page != nil { - page.Close() + if !bing.Browser.LeavePageOpen { + if page != nil { + err := page.Close() + if err != nil { + bing.logger.Debug("Page close error: %v", err) + } + } } } func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Bing search, query: %+v", query) + bing.logger.Debug("Starting search, query: %+v", query) searchResults := []core.SearchResult{} @@ -82,8 +113,10 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { } defer bing.close(page) + page.WaitLoad() + if bing.checkCaptcha(page) { - logrus.Errorf("Bing captcha occurred during: %s", url) + bing.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } @@ -92,20 +125,20 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { organicElements, err := page.Timeout(bing.Timeout).Elements("li.b_algo") if err != nil { - logrus.Errorf("Cannot parse organic results: %s", err) + bing.logger.Error("Cannot parse organic results: %s", err) return nil, core.ErrSearchTimeout } adElements, err := page.Timeout(bing.Timeout).Elements("li.b_ad") if err != nil { - logrus.Debug("No ad results found or error parsing ads") + bing.logger.Debug("No ads found") } totalResults, err := bing.getTotalResults(page) if err != nil { - logrus.Errorf("Error capturing total results: %v", err) + bing.logger.Debug("Failed to get total results: %v", err) } - logrus.Infof("%d SERP results found (%d ads)", totalResults, len(adElements)) + bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements)) rank := 0 for _, result := range organicElements { @@ -113,14 +146,14 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { titleElem, err := result.Element("a") if err != nil { - logrus.Debug("No title found for result") + bing.logger.Debug("Missing title") continue } srchRes.Title, _ = titleElem.Text() href, err := titleElem.Property("href") if err != nil { - logrus.Debug("No URL found for result") + bing.logger.Debug("Missing URL") continue } srchRes.URL = href.String() @@ -150,14 +183,14 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) { titleElem, err := adResult.Element("h2 a") if err != nil { - logrus.Debug("No title found for ad") + bing.logger.Debug("Ad missing title") continue } srchRes.Title, _ = titleElem.Text() href, err := titleElem.Property("href") if err != nil { - logrus.Debug("No URL found for ad") + bing.logger.Debug("Ad missing URL") continue } srchRes.URL = href.String() @@ -188,7 +221,7 @@ type BingImageData struct { // SearchImage performs Bing image search and returns results func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Bing image search, query: %+v", query) + bing.logger.Debug("Starting image search, query: %+v", query) searchResults := []core.SearchResult{} @@ -204,9 +237,11 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) { } defer bing.close(page) + page.WaitLoad() + // Check for captcha if bing.checkCaptcha(page) { - logrus.Errorf("Bing captcha occurred during image search: %s", url) + bing.logger.Error("Captcha detected during image search: %s", url) return nil, core.ErrCaptcha } @@ -220,7 +255,7 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) { // Find all image result containers using CSS selector imageContainers, err := page.Timeout(bing.Timeout).Elements("div.iuscp, div.isv") if err != nil { - logrus.Errorf("Cannot parse image results: %s", err) + bing.logger.Error("Cannot parse image results: %s", err) return nil, core.ErrSearchTimeout } @@ -228,7 +263,7 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) { return nil, errors.New("no image results found") } - logrus.Infof("Found %d image result elements", len(imageContainers)) + bing.logger.Info("Found %d image elements", len(imageContainers)) rank := 0 for _, c := range imageContainers { @@ -237,21 +272,21 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) { // Get the element inside the div linkElem, err := c.Element("a") if err != nil { - logrus.Debug("No element found in image container") + bing.logger.Debug("Missing element") continue } // Extract image metadata from m attribute (contains JSON) mAttr, err := linkElem.Attribute("m") if err != nil || mAttr == nil { - logrus.Debug("No m attribute found in image element or attribute is nil") + bing.logger.Debug("Missing m attribute") continue } // Ensure we have valid JSON data to unmarshal jsonData := []byte(*mAttr) if len(jsonData) == 0 { - logrus.Debug("Empty JSON data in m attribute") + bing.logger.Debug("Empty JSON data") continue } @@ -259,7 +294,7 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) { var imgData BingImageData err = json.Unmarshal(jsonData, &imgData) if err != nil { - logrus.Debugf("Failed to parse image JSON data: %s", err) + bing.logger.Debug("Failed to parse JSON: %s", err) continue } diff --git a/bing/url.go b/bing/url.go index 7385ef1..5bce130 100644 --- a/bing/url.go +++ b/bing/url.go @@ -2,6 +2,7 @@ package bing import ( "errors" + "fmt" "net/url" "strconv" "strings" @@ -47,7 +48,7 @@ func BuildURL(q core.Query) (string, error) { params.Add("count", strconv.Itoa(q.Limit)) } - // Set search date range (convert from Google format to Bing format) + // Set search date range - Bing supports date filtering via query text if q.DateInterval != "" { intervals := strings.Split(q.DateInterval, "..") if len(intervals) != 2 { @@ -65,9 +66,14 @@ func BuildURL(q core.Query) (string, error) { return "", errors.New("invalid end date format, expected YYYYMMDD") } - // Bing uses filters parameter with specific date format - dateFilter := "ex1:\"ez5_" + startDate.Format("2006-01-02") + "_" + endDate.Format("2006-01-02") + "\"" - params.Add("filters", dateFilter) + // Add date range to the search query text (Bing supports this format) + dateRange := fmt.Sprintf(" after:%s before:%s", + startDate.Format("2006-01-02"), + endDate.Format("2006-01-02")) + + // Update the query text to include date range + currentQuery := params.Get("q") + params.Set("q", currentQuery+dateRange) } // Bing-specific parameters for consistent results diff --git a/cmd/root.go b/cmd/root.go index 30c3c6c..ac34a79 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "strconv" "strings" "github.com/karust/openserp/core" @@ -12,18 +13,19 @@ import ( ) const ( - version = "0.4.1" + version = "0.5.1" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) type Config struct { - App AppConfig `mapstructure:"app"` - Config2Capcha Config2Captcha `mapstructure:"2captcha"` - GoogleConfig core.SearchEngineOptions `mapstructure:"google"` - YandexConfig core.SearchEngineOptions `mapstructure:"yandex"` - BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"` - BingConfig core.SearchEngineOptions `mapstructure:"bing"` + App AppConfig `mapstructure:"app"` + Config2Capcha Config2Captcha `mapstructure:"2captcha"` + GoogleConfig core.SearchEngineOptions `mapstructure:"google"` + YandexConfig core.SearchEngineOptions `mapstructure:"yandex"` + BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"` + BingConfig core.SearchEngineOptions `mapstructure:"bing"` + DuckDuckGoConfig core.SearchEngineOptions `mapstructure:"duckduckgo"` } type Config2Captcha struct { @@ -48,6 +50,13 @@ type AppConfig struct { var config = Config{} +var flagToConfigKey = map[string]string{ + "config": "app.config_path", + "leave": "app.leave_head", + "raw": "app.raw_requests", + "2captcha_key": "2captcha.apikey", +} + var RootCmd = &cobra.Command{ Use: "openserp", Short: "Open SERP", @@ -76,15 +85,39 @@ var RootCmd = &cobra.Command{ // Bind each cobra flag to its associated viper configuration (config file and environment variable) func bindFlags(cmd *cobra.Command, vpr *viper.Viper) { cmd.Flags().VisitAll(func(flg *pflag.Flag) { - configName := "app." + flg.Name + configName, ok := flagToConfigKey[flg.Name] + if !ok { + configName = "app." + flg.Name + } + + if err := vpr.BindPFlag(configName, flg); err != nil { + logrus.Errorf("Unable to bind flag %s: %v", flg.Name, err) + } - // Apply viper config value to the flag if viper has a value if flg.Changed { - vpr.Set(configName, flg.Value) + val, err := parseFlagValue(flg) + if err != nil { + logrus.Errorf("Unable to parse flag %s: %v", flg.Name, err) + return + } + vpr.Set(configName, val) } }) } +func parseFlagValue(flg *pflag.Flag) (interface{}, error) { + switch flg.Value.Type() { + case "string": + return flg.Value.String(), nil + case "bool": + return strconv.ParseBool(flg.Value.String()) + case "int": + return strconv.Atoi(flg.Value.String()) + default: + return flg.Value.String(), nil + } +} + // Initialize Viper func initializeConfig(cmd *cobra.Command) error { v := viper.New() @@ -93,14 +126,14 @@ func initializeConfig(cmd *cobra.Command) error { v.SetConfigName(defaultConfigFilename) v.AddConfigPath(".") - // 1. Config. Return an error if we cannot parse the config file. + // 1. Config file (lowest priority). Return an error if we cannot parse the config file. err := v.ReadInConfig() if err != nil { err = fmt.Errorf("cannot read config: %v", err) logrus.Warn(err) } - // 2. Env. Bind environment variables to their equivalent keys with underscores + // 2. Environment variables (medium priority). Bind environment variables to their equivalent keys with underscores for _, key := range v.AllKeys() { envKey := envPrefix + "_" + strings.ToUpper(strings.ReplaceAll(key, ".", "_")) err := v.BindEnv(key, envKey) @@ -109,7 +142,7 @@ func initializeConfig(cmd *cobra.Command) error { } } - // 3. Cmd flags. Bind the current command's flags to viper + // 3. Command flags (highest priority). Bind the current command's flags to viper bindFlags(cmd, v) // Dump Viper values to config struct diff --git a/cmd/search.go b/cmd/search.go index 55c0be2..9b243a2 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -7,7 +7,9 @@ import ( "time" "github.com/karust/openserp/baidu" + "github.com/karust/openserp/bing" "github.com/karust/openserp/core" + "github.com/karust/openserp/duckduckgo" "github.com/karust/openserp/google" "github.com/karust/openserp/yandex" "github.com/sirupsen/logrus" @@ -17,7 +19,7 @@ import ( var searchCMD = &cobra.Command{ Use: "search", Aliases: []string{"find"}, - Short: "Search results using chosen web search engine (google, yandex, baidu)", + Short: "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo)", Args: cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)), Run: search, } @@ -31,19 +33,25 @@ func search(cmd *cobra.Command, args []string) { ProxyURL: config.App.ProxyURL, Insecure: config.App.Insecure, } - results := []core.SearchResult{} + logrus.Infof("Starting SERP search request using %s engine for query: %s", engineType, query.Text) + + var results []core.SearchResult if config.App.IsRawRequests { + logrus.Infof("Using raw requests mode for %s search", engineType) results, err = searchRaw(engineType, query) } else { + logrus.Infof("Using browser mode for %s search", engineType) results, err = searchBrowser(engineType, query) } if err != nil { - logrus.Error(err) + logrus.Errorf("Error during %s search: %s", engineType, err) return } + logrus.Infof("Successfully completed SERP search using %s engine, returned %d results", engineType, len(results)) + b, err := json.MarshalIndent(results, "", " ") if err != nil { logrus.Error(err) @@ -82,6 +90,10 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er engine = google.New(*browser, config.GoogleConfig) case "baidu": engine = baidu.New(*browser, config.BaiduConfig) + case "bing": + engine = bing.New(*browser, config.BingConfig) + case "duck": + engine = duckduckgo.New(*browser, config.DuckDuckGoConfig) default: logrus.Infof("No `%s` search engine found", engineType) } @@ -99,6 +111,12 @@ func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error) return google.Search(query) case "baidu": return baidu.Search(query) + case "bing": + logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.") + return nil, fmt.Errorf("bing does not support raw requests mode") + case "duck": + logrus.Warn("DuckDuckGo does not support raw HTTP requests mode. Please use browser mode instead.") + return nil, fmt.Errorf("duckduckgo does not support raw requests mode") default: logrus.Infof("No `%s` search engine found", engineType) } diff --git a/cmd/serve.go b/cmd/serve.go index 6be1168..342dd0d 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -7,6 +7,7 @@ import ( "github.com/karust/openserp/baidu" "github.com/karust/openserp/bing" "github.com/karust/openserp/core" + "github.com/karust/openserp/duckduckgo" "github.com/karust/openserp/google" "github.com/karust/openserp/yandex" "github.com/sirupsen/logrus" @@ -98,8 +99,9 @@ func serve(cmd *cobra.Command, args []string) { gogl := google.New(*browser, config.GoogleConfig) baidu := baidu.New(*browser, config.BaiduConfig) bing := bing.New(*browser, config.BingConfig) + ddg := duckduckgo.New(*browser, config.DuckDuckGoConfig) - serv := core.NewServer(config.App.Host, config.App.Port, gogl, yand, baidu, bing) + serv := core.NewServer(config.App.Host, config.App.Port, gogl, yand, baidu, bing, ddg) err = serv.Listen() if err != nil { diff --git a/config.yaml b/config.yaml index e1dd30c..0ec5787 100644 --- a/config.yaml +++ b/config.yaml @@ -8,7 +8,7 @@ app: leakless: false leave_head: false stealth: false - insecure: false + insecure: true 2captcha: apikey: "123123123123123" @@ -25,3 +25,11 @@ yandex: baidu: rate_requests: 4 rate_burst: 2 + +bing: + rate_requests: 4 + rate_burst: 2 + +duckduckgo: + rate_requests: 4 + rate_burst: 2 diff --git a/core/logger.go b/core/logger.go index 62eedf5..49b4ad1 100644 --- a/core/logger.go +++ b/core/logger.go @@ -14,20 +14,93 @@ type customFormatter struct { } func (f *customFormatter) Format(entry *logrus.Entry) ([]byte, error) { - // var levelColor int - // switch entry.Level { - // case logrus.DebugLevel, logrus.TraceLevel: - // levelColor = 42 // green highlight - // case logrus.WarnLevel: - // levelColor = 33 // yellow - // case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel: - // levelColor = 41 // red highlight - // default: - // levelColor = 36 // blue - // } - //[]byte(fmt.Sprintf("[%s] - \x1b[%dm%s\x1b[0m - %s\n", entry.Time.Format(f.TimestampFormat), levelColor, strings.ToUpper(entry.Level.String()), entry.Message)) + message := entry.Message - return []byte(fmt.Sprintf("[%s][%s] \t%s\n", entry.Time.Format(f.TimestampFormat), strings.ToUpper(entry.Level.String()), entry.Message)), nil + // Check if engine name is provided as a field + engineName := "" + if engine, exists := entry.Data["engine"]; exists { + if engineStr, ok := engine.(string); ok { + engineName = engineStr + } + } + + // Format: [timestamp][level][engine] message + if engineName != "" { + return []byte(fmt.Sprintf("[%s][%s][%s] %s\n", + entry.Time.Format(f.TimestampFormat), + strings.ToUpper(entry.Level.String()), + engineName, + message)), nil + } + + // Format: [timestamp][level] message (no engine) + return []byte(fmt.Sprintf("[%s][%s] %s\n", + entry.Time.Format(f.TimestampFormat), + strings.ToUpper(entry.Level.String()), + message)), nil +} + +// EngineLogger provides simplified logging for search engines +type EngineLogger struct { + engine string + logger *logrus.Entry +} + +// NewEngineLogger creates a new logger for a specific search engine +func NewEngineLogger(engine string) *EngineLogger { + return &EngineLogger{ + engine: engine, + logger: logrus.WithField("engine", engine), + } +} + +// Debug logs a debug message +func (el *EngineLogger) Debug(message string, args ...interface{}) { + el.logger.Debugf(message, args...) +} + +// Info logs an info message +func (el *EngineLogger) Info(message string, args ...interface{}) { + el.logger.Infof(message, args...) +} + +// Warn logs a warning message +func (el *EngineLogger) Warn(message string, args ...interface{}) { + el.logger.Warnf(message, args...) +} + +// Error logs an error message +func (el *EngineLogger) Error(message string, args ...interface{}) { + el.logger.Errorf(message, args...) +} + +// Fatal logs a fatal message +func (el *EngineLogger) Fatal(message string, args ...interface{}) { + el.logger.Fatalf(message, args...) +} + +// Panic logs a panic message +func (el *EngineLogger) Panic(message string, args ...interface{}) { + el.logger.Panicf(message, args...) +} + +// LogWithEngine logs a message with engine information (deprecated - use EngineLogger instead) +func LogWithEngine(level logrus.Level, engine, message string, args ...interface{}) { + entry := logrus.WithField("engine", engine) + switch level { + case logrus.DebugLevel: + entry.Debugf(message, args...) + case logrus.InfoLevel: + entry.Infof(message, args...) + case logrus.WarnLevel: + entry.Warnf(message, args...) + case logrus.ErrorLevel: + entry.Errorf(message, args...) + case logrus.FatalLevel: + entry.Fatalf(message, args...) + case logrus.PanicLevel: + entry.Panicf(message, args...) + } } func InitLogger(isVerbose, isDebug bool) { diff --git a/duckduckgo/search.go b/duckduckgo/search.go new file mode 100644 index 0000000..aceb263 --- /dev/null +++ b/duckduckgo/search.go @@ -0,0 +1,427 @@ +package duckduckgo + +import ( + "fmt" + "time" + + "github.com/go-rod/rod" + "github.com/karust/openserp/core" + "golang.org/x/time/rate" +) + +type DuckDuckGo struct { + core.Browser + core.SearchEngineOptions + pageSleep time.Duration // Sleep between pages + logger *core.EngineLogger +} + +func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo { + ddg := DuckDuckGo{Browser: browser} + opts.Init() + ddg.SearchEngineOptions = opts + ddg.logger = core.NewEngineLogger("DuckDuckGo") + + ddg.pageSleep = time.Second * 1 + return &ddg +} + +func (ddg *DuckDuckGo) Name() string { + return "duckduckgo" +} + +func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter { + ratelimit := rate.Every(ddg.GetRatelimit()) + return rate.NewLimiter(ratelimit, ddg.RateBurst) +} + +func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool { + // DuckDuckGo rarely shows captchas, but we can check for common patterns + _, err := page.Timeout(ddg.GetSelectorTimeout()).Search("div[class*='captcha']") + return err == nil +} + +// Check if no results are found +func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool { + _, err := page.Timeout(ddg.GetSelectorTimeout()).Search("div[class*='no-results']") + return err == nil +} + +func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult { + searchResults := []core.SearchResult{} + + for i, r := range results { + // Get URL - try multiple selectors + var link *rod.Element + var err error + + linkSelectors := []string{ + "a[data-testid='result-title-a']", + "a.result__a", + "a.result__url", + "h2 a", + "h3 a", + "a[href]", + "a", + } + + for _, selector := range linkSelectors { + link, err = r.Element(selector) + if err == nil { + break + } + } + + if err != nil { + ddg.logger.Debug("Missing link") + continue + } + + linkText, err := link.Property("href") + if err != nil { + ddg.logger.Debug("Missing href") + continue + } + + // Validate that we have a proper URL + hrefStr := linkText.String() + if hrefStr == "" || hrefStr == "#" || hrefStr == "javascript:void(0)" { + ddg.logger.Debug("Invalid href: %s", hrefStr) + continue + } + + // Get title - try multiple selectors + var titleTag *rod.Element + titleSelectors := []string{ + "h2", + ".result__title", + ".result__a", + "span", + "div", + } + + for _, selector := range titleSelectors { + titleTag, err = r.Element(selector) + if err == nil { + break + } + } + + title := "No title" + if titleTag != nil { + title, _ = titleTag.Text() + } + + // Get description - try multiple selectors + desc := "" + descSelectors := []string{ + "div[data-result='snippet']", + ".result__snippet", + ".result__body", + "span[class*='snippet']", + "div[class*='snippet']", + "p", + } + + for _, selector := range descSelectors { + descTag, err := r.Element(selector) + if err == nil { + desc = descTag.MustText() + break + } + } + + // Check if it's an ad + isAd := false + adSelectors := []string{ + "[data-testid='ad-badge']", + ".ad-badge", + ".result--ad", + } + + for _, selector := range adSelectors { + adIndicator, err := r.Element(selector) + if err == nil && adIndicator != nil { + isAd = true + break + } + } + + result := core.SearchResult{ + Rank: (pageNum * 10) + (i + 1), + URL: hrefStr, + Title: title, + Description: desc, + Ad: isAd, + } + searchResults = append(searchResults, result) + } + + return searchResults +} + +func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) { + ddg.logger.Debug("Starting search, query: %+v", query) + + allResults := []core.SearchResult{} + searchPage := 0 + maxPages := 5 // Prevent infinite loops + consecutiveEmptyPages := 0 // Track consecutive pages with no valid results + + for len(allResults) < query.Limit && searchPage < maxPages { + url, err := BuildURL(query) + if err != nil { + return nil, err + } + + page, err := ddg.Navigate(url) + if err != nil { + return nil, err + } + + // Get all search results in page - try multiple selectors + var searchRes *rod.SearchResult + var searchErr error + + // Try different selectors for DuckDuckGo results + selectors := []string{ + "article[data-testid='result']", + "div[data-testid='result']", + "div.result", + "div.web-result", + ".result", + "[data-testid='result']", + } + + for _, selector := range selectors { + searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector) + if searchErr == nil && searchRes != nil { + ddg.logger.Debug("Found results with selector: %s", selector) + break + } + } + if searchErr != nil { + defer page.Close() + ddg.logger.Error("Cannot parse search results: %s", searchErr) + return nil, core.ErrSearchTimeout + } + + // Check why no results, maybe captcha? + if searchRes == nil { + defer page.Close() + + if ddg.isNoResults(page) { + ddg.logger.Warn("No results found") + } else if ddg.isCaptcha(page) { + ddg.logger.Error("Captcha detected: %s", url) + return nil, core.ErrCaptcha + } + break + } + + elements, err := searchRes.All() + if err != nil { + ddg.logger.Error("Cannot get search elements: %s", err) + break + } + + r := ddg.parseResults(elements, searchPage) + + // Track consecutive empty pages + if len(r) == 0 { + consecutiveEmptyPages++ + ddg.logger.Debug("No valid results found on page %d (consecutive empty: %d)", searchPage, consecutiveEmptyPages) + + // Break if we have too many consecutive empty pages + if consecutiveEmptyPages >= 2 { + ddg.logger.Warn("Too many consecutive empty pages, stopping search") + break + } + } else { + consecutiveEmptyPages = 0 // Reset counter when we find results + } + + allResults = append(allResults, r...) + + searchPage++ + + if !ddg.Browser.LeavePageOpen { + // Close tab before opening new one during the cycle + err = page.Close() + if err != nil { + ddg.logger.Debug("Page close error: %v", err) + } + } + + time.Sleep(ddg.pageSleep) + } + + ddg.logger.Info("Search completed: %d results", len(allResults)) + return core.DeduplicateResults(allResults), nil +} + +func (ddg *DuckDuckGo) SearchImage(query core.Query) ([]core.SearchResult, error) { + ddg.logger.Debug("Starting image search, query: %+v", query) + + searchResults := []core.SearchResult{} + + url, err := BuildImageURL(query) + if err != nil { + return nil, err + } + + page, err := ddg.Navigate(url) + if err != nil { + return nil, err + } + + if !ddg.Browser.LeavePageOpen { + defer page.Close() + } + + // Wait for page to load + page.WaitLoad() + time.Sleep(time.Second * 2) // Give time for images to load + + // Try multiple selectors for DuckDuckGo image results + var searchRes *rod.SearchResult + var searchErr error + + selectors := []string{ + "figure", + // "figure.nsogf_Hpj9UUxfhcwQd5", + // "div[data-testid='result']", + // "div.tile--img", + // "div.tile.tile--img", + // "div.js-images-show-more", + // "div.img-result", + } + + ddg.logger.Debug("Trying selectors: %v", selectors) + + for _, selector := range selectors { + searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector) + if searchErr == nil && searchRes != nil { + ddg.logger.Debug("Found image results with selector: %s", selector) + break + } else { + ddg.logger.Debug("Selector '%s' not found: %v", selector, searchErr) + } + } + + if searchErr != nil { + ddg.logger.Error("Cannot find image results: %s", searchErr) + return searchResults, core.ErrSearchTimeout + } + + // Check why no results + if searchRes == nil { + if ddg.isCaptcha(page) { + ddg.logger.Error("Captcha detected: %s", url) + return searchResults, core.ErrCaptcha + } else if ddg.isNoResults(page) { + ddg.logger.Warn("No image results found") + } + return searchResults, core.ErrSearchTimeout + } + + elements, err := searchRes.All() + if err != nil { + ddg.logger.Error("Cannot get search elements: %s", err) + return searchResults, err + } + + ddg.logger.Info("Found %d image elements", len(elements)) + + for i, r := range elements { + // Get image URL - try multiple selectors + var imgTag *rod.Element + var imgErr error + + imgSelectors := []string{ + "img", + "div.SZ76bwIlqO8BBoqOLqYV img", + "img[src*='duckduckgo.com']", + } + + for _, selector := range imgSelectors { + imgTag, imgErr = r.Element(selector) + if imgErr == nil { + break + } + } + + if imgErr != nil { + ddg.logger.Debug("Missing img tag for element %d", i) + continue + } + + imgSrc, err := imgTag.Property("src") + if err != nil { + ddg.logger.Debug("Missing src property for image %d", i) + continue + } + + // Get title - try multiple selectors based on the HTML structure + var titleTag *rod.Element + var titleErr error + + titleSelectors := []string{ + "figcaption a p span", + "figcaption span", + "figcaption p span", + "span.EKtkFWMYpwzMKOYr0GYm", + "h3", + "span", + "p", + } + + for _, selector := range titleSelectors { + titleTag, titleErr = r.Element(selector) + if titleErr == nil { + break + } + } + + title := "No title" + if titleTag != nil { + title, _ = titleTag.Text() + } + + // Get source page URL - try multiple selectors + var linkTag *rod.Element + var linkErr error + + linkSelectors := []string{ + "figcaption a", + "a", + } + + for _, selector := range linkSelectors { + linkTag, linkErr = r.Element(selector) + if linkErr == nil { + break + } + } + + sourceURL := "" + if linkTag != nil { + href, err := linkTag.Property("href") + if err == nil { + sourceURL = href.String() + } + } + + result := core.SearchResult{ + Rank: i + 1, + URL: imgSrc.String(), + Title: title, + Description: fmt.Sprintf("Source: %s", sourceURL), + } + + searchResults = append(searchResults, result) + } + + ddg.logger.Info("Parsed %d image results", len(searchResults)) + return core.DeduplicateResults(searchResults), nil +} diff --git a/duckduckgo/search_test.go b/duckduckgo/search_test.go new file mode 100644 index 0000000..7758e98 --- /dev/null +++ b/duckduckgo/search_test.go @@ -0,0 +1,103 @@ +package duckduckgo + +import ( + "testing" + + "github.com/karust/openserp/core" +) + +func TestBuildURL(t *testing.T) { + tests := []struct { + name string + query core.Query + expected string + wantErr bool + }{ + { + name: "Basic search", + query: core.Query{ + Text: "golang programming", + }, + expected: "https://duckduckgo.com/?q=golang+programming&t=h&ia=web", + wantErr: false, + }, + { + name: "Search with site filter", + query: core.Query{ + Text: "golang", + Site: "github.com", + }, + expected: "https://duckduckgo.com/?q=golang+site%3Agithub.com&t=h&ia=web", + wantErr: false, + }, + { + name: "Search with filetype", + query: core.Query{ + Text: "documentation", + Filetype: "pdf", + }, + expected: "https://duckduckgo.com/?q=documentation+filetype%3Apdf&t=h&ia=web", + wantErr: false, + }, + { + name: "Empty query", + query: core.Query{ + Text: "", + }, + expected: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildURL(tt.query) + if (err != nil) != tt.wantErr { + t.Errorf("BuildURL() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.expected { + t.Errorf("BuildURL() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestBuildImageURL(t *testing.T) { + tests := []struct { + name string + query core.Query + expected string + wantErr bool + }{ + { + name: "Basic image search", + query: core.Query{ + Text: "golang logo", + }, + expected: "https://duckduckgo.com/?q=golang+logo&t=h&iax=images&ia=images", + wantErr: false, + }, + { + name: "Empty query", + query: core.Query{ + Text: "", + }, + expected: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildImageURL(tt.query) + if (err != nil) != tt.wantErr { + t.Errorf("BuildImageURL() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.expected { + t.Errorf("BuildImageURL() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/duckduckgo/url.go b/duckduckgo/url.go new file mode 100644 index 0000000..e581111 --- /dev/null +++ b/duckduckgo/url.go @@ -0,0 +1,99 @@ +package duckduckgo + +import ( + "errors" + "fmt" + "net/url" + "strings" + + "github.com/karust/openserp/core" +) + +const baseURL = "https://duckduckgo.com" + +func BuildURL(q core.Query) (string, error) { + base, err := url.Parse(baseURL) + if err != nil { + return "", err + } + + base.Path += "" + params := url.Values{} + + // Set request text + if q.Text != "" || q.Site != "" || q.Filetype != "" { + text := q.Text + if q.Site != "" { + text += " site:" + q.Site + } + if q.Filetype != "" { + text += " filetype:" + q.Filetype + } + + params.Add("q", text) + } + + if len(params.Get("q")) == 0 { + return "", errors.New("empty query built") + } + + // Set search date range + if q.DateInterval != "" { + intervals := strings.Split(q.DateInterval, "..") + if len(intervals) != 2 { + return "", errors.New("incorrect data interval provided") + } + // DuckDuckGo uses different date format + params.Add("df", fmt.Sprintf("%s..%s", intervals[0], intervals[1])) + } + + // Set language + if q.LangCode != "" { + params.Add("kl", strings.ToLower(q.LangCode)) + } + + // DuckDuckGo specific parameters + params.Add("t", "h") // HTML format + params.Add("ia", "web") // Web search + + base.RawQuery = params.Encode() + return base.String(), nil +} + +func BuildImageURL(q core.Query) (string, error) { + base, err := url.Parse(baseURL) + if err != nil { + return "", err + } + + base.Path += "" + params := url.Values{} + params.Add("t", "h") // HTML format + params.Add("iax", "images") // Image search + params.Add("ia", "images") + + // Set request text + if q.Text != "" || q.Site != "" || q.Filetype != "" { + text := q.Text + if q.Site != "" { + text += " site:" + q.Site + } + if q.Filetype != "" { + text += " filetype:" + q.Filetype + } + + params.Add("q", text) + } + + if len(params.Get("q")) == 0 { + return "", errors.New("empty query built") + } + + // Set language + if q.LangCode != "" { + params.Add("kl", strings.ToLower(q.LangCode)) + } + + base.RawQuery = params.Encode() + return base.String(), nil +} diff --git a/google/search.go b/google/search.go index 3eb2249..0f2a2c9 100644 --- a/google/search.go +++ b/google/search.go @@ -11,7 +11,6 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" "github.com/karust/openserp/core" - "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -19,13 +18,15 @@ type Google struct { core.Browser core.SearchEngineOptions rgxpGetDigits *regexp.Regexp + logger *core.EngineLogger } func New(browser core.Browser, opts core.SearchEngineOptions) *Google { gogl := Google{Browser: browser} opts.Init() gogl.SearchEngineOptions = opts - gogl.rgxpGetDigits = regexp.MustCompile("\\d") + gogl.logger = core.NewEngineLogger("Google") + gogl.rgxpGetDigits = regexp.MustCompile(`\d`) return &gogl } @@ -68,18 +69,18 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) { } func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool { - logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL) + gogl.logger.Debug("Solve captcha: sitekey=%s", sitekey) resp, _, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas) if err != nil { - logrus.Errorf("Error solving google captcha: %s", err) + gogl.logger.Error("Captcha solve failed: %s", err) return false } - logrus.Debug("Resp:", resp) + gogl.logger.Debug("Captcha response received") _, err = page.Eval(fmt.Sprintf(`;(() => { document.getElementById("g-recaptcha-response").innerHTML="%s"; submitCallback(); })();`, resp)) if err != nil { - logrus.Errorf("Error setting captcha response: %s", err) + gogl.logger.Error("Failed to set captcha response: %s", err) return false } @@ -94,13 +95,13 @@ func (gogl *Google) checkCaptcha(page *rod.Page) bool { sitekey, err := captchaDiv.First.Attribute("data-sitekey") if err != nil { - logrus.Errorf("Cannot get Google captcha sitekey: %s", err) + gogl.logger.Error("Cannot get captcha sitekey: %s", err) return false } dataS, err := captchaDiv.First.Attribute("data-s") if err != nil { - logrus.Errorf("Cannot get Google captcha datas: %s", err) + gogl.logger.Error("Cannot get captcha datas: %s", err) return false } @@ -114,19 +115,19 @@ func (gogl *Google) preparePage(page *rod.Page) { // Remove "similar queries" lists _, err := page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove()); })();") if err != nil { - logrus.Errorf("Error preparing the page: %s", err) + gogl.logger.Error("Page preparation failed: %s", err) } } func (gogl *Google) acceptCookies(page *rod.Page) { diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search("div[role='dialog'][aria-modal] button") if err != nil { - logrus.Errorf("Cannot find cookie consent: %s", err) + gogl.logger.Debug("Cookie consent not found: %s", err) return } btnElms, err := diaglogBtns.All() if err != nil { - logrus.Errorf("Cannot get cookie consent buttons: %s", err) + gogl.logger.Debug("Cannot get cookie consent buttons: %s", err) return } btnElms[3].Click(proto.InputMouseButtonLeft, 1) @@ -134,7 +135,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) { } func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Google search, query: %+v", query) + gogl.logger.Debug("Starting search, query: %+v", query) searchResults := []core.SearchResult{} @@ -153,7 +154,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { // Check first if there captcha if gogl.checkCaptcha(page) { - logrus.Errorf("Google captcha occurred during: %s", url) + gogl.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } @@ -165,7 +166,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { // Find all results using stable attributes results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]") if err != nil { - logrus.Errorf("Cannot parse search results: %s", err) + gogl.logger.Error("Cannot parse search results: %s", err) return nil, core.ErrSearchTimeout } @@ -175,9 +176,9 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { totalResults, err := gogl.getTotalResults(page) if err != nil { - logrus.Errorf("Error capturing total results: %v", err) + gogl.logger.Debug("Failed to get total results: %v", err) } - logrus.Infof("%d SERP results", totalResults) + gogl.logger.Info("Found %d total results", totalResults) searchResultElems, err := results.All() if err != nil { @@ -198,14 +199,14 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { // Get URL link, err := resEl.Element("a") if err != nil { - logrus.Debug("No link found") + gogl.logger.Debug("Missing link") continue } link.MoveMouseOut() href, err := link.Property("href") if err != nil { - logrus.Debug("No `href` tag found") + gogl.logger.Debug("Missing href") continue } srchRes.URL = href.String() @@ -223,17 +224,17 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { // 2. Parse answer boxes answerEls, err := resEl.Page().Search("div[data-hveid][data-ulkwtsb] div[data-q]") if err != nil { - logrus.Debugf("Error while parsing answer box 1: %s", err.Error()) + gogl.logger.Debug("Answer parsing failed: %s", err.Error()) continue } answers, err := answerEls.All() if err != nil { - logrus.Debugf("Error while parsing answer box 2: %s", err.Error()) + gogl.logger.Debug("Answer parsing failed: %s", err.Error()) continue } - logrus.Infof("%d answers found", len(answers)) + gogl.logger.Info("Found %d answers", len(answers)) // Unvail answer contents for _, answ := range answers { @@ -246,21 +247,21 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { for i, answ := range answers { answerText := strings.Split(answ.MustText(), "\n") if len(answerText) < 2 { - logrus.Debugf("Short answer text: %s", answerText) + gogl.logger.Debug("Short answer text: %s", answerText) continue } // Get URL link, err := answ.Element("a") if err != nil { - logrus.Debug("No answer link found") + gogl.logger.Debug("Missing answer link") continue } link.MoveMouseOut() href, err := link.Property("href") if err != nil { - logrus.Debug("No answer `href` tag found") + gogl.logger.Debug("Missing answer href") continue } srchRes.URL = href.String() @@ -334,7 +335,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { } func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Google Image search, query: %+v", query) + gogl.logger.Debug("Starting image search, query: %+v", query) searchResultsMap := map[string]core.SearchResult{} url, err := BuildImageURL(query) @@ -356,14 +357,14 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]") if err != nil { - logrus.Errorf("Cannot parse search results: %s", err) + gogl.logger.Error("Cannot parse search results: %s", err) return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout } // Check why no results if results == nil { if gogl.checkCaptcha(page) { - logrus.Errorf("Google captcha occurred during: %s", url) + gogl.logger.Error("Captcha detected: %s", url) return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha } return *core.ConvertSearchResultsMap(searchResultsMap), err @@ -382,13 +383,13 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { // TODO: parse AF_initDataCallback to optimize instead of this? err := r.Click(proto.InputMouseButtonRight, 1) if err != nil { - logrus.Error("Error clicking") + gogl.logger.Error("Click failed") continue } dataVed, err := r.Attribute("data-ved") if err != nil { - logrus.Error("Cannot find `data-ved` attr") + gogl.logger.Error("Missing data-ved attribute") continue } @@ -405,25 +406,25 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { linkText, err := link.Property("href") if err != nil { - logrus.Error("No `href` tag found") + gogl.logger.Error("Missing href") } imgSrc, err := parseSourceImageURL(linkText.String()) if err != nil { - logrus.Errorf("Cannot parse image href: %v", err) + gogl.logger.Error("Failed to parse image URL: %v", err) continue } // Get title titleTag, err := r.Element("h3") if err != nil { - logrus.Error("No `h3` tag found") + gogl.logger.Error("Missing h3 tag") continue } title, err := titleTag.Text() if err != nil { - logrus.Error("Cannot extract text from title") + gogl.logger.Error("Failed to extract title") title = "No title" } @@ -446,7 +447,7 @@ func (gogl *Google) close(page *rod.Page) { if !gogl.Browser.LeavePageOpen { err := page.Close() if err != nil { - logrus.Error(err) + gogl.logger.Debug("Page close error: %v", err) } } } diff --git a/yandex/search.go b/yandex/search.go index d31b349..8807125 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -8,7 +8,6 @@ import ( "github.com/go-rod/rod" "github.com/karust/openserp/core" - "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -38,12 +37,14 @@ type Yandex struct { core.Browser core.SearchEngineOptions pageSleep time.Duration // Sleep between pages + logger *core.EngineLogger } func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex { yand := Yandex{Browser: browser} opts.Init() yand.SearchEngineOptions = opts + yand.logger = core.NewEngineLogger("Yandex") yand.pageSleep = time.Second * 1 return &yand @@ -80,19 +81,19 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc } linkText, err := link.Property("href") if err != nil { - logrus.Error("No `href` tag found") + yand.logger.Error("Missing href") } // Get title titleTag, err := link.Element("h2") if err != nil { - logrus.Error("No title `h2` tag found") + yand.logger.Error("Missing h2 title") continue } title, err := titleTag.Text() if err != nil { - logrus.Error("Cannot extract text from title") + yand.logger.Error("Failed to extract title") title = "No title" } @@ -100,7 +101,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc descTag, err := r.Element(`span.OrganicTextContentSpan`) desc := "" if err != nil { - logrus.Trace("No description `span.OrganicTextContentSpan` tag found") + yand.logger.Debug("No description") } else { desc = descTag.MustText() } @@ -113,7 +114,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc } func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Yandex search, query: %+v", query) + yand.logger.Debug("Starting search, query: %+v", query) allResults := []core.SearchResult{} searchPage := 0 @@ -133,7 +134,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item") if err != nil { defer page.Close() - logrus.Errorf("Cannot parse search results: %s", err) + yand.logger.Error("Cannot parse search results: %s", err) return nil, core.ErrSearchTimeout } @@ -142,9 +143,9 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { defer page.Close() if yand.isNoResults(page) { - logrus.Errorf("No results found") + yand.logger.Warn("No results found") } else if yand.isCaptcha(page) { - logrus.Errorf("Yandex captcha occurred during: %s", url) + yand.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } break @@ -152,7 +153,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { elements, err := searchRes.All() if err != nil { - logrus.Errorf("Cannot get all elements from search results: %s", err) + yand.logger.Error("Cannot get search elements: %s", err) break } @@ -165,18 +166,19 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { // Close tab before opening new one during the cycle err = page.Close() if err != nil { - logrus.Error(err) + yand.logger.Debug("Page close error: %v", err) } } time.Sleep(yand.pageSleep) } + yand.logger.Info("Search completed: %d results", len(allResults)) return core.DeduplicateResults(allResults), nil } func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { - logrus.Tracef("Start Yandex image search, query: %+v", query) + yand.logger.Debug("Starting image search, query: %+v", query) searchResults := []core.SearchResult{} @@ -193,7 +195,7 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { return nil, err } - if !yand.LeavePageOpen { + if !yand.Browser.LeavePageOpen { defer page.Close() } @@ -203,16 +205,16 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { results, err := page.Timeout(yand.Timeout).Search("div[role='main'] div[data-state]") if err != nil { - logrus.Errorf("Cannot find search results: %s", err) + yand.logger.Error("Cannot find search results: %s", err) } // Check why no results if results == nil { if yand.isCaptcha(page) { - logrus.Errorf("Yandex captcha occurred during: %s", url) + yand.logger.Error("Captcha detected: %s", url) return searchResults, core.ErrCaptcha } else if yand.isNoResults(page) { - logrus.Errorf("No results found") + yand.logger.Warn("No results found") } return searchResults, core.ErrSearchTimeout } @@ -239,7 +241,7 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, res) } - if !yand.LeavePageOpen { + if !yand.Browser.LeavePageOpen { page.Close() } }