diff --git a/baidu/search.go b/baidu/search.go
index fd7fffd..f2ecabc 100644
--- a/baidu/search.go
+++ b/baidu/search.go
@@ -8,7 +8,6 @@ import (
"github.com/go-rod/rod"
"github.com/karust/openserp/core"
- "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
@@ -35,12 +34,14 @@ type imageDataJson struct {
type Baidu struct {
core.Browser
core.SearchEngineOptions
+ logger *core.EngineLogger
}
func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu {
baid := Baidu{Browser: browser}
opts.Init()
baid.SearchEngineOptions = opts
+ baid.logger = core.NewEngineLogger("Baidu")
return &baid
}
@@ -64,7 +65,7 @@ func (baid *Baidu) isTimeout(page *rod.Page) bool {
}
func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Baidu search, query: %+v", query)
+ baid.logger.Debug("Starting search, query: %+v", query)
searchResults := []core.SearchResult{}
@@ -82,7 +83,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
if err != nil {
defer page.Close()
- logrus.Errorf("Cannot parse search results: %s", err)
+ baid.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrSearchTimeout
}
@@ -91,10 +92,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
defer page.Close()
if baid.isCaptcha(page) {
- logrus.Errorf("Baidu captcha occurred during: %s", url)
+ baid.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
} else if baid.isTimeout(page) {
- logrus.Errorf("Baidu timeout occurred during: %s", url)
+ baid.logger.Error("Timeout occurred: %s", url)
return nil, core.ErrCaptcha
}
return nil, nil
@@ -113,13 +114,13 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
}
linkText, err := link.Property("href")
if err != nil {
- logrus.Error("No `href` tag found")
+ baid.logger.Error("Missing href tag")
}
// Get title
title, err := link.Text()
if err != nil {
- logrus.Error("Cannot extract text from title")
+ baid.logger.Error("Failed to extract title")
title = "No title"
}
@@ -134,10 +135,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
searchResults = append(searchResults, gR)
}
- if !baid.LeavePageOpen {
+ if !baid.Browser.LeavePageOpen {
err = page.Close()
if err != nil {
- logrus.Error(err)
+ baid.logger.Error("Page close error: %v", err)
}
}
@@ -145,7 +146,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
}
func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Baidu Image search, query: %+v", query)
+ baid.logger.Debug("Starting image search, query: %+v", query)
searchResults := []core.SearchResult{}
searchPage := 0
@@ -162,7 +163,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
return nil, err
}
- if !baid.LeavePageOpen {
+ if !baid.Browser.LeavePageOpen {
defer page.Close()
}
page.Reload()
@@ -171,7 +172,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
result, err := page.Timeout(baid.Timeout).Search("body > pre")
if err != nil {
defer page.Close()
- logrus.Errorf("Cannot parse search results: %s", err)
+ baid.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrSearchTimeout
}
@@ -180,10 +181,10 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
defer page.Close()
if baid.isCaptcha(page) {
- logrus.Errorf("Baidu captcha occurred during: %s", url)
+ baid.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
} else if baid.isTimeout(page) {
- logrus.Errorf("Baidu timeout occurred during: %s", url)
+ baid.logger.Error("Timeout occurred: %s", url)
return nil, core.ErrCaptcha
}
return nil, nil
@@ -207,7 +208,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
err = json.Unmarshal([]byte(fixedJson), &data)
if err != nil {
- logrus.Errorf("Cannot unmarshal JSON: %v\nData: %v", err, jsonText)
+ baid.logger.Error("Failed to unmarshal JSON: %v", err)
return nil, err
}
@@ -233,7 +234,7 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
searchPage += 1
- if !baid.LeavePageOpen {
+ if !baid.Browser.LeavePageOpen {
page.Close()
}
}
diff --git a/baidu/url.go b/baidu/url.go
index 666534b..3e46952 100644
--- a/baidu/url.go
+++ b/baidu/url.go
@@ -13,7 +13,7 @@ import (
)
func dateToTimestamp(date string) (int64, error) {
- layout := "20060201"
+ layout := "20060102"
t, err := time.Parse(layout, date)
if err != nil {
return 0, err
@@ -55,12 +55,12 @@ func BuildURL(q core.Query) (string, error) {
if q.LangCode != "" {
//params.Add("rqlang", q.LangCode)
- logrus.Warn("Baidu's Language specific search not supported yet")
+ logrus.Warn("Language search not supported")
}
if q.Filetype != "" {
//params.Add("ft", q.Filetype)
- logrus.Warn("Baidu's File search not supported yet")
+ logrus.Warn("File search not supported")
}
if q.Limit != 0 {
diff --git a/bing/search.go b/bing/search.go
index e075a4b..bc79d0a 100644
--- a/bing/search.go
+++ b/bing/search.go
@@ -10,19 +10,20 @@ import (
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/karust/openserp/core"
- "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
type Bing struct {
core.Browser
core.SearchEngineOptions
+ logger *core.EngineLogger
}
func New(browser core.Browser, opts core.SearchEngineOptions) *Bing {
bing := Bing{Browser: browser}
opts.Init()
bing.SearchEngineOptions = opts
+ bing.logger = core.NewEngineLogger("Bing")
return &bing
}
@@ -44,11 +45,36 @@ func (bing *Bing) getTotalResults(page *rod.Page) (int, error) {
}
func (bing *Bing) checkCaptcha(page *rod.Page) bool {
- captcha, err := page.Timeout(bing.GetSelectorTimeout() / 2).Element("div#bxc")
- if err != nil {
+ if page == nil {
return false
}
- return captcha != nil
+
+ if info, err := page.Info(); err == nil {
+ url := strings.ToLower(info.URL)
+ if strings.Contains(url, "turing") || strings.Contains(url, "captcha") {
+ return true
+ }
+ }
+
+ timeout := bing.GetSelectorTimeout() / 2
+ if timeout <= 0 {
+ timeout = time.Second * 2
+ }
+
+ selectors := []string{
+ "div.captcha",
+ "div.captcha_header",
+ }
+
+ for _, selector := range selectors {
+ has, err, _ := page.Timeout(timeout).Has(selector)
+ if err == nil && has {
+ bing.logger.Debug("Captcha detected: %s", selector)
+ return true
+ }
+ }
+
+ return false
}
func (bing *Bing) acceptCookies(page *rod.Page) {
@@ -61,13 +87,18 @@ func (bing *Bing) acceptCookies(page *rod.Page) {
}
func (bing *Bing) close(page *rod.Page) {
- if page != nil {
- page.Close()
+ if !bing.Browser.LeavePageOpen {
+ if page != nil {
+ err := page.Close()
+ if err != nil {
+ bing.logger.Debug("Page close error: %v", err)
+ }
+ }
}
}
func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Bing search, query: %+v", query)
+ bing.logger.Debug("Starting search, query: %+v", query)
searchResults := []core.SearchResult{}
@@ -82,8 +113,10 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
}
defer bing.close(page)
+ page.WaitLoad()
+
if bing.checkCaptcha(page) {
- logrus.Errorf("Bing captcha occurred during: %s", url)
+ bing.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
}
@@ -92,20 +125,20 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
organicElements, err := page.Timeout(bing.Timeout).Elements("li.b_algo")
if err != nil {
- logrus.Errorf("Cannot parse organic results: %s", err)
+ bing.logger.Error("Cannot parse organic results: %s", err)
return nil, core.ErrSearchTimeout
}
adElements, err := page.Timeout(bing.Timeout).Elements("li.b_ad")
if err != nil {
- logrus.Debug("No ad results found or error parsing ads")
+ bing.logger.Debug("No ads found")
}
totalResults, err := bing.getTotalResults(page)
if err != nil {
- logrus.Errorf("Error capturing total results: %v", err)
+ bing.logger.Debug("Failed to get total results: %v", err)
}
- logrus.Infof("%d SERP results found (%d ads)", totalResults, len(adElements))
+ bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
rank := 0
for _, result := range organicElements {
@@ -113,14 +146,14 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
titleElem, err := result.Element("a")
if err != nil {
- logrus.Debug("No title found for result")
+ bing.logger.Debug("Missing title")
continue
}
srchRes.Title, _ = titleElem.Text()
href, err := titleElem.Property("href")
if err != nil {
- logrus.Debug("No URL found for result")
+ bing.logger.Debug("Missing URL")
continue
}
srchRes.URL = href.String()
@@ -150,14 +183,14 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
titleElem, err := adResult.Element("h2 a")
if err != nil {
- logrus.Debug("No title found for ad")
+ bing.logger.Debug("Ad missing title")
continue
}
srchRes.Title, _ = titleElem.Text()
href, err := titleElem.Property("href")
if err != nil {
- logrus.Debug("No URL found for ad")
+ bing.logger.Debug("Ad missing URL")
continue
}
srchRes.URL = href.String()
@@ -188,7 +221,7 @@ type BingImageData struct {
// SearchImage performs Bing image search and returns results
func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Bing image search, query: %+v", query)
+ bing.logger.Debug("Starting image search, query: %+v", query)
searchResults := []core.SearchResult{}
@@ -204,9 +237,11 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
}
defer bing.close(page)
+ page.WaitLoad()
+
// Check for captcha
if bing.checkCaptcha(page) {
- logrus.Errorf("Bing captcha occurred during image search: %s", url)
+ bing.logger.Error("Captcha detected during image search: %s", url)
return nil, core.ErrCaptcha
}
@@ -220,7 +255,7 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
// Find all image result containers using CSS selector
imageContainers, err := page.Timeout(bing.Timeout).Elements("div.iuscp, div.isv")
if err != nil {
- logrus.Errorf("Cannot parse image results: %s", err)
+ bing.logger.Error("Cannot parse image results: %s", err)
return nil, core.ErrSearchTimeout
}
@@ -228,7 +263,7 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
return nil, errors.New("no image results found")
}
- logrus.Infof("Found %d image result elements", len(imageContainers))
+ bing.logger.Info("Found %d image elements", len(imageContainers))
rank := 0
for _, c := range imageContainers {
@@ -237,21 +272,21 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
// Get the element inside the div
linkElem, err := c.Element("a")
if err != nil {
- logrus.Debug("No element found in image container")
+ bing.logger.Debug("Missing element")
continue
}
// Extract image metadata from m attribute (contains JSON)
mAttr, err := linkElem.Attribute("m")
if err != nil || mAttr == nil {
- logrus.Debug("No m attribute found in image element or attribute is nil")
+ bing.logger.Debug("Missing m attribute")
continue
}
// Ensure we have valid JSON data to unmarshal
jsonData := []byte(*mAttr)
if len(jsonData) == 0 {
- logrus.Debug("Empty JSON data in m attribute")
+ bing.logger.Debug("Empty JSON data")
continue
}
@@ -259,7 +294,7 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
var imgData BingImageData
err = json.Unmarshal(jsonData, &imgData)
if err != nil {
- logrus.Debugf("Failed to parse image JSON data: %s", err)
+ bing.logger.Debug("Failed to parse JSON: %s", err)
continue
}
diff --git a/bing/url.go b/bing/url.go
index 7385ef1..5bce130 100644
--- a/bing/url.go
+++ b/bing/url.go
@@ -2,6 +2,7 @@ package bing
import (
"errors"
+ "fmt"
"net/url"
"strconv"
"strings"
@@ -47,7 +48,7 @@ func BuildURL(q core.Query) (string, error) {
params.Add("count", strconv.Itoa(q.Limit))
}
- // Set search date range (convert from Google format to Bing format)
+ // Set search date range - Bing supports date filtering via query text
if q.DateInterval != "" {
intervals := strings.Split(q.DateInterval, "..")
if len(intervals) != 2 {
@@ -65,9 +66,14 @@ func BuildURL(q core.Query) (string, error) {
return "", errors.New("invalid end date format, expected YYYYMMDD")
}
- // Bing uses filters parameter with specific date format
- dateFilter := "ex1:\"ez5_" + startDate.Format("2006-01-02") + "_" + endDate.Format("2006-01-02") + "\""
- params.Add("filters", dateFilter)
+ // Add date range to the search query text (Bing supports this format)
+ dateRange := fmt.Sprintf(" after:%s before:%s",
+ startDate.Format("2006-01-02"),
+ endDate.Format("2006-01-02"))
+
+ // Update the query text to include date range
+ currentQuery := params.Get("q")
+ params.Set("q", currentQuery+dateRange)
}
// Bing-specific parameters for consistent results
diff --git a/cmd/root.go b/cmd/root.go
index 30c3c6c..ac34a79 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
+ "strconv"
"strings"
"github.com/karust/openserp/core"
@@ -12,18 +13,19 @@ import (
)
const (
- version = "0.4.1"
+ version = "0.5.1"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
type Config struct {
- App AppConfig `mapstructure:"app"`
- Config2Capcha Config2Captcha `mapstructure:"2captcha"`
- GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
- YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
- BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"`
- BingConfig core.SearchEngineOptions `mapstructure:"bing"`
+ App AppConfig `mapstructure:"app"`
+ Config2Capcha Config2Captcha `mapstructure:"2captcha"`
+ GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
+ YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
+ BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"`
+ BingConfig core.SearchEngineOptions `mapstructure:"bing"`
+ DuckDuckGoConfig core.SearchEngineOptions `mapstructure:"duckduckgo"`
}
type Config2Captcha struct {
@@ -48,6 +50,13 @@ type AppConfig struct {
var config = Config{}
+var flagToConfigKey = map[string]string{
+ "config": "app.config_path",
+ "leave": "app.leave_head",
+ "raw": "app.raw_requests",
+ "2captcha_key": "2captcha.apikey",
+}
+
var RootCmd = &cobra.Command{
Use: "openserp",
Short: "Open SERP",
@@ -76,15 +85,39 @@ var RootCmd = &cobra.Command{
// Bind each cobra flag to its associated viper configuration (config file and environment variable)
func bindFlags(cmd *cobra.Command, vpr *viper.Viper) {
cmd.Flags().VisitAll(func(flg *pflag.Flag) {
- configName := "app." + flg.Name
+ configName, ok := flagToConfigKey[flg.Name]
+ if !ok {
+ configName = "app." + flg.Name
+ }
+
+ if err := vpr.BindPFlag(configName, flg); err != nil {
+ logrus.Errorf("Unable to bind flag %s: %v", flg.Name, err)
+ }
- // Apply viper config value to the flag if viper has a value
if flg.Changed {
- vpr.Set(configName, flg.Value)
+ val, err := parseFlagValue(flg)
+ if err != nil {
+ logrus.Errorf("Unable to parse flag %s: %v", flg.Name, err)
+ return
+ }
+ vpr.Set(configName, val)
}
})
}
+func parseFlagValue(flg *pflag.Flag) (interface{}, error) {
+ switch flg.Value.Type() {
+ case "string":
+ return flg.Value.String(), nil
+ case "bool":
+ return strconv.ParseBool(flg.Value.String())
+ case "int":
+ return strconv.Atoi(flg.Value.String())
+ default:
+ return flg.Value.String(), nil
+ }
+}
+
// Initialize Viper
func initializeConfig(cmd *cobra.Command) error {
v := viper.New()
@@ -93,14 +126,14 @@ func initializeConfig(cmd *cobra.Command) error {
v.SetConfigName(defaultConfigFilename)
v.AddConfigPath(".")
- // 1. Config. Return an error if we cannot parse the config file.
+ // 1. Config file (lowest priority). Return an error if we cannot parse the config file.
err := v.ReadInConfig()
if err != nil {
err = fmt.Errorf("cannot read config: %v", err)
logrus.Warn(err)
}
- // 2. Env. Bind environment variables to their equivalent keys with underscores
+ // 2. Environment variables (medium priority). Bind environment variables to their equivalent keys with underscores
for _, key := range v.AllKeys() {
envKey := envPrefix + "_" + strings.ToUpper(strings.ReplaceAll(key, ".", "_"))
err := v.BindEnv(key, envKey)
@@ -109,7 +142,7 @@ func initializeConfig(cmd *cobra.Command) error {
}
}
- // 3. Cmd flags. Bind the current command's flags to viper
+ // 3. Command flags (highest priority). Bind the current command's flags to viper
bindFlags(cmd, v)
// Dump Viper values to config struct
diff --git a/cmd/search.go b/cmd/search.go
index 55c0be2..9b243a2 100644
--- a/cmd/search.go
+++ b/cmd/search.go
@@ -7,7 +7,9 @@ import (
"time"
"github.com/karust/openserp/baidu"
+ "github.com/karust/openserp/bing"
"github.com/karust/openserp/core"
+ "github.com/karust/openserp/duckduckgo"
"github.com/karust/openserp/google"
"github.com/karust/openserp/yandex"
"github.com/sirupsen/logrus"
@@ -17,7 +19,7 @@ import (
var searchCMD = &cobra.Command{
Use: "search",
Aliases: []string{"find"},
- Short: "Search results using chosen web search engine (google, yandex, baidu)",
+ Short: "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo)",
Args: cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)),
Run: search,
}
@@ -31,19 +33,25 @@ func search(cmd *cobra.Command, args []string) {
ProxyURL: config.App.ProxyURL,
Insecure: config.App.Insecure,
}
- results := []core.SearchResult{}
+ logrus.Infof("Starting SERP search request using %s engine for query: %s", engineType, query.Text)
+
+ var results []core.SearchResult
if config.App.IsRawRequests {
+ logrus.Infof("Using raw requests mode for %s search", engineType)
results, err = searchRaw(engineType, query)
} else {
+ logrus.Infof("Using browser mode for %s search", engineType)
results, err = searchBrowser(engineType, query)
}
if err != nil {
- logrus.Error(err)
+ logrus.Errorf("Error during %s search: %s", engineType, err)
return
}
+ logrus.Infof("Successfully completed SERP search using %s engine, returned %d results", engineType, len(results))
+
b, err := json.MarshalIndent(results, "", " ")
if err != nil {
logrus.Error(err)
@@ -82,6 +90,10 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
engine = google.New(*browser, config.GoogleConfig)
case "baidu":
engine = baidu.New(*browser, config.BaiduConfig)
+ case "bing":
+ engine = bing.New(*browser, config.BingConfig)
+ case "duck":
+ engine = duckduckgo.New(*browser, config.DuckDuckGoConfig)
default:
logrus.Infof("No `%s` search engine found", engineType)
}
@@ -99,6 +111,12 @@ func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error)
return google.Search(query)
case "baidu":
return baidu.Search(query)
+ case "bing":
+ logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.")
+ return nil, fmt.Errorf("bing does not support raw requests mode")
+ case "duck":
+ logrus.Warn("DuckDuckGo does not support raw HTTP requests mode. Please use browser mode instead.")
+ return nil, fmt.Errorf("duckduckgo does not support raw requests mode")
default:
logrus.Infof("No `%s` search engine found", engineType)
}
diff --git a/cmd/serve.go b/cmd/serve.go
index 6be1168..342dd0d 100644
--- a/cmd/serve.go
+++ b/cmd/serve.go
@@ -7,6 +7,7 @@ import (
"github.com/karust/openserp/baidu"
"github.com/karust/openserp/bing"
"github.com/karust/openserp/core"
+ "github.com/karust/openserp/duckduckgo"
"github.com/karust/openserp/google"
"github.com/karust/openserp/yandex"
"github.com/sirupsen/logrus"
@@ -98,8 +99,9 @@ func serve(cmd *cobra.Command, args []string) {
gogl := google.New(*browser, config.GoogleConfig)
baidu := baidu.New(*browser, config.BaiduConfig)
bing := bing.New(*browser, config.BingConfig)
+ ddg := duckduckgo.New(*browser, config.DuckDuckGoConfig)
- serv := core.NewServer(config.App.Host, config.App.Port, gogl, yand, baidu, bing)
+ serv := core.NewServer(config.App.Host, config.App.Port, gogl, yand, baidu, bing, ddg)
err = serv.Listen()
if err != nil {
diff --git a/config.yaml b/config.yaml
index e1dd30c..0ec5787 100644
--- a/config.yaml
+++ b/config.yaml
@@ -8,7 +8,7 @@ app:
leakless: false
leave_head: false
stealth: false
- insecure: false
+ insecure: true
2captcha:
apikey: "123123123123123"
@@ -25,3 +25,11 @@ yandex:
baidu:
rate_requests: 4
rate_burst: 2
+
+bing:
+ rate_requests: 4
+ rate_burst: 2
+
+duckduckgo:
+ rate_requests: 4
+ rate_burst: 2
diff --git a/core/logger.go b/core/logger.go
index 62eedf5..49b4ad1 100644
--- a/core/logger.go
+++ b/core/logger.go
@@ -14,20 +14,93 @@ type customFormatter struct {
}
func (f *customFormatter) Format(entry *logrus.Entry) ([]byte, error) {
- // var levelColor int
- // switch entry.Level {
- // case logrus.DebugLevel, logrus.TraceLevel:
- // levelColor = 42 // green highlight
- // case logrus.WarnLevel:
- // levelColor = 33 // yellow
- // case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
- // levelColor = 41 // red highlight
- // default:
- // levelColor = 36 // blue
- // }
- //[]byte(fmt.Sprintf("[%s] - \x1b[%dm%s\x1b[0m - %s\n", entry.Time.Format(f.TimestampFormat), levelColor, strings.ToUpper(entry.Level.String()), entry.Message))
+ message := entry.Message
- return []byte(fmt.Sprintf("[%s][%s] \t%s\n", entry.Time.Format(f.TimestampFormat), strings.ToUpper(entry.Level.String()), entry.Message)), nil
+ // Check if engine name is provided as a field
+ engineName := ""
+ if engine, exists := entry.Data["engine"]; exists {
+ if engineStr, ok := engine.(string); ok {
+ engineName = engineStr
+ }
+ }
+
+ // Format: [timestamp][level][engine] message
+ if engineName != "" {
+ return []byte(fmt.Sprintf("[%s][%s][%s] %s\n",
+ entry.Time.Format(f.TimestampFormat),
+ strings.ToUpper(entry.Level.String()),
+ engineName,
+ message)), nil
+ }
+
+ // Format: [timestamp][level] message (no engine)
+ return []byte(fmt.Sprintf("[%s][%s] %s\n",
+ entry.Time.Format(f.TimestampFormat),
+ strings.ToUpper(entry.Level.String()),
+ message)), nil
+}
+
+// EngineLogger provides simplified logging for search engines
+type EngineLogger struct {
+ engine string
+ logger *logrus.Entry
+}
+
+// NewEngineLogger creates a new logger for a specific search engine
+func NewEngineLogger(engine string) *EngineLogger {
+ return &EngineLogger{
+ engine: engine,
+ logger: logrus.WithField("engine", engine),
+ }
+}
+
+// Debug logs a debug message
+func (el *EngineLogger) Debug(message string, args ...interface{}) {
+ el.logger.Debugf(message, args...)
+}
+
+// Info logs an info message
+func (el *EngineLogger) Info(message string, args ...interface{}) {
+ el.logger.Infof(message, args...)
+}
+
+// Warn logs a warning message
+func (el *EngineLogger) Warn(message string, args ...interface{}) {
+ el.logger.Warnf(message, args...)
+}
+
+// Error logs an error message
+func (el *EngineLogger) Error(message string, args ...interface{}) {
+ el.logger.Errorf(message, args...)
+}
+
+// Fatal logs a fatal message
+func (el *EngineLogger) Fatal(message string, args ...interface{}) {
+ el.logger.Fatalf(message, args...)
+}
+
+// Panic logs a panic message
+func (el *EngineLogger) Panic(message string, args ...interface{}) {
+ el.logger.Panicf(message, args...)
+}
+
+// LogWithEngine logs a message with engine information (deprecated - use EngineLogger instead)
+func LogWithEngine(level logrus.Level, engine, message string, args ...interface{}) {
+ entry := logrus.WithField("engine", engine)
+ switch level {
+ case logrus.DebugLevel:
+ entry.Debugf(message, args...)
+ case logrus.InfoLevel:
+ entry.Infof(message, args...)
+ case logrus.WarnLevel:
+ entry.Warnf(message, args...)
+ case logrus.ErrorLevel:
+ entry.Errorf(message, args...)
+ case logrus.FatalLevel:
+ entry.Fatalf(message, args...)
+ case logrus.PanicLevel:
+ entry.Panicf(message, args...)
+ }
}
func InitLogger(isVerbose, isDebug bool) {
diff --git a/duckduckgo/search.go b/duckduckgo/search.go
new file mode 100644
index 0000000..aceb263
--- /dev/null
+++ b/duckduckgo/search.go
@@ -0,0 +1,427 @@
+package duckduckgo
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/go-rod/rod"
+ "github.com/karust/openserp/core"
+ "golang.org/x/time/rate"
+)
+
+type DuckDuckGo struct {
+ core.Browser
+ core.SearchEngineOptions
+ pageSleep time.Duration // Sleep between pages
+ logger *core.EngineLogger
+}
+
+func New(browser core.Browser, opts core.SearchEngineOptions) *DuckDuckGo {
+ ddg := DuckDuckGo{Browser: browser}
+ opts.Init()
+ ddg.SearchEngineOptions = opts
+ ddg.logger = core.NewEngineLogger("DuckDuckGo")
+
+ ddg.pageSleep = time.Second * 1
+ return &ddg
+}
+
+func (ddg *DuckDuckGo) Name() string {
+ return "duckduckgo"
+}
+
+func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter {
+ ratelimit := rate.Every(ddg.GetRatelimit())
+ return rate.NewLimiter(ratelimit, ddg.RateBurst)
+}
+
+func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool {
+ // DuckDuckGo rarely shows captchas, but we can check for common patterns
+ _, err := page.Timeout(ddg.GetSelectorTimeout()).Search("div[class*='captcha']")
+ return err == nil
+}
+
+// Check if no results are found
+func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool {
+ _, err := page.Timeout(ddg.GetSelectorTimeout()).Search("div[class*='no-results']")
+ return err == nil
+}
+
+func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
+ searchResults := []core.SearchResult{}
+
+ for i, r := range results {
+ // Get URL - try multiple selectors
+ var link *rod.Element
+ var err error
+
+ linkSelectors := []string{
+ "a[data-testid='result-title-a']",
+ "a.result__a",
+ "a.result__url",
+ "h2 a",
+ "h3 a",
+ "a[href]",
+ "a",
+ }
+
+ for _, selector := range linkSelectors {
+ link, err = r.Element(selector)
+ if err == nil {
+ break
+ }
+ }
+
+ if err != nil {
+ ddg.logger.Debug("Missing link")
+ continue
+ }
+
+ linkText, err := link.Property("href")
+ if err != nil {
+ ddg.logger.Debug("Missing href")
+ continue
+ }
+
+ // Validate that we have a proper URL
+ hrefStr := linkText.String()
+ if hrefStr == "" || hrefStr == "#" || hrefStr == "javascript:void(0)" {
+ ddg.logger.Debug("Invalid href: %s", hrefStr)
+ continue
+ }
+
+ // Get title - try multiple selectors
+ var titleTag *rod.Element
+ titleSelectors := []string{
+ "h2",
+ ".result__title",
+ ".result__a",
+ "span",
+ "div",
+ }
+
+ for _, selector := range titleSelectors {
+ titleTag, err = r.Element(selector)
+ if err == nil {
+ break
+ }
+ }
+
+ title := "No title"
+ if titleTag != nil {
+ title, _ = titleTag.Text()
+ }
+
+ // Get description - try multiple selectors
+ desc := ""
+ descSelectors := []string{
+ "div[data-result='snippet']",
+ ".result__snippet",
+ ".result__body",
+ "span[class*='snippet']",
+ "div[class*='snippet']",
+ "p",
+ }
+
+ for _, selector := range descSelectors {
+ descTag, err := r.Element(selector)
+ if err == nil {
+ desc = descTag.MustText()
+ break
+ }
+ }
+
+ // Check if it's an ad
+ isAd := false
+ adSelectors := []string{
+ "[data-testid='ad-badge']",
+ ".ad-badge",
+ ".result--ad",
+ }
+
+ for _, selector := range adSelectors {
+ adIndicator, err := r.Element(selector)
+ if err == nil && adIndicator != nil {
+ isAd = true
+ break
+ }
+ }
+
+ result := core.SearchResult{
+ Rank: (pageNum * 10) + (i + 1),
+ URL: hrefStr,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
+ }
+ searchResults = append(searchResults, result)
+ }
+
+ return searchResults
+}
+
+func (ddg *DuckDuckGo) Search(query core.Query) ([]core.SearchResult, error) {
+ ddg.logger.Debug("Starting search, query: %+v", query)
+
+ allResults := []core.SearchResult{}
+ searchPage := 0
+ maxPages := 5 // Prevent infinite loops
+ consecutiveEmptyPages := 0 // Track consecutive pages with no valid results
+
+ for len(allResults) < query.Limit && searchPage < maxPages {
+ url, err := BuildURL(query)
+ if err != nil {
+ return nil, err
+ }
+
+ page, err := ddg.Navigate(url)
+ if err != nil {
+ return nil, err
+ }
+
+ // Get all search results in page - try multiple selectors
+ var searchRes *rod.SearchResult
+ var searchErr error
+
+ // Try different selectors for DuckDuckGo results
+ selectors := []string{
+ "article[data-testid='result']",
+ "div[data-testid='result']",
+ "div.result",
+ "div.web-result",
+ ".result",
+ "[data-testid='result']",
+ }
+
+ for _, selector := range selectors {
+ searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector)
+ if searchErr == nil && searchRes != nil {
+ ddg.logger.Debug("Found results with selector: %s", selector)
+ break
+ }
+ }
+ if searchErr != nil {
+ defer page.Close()
+ ddg.logger.Error("Cannot parse search results: %s", searchErr)
+ return nil, core.ErrSearchTimeout
+ }
+
+ // Check why no results, maybe captcha?
+ if searchRes == nil {
+ defer page.Close()
+
+ if ddg.isNoResults(page) {
+ ddg.logger.Warn("No results found")
+ } else if ddg.isCaptcha(page) {
+ ddg.logger.Error("Captcha detected: %s", url)
+ return nil, core.ErrCaptcha
+ }
+ break
+ }
+
+ elements, err := searchRes.All()
+ if err != nil {
+ ddg.logger.Error("Cannot get search elements: %s", err)
+ break
+ }
+
+ r := ddg.parseResults(elements, searchPage)
+
+ // Track consecutive empty pages
+ if len(r) == 0 {
+ consecutiveEmptyPages++
+ ddg.logger.Debug("No valid results found on page %d (consecutive empty: %d)", searchPage, consecutiveEmptyPages)
+
+ // Break if we have too many consecutive empty pages
+ if consecutiveEmptyPages >= 2 {
+ ddg.logger.Warn("Too many consecutive empty pages, stopping search")
+ break
+ }
+ } else {
+ consecutiveEmptyPages = 0 // Reset counter when we find results
+ }
+
+ allResults = append(allResults, r...)
+
+ searchPage++
+
+ if !ddg.Browser.LeavePageOpen {
+ // Close tab before opening new one during the cycle
+ err = page.Close()
+ if err != nil {
+ ddg.logger.Debug("Page close error: %v", err)
+ }
+ }
+
+ time.Sleep(ddg.pageSleep)
+ }
+
+ ddg.logger.Info("Search completed: %d results", len(allResults))
+ return core.DeduplicateResults(allResults), nil
+}
+
+func (ddg *DuckDuckGo) SearchImage(query core.Query) ([]core.SearchResult, error) {
+ ddg.logger.Debug("Starting image search, query: %+v", query)
+
+ searchResults := []core.SearchResult{}
+
+ url, err := BuildImageURL(query)
+ if err != nil {
+ return nil, err
+ }
+
+ page, err := ddg.Navigate(url)
+ if err != nil {
+ return nil, err
+ }
+
+ if !ddg.Browser.LeavePageOpen {
+ defer page.Close()
+ }
+
+ // Wait for page to load
+ page.WaitLoad()
+ time.Sleep(time.Second * 2) // Give time for images to load
+
+ // Try multiple selectors for DuckDuckGo image results
+ var searchRes *rod.SearchResult
+ var searchErr error
+
+ selectors := []string{
+ "figure",
+ // "figure.nsogf_Hpj9UUxfhcwQd5",
+ // "div[data-testid='result']",
+ // "div.tile--img",
+ // "div.tile.tile--img",
+ // "div.js-images-show-more",
+ // "div.img-result",
+ }
+
+ ddg.logger.Debug("Trying selectors: %v", selectors)
+
+ for _, selector := range selectors {
+ searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector)
+ if searchErr == nil && searchRes != nil {
+ ddg.logger.Debug("Found image results with selector: %s", selector)
+ break
+ } else {
+ ddg.logger.Debug("Selector '%s' not found: %v", selector, searchErr)
+ }
+ }
+
+ if searchErr != nil {
+ ddg.logger.Error("Cannot find image results: %s", searchErr)
+ return searchResults, core.ErrSearchTimeout
+ }
+
+ // Check why no results
+ if searchRes == nil {
+ if ddg.isCaptcha(page) {
+ ddg.logger.Error("Captcha detected: %s", url)
+ return searchResults, core.ErrCaptcha
+ } else if ddg.isNoResults(page) {
+ ddg.logger.Warn("No image results found")
+ }
+ return searchResults, core.ErrSearchTimeout
+ }
+
+ elements, err := searchRes.All()
+ if err != nil {
+ ddg.logger.Error("Cannot get search elements: %s", err)
+ return searchResults, err
+ }
+
+ ddg.logger.Info("Found %d image elements", len(elements))
+
+ for i, r := range elements {
+ // Get image URL - try multiple selectors
+ var imgTag *rod.Element
+ var imgErr error
+
+ imgSelectors := []string{
+ "img",
+ "div.SZ76bwIlqO8BBoqOLqYV img",
+ "img[src*='duckduckgo.com']",
+ }
+
+ for _, selector := range imgSelectors {
+ imgTag, imgErr = r.Element(selector)
+ if imgErr == nil {
+ break
+ }
+ }
+
+ if imgErr != nil {
+ ddg.logger.Debug("Missing img tag for element %d", i)
+ continue
+ }
+
+ imgSrc, err := imgTag.Property("src")
+ if err != nil {
+ ddg.logger.Debug("Missing src property for image %d", i)
+ continue
+ }
+
+ // Get title - try multiple selectors based on the HTML structure
+ var titleTag *rod.Element
+ var titleErr error
+
+ titleSelectors := []string{
+ "figcaption a p span",
+ "figcaption span",
+ "figcaption p span",
+ "span.EKtkFWMYpwzMKOYr0GYm",
+ "h3",
+ "span",
+ "p",
+ }
+
+ for _, selector := range titleSelectors {
+ titleTag, titleErr = r.Element(selector)
+ if titleErr == nil {
+ break
+ }
+ }
+
+ title := "No title"
+ if titleTag != nil {
+ title, _ = titleTag.Text()
+ }
+
+ // Get source page URL - try multiple selectors
+ var linkTag *rod.Element
+ var linkErr error
+
+ linkSelectors := []string{
+ "figcaption a",
+ "a",
+ }
+
+ for _, selector := range linkSelectors {
+ linkTag, linkErr = r.Element(selector)
+ if linkErr == nil {
+ break
+ }
+ }
+
+ sourceURL := ""
+ if linkTag != nil {
+ href, err := linkTag.Property("href")
+ if err == nil {
+ sourceURL = href.String()
+ }
+ }
+
+ result := core.SearchResult{
+ Rank: i + 1,
+ URL: imgSrc.String(),
+ Title: title,
+ Description: fmt.Sprintf("Source: %s", sourceURL),
+ }
+
+ searchResults = append(searchResults, result)
+ }
+
+ ddg.logger.Info("Parsed %d image results", len(searchResults))
+ return core.DeduplicateResults(searchResults), nil
+}
diff --git a/duckduckgo/search_test.go b/duckduckgo/search_test.go
new file mode 100644
index 0000000..7758e98
--- /dev/null
+++ b/duckduckgo/search_test.go
@@ -0,0 +1,103 @@
+package duckduckgo
+
+import (
+ "testing"
+
+ "github.com/karust/openserp/core"
+)
+
+func TestBuildURL(t *testing.T) {
+ tests := []struct {
+ name string
+ query core.Query
+ expected string
+ wantErr bool
+ }{
+ {
+ name: "Basic search",
+ query: core.Query{
+ Text: "golang programming",
+ },
+ expected: "https://duckduckgo.com/?q=golang+programming&t=h&ia=web",
+ wantErr: false,
+ },
+ {
+ name: "Search with site filter",
+ query: core.Query{
+ Text: "golang",
+ Site: "github.com",
+ },
+ expected: "https://duckduckgo.com/?q=golang+site%3Agithub.com&t=h&ia=web",
+ wantErr: false,
+ },
+ {
+ name: "Search with filetype",
+ query: core.Query{
+ Text: "documentation",
+ Filetype: "pdf",
+ },
+ expected: "https://duckduckgo.com/?q=documentation+filetype%3Apdf&t=h&ia=web",
+ wantErr: false,
+ },
+ {
+ name: "Empty query",
+ query: core.Query{
+ Text: "",
+ },
+ expected: "",
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := BuildURL(tt.query)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("BuildURL() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.expected {
+ t.Errorf("BuildURL() = %v, want %v", got, tt.expected)
+ }
+ })
+ }
+}
+
+func TestBuildImageURL(t *testing.T) {
+ tests := []struct {
+ name string
+ query core.Query
+ expected string
+ wantErr bool
+ }{
+ {
+ name: "Basic image search",
+ query: core.Query{
+ Text: "golang logo",
+ },
+ expected: "https://duckduckgo.com/?q=golang+logo&t=h&iax=images&ia=images",
+ wantErr: false,
+ },
+ {
+ name: "Empty query",
+ query: core.Query{
+ Text: "",
+ },
+ expected: "",
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := BuildImageURL(tt.query)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("BuildImageURL() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.expected {
+ t.Errorf("BuildImageURL() = %v, want %v", got, tt.expected)
+ }
+ })
+ }
+}
diff --git a/duckduckgo/url.go b/duckduckgo/url.go
new file mode 100644
index 0000000..e581111
--- /dev/null
+++ b/duckduckgo/url.go
@@ -0,0 +1,99 @@
+package duckduckgo
+
+import (
+ "errors"
+ "fmt"
+ "net/url"
+ "strings"
+
+ "github.com/karust/openserp/core"
+)
+
+const baseURL = "https://duckduckgo.com"
+
+func BuildURL(q core.Query) (string, error) {
+ base, err := url.Parse(baseURL)
+ if err != nil {
+ return "", err
+ }
+
+ base.Path += ""
+ params := url.Values{}
+
+ // Set request text
+ if q.Text != "" || q.Site != "" || q.Filetype != "" {
+ text := q.Text
+ if q.Site != "" {
+ text += " site:" + q.Site
+ }
+ if q.Filetype != "" {
+ text += " filetype:" + q.Filetype
+ }
+
+ params.Add("q", text)
+ }
+
+ if len(params.Get("q")) == 0 {
+ return "", errors.New("empty query built")
+ }
+
+ // Set search date range
+ if q.DateInterval != "" {
+ intervals := strings.Split(q.DateInterval, "..")
+ if len(intervals) != 2 {
+ return "", errors.New("incorrect data interval provided")
+ }
+ // DuckDuckGo uses different date format
+ params.Add("df", fmt.Sprintf("%s..%s", intervals[0], intervals[1]))
+ }
+
+ // Set language
+ if q.LangCode != "" {
+ params.Add("kl", strings.ToLower(q.LangCode))
+ }
+
+ // DuckDuckGo specific parameters
+ params.Add("t", "h") // HTML format
+ params.Add("ia", "web") // Web search
+
+ base.RawQuery = params.Encode()
+ return base.String(), nil
+}
+
+func BuildImageURL(q core.Query) (string, error) {
+ base, err := url.Parse(baseURL)
+ if err != nil {
+ return "", err
+ }
+
+ base.Path += ""
+ params := url.Values{}
+ params.Add("t", "h") // HTML format
+ params.Add("iax", "images") // Image search
+ params.Add("ia", "images")
+
+ // Set request text
+ if q.Text != "" || q.Site != "" || q.Filetype != "" {
+ text := q.Text
+ if q.Site != "" {
+ text += " site:" + q.Site
+ }
+ if q.Filetype != "" {
+ text += " filetype:" + q.Filetype
+ }
+
+ params.Add("q", text)
+ }
+
+ if len(params.Get("q")) == 0 {
+ return "", errors.New("empty query built")
+ }
+
+ // Set language
+ if q.LangCode != "" {
+ params.Add("kl", strings.ToLower(q.LangCode))
+ }
+
+ base.RawQuery = params.Encode()
+ return base.String(), nil
+}
diff --git a/google/search.go b/google/search.go
index 3eb2249..0f2a2c9 100644
--- a/google/search.go
+++ b/google/search.go
@@ -11,7 +11,6 @@ import (
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/karust/openserp/core"
- "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
@@ -19,13 +18,15 @@ type Google struct {
core.Browser
core.SearchEngineOptions
rgxpGetDigits *regexp.Regexp
+ logger *core.EngineLogger
}
func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
gogl := Google{Browser: browser}
opts.Init()
gogl.SearchEngineOptions = opts
- gogl.rgxpGetDigits = regexp.MustCompile("\\d")
+ gogl.logger = core.NewEngineLogger("Google")
+ gogl.rgxpGetDigits = regexp.MustCompile(`\d`)
return &gogl
}
@@ -68,18 +69,18 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
}
func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
- logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL)
+ gogl.logger.Debug("Solve captcha: sitekey=%s", sitekey)
resp, _, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
if err != nil {
- logrus.Errorf("Error solving google captcha: %s", err)
+ gogl.logger.Error("Captcha solve failed: %s", err)
return false
}
- logrus.Debug("Resp:", resp)
+ gogl.logger.Debug("Captcha response received")
_, err = page.Eval(fmt.Sprintf(`;(() => { document.getElementById("g-recaptcha-response").innerHTML="%s"; submitCallback(); })();`, resp))
if err != nil {
- logrus.Errorf("Error setting captcha response: %s", err)
+ gogl.logger.Error("Failed to set captcha response: %s", err)
return false
}
@@ -94,13 +95,13 @@ func (gogl *Google) checkCaptcha(page *rod.Page) bool {
sitekey, err := captchaDiv.First.Attribute("data-sitekey")
if err != nil {
- logrus.Errorf("Cannot get Google captcha sitekey: %s", err)
+ gogl.logger.Error("Cannot get captcha sitekey: %s", err)
return false
}
dataS, err := captchaDiv.First.Attribute("data-s")
if err != nil {
- logrus.Errorf("Cannot get Google captcha datas: %s", err)
+ gogl.logger.Error("Cannot get captcha datas: %s", err)
return false
}
@@ -114,19 +115,19 @@ func (gogl *Google) preparePage(page *rod.Page) {
// Remove "similar queries" lists
_, err := page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove()); })();")
if err != nil {
- logrus.Errorf("Error preparing the page: %s", err)
+ gogl.logger.Error("Page preparation failed: %s", err)
}
}
func (gogl *Google) acceptCookies(page *rod.Page) {
diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search("div[role='dialog'][aria-modal] button")
if err != nil {
- logrus.Errorf("Cannot find cookie consent: %s", err)
+ gogl.logger.Debug("Cookie consent not found: %s", err)
return
}
btnElms, err := diaglogBtns.All()
if err != nil {
- logrus.Errorf("Cannot get cookie consent buttons: %s", err)
+ gogl.logger.Debug("Cannot get cookie consent buttons: %s", err)
return
}
btnElms[3].Click(proto.InputMouseButtonLeft, 1)
@@ -134,7 +135,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
}
func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Google search, query: %+v", query)
+ gogl.logger.Debug("Starting search, query: %+v", query)
searchResults := []core.SearchResult{}
@@ -153,7 +154,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
// Check first if there captcha
if gogl.checkCaptcha(page) {
- logrus.Errorf("Google captcha occurred during: %s", url)
+ gogl.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
}
@@ -165,7 +166,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
// Find all results using stable attributes
results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]")
if err != nil {
- logrus.Errorf("Cannot parse search results: %s", err)
+ gogl.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrSearchTimeout
}
@@ -175,9 +176,9 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
totalResults, err := gogl.getTotalResults(page)
if err != nil {
- logrus.Errorf("Error capturing total results: %v", err)
+ gogl.logger.Debug("Failed to get total results: %v", err)
}
- logrus.Infof("%d SERP results", totalResults)
+ gogl.logger.Info("Found %d total results", totalResults)
searchResultElems, err := results.All()
if err != nil {
@@ -198,14 +199,14 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
// Get URL
link, err := resEl.Element("a")
if err != nil {
- logrus.Debug("No link found")
+ gogl.logger.Debug("Missing link")
continue
}
link.MoveMouseOut()
href, err := link.Property("href")
if err != nil {
- logrus.Debug("No `href` tag found")
+ gogl.logger.Debug("Missing href")
continue
}
srchRes.URL = href.String()
@@ -223,17 +224,17 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
// 2. Parse answer boxes
answerEls, err := resEl.Page().Search("div[data-hveid][data-ulkwtsb] div[data-q]")
if err != nil {
- logrus.Debugf("Error while parsing answer box 1: %s", err.Error())
+ gogl.logger.Debug("Answer parsing failed: %s", err.Error())
continue
}
answers, err := answerEls.All()
if err != nil {
- logrus.Debugf("Error while parsing answer box 2: %s", err.Error())
+ gogl.logger.Debug("Answer parsing failed: %s", err.Error())
continue
}
- logrus.Infof("%d answers found", len(answers))
+ gogl.logger.Info("Found %d answers", len(answers))
// Unvail answer contents
for _, answ := range answers {
@@ -246,21 +247,21 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
for i, answ := range answers {
answerText := strings.Split(answ.MustText(), "\n")
if len(answerText) < 2 {
- logrus.Debugf("Short answer text: %s", answerText)
+ gogl.logger.Debug("Short answer text: %s", answerText)
continue
}
// Get URL
link, err := answ.Element("a")
if err != nil {
- logrus.Debug("No answer link found")
+ gogl.logger.Debug("Missing answer link")
continue
}
link.MoveMouseOut()
href, err := link.Property("href")
if err != nil {
- logrus.Debug("No answer `href` tag found")
+ gogl.logger.Debug("Missing answer href")
continue
}
srchRes.URL = href.String()
@@ -334,7 +335,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
}
func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Google Image search, query: %+v", query)
+ gogl.logger.Debug("Starting image search, query: %+v", query)
searchResultsMap := map[string]core.SearchResult{}
url, err := BuildImageURL(query)
@@ -356,14 +357,14 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
if err != nil {
- logrus.Errorf("Cannot parse search results: %s", err)
+ gogl.logger.Error("Cannot parse search results: %s", err)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
}
// Check why no results
if results == nil {
if gogl.checkCaptcha(page) {
- logrus.Errorf("Google captcha occurred during: %s", url)
+ gogl.logger.Error("Captcha detected: %s", url)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
}
return *core.ConvertSearchResultsMap(searchResultsMap), err
@@ -382,13 +383,13 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
// TODO: parse AF_initDataCallback to optimize instead of this?
err := r.Click(proto.InputMouseButtonRight, 1)
if err != nil {
- logrus.Error("Error clicking")
+ gogl.logger.Error("Click failed")
continue
}
dataVed, err := r.Attribute("data-ved")
if err != nil {
- logrus.Error("Cannot find `data-ved` attr")
+ gogl.logger.Error("Missing data-ved attribute")
continue
}
@@ -405,25 +406,25 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
linkText, err := link.Property("href")
if err != nil {
- logrus.Error("No `href` tag found")
+ gogl.logger.Error("Missing href")
}
imgSrc, err := parseSourceImageURL(linkText.String())
if err != nil {
- logrus.Errorf("Cannot parse image href: %v", err)
+ gogl.logger.Error("Failed to parse image URL: %v", err)
continue
}
// Get title
titleTag, err := r.Element("h3")
if err != nil {
- logrus.Error("No `h3` tag found")
+ gogl.logger.Error("Missing h3 tag")
continue
}
title, err := titleTag.Text()
if err != nil {
- logrus.Error("Cannot extract text from title")
+ gogl.logger.Error("Failed to extract title")
title = "No title"
}
@@ -446,7 +447,7 @@ func (gogl *Google) close(page *rod.Page) {
if !gogl.Browser.LeavePageOpen {
err := page.Close()
if err != nil {
- logrus.Error(err)
+ gogl.logger.Debug("Page close error: %v", err)
}
}
}
diff --git a/yandex/search.go b/yandex/search.go
index d31b349..8807125 100644
--- a/yandex/search.go
+++ b/yandex/search.go
@@ -8,7 +8,6 @@ import (
"github.com/go-rod/rod"
"github.com/karust/openserp/core"
- "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
@@ -38,12 +37,14 @@ type Yandex struct {
core.Browser
core.SearchEngineOptions
pageSleep time.Duration // Sleep between pages
+ logger *core.EngineLogger
}
func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex {
yand := Yandex{Browser: browser}
opts.Init()
yand.SearchEngineOptions = opts
+ yand.logger = core.NewEngineLogger("Yandex")
yand.pageSleep = time.Second * 1
return &yand
@@ -80,19 +81,19 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
}
linkText, err := link.Property("href")
if err != nil {
- logrus.Error("No `href` tag found")
+ yand.logger.Error("Missing href")
}
// Get title
titleTag, err := link.Element("h2")
if err != nil {
- logrus.Error("No title `h2` tag found")
+ yand.logger.Error("Missing h2 title")
continue
}
title, err := titleTag.Text()
if err != nil {
- logrus.Error("Cannot extract text from title")
+ yand.logger.Error("Failed to extract title")
title = "No title"
}
@@ -100,7 +101,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
descTag, err := r.Element(`span.OrganicTextContentSpan`)
desc := ""
if err != nil {
- logrus.Trace("No description `span.OrganicTextContentSpan` tag found")
+ yand.logger.Debug("No description")
} else {
desc = descTag.MustText()
}
@@ -113,7 +114,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
}
func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Yandex search, query: %+v", query)
+ yand.logger.Debug("Starting search, query: %+v", query)
allResults := []core.SearchResult{}
searchPage := 0
@@ -133,7 +134,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item")
if err != nil {
defer page.Close()
- logrus.Errorf("Cannot parse search results: %s", err)
+ yand.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrSearchTimeout
}
@@ -142,9 +143,9 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
defer page.Close()
if yand.isNoResults(page) {
- logrus.Errorf("No results found")
+ yand.logger.Warn("No results found")
} else if yand.isCaptcha(page) {
- logrus.Errorf("Yandex captcha occurred during: %s", url)
+ yand.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
}
break
@@ -152,7 +153,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
elements, err := searchRes.All()
if err != nil {
- logrus.Errorf("Cannot get all elements from search results: %s", err)
+ yand.logger.Error("Cannot get search elements: %s", err)
break
}
@@ -165,18 +166,19 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
// Close tab before opening new one during the cycle
err = page.Close()
if err != nil {
- logrus.Error(err)
+ yand.logger.Debug("Page close error: %v", err)
}
}
time.Sleep(yand.pageSleep)
}
+ yand.logger.Info("Search completed: %d results", len(allResults))
return core.DeduplicateResults(allResults), nil
}
func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
- logrus.Tracef("Start Yandex image search, query: %+v", query)
+ yand.logger.Debug("Starting image search, query: %+v", query)
searchResults := []core.SearchResult{}
@@ -193,7 +195,7 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
return nil, err
}
- if !yand.LeavePageOpen {
+ if !yand.Browser.LeavePageOpen {
defer page.Close()
}
@@ -203,16 +205,16 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
results, err := page.Timeout(yand.Timeout).Search("div[role='main'] div[data-state]")
if err != nil {
- logrus.Errorf("Cannot find search results: %s", err)
+ yand.logger.Error("Cannot find search results: %s", err)
}
// Check why no results
if results == nil {
if yand.isCaptcha(page) {
- logrus.Errorf("Yandex captcha occurred during: %s", url)
+ yand.logger.Error("Captcha detected: %s", url)
return searchResults, core.ErrCaptcha
} else if yand.isNoResults(page) {
- logrus.Errorf("No results found")
+ yand.logger.Warn("No results found")
}
return searchResults, core.ErrSearchTimeout
}
@@ -239,7 +241,7 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
searchResults = append(searchResults, res)
}
- if !yand.LeavePageOpen {
+ if !yand.Browser.LeavePageOpen {
page.Close()
}
}