From be022a3bb85dca34336896b8263e4bcbac1a4119 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Sat, 24 Jun 2023 22:26:36 +0300 Subject: [PATCH] Add leave head cmd --- baidu/search.go | 24 ++++++++++++++---------- cmd/root.go | 10 ++++++---- cmd/search.go | 16 ++++++++-------- cmd/serve.go | 11 ++++++++--- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/baidu/search.go b/baidu/search.go index 0da9552..cdb02e0 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -1,7 +1,6 @@ package baidu import ( - "fmt" "strings" "time" @@ -33,9 +32,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { if err != nil { return nil, err } + page := baid.Navigate(url) - results, err := page.Search("div.c-container.new-pmd") + results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd") if err != nil { return nil, err } @@ -52,29 +52,33 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { continue } linkText, err := link.Property("href") + if err != nil { + logrus.Error("No `href` tag found") + } // Get title title, err := link.Text() if err != nil { + logrus.Error("Cannot extract text from title") title = "No title" - fmt.Println(2, err) } // Get description - desc := "No description found" - desc, err = r.Text() + desc, err := r.Text() if err != nil { - continue + desc = "" } desc = strings.ReplaceAll(desc, title, "") - gR := core.SearchResult{Rank: i, URL: linkText.String(), Title: title, Description: desc} + gR := core.SearchResult{Rank: i + 1, URL: linkText.String(), Title: title, Description: desc} searchResults = append(searchResults, gR) } - err = page.Close() - if err != nil { - logrus.Error(err) + if !baid.LeavePageOpen { + err = page.Close() + if err != nil { + logrus.Error(err) + } } return searchResults, nil diff --git a/cmd/root.go b/cmd/root.go index 5f2dba7..11d5725 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( ) const ( - version = "0.1.0" + version = "0.1.1" defaultConfigFilename = "config" envPrefix = "OPENSERP" replaceHyphenWithCamelCase = false @@ -23,6 +23,7 @@ type AppConfig struct { Timeout int ConfigPath string IsBrowserHead bool `mapstructure:"head"` + IsLeaveHead bool `mapstructure:"leave_head"` IsLeakless bool `mapstructure:"leakless"` IsDebug bool `mapstructure:"debug"` IsVerbose bool `mapstructure:"verbose"` @@ -38,10 +39,9 @@ var RootCmd = &cobra.Command{ Version: version, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { core.InitLogger(appConf.IsVerbose, appConf.IsDebug) + err := initializeConfig(cmd) logrus.Debugf("Config: %+v", appConf) - - // Bind cobra and viper - return initializeConfig(cmd) + return err }, // Run: func(cmd *cobra.Command, args []string) { // // Working with OutOrStdout/OutOrStderr allows us to unit test our command easier @@ -96,10 +96,12 @@ func initializeConfig(cmd *cobra.Command) error { func init() { RootCmd.PersistentFlags().IntVarP(&appConf.Port, "port", "p", 7070, "Port number to run server") RootCmd.PersistentFlags().StringVarP(&appConf.Host, "host", "a", "127.0.0.1", "Host address to run server") + RootCmd.PersistentFlags().IntVarP(&appConf.Timeout, "timeout", "t", 30, "Timeout to fail request") RootCmd.PersistentFlags().StringVarP(&appConf.ConfigPath, "config", "c", "./config.yaml", "Configuration file path") RootCmd.PersistentFlags().BoolVarP(&appConf.IsVerbose, "verbose", "v", false, "Use verbose output") RootCmd.PersistentFlags().BoolVarP(&appConf.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser") RootCmd.PersistentFlags().BoolVarP(&appConf.IsBrowserHead, "head", "", false, "Enable browser UI") RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search") RootCmd.PersistentFlags().BoolVarP(&appConf.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") + RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") } diff --git a/cmd/search.go b/cmd/search.go index 96c61b1..a4c6c24 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -25,12 +25,11 @@ var searchCMD = &cobra.Command{ func search(cmd *cobra.Command, args []string) { var err error engineType := args[0] - results := []core.SearchResult{} - query := core.Query{ Text: args[1], Limit: 10, } + results := []core.SearchResult{} if appConf.IsRawRequests { results, err = searchRaw(engineType, query) @@ -55,14 +54,15 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er var engine core.SearchEngine opts := core.BrowserOpts{ - IsHeadless: !appConf.IsBrowserHead, // Disable headless if browser head mode is set - IsLeakless: appConf.IsLeakless, - Timeout: time.Second * time.Duration(appConf.Timeout), + IsHeadless: !appConf.IsBrowserHead, // Disable headless if browser head mode is set + IsLeakless: appConf.IsLeakless, + Timeout: time.Second * time.Duration(appConf.Timeout), + LeavePageOpen: appConf.IsLeaveHead, } - //if appConf.IsDebug { - // opts.IsHeadless = false - //} + if appConf.IsDebug { + opts.IsHeadless = false + } browser, err := core.NewBrowser(opts) if err != nil { diff --git a/cmd/serve.go b/cmd/serve.go index 1632a62..006bdd3 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -21,9 +21,14 @@ var serveCMD = &cobra.Command{ func serve(cmd *cobra.Command, args []string) { opts := core.BrowserOpts{ - IsHeadless: !appConf.IsBrowserHead, - IsLeakless: appConf.IsLeakless, - Timeout: time.Second * time.Duration(appConf.Timeout), + IsHeadless: !appConf.IsBrowserHead, // Disable headless if browser head mode is set + IsLeakless: appConf.IsLeakless, + Timeout: time.Second * time.Duration(appConf.Timeout), + LeavePageOpen: appConf.IsLeaveHead, + } + + if appConf.IsDebug { + opts.IsHeadless = false } browser, err := core.NewBrowser(opts)