diff --git a/Dockerfile b/Dockerfile index e6571be..707f9ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ RUN go build -o /app/openserp . FROM zenika/alpine-chrome:with-chromedriver COPY --from=builder /app/openserp /usr/local/bin/openserp +ADD config.yaml /usr/src/app ENTRYPOINT ["openserp"] diff --git a/cmd/root.go b/cmd/root.go index 05042f2..e176545 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,7 +1,9 @@ package cmd import ( + "errors" "fmt" + "strings" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" @@ -11,45 +13,55 @@ import ( ) const ( - version = "0.2.1" - defaultConfigFilename = "config" - envPrefix = "OPENSERP" - replaceHyphenWithCamelCase = false + version = "0.2.1" + defaultConfigFilename = "config" + envPrefix = "OPENSERP" ) -type AppConfig struct { - Host string - Port int - Timeout int - ConfigPath string - IsBrowserHead bool `mapstructure:"head"` - IsLeaveHead bool `mapstructure:"leave_head"` - IsLeakless bool `mapstructure:"leakless"` - IsDebug bool `mapstructure:"debug"` - IsVerbose bool `mapstructure:"verbose"` - IsRawRequests bool `mapstructure:"raw_requests"` - GoogleConfig core.SearchEngineOptions `mapstructure:"google"` - YandexConfig core.SearchEngineOptions `mapstructure:"yandex"` - BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"` +type Config struct { + App AppConfig `mapstructure:"app"` + GoogleConfig core.SearchEngineOptions `mapstructure:"google"` + YandexConfig core.SearchEngineOptions `mapstructure:"yandex"` + BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"` } -var appConf = AppConfig{} +type AppConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Timeout int `mapstructure:"timeout"` + ConfigPath string `mapstructure:"config_path"` + IsBrowserHead bool `mapstructure:"head"` + IsLeaveHead bool `mapstructure:"leave_head"` + IsLeakless bool `mapstructure:"leakless"` + IsDebug bool `mapstructure:"debug"` + IsVerbose bool `mapstructure:"verbose"` + IsRawRequests bool `mapstructure:"raw_requests"` +} + +var config = Config{} var RootCmd = &cobra.Command{ - Use: "openserp", - Short: "Open SERP", - Long: `Search via Google, Yandex and Baidu`, - Version: version, + Use: "openserp", + Short: "Open SERP", + Long: `Get [Google, Yandex, Baidu] search engine results via API or CLI.`, + Version: version, + SilenceUsage: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - core.InitLogger(appConf.IsVerbose, appConf.IsDebug) + core.InitLogger(config.App.IsVerbose, config.App.IsDebug) + err := initializeConfig(cmd) - logrus.Debugf("Config: %+v", appConf) - return err + if err != nil { + return err + } + + logrus.Debugf("Final config: %+v", config) + return nil }, + // Run: func(cmd *cobra.Command, args []string) { // // Working with OutOrStdout/OutOrStderr allows us to unit test our command easier // //out := cmd.OutOrStdout() - // logrus.Trace("Config:", appConf) + // logrus.Trace("Config:", config) // }, } @@ -57,14 +69,10 @@ var RootCmd = &cobra.Command{ func bindFlags(cmd *cobra.Command, vpr *viper.Viper) { cmd.Flags().VisitAll(func(flg *pflag.Flag) { configName := "app." + flg.Name - //if replaceHyphenWithCamelCase { - // configName = strings.ReplaceAll(f.Name, "-", "") - //} // Apply viper config value to the flag if viper has a value - if !flg.Changed && vpr.IsSet(configName) { - val := vpr.Get(configName) - cmd.Flags().Set(flg.Name, fmt.Sprintf("%v", val)) + if flg.Changed { + vpr.Set(configName, flg.Value) } }) } @@ -77,34 +85,47 @@ func initializeConfig(cmd *cobra.Command) error { v.SetConfigName(defaultConfigFilename) v.AddConfigPath(".") - // Return an error if we cannot parse the config file. - if err := v.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); !ok { - return err + // 1. Config. Return an error if we cannot parse the config file. + err := v.ReadInConfig() + if err != nil { + err = errors.New(fmt.Sprintf("Cannot read config: %v", err)) + logrus.Warn(err) + } + + // 2. Env. Bind environment variables to their equivalent keys with underscores + for _, key := range v.AllKeys() { + envKey := envPrefix + "_" + strings.ToUpper(strings.ReplaceAll(key, ".", "_")) + err := v.BindEnv(key, envKey) + if err != nil { + logrus.Errorf("Unable to bind ENV valye: %v", err) } } - v.SetEnvPrefix(envPrefix) - - // Bind environment variables to their equivalent keys with underscores - //v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - v.AutomaticEnv() - - // Bind the current command's flags to viper + // 3. Cmd flags. Bind the current command's flags to viper bindFlags(cmd, v) + // Dump Viper values to config struct + err = v.Unmarshal(&config) + if err != nil { + return errors.New(fmt.Sprintf("Cannot unmarshall config: %v", err)) + } + + if config.App.IsDebug { + logrus.Debug("Viper config:") + v.Debug() + } return nil } func init() { - RootCmd.PersistentFlags().IntVarP(&appConf.Port, "port", "p", 7070, "Port number to run server") - RootCmd.PersistentFlags().StringVarP(&appConf.Host, "host", "a", "127.0.0.1", "Host address to run server") - RootCmd.PersistentFlags().IntVarP(&appConf.Timeout, "timeout", "t", 30, "Timeout to fail request") - RootCmd.PersistentFlags().StringVarP(&appConf.ConfigPath, "config", "c", "./config.yaml", "Configuration file path") - RootCmd.PersistentFlags().BoolVarP(&appConf.IsVerbose, "verbose", "v", false, "Use verbose output") - RootCmd.PersistentFlags().BoolVarP(&appConf.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser") - RootCmd.PersistentFlags().BoolVarP(&appConf.IsBrowserHead, "head", "", false, "Enable browser UI") - RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search") - RootCmd.PersistentFlags().BoolVarP(&appConf.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") - RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") + RootCmd.PersistentFlags().IntVarP(&config.App.Port, "port", "p", 7070, "Port number to run server") + RootCmd.PersistentFlags().StringVarP(&config.App.Host, "host", "a", "127.0.0.1", "Host address to run server") + RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request") + RootCmd.PersistentFlags().StringVarP(&config.App.ConfigPath, "config", "c", "", "Configuration file path") + RootCmd.PersistentFlags().BoolVarP(&config.App.IsVerbose, "verbose", "v", false, "Use verbose output") + RootCmd.PersistentFlags().BoolVarP(&config.App.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser") + RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI") + RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search") + RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") + RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") } diff --git a/cmd/search.go b/cmd/search.go index 8fe5090..60fef09 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -31,7 +31,7 @@ func search(cmd *cobra.Command, args []string) { } results := []core.SearchResult{} - if appConf.IsRawRequests { + if config.App.IsRawRequests { results, err = searchRaw(engineType, query) } else { results, err = searchBrowser(engineType, query) @@ -54,13 +54,13 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er var engine core.SearchEngine opts := core.BrowserOpts{ - IsHeadless: !appConf.IsBrowserHead, // Disable headless if browser head mode is set - IsLeakless: appConf.IsLeakless, - Timeout: time.Second * time.Duration(appConf.Timeout), - LeavePageOpen: appConf.IsLeaveHead, + IsHeadless: !config.App.IsBrowserHead, // Disable headless if browser head mode is set + IsLeakless: config.App.IsLeakless, + Timeout: time.Second * time.Duration(config.App.Timeout), + LeavePageOpen: config.App.IsLeaveHead, } - if appConf.IsDebug { + if config.App.IsDebug { opts.IsHeadless = false } @@ -71,11 +71,11 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er switch strings.ToLower(engineType) { case "yandex": - engine = yandex.New(*browser, appConf.YandexConfig) + engine = yandex.New(*browser, config.YandexConfig) case "google": - engine = google.New(*browser, appConf.GoogleConfig) + engine = google.New(*browser, config.GoogleConfig) case "baidu": - engine = baidu.New(*browser, appConf.BaiduConfig) + engine = baidu.New(*browser, config.BaiduConfig) default: logrus.Infof("No `%s` search engine found", engineType) } diff --git a/cmd/serve.go b/cmd/serve.go index 2c5503a..00d34fd 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -21,13 +21,13 @@ var serveCMD = &cobra.Command{ func serve(cmd *cobra.Command, args []string) { opts := core.BrowserOpts{ - IsHeadless: !appConf.IsBrowserHead, // Disable headless if browser head mode is set - IsLeakless: appConf.IsLeakless, - Timeout: time.Second * time.Duration(appConf.Timeout), - LeavePageOpen: appConf.IsLeaveHead, + IsHeadless: !config.App.IsBrowserHead, // Disable headless if browser head mode is set + IsLeakless: config.App.IsLeakless, + Timeout: time.Second * time.Duration(config.App.Timeout), + LeavePageOpen: config.App.IsLeaveHead, } - if appConf.IsDebug { + if config.App.IsDebug { opts.IsHeadless = false } @@ -36,11 +36,11 @@ func serve(cmd *cobra.Command, args []string) { logrus.Error(err) } - yand := yandex.New(*browser, appConf.YandexConfig) - gogl := google.New(*browser, appConf.GoogleConfig) - baidu := baidu.New(*browser, appConf.BaiduConfig) + yand := yandex.New(*browser, config.YandexConfig) + gogl := google.New(*browser, config.GoogleConfig) + baidu := baidu.New(*browser, config.BaiduConfig) - serv := core.NewServer(appConf.Host, appConf.Port, gogl, yand, baidu) + serv := core.NewServer(config.App.Host, config.App.Port, gogl, yand, baidu) serv.Listen() } diff --git a/config.yaml b/config.yaml index 42507d6..57bc6d1 100644 --- a/config.yaml +++ b/config.yaml @@ -6,3 +6,15 @@ app: timeout: 15 head: false leakless: false + +google: + rate_requests: 4 # Number of requests per Minute + rate_burst: 2 # Number of non-ratelimited requests per Minute + +yandex: + rate_requests: 4 + rate_burst: 2 + +baidu: + rate_requests: 4 + rate_burst: 2 diff --git a/docker-compose.yaml b/docker-compose.yaml index 65192bd..9bb7319 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,4 +7,12 @@ services: context: . ports: - 7000:7000 - command: serve -a 0.0.0.0 -p 7000 -v -l \ No newline at end of file + command: serve -l + #volumes: + # - ./config.yaml:/usr/src/app/config.yaml + environment: + OPENSERP_APP_HOST: "0.0.0.0" + OPENSERP_APP_PORT: 7000 + OPENSERP_BAIDU_RATE_REQUESTS: 6 # Number of requests per Minute + OPENSERP_BAIDU_RATE_BURST: 2 # Number of non-ratelimited requests per Minute + \ No newline at end of file