zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-14 04:36:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Support http and socks5 proxy with auth for raw search

Browse Source
This commit is contained in:
Pachakutiq
2025-04-08 16:06:41 +08:00
parent 205f722ddc
commit 84e2d46c4b
6 changed files with 87 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
baidu/search_raw.go
View File

@@ -1,9 +1,12 @@
package baidu
import (
"crypto/tls"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
@@ -11,8 +14,27 @@ import (
"github.com/sirupsen/logrus"
)
func baiduRequest(searchURL string) (*http.Response, error) {
baseClient := &http.Client{}
func baiduRequest(searchURL string, query core.Query) (*http.Response, error) {
// Create HTTP transport with proxy
transport := &http.Transport{}
if query.ProxyURL != "" {
proxyUrl, err := url.Parse(query.ProxyURL)
if err != nil {
return nil, err
}
transport.Proxy = http.ProxyURL(proxyUrl)
}
// Set insecure TLS
if query.Insecure {
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
baseClient := &http.Client{
Transport: transport,
Timeout: time.Second * 10,
}
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
return nil, err
@@ -27,7 +49,7 @@ func baiduRequest(searchURL string) (*http.Response, error) {
}
func baiduResultParser(response *http.Response) ([]core.SearchResult, error) {
doc, err := goquery.NewDocumentFromResponse(response)
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return nil, err
}
@@ -79,7 +101,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
}
logrus.Debugf("Baidu URL built: %s", googleURL)
res, err := baiduRequest(googleURL)
res, err := baiduRequest(googleURL, query)
if err != nil {
return nil, err
}

4
cmd/root.go
View File

@@ -40,6 +40,8 @@ type AppConfig struct {
IsDebug bool `mapstructure:"debug"`
IsVerbose bool `mapstructure:"verbose"`
IsRawRequests bool `mapstructure:"raw_requests"`
ProxyURL string `mapstructure:"proxy"`
Insecure bool `mapstructure:"insecure"`
}
var config = Config{}
@@ -133,4 +135,6 @@ func init() {
RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "", "", "HTTP proxy URL (e.g. http://user:pass@proxy:8080)")
RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
}

6
cmd/search.go
View File

@@ -26,8 +26,10 @@ func search(cmd *cobra.Command, args []string) {
var err error
engineType := args[0]
query := core.Query{
Text: args[1],
Limit: 10,
Text: args[1],
Limit: 10,
ProxyURL: config.App.ProxyURL,
Insecure: config.App.Insecure,
}
results := []core.SearchResult{}

2
core/common.go
View File

@@ -41,6 +41,8 @@ type Query struct {
Site string // Search site
Limit int // Limit the number of results
Answers bool // Include question and answers from SERP page to results with negative indexes
ProxyURL string // Proxy URL for raw requests
Insecure bool // Allow insecure TLS connections
}
func (q Query) IsEmpty() bool {

29
google/search_raw.go
View File

@@ -1,8 +1,11 @@
package google
import (
"crypto/tls"
"net/http"
"net/url"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
@@ -10,8 +13,26 @@ import (
"github.com/sirupsen/logrus"
)
func googleRequest(searchURL string) (*http.Response, error) {
baseClient := &http.Client{}
func googleRequest(searchURL string, query core.Query) (*http.Response, error) {
// Create HTTP transport with proxy
transport := &http.Transport{}
if query.ProxyURL != "" {
proxyUrl, err := url.Parse(query.ProxyURL)
if err != nil {
return nil, err
}
transport.Proxy = http.ProxyURL(proxyUrl)
}
// Set insecure TLS
if query.Insecure {
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
baseClient := &http.Client{
Transport: transport,
Timeout: time.Second * 10,
}
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
return nil, err
@@ -26,7 +47,7 @@ func googleRequest(searchURL string) (*http.Response, error) {
}
func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
doc, err := goquery.NewDocumentFromResponse(response)
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return nil, err
}
@@ -77,7 +98,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
}
logrus.Debugf("Google URL built: %s", googleURL)
res, err := googleRequest(googleURL)
res, err := googleRequest(googleURL, query)
if err != nil {
return nil, err
}

30
yandex/search_raw.go
View File

@@ -1,8 +1,11 @@
package yandex
import (
"crypto/tls"
"net/http"
"net/url"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
@@ -10,8 +13,27 @@ import (
"github.com/sirupsen/logrus"
)
func yandexRequest(searchURL string) (*http.Response, error) {
baseClient := &http.Client{}
func yandexRequest(searchURL string, query core.Query) (*http.Response, error) {
// Create HTTP transport with proxy
transport := &http.Transport{}
if query.ProxyURL != "" {
proxyUrl, err := url.Parse(query.ProxyURL)
if err != nil {
return nil, err
}
transport.Proxy = http.ProxyURL(proxyUrl)
}
// Set insecure TLS
if query.Insecure {
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
baseClient := &http.Client{
Transport: transport,
Timeout: time.Second * 10,
}
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
return nil, err
@@ -26,7 +48,7 @@ func yandexRequest(searchURL string) (*http.Response, error) {
}
func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
doc, err := goquery.NewDocumentFromResponse(response)
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return nil, err
}
@@ -77,7 +99,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
}
logrus.Debugf("Yandex URL built: %s", googleURL)
res, err := yandexRequest(googleURL)
res, err := yandexRequest(googleURL, query)
if err != nil {
return nil, err
}