package core import ( "context" "crypto/tls" "fmt" "io" "net" "net/http" "net/url" "strings" "time" "github.com/corpix/uarand" utls "github.com/refraction-networking/utls" ) const rawHTTPTimeout = 30 * time.Second // SetAcceptLanguageHeader sets the Accept-Language header from a lang code. // No-op when the code has no language subtag. func SetAcceptLanguageHeader(req *http.Request, langCode string) { if req == nil { return } if value := BuildAcceptLanguageHeader(langCode); value != "" { req.Header.Set("Accept-Language", value) } } // DrainAndCloseResponse drains unread bytes before closing so HTTP transports // can safely reuse connections when callers don't consume the full body. func DrainAndCloseResponse(resp *http.Response) { if resp == nil || resp.Body == nil { return } _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() } // RawSearchRequest builds and executes a raw-mode SERP HTTP GET. It uses the // shared raw HTTP client (TLS fingerprinting, network usage tracking, proxy // support), randomizes the User-Agent, and applies the Accept-Language header // derived from the query locale. The caller owns the returned response and // must drain/close it (see DrainAndCloseResponse). func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) { client, err := NewRawHTTPClient(query) if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", uarand.GetRandom()) SetAcceptLanguageHeader(req, query.LangCode) return client.Do(req) } func ReadRawSearchBody(resp *http.Response) ([]byte, error) { if resp == nil { return nil, fmt.Errorf("%w: nil raw search response", ErrEngineInternal) } if err := ClassifySearchHTTPStatus(resp.StatusCode); err != nil { return nil, err } return io.ReadAll(resp.Body) } func ClassifySearchHTTPStatus(status int) error { switch status { case 0: return nil case http.StatusForbidden, http.StatusUnauthorized: return ErrBlocked case http.StatusTooManyRequests: return ErrRateLimited } if status >= 500 { return fmt.Errorf("%w: search engine returned HTTP %d", ErrBlocked, status) } if status < 200 || status >= 300 { return fmt.Errorf("%w: search engine returned HTTP %d", ErrParser, status) } return nil } func NewRawHTTPClient(query Query) (*http.Client, error) { transport, err := newRawTransport(query) if err != nil { return nil, err } roundTripper := http.RoundTripper(transport) if transport.Proxy != nil { roundTripper = proxyErrorTransport{base: roundTripper} } return &http.Client{ Transport: roundTripper, Timeout: rawHTTPTimeout, }, nil } func newRawTransport(query Query) (*http.Transport, error) { transport := &http.Transport{ DialContext: dialNetworkUsageConn, } if query.Insecure { transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} } proxyURL, err := NormalizeProxyURL(query.ProxyURL) if err != nil { return nil, err } if proxyURL != "" { parsed, err := url.Parse(proxyURL) if err != nil { return nil, err } // Keep proxied requests on the standard transport path so SOCKS5/SOCKS5H // resolution and routing are handled by the configured proxy correctly. transport.Proxy = http.ProxyURL(parsed) return transport, nil } transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { rawConn, err := dialNetworkUsageConn(ctx, network, addr) if err != nil { return nil, err } hostname := strings.Split(addr, ":")[0] config := &utls.Config{ ServerName: hostname, InsecureSkipVerify: query.Insecure, NextProtos: []string{"http/1.1"}, } uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto) if err := uconn.BuildHandshakeState(); err != nil { rawConn.Close() return nil, err } forceHTTP1ALPN(uconn) if err := uconn.Handshake(); err != nil { rawConn.Close() return nil, err } return uconn, nil } return transport, nil } func forceHTTP1ALPN(conn *utls.UConn) { for _, ext := range conn.Extensions { if alpn, ok := ext.(*utls.ALPNExtension); ok { alpn.AlpnProtocols = []string{"http/1.1"} return } } conn.Extensions = append(conn.Extensions, &utls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}}) } func dialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{} conn, err := dialer.DialContext(ctx, network, addr) if err != nil { return nil, err } return networkUsageConn{Conn: conn, ctx: ctx}, nil } type networkUsageConn struct { net.Conn ctx context.Context } func (c networkUsageConn) Read(p []byte) (int, error) { n, err := c.Conn.Read(p) AddNetworkBytes(c.ctx, int64(n)) return n, err } type proxyErrorTransport struct { base http.RoundTripper } func (t proxyErrorTransport) RoundTrip(req *http.Request) (*http.Response, error) { resp, err := t.base.RoundTrip(req) if err != nil { return nil, classifyProxyNetworkError(err) } if resp != nil && resp.StatusCode == http.StatusProxyAuthRequired { DrainAndCloseResponse(resp) return nil, classifyProxyNetworkError(ErrProxyAuth) } return resp, nil }