openserp/core/http_client.go

package core

import (
	"bytes"
	"context"
	"fmt"
	"hash/fnv"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	fhttp "github.com/bogdanfinn/fhttp"
	tlsclient "github.com/bogdanfinn/tls-client"
	"github.com/bogdanfinn/tls-client/profiles"
	browserprofile "github.com/karust/openserp/core/browser"
)

const rawHTTPTimeout = 30 * time.Second
const rawHTTPClientCacheMaxEntries = 64

// fallbackRawUserAgent guards against tls-client's "Go-http-client" UA leaking.
const fallbackRawUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"

// rawChromeProfiles pairs each TLS fingerprint with its Chrome major so the UA
// and Sec-CH-UA stay coherent. We round-robin the major here. Add presets as
// tls-client ships them.
var rawChromeProfiles = []struct {
	major int
	tls   profiles.ClientProfile
}{
	{133, profiles.Chrome_133},
	{144, profiles.Chrome_144},
	{146, profiles.Chrome_146},
}

// pickRawChromeProfile hashes salt to a stable but varied fingerprint.
func pickRawChromeProfile(salt string) (int, profiles.ClientProfile) {
	h := fnv.New32a()
	_, _ = h.Write([]byte(salt))
	p := rawChromeProfiles[int(h.Sum32())%len(rawChromeProfiles)]
	return p.major, p.tls
}

// rawHeaderOrder controls request header order; tls-client profiles do not.
var rawHeaderOrder = []string{
	"host",
	"user-agent",
	"accept",
	"accept-language",
	"accept-encoding",
	"upgrade-insecure-requests",
	"sec-ch-ua",
	"sec-ch-ua-mobile",
	"sec-ch-ua-platform",
	"sec-fetch-site",
	"sec-fetch-mode",
	"sec-fetch-user",
	"sec-fetch-dest",
}

var rawHTTPClientCache = struct {
	sync.Mutex
	clients map[rawHTTPClientKey]*rawHTTPClientEntry
}{
	clients: map[rawHTTPClientKey]*rawHTTPClientEntry{},
}

type rawHTTPClientKey struct {
	proxyURL             string
	profile              string
	insecure             bool
	guardPrivateNetworks bool
}

type rawHTTPClientEntry struct {
	client   tlsclient.HttpClient
	lastUsed time.Time
}

type rawRequestProfile struct {
	id             string
	userAgent      string
	acceptLanguage string
	secCHUA        string
	platform       string
	mobile         bool
	tlsProfile     profiles.ClientProfile
}

// DrainAndCloseResponse drains then closes the body so the connection can be reused.
func DrainAndCloseResponse(resp *http.Response) {
	if resp == nil || resp.Body == nil {
		return
	}

	_, _ = io.Copy(io.Discard, resp.Body)
	_ = resp.Body.Close()
}

// RawSearchRequest executes a raw-mode GET and returns a stdlib response.
func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) {
	profile := rawRequestProfileFor(ctx, query)
	client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile)
	if err != nil {
		return nil, err
	}
	SetBrowserProfileID(ctx, profile.id)

	// Guarded path validates every hop, including the first.
	if query.GuardPrivateNetworks {
		return doGuardedRawRequest(ctx, client, searchURL, profile, query)
	}
	return doRawRequest(ctx, client, searchURL, profile, query)
}

// doRawRequest issues one GET and converts the response at the boundary.
func doRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) {
	req, err := fhttp.NewRequestWithContext(ctx, fhttp.MethodGet, searchURL, nil)
	if err != nil {
		return nil, err
	}
	applyRawRequestHeaders(req, profile)
	return execRawRequest(ctx, client, req, rawRequestUsesProxy(query))
}

// execRawRequest runs the request and converts proxy errors and the response.
func execRawRequest(ctx context.Context, client tlsclient.HttpClient, req *fhttp.Request, proxied bool) (*http.Response, error) {
	resp, err := client.Do(req)
	if err != nil {
		if proxied {
			return nil, classifyProxyNetworkError(err)
		}
		return nil, err
	}
	return convertRawResponse(ctx, resp), nil
}

const maxGuardedRedirects = 10

// doGuardedRawRequest validates every redirect hop before fetching it.
func doGuardedRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) {
	current := searchURL
	for hop := 0; ; hop++ {
		if err := ValidatePublicHTTPURL(ctx, current); err != nil {
			return nil, err
		}
		resp, err := doRawRequest(ctx, client, current, profile, query)
		if err != nil {
			return nil, err
		}
		location, ok := redirectLocation(resp)
		if !ok {
			return resp, nil
		}
		if hop >= maxGuardedRedirects {
			DrainAndCloseResponse(resp)
			return nil, fmt.Errorf("%w: stopped after %d redirects", ErrEngineInternal, maxGuardedRedirects)
		}
		next, err := resolveRedirectURL(current, location)
		if err != nil {
			DrainAndCloseResponse(resp)
			return nil, err
		}
		DrainAndCloseResponse(resp)
		current = next
	}
}

func redirectLocation(resp *http.Response) (string, bool) {
	if resp == nil {
		return "", false
	}
	switch resp.StatusCode {
	case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
		http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
		location := strings.TrimSpace(resp.Header.Get("Location"))
		return location, location != ""
	default:
		return "", false
	}
}

func resolveRedirectURL(base, location string) (string, error) {
	baseURL, err := url.Parse(base)
	if err != nil {
		return "", err
	}
	locURL, err := url.Parse(location)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(locURL).String(), nil
}

func ReadRawSearchBody(resp *http.Response) ([]byte, error) {
	if resp == nil {
		return nil, fmt.Errorf("%w: nil raw search response", ErrEngineInternal)
	}
	if err := ClassifySearchHTTPStatus(resp.StatusCode); err != nil {
		return nil, err
	}
	return io.ReadAll(resp.Body)
}

func ClassifySearchHTTPStatus(status int) error {
	switch status {
	case 0:
		return nil
	case http.StatusForbidden, http.StatusUnauthorized:
		return ErrBlocked
	case http.StatusTooManyRequests:
		return ErrRateLimited
	}
	if status >= 500 {
		return fmt.Errorf("%w: search engine returned HTTP %d", ErrBlocked, status)
	}
	if status < 200 || status >= 300 {
		return fmt.Errorf("%w: search engine returned HTTP %d", ErrParser, status)
	}
	return nil
}

// NewRawHTTPClient returns a stdlib client backed by tls-client.
func NewRawHTTPClient(query Query) (*http.Client, error) {
	profile := rawRequestProfileFor(context.Background(), query)
	client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile)
	if err != nil {
		return nil, err
	}

	stdClient := &http.Client{
		Transport: rawTLSRoundTripper{
			client:               client,
			proxied:              rawRequestUsesProxy(query),
			guardPrivateNetworks: query.GuardPrivateNetworks,
			profile:              profile,
		},
		Timeout: rawHTTPTimeout,
	}
	if query.GuardPrivateNetworks {
		stdClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return ValidatePublicHTTPURL(req.Context(), req.URL.String())
		}
	}
	return stdClient, nil
}

func cachedRawHTTPClient(query Query, profileKey string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) {
	proxyURL, err := NormalizeProxyURL(query.ProxyURL)
	if err != nil {
		return nil, err
	}

	key := rawHTTPClientKey{
		proxyURL:             proxyURL,
		profile:              profileKey,
		insecure:             query.Insecure,
		guardPrivateNetworks: query.GuardPrivateNetworks,
	}

	rawHTTPClientCache.Lock()
	defer rawHTTPClientCache.Unlock()

	now := time.Now()
	if entry := rawHTTPClientCache.clients[key]; entry != nil {
		entry.lastUsed = now
		return entry.client, nil
	}

	client, err := newRawTLSClient(query, proxyURL, tlsProfile)
	if err != nil {
		return nil, err
	}
	rawHTTPClientCache.clients[key] = &rawHTTPClientEntry{
		client:   client,
		lastUsed: now,
	}
	evictRawHTTPClientCacheLocked()
	return client, nil
}

func rawRequestUsesProxy(query Query) bool {
	return strings.TrimSpace(query.ProxyURL) != ""
}

func evictRawHTTPClientCacheLocked() {
	for len(rawHTTPClientCache.clients) > rawHTTPClientCacheMaxEntries {
		var (
			oldestKey   rawHTTPClientKey
			oldestEntry *rawHTTPClientEntry
		)
		for key, entry := range rawHTTPClientCache.clients {
			if oldestEntry == nil || entry.lastUsed.Before(oldestEntry.lastUsed) {
				oldestKey = key
				oldestEntry = entry
			}
		}
		if oldestEntry == nil {
			return
		}
		delete(rawHTTPClientCache.clients, oldestKey)
		oldestEntry.client.CloseIdleConnections()
	}
}

// newRawTLSClient builds a pooled Chrome-profile transport; proxyURL must be normalized.
func newRawTLSClient(query Query, proxyURL string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) {
	options := []tlsclient.HttpClientOption{
		tlsclient.WithClientProfile(tlsProfile),
		tlsclient.WithTimeout(int(rawHTTPTimeout / time.Second)),
		tlsclient.WithNotFollowRedirects(),
	}
	if query.Insecure {
		options = append(options, tlsclient.WithInsecureSkipVerify())
	}

	if proxyURL != "" {
		options = append(options, tlsclient.WithProxyUrl(proxyURL))
	} else if query.GuardPrivateNetworks {
		options = append(options, tlsclient.WithDialContext(GuardedDialContext))
	}

	return tlsclient.NewHttpClient(tlsclient.NewNoopLogger(), options...)
}

// convertRawResponse keeps fhttp from leaking past this file.
func convertRawResponse(ctx context.Context, resp *fhttp.Response) *http.Response {
	if resp == nil {
		return nil
	}
	std := &http.Response{
		Status:        resp.Status,
		StatusCode:    resp.StatusCode,
		Proto:         resp.Proto,
		ProtoMajor:    resp.ProtoMajor,
		ProtoMinor:    resp.ProtoMinor,
		Header:        http.Header(resp.Header),
		ContentLength: resp.ContentLength,
		Body:          resp.Body,
	}
	if std.Body == nil {
		std.Body = io.NopCloser(bytes.NewReader(nil))
	}
	std.Body = networkUsageReadCloser{ReadCloser: std.Body, ctx: ctx}
	return std
}

type rawTLSRoundTripper struct {
	client               tlsclient.HttpClient
	proxied              bool
	guardPrivateNetworks bool
	profile              rawRequestProfile
}

func (rt rawTLSRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	if rt.guardPrivateNetworks {
		if err := ValidatePublicHTTPURL(req.Context(), req.URL.String()); err != nil {
			return nil, err
		}
	}

	freq, err := fhttp.NewRequestWithContext(req.Context(), req.Method, req.URL.String(), req.Body)
	if err != nil {
		return nil, err
	}
	for key, values := range req.Header {
		freq.Header[key] = values
	}
	applyRawRequestHeaders(freq, rt.profile)
	SetBrowserProfileID(req.Context(), rt.profile.id)
	return execRawRequest(req.Context(), rt.client, freq, rt.proxied)
}

func rawRequestProfileFor(ctx context.Context, query Query) rawRequestProfile {
	engine := engineFromContext(ctx)
	region := rawProfileRegion(ctx, query)
	salt := rawProfileSalt(ctx, engine, region)

	profile := browserprofile.Profile{}
	if forcedID := forcedProfileIDFromContext(ctx); forcedID != "" {
		if forced, ok := browserprofile.ProfileByID(forcedID); ok {
			profile = forced
		}
	}
	if strings.TrimSpace(profile.ID) == "" {
		profile = browserprofile.SelectProfileForSession(engine, region, salt)
	}
	profile = applyProfileLanguageHint(profile, region)

	major, tlsProfile := pickRawChromeProfile(salt + "\x00" + strings.TrimSpace(profile.ID))
	profile = applyRawChromeMajor(profile, major)

	userAgent := strings.TrimSpace(profile.UserAgent)
	if userAgent == "" {
		userAgent = fallbackRawUserAgent
	}
	acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
	if acceptLanguage == "" {
		acceptLanguage = BuildAcceptLanguageHeader(region)
	}
	if acceptLanguage == "" {
		acceptLanguage = BuildAcceptLanguageHeader(query.LangCode)
	}

	return rawRequestProfile{
		id:             strings.TrimSpace(profile.ID),
		userAgent:      userAgent,
		acceptLanguage: acceptLanguage,
		secCHUA:        formatSecCHUA(profile.UACHBrands),
		platform:       strings.TrimSpace(profile.Platform),
		mobile:         profile.Mobile,
		tlsProfile:     tlsProfile,
	}
}

func applyRawChromeMajor(profile browserprofile.Profile, major int) browserprofile.Profile {
	version := strconv.Itoa(major)
	if template := strings.TrimSpace(profile.UserAgentTemplate); template != "" {
		profile.UserAgent = strings.ReplaceAll(template, "{chrome_major}", version)
	} else if extractChromeVersion(profile.UserAgent) == "" {
		profile.UserAgent = fallbackRawUserAgent
	} else {
		profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0")
	}
	if len(profile.UACHBrands) == 0 {
		profile.UACHBrands = rawUACHBrands(version, false)
	} else {
		profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false)
	}
	if len(profile.UACHFullVerList) == 0 {
		profile.UACHFullVerList = rawUACHBrands(version+".0.0.0", true)
	} else {
		profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true)
	}
	return profile
}

func rawUACHBrands(version string, full bool) []browserprofile.BrandVersion {
	notABrandVersion := "24"
	if full {
		notABrandVersion = "24.0.0.0"
	}
	return []browserprofile.BrandVersion{
		{Brand: "Not_A Brand", Version: notABrandVersion},
		{Brand: "Chromium", Version: version},
		{Brand: "Google Chrome", Version: version},
	}
}

func rawProfileRegion(ctx context.Context, query Query) string {
	if region := profileRegionFromContext(ctx); region != "" {
		return region
	}
	if query.ProxyCountry != "" {
		return query.ProxyCountry
	}
	return profileRegionHint(query)
}

func rawProfileSalt(ctx context.Context, engine, region string) string {
	if laneKey := proxyLaneKeyFromContext(ctx); !laneKey.Empty() {
		return laneKey.SessionID
	}
	return browserprofile.LaneKey(engine, region)
}

// cacheKey includes all headers that affect the pooled fingerprint.
func (p rawRequestProfile) cacheKey() string {
	return strings.Join([]string{p.id, p.userAgent, p.acceptLanguage, p.secCHUA, p.platform, fmt.Sprint(p.mobile)}, "\x00")
}

// applyRawRequestHeaders sets the Chrome identity headers and order; tls-client
// owns Host and Accept-Encoding.
func applyRawRequestHeaders(req *fhttp.Request, profile rawRequestProfile) {
	if req == nil {
		return
	}
	secCHUAMobile := "?0"
	if profile.mobile {
		secCHUAMobile = "?1"
	}
	platform := ""
	if profile.platform != "" {
		platform = quoteSecCHValue(profile.platform)
	}
	for _, h := range [][2]string{
		{"User-Agent", profile.userAgent},
		{"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"},
		{"Accept-Language", profile.acceptLanguage},
		{"Upgrade-Insecure-Requests", "1"},
		{"Sec-CH-UA", profile.secCHUA},
		{"Sec-CH-UA-Mobile", secCHUAMobile},
		{"Sec-CH-UA-Platform", platform},
		{"Sec-Fetch-Site", "none"},
		{"Sec-Fetch-Mode", "navigate"},
		{"Sec-Fetch-User", "?1"},
		{"Sec-Fetch-Dest", "document"},
	} {
		if h[1] != "" {
			req.Header.Set(h[0], h[1])
		}
	}
	req.Header[fhttp.HeaderOrderKey] = rawHeaderOrder
}

func formatSecCHUA(brands []browserprofile.BrandVersion) string {
	parts := make([]string, 0, len(brands))
	for _, brand := range brands {
		name := strings.TrimSpace(brand.Brand)
		version := strings.TrimSpace(brand.Version)
		if name == "" || version == "" {
			continue
		}
		parts = append(parts, quoteSecCHValue(name)+`;v=`+quoteSecCHValue(version))
	}
	return strings.Join(parts, ", ")
}

func quoteSecCHValue(value string) string {
	value = strings.ReplaceAll(value, `\`, `\\`)
	value = strings.ReplaceAll(value, `"`, `\"`)
	return `"` + value + `"`
}

type networkUsageReadCloser struct {
	io.ReadCloser
	ctx context.Context
}

func (r networkUsageReadCloser) Read(p []byte) (int, error) {
	n, err := r.ReadCloser.Read(p)
	AddNetworkBytes(r.ctx, int64(n))
	return n, err
}