core/http: adopt tls-client for raw+extract fingerprinting. Update docs

cmd/root.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	version               = "0.8.4"
+	version               = "0.8.5"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )

core/http_client.go

@@ -1,35 +1,97 @@
 package core
 
 import (
+	"bytes"
 	"context"
-	"crypto/tls"
 	"fmt"
+	"hash/fnv"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/corpix/uarand"
-	utls "github.com/refraction-networking/utls"
+	fhttp "github.com/bogdanfinn/fhttp"
+	tlsclient "github.com/bogdanfinn/tls-client"
+	"github.com/bogdanfinn/tls-client/profiles"
+	browserprofile "github.com/karust/openserp/core/browser"
 )
 
 const rawHTTPTimeout = 30 * time.Second
+const rawHTTPClientCacheMaxEntries = 64
 
-// SetAcceptLanguageHeader sets the Accept-Language header from a lang code.
-// No-op when the code has no language subtag.
-func SetAcceptLanguageHeader(req *http.Request, langCode string) {
-	if req == nil {
-		return
-	}
-	if value := BuildAcceptLanguageHeader(langCode); value != "" {
-		req.Header.Set("Accept-Language", value)
-	}
+// fallbackRawUserAgent guards against tls-client's "Go-http-client" UA leaking.
+const fallbackRawUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"
+
+// rawChromeProfiles pairs each TLS fingerprint with its Chrome major so the UA
+// and Sec-CH-UA stay coherent. We round-robin the major here. Add presets as
+// tls-client ships them.
+var rawChromeProfiles = []struct {
+	major int
+	tls   profiles.ClientProfile
+}{
+	{133, profiles.Chrome_133},
+	{144, profiles.Chrome_144},
+	{146, profiles.Chrome_146},
 }
 
-// DrainAndCloseResponse drains unread bytes before closing so HTTP transports
-// can safely reuse connections when callers don't consume the full body.
+// pickRawChromeProfile hashes salt to a stable but varied fingerprint.
+func pickRawChromeProfile(salt string) (int, profiles.ClientProfile) {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(salt))
+	p := rawChromeProfiles[int(h.Sum32())%len(rawChromeProfiles)]
+	return p.major, p.tls
+}
+
+// rawHeaderOrder controls request header order; tls-client profiles do not.
+var rawHeaderOrder = []string{
+	"host",
+	"user-agent",
+	"accept",
+	"accept-language",
+	"accept-encoding",
+	"upgrade-insecure-requests",
+	"sec-ch-ua",
+	"sec-ch-ua-mobile",
+	"sec-ch-ua-platform",
+	"sec-fetch-site",
+	"sec-fetch-mode",
+	"sec-fetch-user",
+	"sec-fetch-dest",
+}
+
+var rawHTTPClientCache = struct {
+	sync.Mutex
+	clients map[rawHTTPClientKey]*rawHTTPClientEntry
+}{
+	clients: map[rawHTTPClientKey]*rawHTTPClientEntry{},
+}
+
+type rawHTTPClientKey struct {
+	proxyURL             string
+	profile              string
+	insecure             bool
+	guardPrivateNetworks bool
+}
+
+type rawHTTPClientEntry struct {
+	client   tlsclient.HttpClient
+	lastUsed time.Time
+}
+
+type rawRequestProfile struct {
+	id             string
+	userAgent      string
+	acceptLanguage string
+	secCHUA        string
+	platform       string
+	mobile         bool
+	tlsProfile     profiles.ClientProfile
+}
+
+// DrainAndCloseResponse drains then closes the body so the connection can be reused.
 func DrainAndCloseResponse(resp *http.Response) {
 	if resp == nil || resp.Body == nil {
 		return
@@ -39,28 +101,99 @@ func DrainAndCloseResponse(resp *http.Response) {
 	_ = resp.Body.Close()
 }
 
-// RawSearchRequest builds and executes a raw-mode SERP HTTP GET. It uses the
-// shared raw HTTP client (TLS fingerprinting, network usage tracking, proxy
-// support), randomizes the User-Agent, and applies the Accept-Language header
-// derived from the query locale. The caller owns the returned response and
-// must drain/close it (see DrainAndCloseResponse).
+// RawSearchRequest executes a raw-mode GET and returns a stdlib response.
 func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) {
+	profile := rawRequestProfileFor(ctx, query)
+	client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile)
+	if err != nil {
+		return nil, err
+	}
+	SetBrowserProfileID(ctx, profile.id)
+
+	// Guarded path validates every hop, including the first.
 	if query.GuardPrivateNetworks {
-		if err := ValidatePublicHTTPURL(ctx, searchURL); err != nil {
+		return doGuardedRawRequest(ctx, client, searchURL, profile, query)
+	}
+	return doRawRequest(ctx, client, searchURL, profile, query)
+}
+
+// doRawRequest issues one GET and converts the response at the boundary.
+func doRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) {
+	req, err := fhttp.NewRequestWithContext(ctx, fhttp.MethodGet, searchURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	applyRawRequestHeaders(req, profile)
+	return execRawRequest(ctx, client, req, rawRequestUsesProxy(query))
+}
+
+// execRawRequest runs the request and converts proxy errors and the response.
+func execRawRequest(ctx context.Context, client tlsclient.HttpClient, req *fhttp.Request, proxied bool) (*http.Response, error) {
+	resp, err := client.Do(req)
+	if err != nil {
+		if proxied {
+			return nil, classifyProxyNetworkError(err)
+		}
+		return nil, err
+	}
+	return convertRawResponse(ctx, resp), nil
+}
+
+const maxGuardedRedirects = 10
+
+// doGuardedRawRequest validates every redirect hop before fetching it.
+func doGuardedRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) {
+	current := searchURL
+	for hop := 0; ; hop++ {
+		if err := ValidatePublicHTTPURL(ctx, current); err != nil {
 			return nil, err
 		}
+		resp, err := doRawRequest(ctx, client, current, profile, query)
+		if err != nil {
+			return nil, err
+		}
+		location, ok := redirectLocation(resp)
+		if !ok {
+			return resp, nil
+		}
+		if hop >= maxGuardedRedirects {
+			DrainAndCloseResponse(resp)
+			return nil, fmt.Errorf("%w: stopped after %d redirects", ErrEngineInternal, maxGuardedRedirects)
+		}
+		next, err := resolveRedirectURL(current, location)
+		if err != nil {
+			DrainAndCloseResponse(resp)
+			return nil, err
+		}
+		DrainAndCloseResponse(resp)
+		current = next
 	}
-	client, err := NewRawHTTPClient(query)
+}
+
+func redirectLocation(resp *http.Response) (string, bool) {
+	if resp == nil {
+		return "", false
+	}
+	switch resp.StatusCode {
+	case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
+		http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+		location := strings.TrimSpace(resp.Header.Get("Location"))
+		return location, location != ""
+	default:
+		return "", false
+	}
+}
+
+func resolveRedirectURL(base, location string) (string, error) {
+	baseURL, err := url.Parse(base)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
+	locURL, err := url.Parse(location)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
-	req.Header.Set("User-Agent", uarand.GetRandom())
-	SetAcceptLanguageHeader(req, query.LangCode)
-	return client.Do(req)
+	return baseURL.ResolveReference(locURL).String(), nil
 }
 
 func ReadRawSearchBody(resp *http.Response) ([]byte, error) {
@@ -91,140 +224,292 @@ func ClassifySearchHTTPStatus(status int) error {
 	return nil
 }
 
+// NewRawHTTPClient returns a stdlib client backed by tls-client.
 func NewRawHTTPClient(query Query) (*http.Client, error) {
-	transport, err := newRawTransport(query)
+	profile := rawRequestProfileFor(context.Background(), query)
+	client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile)
 	if err != nil {
 		return nil, err
 	}
-	roundTripper := http.RoundTripper(transport)
-	if transport.Proxy != nil {
-		roundTripper = proxyErrorTransport{base: roundTripper}
-	}
 
-	client := &http.Client{
-		Transport: roundTripper,
-		Timeout:   rawHTTPTimeout,
+	stdClient := &http.Client{
+		Transport: rawTLSRoundTripper{
+			client:               client,
+			proxied:              rawRequestUsesProxy(query),
+			guardPrivateNetworks: query.GuardPrivateNetworks,
+			profile:              profile,
+		},
+		Timeout: rawHTTPTimeout,
 	}
 	if query.GuardPrivateNetworks {
-		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		stdClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 			return ValidatePublicHTTPURL(req.Context(), req.URL.String())
 		}
 	}
-	return client, nil
+	return stdClient, nil
 }
 
-func newRawTransport(query Query) (*http.Transport, error) {
-	dialContext := dialNetworkUsageConn
-	if query.GuardPrivateNetworks {
-		dialContext = guardedDialNetworkUsageConn
-	}
-
-	transport := &http.Transport{
-		DialContext: dialContext,
-	}
-	if query.Insecure {
-		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	}
-
+func cachedRawHTTPClient(query Query, profileKey string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) {
 	proxyURL, err := NormalizeProxyURL(query.ProxyURL)
 	if err != nil {
 		return nil, err
 	}
-	if proxyURL != "" {
-		parsed, err := url.Parse(proxyURL)
-		if err != nil {
-			return nil, err
-		}
 
-		// Keep proxied requests on the standard transport path so SOCKS5/SOCKS5H
-		// resolution and routing are handled by the configured proxy correctly.
-		// The extract SSRF guard validates the target URL before the request and
-		// on redirects; the proxy address itself may legitimately be local.
-		transport.DialContext = dialNetworkUsageConn
-		transport.Proxy = http.ProxyURL(parsed)
-		return transport, nil
+	key := rawHTTPClientKey{
+		proxyURL:             proxyURL,
+		profile:              profileKey,
+		insecure:             query.Insecure,
+		guardPrivateNetworks: query.GuardPrivateNetworks,
 	}
 
-	transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-		rawConn, err := dialContext(ctx, network, addr)
-		if err != nil {
-			return nil, err
-		}
+	rawHTTPClientCache.Lock()
+	defer rawHTTPClientCache.Unlock()
 
-		hostname := strings.Split(addr, ":")[0]
-		config := &utls.Config{
-			ServerName:         hostname,
-			InsecureSkipVerify: query.Insecure,
-			NextProtos:         []string{"http/1.1"},
-		}
-
-		uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto)
-		if err := uconn.BuildHandshakeState(); err != nil {
-			rawConn.Close()
-			return nil, err
-		}
-		forceHTTP1ALPN(uconn)
-		if err := uconn.Handshake(); err != nil {
-			rawConn.Close()
-			return nil, err
-		}
-
-		return uconn, nil
+	now := time.Now()
+	if entry := rawHTTPClientCache.clients[key]; entry != nil {
+		entry.lastUsed = now
+		return entry.client, nil
 	}
 
-	return transport, nil
+	client, err := newRawTLSClient(query, proxyURL, tlsProfile)
+	if err != nil {
+		return nil, err
+	}
+	rawHTTPClientCache.clients[key] = &rawHTTPClientEntry{
+		client:   client,
+		lastUsed: now,
+	}
+	evictRawHTTPClientCacheLocked()
+	return client, nil
 }
 
-func forceHTTP1ALPN(conn *utls.UConn) {
-	for _, ext := range conn.Extensions {
-		if alpn, ok := ext.(*utls.ALPNExtension); ok {
-			alpn.AlpnProtocols = []string{"http/1.1"}
+func rawRequestUsesProxy(query Query) bool {
+	return strings.TrimSpace(query.ProxyURL) != ""
+}
+
+func evictRawHTTPClientCacheLocked() {
+	for len(rawHTTPClientCache.clients) > rawHTTPClientCacheMaxEntries {
+		var (
+			oldestKey   rawHTTPClientKey
+			oldestEntry *rawHTTPClientEntry
+		)
+		for key, entry := range rawHTTPClientCache.clients {
+			if oldestEntry == nil || entry.lastUsed.Before(oldestEntry.lastUsed) {
+				oldestKey = key
+				oldestEntry = entry
+			}
+		}
+		if oldestEntry == nil {
 			return
 		}
+		delete(rawHTTPClientCache.clients, oldestKey)
+		oldestEntry.client.CloseIdleConnections()
 	}
-	conn.Extensions = append(conn.Extensions, &utls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}})
 }
 
-func dialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) {
-	dialer := &net.Dialer{}
-	conn, err := dialer.DialContext(ctx, network, addr)
+// newRawTLSClient builds a pooled Chrome-profile transport; proxyURL must be normalized.
+func newRawTLSClient(query Query, proxyURL string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) {
+	options := []tlsclient.HttpClientOption{
+		tlsclient.WithClientProfile(tlsProfile),
+		tlsclient.WithTimeout(int(rawHTTPTimeout / time.Second)),
+		tlsclient.WithNotFollowRedirects(),
+	}
+	if query.Insecure {
+		options = append(options, tlsclient.WithInsecureSkipVerify())
+	}
+
+	if proxyURL != "" {
+		options = append(options, tlsclient.WithProxyUrl(proxyURL))
+	} else if query.GuardPrivateNetworks {
+		options = append(options, tlsclient.WithDialContext(GuardedDialContext))
+	}
+
+	return tlsclient.NewHttpClient(tlsclient.NewNoopLogger(), options...)
+}
+
+// convertRawResponse keeps fhttp from leaking past this file.
+func convertRawResponse(ctx context.Context, resp *fhttp.Response) *http.Response {
+	if resp == nil {
+		return nil
+	}
+	std := &http.Response{
+		Status:        resp.Status,
+		StatusCode:    resp.StatusCode,
+		Proto:         resp.Proto,
+		ProtoMajor:    resp.ProtoMajor,
+		ProtoMinor:    resp.ProtoMinor,
+		Header:        http.Header(resp.Header),
+		ContentLength: resp.ContentLength,
+		Body:          resp.Body,
+	}
+	if std.Body == nil {
+		std.Body = io.NopCloser(bytes.NewReader(nil))
+	}
+	std.Body = networkUsageReadCloser{ReadCloser: std.Body, ctx: ctx}
+	return std
+}
+
+type rawTLSRoundTripper struct {
+	client               tlsclient.HttpClient
+	proxied              bool
+	guardPrivateNetworks bool
+	profile              rawRequestProfile
+}
+
+func (rt rawTLSRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if rt.guardPrivateNetworks {
+		if err := ValidatePublicHTTPURL(req.Context(), req.URL.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	freq, err := fhttp.NewRequestWithContext(req.Context(), req.Method, req.URL.String(), req.Body)
 	if err != nil {
 		return nil, err
 	}
-	return networkUsageConn{Conn: conn, ctx: ctx}, nil
-}
-
-func guardedDialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) {
-	conn, err := GuardedDialContext(ctx, network, addr)
-	if err != nil {
-		return nil, err
+	for key, values := range req.Header {
+		freq.Header[key] = values
 	}
-	return networkUsageConn{Conn: conn, ctx: ctx}, nil
+	applyRawRequestHeaders(freq, rt.profile)
+	SetBrowserProfileID(req.Context(), rt.profile.id)
+	return execRawRequest(req.Context(), rt.client, freq, rt.proxied)
 }
 
-type networkUsageConn struct {
-	net.Conn
+func rawRequestProfileFor(ctx context.Context, query Query) rawRequestProfile {
+	engine := engineFromContext(ctx)
+	region := rawProfileRegion(ctx, query)
+	salt := rawProfileSalt(ctx, engine, region)
+
+	profile := browserprofile.Profile{}
+	if forcedID := forcedProfileIDFromContext(ctx); forcedID != "" {
+		if forced, ok := browserprofile.ProfileByID(forcedID); ok {
+			profile = forced
+		}
+	}
+	if strings.TrimSpace(profile.ID) == "" {
+		profile = browserprofile.SelectProfileForSession(engine, region, salt)
+	}
+	profile = applyProfileLanguageHint(profile, region)
+
+	major, tlsProfile := pickRawChromeProfile(salt + "\x00" + strings.TrimSpace(profile.ID))
+	profile = applyRawChromeMajor(profile, major)
+
+	userAgent := strings.TrimSpace(profile.UserAgent)
+	if userAgent == "" {
+		userAgent = fallbackRawUserAgent
+	}
+	acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
+	if acceptLanguage == "" {
+		acceptLanguage = BuildAcceptLanguageHeader(region)
+	}
+	if acceptLanguage == "" {
+		acceptLanguage = BuildAcceptLanguageHeader(query.LangCode)
+	}
+
+	return rawRequestProfile{
+		id:             strings.TrimSpace(profile.ID),
+		userAgent:      userAgent,
+		acceptLanguage: acceptLanguage,
+		secCHUA:        formatSecCHUA(profile.UACHBrands),
+		platform:       strings.TrimSpace(profile.Platform),
+		mobile:         profile.Mobile,
+		tlsProfile:     tlsProfile,
+	}
+}
+
+func applyRawChromeMajor(profile browserprofile.Profile, major int) browserprofile.Profile {
+	version := strconv.Itoa(major)
+	if extractChromeVersion(profile.UserAgent) == "" {
+		profile.UserAgent = fallbackRawUserAgent
+	}
+	profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0")
+	profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false)
+	profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true)
+	return profile
+}
+
+func rawProfileRegion(ctx context.Context, query Query) string {
+	if region := profileRegionFromContext(ctx); region != "" {
+		return region
+	}
+	if query.ProxyCountry != "" {
+		return query.ProxyCountry
+	}
+	return profileRegionHint(query)
+}
+
+func rawProfileSalt(ctx context.Context, engine, region string) string {
+	if laneKey := proxyLaneKeyFromContext(ctx); !laneKey.Empty() {
+		return laneKey.SessionID
+	}
+	return browserprofile.LaneKey(engine, region)
+}
+
+// cacheKey includes all headers that affect the pooled fingerprint.
+func (p rawRequestProfile) cacheKey() string {
+	return strings.Join([]string{p.id, p.userAgent, p.acceptLanguage, p.secCHUA, p.platform, fmt.Sprint(p.mobile)}, "\x00")
+}
+
+// applyRawRequestHeaders sets the Chrome identity headers and order; tls-client
+// owns Host and Accept-Encoding.
+func applyRawRequestHeaders(req *fhttp.Request, profile rawRequestProfile) {
+	if req == nil {
+		return
+	}
+	secCHUAMobile := "?0"
+	if profile.mobile {
+		secCHUAMobile = "?1"
+	}
+	platform := ""
+	if profile.platform != "" {
+		platform = quoteSecCHValue(profile.platform)
+	}
+	for _, h := range [][2]string{
+		{"User-Agent", profile.userAgent},
+		{"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"},
+		{"Accept-Language", profile.acceptLanguage},
+		{"Upgrade-Insecure-Requests", "1"},
+		{"Sec-CH-UA", profile.secCHUA},
+		{"Sec-CH-UA-Mobile", secCHUAMobile},
+		{"Sec-CH-UA-Platform", platform},
+		{"Sec-Fetch-Site", "none"},
+		{"Sec-Fetch-Mode", "navigate"},
+		{"Sec-Fetch-User", "?1"},
+		{"Sec-Fetch-Dest", "document"},
+	} {
+		if h[1] != "" {
+			req.Header.Set(h[0], h[1])
+		}
+	}
+	req.Header[fhttp.HeaderOrderKey] = rawHeaderOrder
+}
+
+func formatSecCHUA(brands []browserprofile.BrandVersion) string {
+	parts := make([]string, 0, len(brands))
+	for _, brand := range brands {
+		name := strings.TrimSpace(brand.Brand)
+		version := strings.TrimSpace(brand.Version)
+		if name == "" || version == "" {
+			continue
+		}
+		parts = append(parts, quoteSecCHValue(name)+`;v=`+quoteSecCHValue(version))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func quoteSecCHValue(value string) string {
+	value = strings.ReplaceAll(value, `\`, `\\`)
+	value = strings.ReplaceAll(value, `"`, `\"`)
+	return `"` + value + `"`
+}
+
+type networkUsageReadCloser struct {
+	io.ReadCloser
 	ctx context.Context
 }
 
-func (c networkUsageConn) Read(p []byte) (int, error) {
-	n, err := c.Conn.Read(p)
-	AddNetworkBytes(c.ctx, int64(n))
+func (r networkUsageReadCloser) Read(p []byte) (int, error) {
+	n, err := r.ReadCloser.Read(p)
+	AddNetworkBytes(r.ctx, int64(n))
 	return n, err
 }
-
-type proxyErrorTransport struct {
-	base http.RoundTripper
-}
-
-func (t proxyErrorTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	resp, err := t.base.RoundTrip(req)
-	if err != nil {
-		return nil, classifyProxyNetworkError(err)
-	}
-	if resp != nil && resp.StatusCode == http.StatusProxyAuthRequired {
-		DrainAndCloseResponse(resp)
-		return nil, classifyProxyNetworkError(ErrProxyAuth)
-	}
-	return resp, nil
-}

core/http_client_test.go

@@ -4,9 +4,13 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 )
 
@@ -106,11 +110,68 @@ func TestRawHTTPClientTracksNetworkBytes(t *testing.T) {
 	}
 }
 
-func TestRawHTTPClientTracksProxyErrorBytes(t *testing.T) {
-	proxyBody := "proxy auth required"
+func TestRawHTTPClientAppliesBrowserHeaderDefaults(t *testing.T) {
+	resetRawHTTPClientCache(t)
+
+	var userAgent, acceptLanguage, secCHUA string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		userAgent = r.Header.Get("User-Agent")
+		acceptLanguage = r.Header.Get("Accept-Language")
+		secCHUA = r.Header.Get("Sec-CH-UA")
+		_, _ = w.Write([]byte("ok"))
+	}))
+	defer server.Close()
+
+	client, err := NewRawHTTPClient(Query{LangCode: "fr", Region: "FR"})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	ctx := WithBrowserProfileUsage(context.Background())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	DrainAndCloseResponse(resp)
+
+	if userAgent == "" || strings.Contains(userAgent, "Go-http-client") {
+		t.Fatalf("unexpected User-Agent %q", userAgent)
+	}
+	if want := BuildAcceptLanguageHeader("fr-FR"); acceptLanguage != want {
+		t.Fatalf("Accept-Language = %q, want %q", acceptLanguage, want)
+	}
+	if secCHUA == "" {
+		t.Fatal("expected Sec-CH-UA to be set")
+	}
+	if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" {
+		t.Fatalf("expected one recorded profile id, got %v", ids)
+	}
+}
+
+func TestRawHTTPClientGuardRejectsInitialPrivateURLWithProxy(t *testing.T) {
+	client, err := NewRawHTTPClient(Query{
+		ProxyURL:             "http://127.0.0.1:1",
+		GuardPrivateNetworks: true,
+	})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	resp, err := client.Get("http://127.0.0.1/")
+	DrainAndCloseResponse(resp)
+	if !errors.Is(err, ErrTargetNotAllowed) {
+		t.Fatalf("expected target guard error, got %v", err)
+	}
+}
+
+func TestRawHTTPClientProxyAuthError(t *testing.T) {
 	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusProxyAuthRequired)
-		_, _ = w.Write([]byte(proxyBody))
+		_, _ = w.Write([]byte("proxy auth required"))
 	}))
 	defer proxy.Close()
 
@@ -129,7 +190,117 @@ func TestRawHTTPClientTracksProxyErrorBytes(t *testing.T) {
 	if !errors.Is(err, ErrProxyAuth) {
 		t.Fatalf("expected proxy auth error, got %v", err)
 	}
-	if got := NetworkBytesFromContext(ctx); got < int64(len(proxyBody)) {
-		t.Fatalf("expected tracked bytes >= proxy body length, got %d", got)
+}
+
+// TestRawSearchRequestReusesPooledClient checks that same-profile calls share
+// one connection and a stable Chrome UA. Bytes/headers are covered elsewhere.
+func TestRawSearchRequestReusesPooledClient(t *testing.T) {
+	resetRawHTTPClientCache(t)
+
+	var connCount atomic.Int32
+	var mu sync.Mutex
+	var userAgents []string
+
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		userAgents = append(userAgents, r.Header.Get("User-Agent"))
+		mu.Unlock()
+		_, _ = w.Write([]byte("ok"))
+	}))
+	server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
+		if state == http.StateNew {
+			connCount.Add(1)
+		}
+	}
+	server.Start()
+	defer server.Close()
+
+	query := Query{LangCode: "de", Region: "DE"}
+	for i := 0; i < 2; i++ {
+		ctx := WithEngine(WithBrowserProfileUsage(context.Background()), "google")
+		readRawSearchBodyForTest(t, ctx, server.URL, query)
+		if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" {
+			t.Fatalf("request %d recorded profile ids %v, want exactly one", i, ids)
+		}
+	}
+
+	if got := connCount.Load(); got != 1 {
+		t.Fatalf("expected one reused TCP connection, got %d", got)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	if len(userAgents) != 2 {
+		t.Fatalf("expected two captured User-Agents, got %d", len(userAgents))
+	}
+	if userAgents[0] == "" || strings.Contains(userAgents[0], "Go-http-client") || userAgents[0] != userAgents[1] {
+		t.Fatalf("expected stable Chrome User-Agent, got %q then %q", userAgents[0], userAgents[1])
+	}
+}
+
+// TestRawRequestProfilesRoundRobinCoherently checks rotation hits every Chrome
+// major and keeps UA major == Sec-CH-UA major == TLS fingerprint.
+func TestRawRequestProfilesRoundRobinCoherently(t *testing.T) {
+	tlsByMajor := map[int]string{}
+	for _, p := range rawChromeProfiles {
+		tlsByMajor[p.major] = p.tls.GetClientHelloStr()
+	}
+
+	seen := map[int]bool{}
+	for i := 0; i < 200; i++ {
+		ctx := WithProxyLaneKey(WithEngine(context.Background(), "google"),
+			ProxyLaneKey{Engine: "google", SessionID: "sid-" + strconv.Itoa(i)})
+
+		profile := rawRequestProfileFor(ctx, Query{Region: "US"})
+		major, err := strconv.Atoi(chromeMajorVersion(extractChromeVersion(profile.userAgent)))
+		if err != nil {
+			t.Fatalf("raw User-Agent has no Chrome major: %q", profile.userAgent)
+		}
+		wantTLS, ok := tlsByMajor[major]
+		if !ok {
+			t.Fatalf("UA major %d has no configured tls-client profile", major)
+		}
+		if profile.tlsProfile.GetClientHelloStr() != wantTLS {
+			t.Fatalf("major %d: TLS fingerprint does not match UA", major)
+		}
+		if !strings.Contains(profile.secCHUA, `;v="`+strconv.Itoa(major)+`"`) {
+			t.Fatalf("Sec-CH-UA %q does not match UA major %d", profile.secCHUA, major)
+		}
+		seen[major] = true
+	}
+
+	for _, p := range rawChromeProfiles {
+		if !seen[p.major] {
+			t.Fatalf("configured Chrome major %d never selected; seen=%v", p.major, seen)
+		}
+	}
+}
+
+func readRawSearchBodyForTest(t *testing.T, ctx context.Context, searchURL string, query Query) string {
+	t.Helper()
+
+	resp, err := RawSearchRequest(ctx, searchURL, query)
+	if err != nil {
+		t.Fatalf("raw search request: %v", err)
+	}
+	defer DrainAndCloseResponse(resp)
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read raw search body: %v", err)
+	}
+	return string(body)
+}
+
+func resetRawHTTPClientCache(t *testing.T) {
+	t.Helper()
+
+	rawHTTPClientCache.Lock()
+	entries := rawHTTPClientCache.clients
+	rawHTTPClientCache.clients = map[rawHTTPClientKey]*rawHTTPClientEntry{}
+	rawHTTPClientCache.Unlock()
+
+	for _, entry := range entries {
+		entry.client.CloseIdleConnections()
 	}
 }

core/proxy_test.go

@@ -344,7 +344,7 @@ func TestNewRawHTTPClientSocks5hUsesProxyDNS(t *testing.T) {
 	}
 }
 
-func TestNewRawHTTPClientDirectTLSUsesHTTP1(t *testing.T) {
+func TestNewRawHTTPClientDirectTLSNegotiatesHTTP2(t *testing.T) {
 	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		_, _ = w.Write([]byte(r.Proto))
 	}))
@@ -367,8 +367,8 @@ func TestNewRawHTTPClientDirectTLSUsesHTTP1(t *testing.T) {
 	if err != nil {
 		t.Fatalf("read response body: %v", err)
 	}
-	if string(body) != "HTTP/1.1" {
-		t.Fatalf("expected raw client to use HTTP/1.1, got %q", string(body))
+	if string(body) != "HTTP/2.0" {
+		t.Fatalf("expected raw client to negotiate HTTP/2, got %q", string(body))
 	}
 }
 

docs/ADDING_AN_ENGINE.md

@@ -36,11 +36,16 @@ Raw engines should expose a `ParseHTML(io.Reader)` path so tests and
 
 Update:
 
-- `cmd/serve.go` for server wiring.
-- CLI search dispatch when the engine is CLI-visible.
+- `cmd/engines.go` with one `engineSpec` row. This central registry drives
+  server wiring, CLI dispatch, aliases, parser endpoints, and raw-mode support.
 - `config.yaml` with rate limits and optional proxy tag.
 - `README.md` and `docs/openapi.yaml` when public endpoints or parameters change.
 
+Set `rawSearchFn` only when the engine has raw HTTP support, and set
+`parseHTMLFn` when `POST /{engine}/parse` should be available. Touch
+`cmd/serve.go` or `cmd/root.go` only for new global behavior or flags, not for
+ordinary engine registration.
+
 ## 4. Add tests
 
 Required for the first PR:

docs/ARCHITECTURE.md

@@ -2,12 +2,12 @@
 
 ## Overview
 
-OpenSERP is a Go API + CLI for search result extraction from Google, Yandex, Baidu, Bing, and DuckDuckGo.
+OpenSERP is a Go API + CLI for search result extraction from Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia.
 
 Execution modes:
 
 - **Browser mode**: default path, headless Chromium via `go-rod`, supported by all engines.
-- **Raw HTTP mode**: direct HTTP + `goquery`, currently supported by Google, Yandex, and Baidu.
+- **Raw HTTP mode**: direct HTTP + `goquery`, currently supported by Google, Yandex, Baidu, and Ecosia.
 
 Browser mode is the primary compatibility path.
 
@@ -54,6 +54,7 @@ openserp/
 ├── baidu/
 ├── bing/
 ├── duckduckgo/
+├── ecosia/
 └── testutil/
 ```
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/karust/openserp
 
-go 1.24
+go 1.24.1
 
 toolchain go1.24.6
 
@@ -9,19 +9,19 @@ require (
 	github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0
 	github.com/PuerkitoBio/goquery v1.10.3
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
-	github.com/corpix/uarand v0.2.0
+	github.com/bogdanfinn/fhttp v0.6.8
+	github.com/bogdanfinn/tls-client v1.15.1
 	github.com/go-rod/rod v0.116.2
-	github.com/gofiber/fiber/v2 v2.52.9
+	github.com/gofiber/fiber/v2 v2.52.13
 	github.com/google/uuid v1.6.0
 	github.com/markusmobius/go-trafilatura v1.12.2
-	github.com/refraction-networking/utls v1.8.0
-	github.com/sirupsen/logrus v1.9.3
+	github.com/sirupsen/logrus v1.9.4
 	github.com/spf13/cobra v1.9.1
-	github.com/spf13/pflag v1.0.7
+	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.20.1
 	github.com/ysmood/gson v0.7.3
-	golang.org/x/net v0.43.0
-	golang.org/x/time v0.12.0
+	golang.org/x/net v0.50.0
+	golang.org/x/time v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -31,6 +31,11 @@ require (
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
+	github.com/bdandy/go-errors v1.2.2 // indirect
+	github.com/bdandy/go-socks4 v1.2.3 // indirect
+	github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect
+	github.com/bogdanfinn/utls v1.7.7-barnius // indirect
+	github.com/bogdanfinn/websocket v1.5.5-barnius // indirect
 	github.com/elliotchance/pie/v2 v2.9.0 // indirect
 	github.com/forPelevin/gomoji v1.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
@@ -42,7 +47,7 @@ require (
 	github.com/hablullah/go-juliandays v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.2 // indirect
 	github.com/markusmobius/go-dateparser v1.2.3 // indirect
 	github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 // indirect
 	github.com/markusmobius/go-htmldate v1.9.1 // indirect
@@ -50,14 +55,15 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/quic-go/qpack v0.6.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/rs/zerolog v1.33.0 // indirect
 	github.com/sagikazarmark/locafero v0.10.0 // indirect
 	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.14.0 // indirect
 	github.com/spf13/cast v1.9.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
 	github.com/tetratelabs/wazero v1.8.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.65.0 // indirect
@@ -68,9 +74,8 @@ require (
 	github.com/ysmood/goob v0.4.0 // indirect
 	github.com/ysmood/got v0.41.0 // indirect
 	github.com/ysmood/leakless v0.9.0 // indirect
-	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/crypto v0.48.0 // indirect
 	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
-	golang.org/x/sys v0.35.0 // indirect
-	golang.org/x/text v0.28.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
 )

go.sum

@@ -16,9 +16,21 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhP
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/bdandy/go-errors v1.2.2 h1:WdFv/oukjTJCLa79UfkGmwX7ZxONAihKu4V0mLIs11Q=
+github.com/bdandy/go-errors v1.2.2/go.mod h1:NkYHl4Fey9oRRdbB1CoC6e84tuqQHiqrOcZpqFEkBxM=
+github.com/bdandy/go-socks4 v1.2.3 h1:Q6Y2heY1GRjCtHbmlKfnwrKVU/k81LS8mRGLRlmDlic=
+github.com/bdandy/go-socks4 v1.2.3/go.mod h1:98kiVFgpdogR8aIGLWLvjDVZ8XcKPsSI/ypGrO+bqHI=
+github.com/bogdanfinn/fhttp v0.6.8 h1:LiQyHOY3i0QoxxNB7nq27/nGNNbtPj0fuBPozhR7Ws4=
+github.com/bogdanfinn/fhttp v0.6.8/go.mod h1:A+EKDzMx2hb4IUbMx4TlkoHnaJEiLl8r/1Ss1Y+5e5M=
+github.com/bogdanfinn/quic-go-utls v1.0.9-utls h1:tV6eDEiRbRCcepALSzxR94JUVD3N3ACIiRLgyc2Ep8s=
+github.com/bogdanfinn/quic-go-utls v1.0.9-utls/go.mod h1:aHph9B9H9yPOt5xnhWKSOum27DJAqpiHzwX+gjvaXcg=
+github.com/bogdanfinn/tls-client v1.15.1 h1:KiFAlED55DJ8Fcocn+/1nX6PrDFcttIHAf/GDkV6KN8=
+github.com/bogdanfinn/tls-client v1.15.1/go.mod h1:LsU6mXVn8MOFDwTkyRfI7V1BZM1p0wf2ZfZsICW/1fM=
+github.com/bogdanfinn/utls v1.7.7-barnius h1:OuJ497cc7F3yKNVHRsYPQdGggmk5x6+V5ZlrCR7fOLU=
+github.com/bogdanfinn/utls v1.7.7-barnius/go.mod h1:aAK1VZQlpKZClF1WEQeq6kyclbkPq4hz6xTbB5xSlmg=
+github.com/bogdanfinn/websocket v1.5.5-barnius h1:bY+qnxpai1qe7Jmjx+Sds/cmOSpuuLoR8x61rWltjOI=
+github.com/bogdanfinn/websocket v1.5.5-barnius/go.mod h1:gvvEw6pTKHb7yOiFvIfAFTStQWyrm25BMVCTj5wRSsI=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE=
-github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -40,8 +52,8 @@ github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f/go.mod h1
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw=
-github.com/gofiber/fiber/v2 v2.52.9/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
+github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk=
+github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -56,13 +68,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE=
 github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
+github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho=
@@ -90,8 +99,8 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/refraction-networking/utls v1.8.0 h1:L38krhiTAyj9EeiQQa2sg+hYb4qwLCqdMcpZrRfbONE=
-github.com/refraction-networking/utls v1.8.0/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -109,8 +118,8 @@ github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5
 github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
 github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA=
@@ -120,16 +129,18 @@ github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqe
 github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
 github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M=
-github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
 github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=
+github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng=
 github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
 github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -163,14 +174,16 @@ github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
 github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
-golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -180,6 +193,7 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211104170005-ce137452f963/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
@@ -187,8 +201,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
-golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -198,9 +212,9 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -210,8 +224,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
-golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -223,6 +237,7 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
@@ -230,10 +245,10 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
-golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
-golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
-golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=