From 4a81ab774cb66c8b93bd1802a5421e5d3fa2dcbc Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Sat, 27 Jun 2026 06:10:11 +0300 Subject: [PATCH] core/http: adopt tls-client for raw+extract fingerprinting. Update docs --- cmd/root.go | 2 +- core/http_client.go | 535 ++++++++++++++++++++++++++++++--------- core/http_client_test.go | 181 ++++++++++++- core/proxy_test.go | 6 +- docs/ADDING_AN_ENGINE.md | 9 +- docs/ARCHITECTURE.md | 5 +- go.mod | 33 ++- go.sum | 71 ++++-- 8 files changed, 662 insertions(+), 180 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 97fb1e0..741ffcf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( ) const ( - version = "0.8.4" + version = "0.8.5" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/http_client.go b/core/http_client.go index 3e5f752..ad73d54 100644 --- a/core/http_client.go +++ b/core/http_client.go @@ -1,35 +1,97 @@ package core import ( + "bytes" "context" - "crypto/tls" "fmt" + "hash/fnv" "io" - "net" "net/http" "net/url" + "strconv" "strings" + "sync" "time" - "github.com/corpix/uarand" - utls "github.com/refraction-networking/utls" + fhttp "github.com/bogdanfinn/fhttp" + tlsclient "github.com/bogdanfinn/tls-client" + "github.com/bogdanfinn/tls-client/profiles" + browserprofile "github.com/karust/openserp/core/browser" ) const rawHTTPTimeout = 30 * time.Second +const rawHTTPClientCacheMaxEntries = 64 -// SetAcceptLanguageHeader sets the Accept-Language header from a lang code. -// No-op when the code has no language subtag. -func SetAcceptLanguageHeader(req *http.Request, langCode string) { - if req == nil { - return - } - if value := BuildAcceptLanguageHeader(langCode); value != "" { - req.Header.Set("Accept-Language", value) - } +// fallbackRawUserAgent guards against tls-client's "Go-http-client" UA leaking. +const fallbackRawUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36" + +// rawChromeProfiles pairs each TLS fingerprint with its Chrome major so the UA +// and Sec-CH-UA stay coherent. We round-robin the major here. Add presets as +// tls-client ships them. +var rawChromeProfiles = []struct { + major int + tls profiles.ClientProfile +}{ + {133, profiles.Chrome_133}, + {144, profiles.Chrome_144}, + {146, profiles.Chrome_146}, } -// DrainAndCloseResponse drains unread bytes before closing so HTTP transports -// can safely reuse connections when callers don't consume the full body. +// pickRawChromeProfile hashes salt to a stable but varied fingerprint. +func pickRawChromeProfile(salt string) (int, profiles.ClientProfile) { + h := fnv.New32a() + _, _ = h.Write([]byte(salt)) + p := rawChromeProfiles[int(h.Sum32())%len(rawChromeProfiles)] + return p.major, p.tls +} + +// rawHeaderOrder controls request header order; tls-client profiles do not. +var rawHeaderOrder = []string{ + "host", + "user-agent", + "accept", + "accept-language", + "accept-encoding", + "upgrade-insecure-requests", + "sec-ch-ua", + "sec-ch-ua-mobile", + "sec-ch-ua-platform", + "sec-fetch-site", + "sec-fetch-mode", + "sec-fetch-user", + "sec-fetch-dest", +} + +var rawHTTPClientCache = struct { + sync.Mutex + clients map[rawHTTPClientKey]*rawHTTPClientEntry +}{ + clients: map[rawHTTPClientKey]*rawHTTPClientEntry{}, +} + +type rawHTTPClientKey struct { + proxyURL string + profile string + insecure bool + guardPrivateNetworks bool +} + +type rawHTTPClientEntry struct { + client tlsclient.HttpClient + lastUsed time.Time +} + +type rawRequestProfile struct { + id string + userAgent string + acceptLanguage string + secCHUA string + platform string + mobile bool + tlsProfile profiles.ClientProfile +} + +// DrainAndCloseResponse drains then closes the body so the connection can be reused. func DrainAndCloseResponse(resp *http.Response) { if resp == nil || resp.Body == nil { return @@ -39,28 +101,99 @@ func DrainAndCloseResponse(resp *http.Response) { _ = resp.Body.Close() } -// RawSearchRequest builds and executes a raw-mode SERP HTTP GET. It uses the -// shared raw HTTP client (TLS fingerprinting, network usage tracking, proxy -// support), randomizes the User-Agent, and applies the Accept-Language header -// derived from the query locale. The caller owns the returned response and -// must drain/close it (see DrainAndCloseResponse). +// RawSearchRequest executes a raw-mode GET and returns a stdlib response. func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) { + profile := rawRequestProfileFor(ctx, query) + client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile) + if err != nil { + return nil, err + } + SetBrowserProfileID(ctx, profile.id) + + // Guarded path validates every hop, including the first. if query.GuardPrivateNetworks { - if err := ValidatePublicHTTPURL(ctx, searchURL); err != nil { + return doGuardedRawRequest(ctx, client, searchURL, profile, query) + } + return doRawRequest(ctx, client, searchURL, profile, query) +} + +// doRawRequest issues one GET and converts the response at the boundary. +func doRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) { + req, err := fhttp.NewRequestWithContext(ctx, fhttp.MethodGet, searchURL, nil) + if err != nil { + return nil, err + } + applyRawRequestHeaders(req, profile) + return execRawRequest(ctx, client, req, rawRequestUsesProxy(query)) +} + +// execRawRequest runs the request and converts proxy errors and the response. +func execRawRequest(ctx context.Context, client tlsclient.HttpClient, req *fhttp.Request, proxied bool) (*http.Response, error) { + resp, err := client.Do(req) + if err != nil { + if proxied { + return nil, classifyProxyNetworkError(err) + } + return nil, err + } + return convertRawResponse(ctx, resp), nil +} + +const maxGuardedRedirects = 10 + +// doGuardedRawRequest validates every redirect hop before fetching it. +func doGuardedRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) { + current := searchURL + for hop := 0; ; hop++ { + if err := ValidatePublicHTTPURL(ctx, current); err != nil { return nil, err } + resp, err := doRawRequest(ctx, client, current, profile, query) + if err != nil { + return nil, err + } + location, ok := redirectLocation(resp) + if !ok { + return resp, nil + } + if hop >= maxGuardedRedirects { + DrainAndCloseResponse(resp) + return nil, fmt.Errorf("%w: stopped after %d redirects", ErrEngineInternal, maxGuardedRedirects) + } + next, err := resolveRedirectURL(current, location) + if err != nil { + DrainAndCloseResponse(resp) + return nil, err + } + DrainAndCloseResponse(resp) + current = next } - client, err := NewRawHTTPClient(query) +} + +func redirectLocation(resp *http.Response) (string, bool) { + if resp == nil { + return "", false + } + switch resp.StatusCode { + case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, + http.StatusTemporaryRedirect, http.StatusPermanentRedirect: + location := strings.TrimSpace(resp.Header.Get("Location")) + return location, location != "" + default: + return "", false + } +} + +func resolveRedirectURL(base, location string) (string, error) { + baseURL, err := url.Parse(base) if err != nil { - return nil, err + return "", err } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil) + locURL, err := url.Parse(location) if err != nil { - return nil, err + return "", err } - req.Header.Set("User-Agent", uarand.GetRandom()) - SetAcceptLanguageHeader(req, query.LangCode) - return client.Do(req) + return baseURL.ResolveReference(locURL).String(), nil } func ReadRawSearchBody(resp *http.Response) ([]byte, error) { @@ -91,140 +224,292 @@ func ClassifySearchHTTPStatus(status int) error { return nil } +// NewRawHTTPClient returns a stdlib client backed by tls-client. func NewRawHTTPClient(query Query) (*http.Client, error) { - transport, err := newRawTransport(query) + profile := rawRequestProfileFor(context.Background(), query) + client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile) if err != nil { return nil, err } - roundTripper := http.RoundTripper(transport) - if transport.Proxy != nil { - roundTripper = proxyErrorTransport{base: roundTripper} - } - client := &http.Client{ - Transport: roundTripper, - Timeout: rawHTTPTimeout, + stdClient := &http.Client{ + Transport: rawTLSRoundTripper{ + client: client, + proxied: rawRequestUsesProxy(query), + guardPrivateNetworks: query.GuardPrivateNetworks, + profile: profile, + }, + Timeout: rawHTTPTimeout, } if query.GuardPrivateNetworks { - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + stdClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { return ValidatePublicHTTPURL(req.Context(), req.URL.String()) } } - return client, nil + return stdClient, nil } -func newRawTransport(query Query) (*http.Transport, error) { - dialContext := dialNetworkUsageConn - if query.GuardPrivateNetworks { - dialContext = guardedDialNetworkUsageConn - } - - transport := &http.Transport{ - DialContext: dialContext, - } - if query.Insecure { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - +func cachedRawHTTPClient(query Query, profileKey string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) { proxyURL, err := NormalizeProxyURL(query.ProxyURL) if err != nil { return nil, err } - if proxyURL != "" { - parsed, err := url.Parse(proxyURL) - if err != nil { - return nil, err - } - // Keep proxied requests on the standard transport path so SOCKS5/SOCKS5H - // resolution and routing are handled by the configured proxy correctly. - // The extract SSRF guard validates the target URL before the request and - // on redirects; the proxy address itself may legitimately be local. - transport.DialContext = dialNetworkUsageConn - transport.Proxy = http.ProxyURL(parsed) - return transport, nil + key := rawHTTPClientKey{ + proxyURL: proxyURL, + profile: profileKey, + insecure: query.Insecure, + guardPrivateNetworks: query.GuardPrivateNetworks, } - transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - rawConn, err := dialContext(ctx, network, addr) - if err != nil { - return nil, err - } + rawHTTPClientCache.Lock() + defer rawHTTPClientCache.Unlock() - hostname := strings.Split(addr, ":")[0] - config := &utls.Config{ - ServerName: hostname, - InsecureSkipVerify: query.Insecure, - NextProtos: []string{"http/1.1"}, - } - - uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto) - if err := uconn.BuildHandshakeState(); err != nil { - rawConn.Close() - return nil, err - } - forceHTTP1ALPN(uconn) - if err := uconn.Handshake(); err != nil { - rawConn.Close() - return nil, err - } - - return uconn, nil + now := time.Now() + if entry := rawHTTPClientCache.clients[key]; entry != nil { + entry.lastUsed = now + return entry.client, nil } - return transport, nil + client, err := newRawTLSClient(query, proxyURL, tlsProfile) + if err != nil { + return nil, err + } + rawHTTPClientCache.clients[key] = &rawHTTPClientEntry{ + client: client, + lastUsed: now, + } + evictRawHTTPClientCacheLocked() + return client, nil } -func forceHTTP1ALPN(conn *utls.UConn) { - for _, ext := range conn.Extensions { - if alpn, ok := ext.(*utls.ALPNExtension); ok { - alpn.AlpnProtocols = []string{"http/1.1"} +func rawRequestUsesProxy(query Query) bool { + return strings.TrimSpace(query.ProxyURL) != "" +} + +func evictRawHTTPClientCacheLocked() { + for len(rawHTTPClientCache.clients) > rawHTTPClientCacheMaxEntries { + var ( + oldestKey rawHTTPClientKey + oldestEntry *rawHTTPClientEntry + ) + for key, entry := range rawHTTPClientCache.clients { + if oldestEntry == nil || entry.lastUsed.Before(oldestEntry.lastUsed) { + oldestKey = key + oldestEntry = entry + } + } + if oldestEntry == nil { return } + delete(rawHTTPClientCache.clients, oldestKey) + oldestEntry.client.CloseIdleConnections() } - conn.Extensions = append(conn.Extensions, &utls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}}) } -func dialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) { - dialer := &net.Dialer{} - conn, err := dialer.DialContext(ctx, network, addr) +// newRawTLSClient builds a pooled Chrome-profile transport; proxyURL must be normalized. +func newRawTLSClient(query Query, proxyURL string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) { + options := []tlsclient.HttpClientOption{ + tlsclient.WithClientProfile(tlsProfile), + tlsclient.WithTimeout(int(rawHTTPTimeout / time.Second)), + tlsclient.WithNotFollowRedirects(), + } + if query.Insecure { + options = append(options, tlsclient.WithInsecureSkipVerify()) + } + + if proxyURL != "" { + options = append(options, tlsclient.WithProxyUrl(proxyURL)) + } else if query.GuardPrivateNetworks { + options = append(options, tlsclient.WithDialContext(GuardedDialContext)) + } + + return tlsclient.NewHttpClient(tlsclient.NewNoopLogger(), options...) +} + +// convertRawResponse keeps fhttp from leaking past this file. +func convertRawResponse(ctx context.Context, resp *fhttp.Response) *http.Response { + if resp == nil { + return nil + } + std := &http.Response{ + Status: resp.Status, + StatusCode: resp.StatusCode, + Proto: resp.Proto, + ProtoMajor: resp.ProtoMajor, + ProtoMinor: resp.ProtoMinor, + Header: http.Header(resp.Header), + ContentLength: resp.ContentLength, + Body: resp.Body, + } + if std.Body == nil { + std.Body = io.NopCloser(bytes.NewReader(nil)) + } + std.Body = networkUsageReadCloser{ReadCloser: std.Body, ctx: ctx} + return std +} + +type rawTLSRoundTripper struct { + client tlsclient.HttpClient + proxied bool + guardPrivateNetworks bool + profile rawRequestProfile +} + +func (rt rawTLSRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if rt.guardPrivateNetworks { + if err := ValidatePublicHTTPURL(req.Context(), req.URL.String()); err != nil { + return nil, err + } + } + + freq, err := fhttp.NewRequestWithContext(req.Context(), req.Method, req.URL.String(), req.Body) if err != nil { return nil, err } - return networkUsageConn{Conn: conn, ctx: ctx}, nil -} - -func guardedDialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) { - conn, err := GuardedDialContext(ctx, network, addr) - if err != nil { - return nil, err + for key, values := range req.Header { + freq.Header[key] = values } - return networkUsageConn{Conn: conn, ctx: ctx}, nil + applyRawRequestHeaders(freq, rt.profile) + SetBrowserProfileID(req.Context(), rt.profile.id) + return execRawRequest(req.Context(), rt.client, freq, rt.proxied) } -type networkUsageConn struct { - net.Conn +func rawRequestProfileFor(ctx context.Context, query Query) rawRequestProfile { + engine := engineFromContext(ctx) + region := rawProfileRegion(ctx, query) + salt := rawProfileSalt(ctx, engine, region) + + profile := browserprofile.Profile{} + if forcedID := forcedProfileIDFromContext(ctx); forcedID != "" { + if forced, ok := browserprofile.ProfileByID(forcedID); ok { + profile = forced + } + } + if strings.TrimSpace(profile.ID) == "" { + profile = browserprofile.SelectProfileForSession(engine, region, salt) + } + profile = applyProfileLanguageHint(profile, region) + + major, tlsProfile := pickRawChromeProfile(salt + "\x00" + strings.TrimSpace(profile.ID)) + profile = applyRawChromeMajor(profile, major) + + userAgent := strings.TrimSpace(profile.UserAgent) + if userAgent == "" { + userAgent = fallbackRawUserAgent + } + acceptLanguage := strings.TrimSpace(profile.AcceptLanguage) + if acceptLanguage == "" { + acceptLanguage = BuildAcceptLanguageHeader(region) + } + if acceptLanguage == "" { + acceptLanguage = BuildAcceptLanguageHeader(query.LangCode) + } + + return rawRequestProfile{ + id: strings.TrimSpace(profile.ID), + userAgent: userAgent, + acceptLanguage: acceptLanguage, + secCHUA: formatSecCHUA(profile.UACHBrands), + platform: strings.TrimSpace(profile.Platform), + mobile: profile.Mobile, + tlsProfile: tlsProfile, + } +} + +func applyRawChromeMajor(profile browserprofile.Profile, major int) browserprofile.Profile { + version := strconv.Itoa(major) + if extractChromeVersion(profile.UserAgent) == "" { + profile.UserAgent = fallbackRawUserAgent + } + profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0") + profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false) + profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true) + return profile +} + +func rawProfileRegion(ctx context.Context, query Query) string { + if region := profileRegionFromContext(ctx); region != "" { + return region + } + if query.ProxyCountry != "" { + return query.ProxyCountry + } + return profileRegionHint(query) +} + +func rawProfileSalt(ctx context.Context, engine, region string) string { + if laneKey := proxyLaneKeyFromContext(ctx); !laneKey.Empty() { + return laneKey.SessionID + } + return browserprofile.LaneKey(engine, region) +} + +// cacheKey includes all headers that affect the pooled fingerprint. +func (p rawRequestProfile) cacheKey() string { + return strings.Join([]string{p.id, p.userAgent, p.acceptLanguage, p.secCHUA, p.platform, fmt.Sprint(p.mobile)}, "\x00") +} + +// applyRawRequestHeaders sets the Chrome identity headers and order; tls-client +// owns Host and Accept-Encoding. +func applyRawRequestHeaders(req *fhttp.Request, profile rawRequestProfile) { + if req == nil { + return + } + secCHUAMobile := "?0" + if profile.mobile { + secCHUAMobile = "?1" + } + platform := "" + if profile.platform != "" { + platform = quoteSecCHValue(profile.platform) + } + for _, h := range [][2]string{ + {"User-Agent", profile.userAgent}, + {"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"}, + {"Accept-Language", profile.acceptLanguage}, + {"Upgrade-Insecure-Requests", "1"}, + {"Sec-CH-UA", profile.secCHUA}, + {"Sec-CH-UA-Mobile", secCHUAMobile}, + {"Sec-CH-UA-Platform", platform}, + {"Sec-Fetch-Site", "none"}, + {"Sec-Fetch-Mode", "navigate"}, + {"Sec-Fetch-User", "?1"}, + {"Sec-Fetch-Dest", "document"}, + } { + if h[1] != "" { + req.Header.Set(h[0], h[1]) + } + } + req.Header[fhttp.HeaderOrderKey] = rawHeaderOrder +} + +func formatSecCHUA(brands []browserprofile.BrandVersion) string { + parts := make([]string, 0, len(brands)) + for _, brand := range brands { + name := strings.TrimSpace(brand.Brand) + version := strings.TrimSpace(brand.Version) + if name == "" || version == "" { + continue + } + parts = append(parts, quoteSecCHValue(name)+`;v=`+quoteSecCHValue(version)) + } + return strings.Join(parts, ", ") +} + +func quoteSecCHValue(value string) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, `"`, `\"`) + return `"` + value + `"` +} + +type networkUsageReadCloser struct { + io.ReadCloser ctx context.Context } -func (c networkUsageConn) Read(p []byte) (int, error) { - n, err := c.Conn.Read(p) - AddNetworkBytes(c.ctx, int64(n)) +func (r networkUsageReadCloser) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + AddNetworkBytes(r.ctx, int64(n)) return n, err } - -type proxyErrorTransport struct { - base http.RoundTripper -} - -func (t proxyErrorTransport) RoundTrip(req *http.Request) (*http.Response, error) { - resp, err := t.base.RoundTrip(req) - if err != nil { - return nil, classifyProxyNetworkError(err) - } - if resp != nil && resp.StatusCode == http.StatusProxyAuthRequired { - DrainAndCloseResponse(resp) - return nil, classifyProxyNetworkError(ErrProxyAuth) - } - return resp, nil -} diff --git a/core/http_client_test.go b/core/http_client_test.go index 88f92cd..77860d8 100644 --- a/core/http_client_test.go +++ b/core/http_client_test.go @@ -4,9 +4,13 @@ import ( "context" "errors" "io" + "net" "net/http" "net/http/httptest" + "strconv" "strings" + "sync" + "sync/atomic" "testing" ) @@ -106,11 +110,68 @@ func TestRawHTTPClientTracksNetworkBytes(t *testing.T) { } } -func TestRawHTTPClientTracksProxyErrorBytes(t *testing.T) { - proxyBody := "proxy auth required" +func TestRawHTTPClientAppliesBrowserHeaderDefaults(t *testing.T) { + resetRawHTTPClientCache(t) + + var userAgent, acceptLanguage, secCHUA string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userAgent = r.Header.Get("User-Agent") + acceptLanguage = r.Header.Get("Accept-Language") + secCHUA = r.Header.Get("Sec-CH-UA") + _, _ = w.Write([]byte("ok")) + })) + defer server.Close() + + client, err := NewRawHTTPClient(Query{LangCode: "fr", Region: "FR"}) + if err != nil { + t.Fatalf("new raw client: %v", err) + } + + ctx := WithBrowserProfileUsage(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + DrainAndCloseResponse(resp) + + if userAgent == "" || strings.Contains(userAgent, "Go-http-client") { + t.Fatalf("unexpected User-Agent %q", userAgent) + } + if want := BuildAcceptLanguageHeader("fr-FR"); acceptLanguage != want { + t.Fatalf("Accept-Language = %q, want %q", acceptLanguage, want) + } + if secCHUA == "" { + t.Fatal("expected Sec-CH-UA to be set") + } + if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" { + t.Fatalf("expected one recorded profile id, got %v", ids) + } +} + +func TestRawHTTPClientGuardRejectsInitialPrivateURLWithProxy(t *testing.T) { + client, err := NewRawHTTPClient(Query{ + ProxyURL: "http://127.0.0.1:1", + GuardPrivateNetworks: true, + }) + if err != nil { + t.Fatalf("new raw client: %v", err) + } + + resp, err := client.Get("http://127.0.0.1/") + DrainAndCloseResponse(resp) + if !errors.Is(err, ErrTargetNotAllowed) { + t.Fatalf("expected target guard error, got %v", err) + } +} + +func TestRawHTTPClientProxyAuthError(t *testing.T) { proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusProxyAuthRequired) - _, _ = w.Write([]byte(proxyBody)) + _, _ = w.Write([]byte("proxy auth required")) })) defer proxy.Close() @@ -129,7 +190,117 @@ func TestRawHTTPClientTracksProxyErrorBytes(t *testing.T) { if !errors.Is(err, ErrProxyAuth) { t.Fatalf("expected proxy auth error, got %v", err) } - if got := NetworkBytesFromContext(ctx); got < int64(len(proxyBody)) { - t.Fatalf("expected tracked bytes >= proxy body length, got %d", got) +} + +// TestRawSearchRequestReusesPooledClient checks that same-profile calls share +// one connection and a stable Chrome UA. Bytes/headers are covered elsewhere. +func TestRawSearchRequestReusesPooledClient(t *testing.T) { + resetRawHTTPClientCache(t) + + var connCount atomic.Int32 + var mu sync.Mutex + var userAgents []string + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + userAgents = append(userAgents, r.Header.Get("User-Agent")) + mu.Unlock() + _, _ = w.Write([]byte("ok")) + })) + server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + connCount.Add(1) + } + } + server.Start() + defer server.Close() + + query := Query{LangCode: "de", Region: "DE"} + for i := 0; i < 2; i++ { + ctx := WithEngine(WithBrowserProfileUsage(context.Background()), "google") + readRawSearchBodyForTest(t, ctx, server.URL, query) + if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" { + t.Fatalf("request %d recorded profile ids %v, want exactly one", i, ids) + } + } + + if got := connCount.Load(); got != 1 { + t.Fatalf("expected one reused TCP connection, got %d", got) + } + + mu.Lock() + defer mu.Unlock() + if len(userAgents) != 2 { + t.Fatalf("expected two captured User-Agents, got %d", len(userAgents)) + } + if userAgents[0] == "" || strings.Contains(userAgents[0], "Go-http-client") || userAgents[0] != userAgents[1] { + t.Fatalf("expected stable Chrome User-Agent, got %q then %q", userAgents[0], userAgents[1]) + } +} + +// TestRawRequestProfilesRoundRobinCoherently checks rotation hits every Chrome +// major and keeps UA major == Sec-CH-UA major == TLS fingerprint. +func TestRawRequestProfilesRoundRobinCoherently(t *testing.T) { + tlsByMajor := map[int]string{} + for _, p := range rawChromeProfiles { + tlsByMajor[p.major] = p.tls.GetClientHelloStr() + } + + seen := map[int]bool{} + for i := 0; i < 200; i++ { + ctx := WithProxyLaneKey(WithEngine(context.Background(), "google"), + ProxyLaneKey{Engine: "google", SessionID: "sid-" + strconv.Itoa(i)}) + + profile := rawRequestProfileFor(ctx, Query{Region: "US"}) + major, err := strconv.Atoi(chromeMajorVersion(extractChromeVersion(profile.userAgent))) + if err != nil { + t.Fatalf("raw User-Agent has no Chrome major: %q", profile.userAgent) + } + wantTLS, ok := tlsByMajor[major] + if !ok { + t.Fatalf("UA major %d has no configured tls-client profile", major) + } + if profile.tlsProfile.GetClientHelloStr() != wantTLS { + t.Fatalf("major %d: TLS fingerprint does not match UA", major) + } + if !strings.Contains(profile.secCHUA, `;v="`+strconv.Itoa(major)+`"`) { + t.Fatalf("Sec-CH-UA %q does not match UA major %d", profile.secCHUA, major) + } + seen[major] = true + } + + for _, p := range rawChromeProfiles { + if !seen[p.major] { + t.Fatalf("configured Chrome major %d never selected; seen=%v", p.major, seen) + } + } +} + +func readRawSearchBodyForTest(t *testing.T, ctx context.Context, searchURL string, query Query) string { + t.Helper() + + resp, err := RawSearchRequest(ctx, searchURL, query) + if err != nil { + t.Fatalf("raw search request: %v", err) + } + defer DrainAndCloseResponse(resp) + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read raw search body: %v", err) + } + return string(body) +} + +func resetRawHTTPClientCache(t *testing.T) { + t.Helper() + + rawHTTPClientCache.Lock() + entries := rawHTTPClientCache.clients + rawHTTPClientCache.clients = map[rawHTTPClientKey]*rawHTTPClientEntry{} + rawHTTPClientCache.Unlock() + + for _, entry := range entries { + entry.client.CloseIdleConnections() } } diff --git a/core/proxy_test.go b/core/proxy_test.go index 62cfb30..a9c6649 100644 --- a/core/proxy_test.go +++ b/core/proxy_test.go @@ -344,7 +344,7 @@ func TestNewRawHTTPClientSocks5hUsesProxyDNS(t *testing.T) { } } -func TestNewRawHTTPClientDirectTLSUsesHTTP1(t *testing.T) { +func TestNewRawHTTPClientDirectTLSNegotiatesHTTP2(t *testing.T) { server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(r.Proto)) })) @@ -367,8 +367,8 @@ func TestNewRawHTTPClientDirectTLSUsesHTTP1(t *testing.T) { if err != nil { t.Fatalf("read response body: %v", err) } - if string(body) != "HTTP/1.1" { - t.Fatalf("expected raw client to use HTTP/1.1, got %q", string(body)) + if string(body) != "HTTP/2.0" { + t.Fatalf("expected raw client to negotiate HTTP/2, got %q", string(body)) } } diff --git a/docs/ADDING_AN_ENGINE.md b/docs/ADDING_AN_ENGINE.md index d87c6c5..0a92b7f 100644 --- a/docs/ADDING_AN_ENGINE.md +++ b/docs/ADDING_AN_ENGINE.md @@ -36,11 +36,16 @@ Raw engines should expose a `ParseHTML(io.Reader)` path so tests and Update: -- `cmd/serve.go` for server wiring. -- CLI search dispatch when the engine is CLI-visible. +- `cmd/engines.go` with one `engineSpec` row. This central registry drives + server wiring, CLI dispatch, aliases, parser endpoints, and raw-mode support. - `config.yaml` with rate limits and optional proxy tag. - `README.md` and `docs/openapi.yaml` when public endpoints or parameters change. +Set `rawSearchFn` only when the engine has raw HTTP support, and set +`parseHTMLFn` when `POST /{engine}/parse` should be available. Touch +`cmd/serve.go` or `cmd/root.go` only for new global behavior or flags, not for +ordinary engine registration. + ## 4. Add tests Required for the first PR: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8482f5b..7fd5b83 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,12 +2,12 @@ ## Overview -OpenSERP is a Go API + CLI for search result extraction from Google, Yandex, Baidu, Bing, and DuckDuckGo. +OpenSERP is a Go API + CLI for search result extraction from Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia. Execution modes: - **Browser mode**: default path, headless Chromium via `go-rod`, supported by all engines. -- **Raw HTTP mode**: direct HTTP + `goquery`, currently supported by Google, Yandex, and Baidu. +- **Raw HTTP mode**: direct HTTP + `goquery`, currently supported by Google, Yandex, Baidu, and Ecosia. Browser mode is the primary compatibility path. @@ -54,6 +54,7 @@ openserp/ ├── baidu/ ├── bing/ ├── duckduckgo/ +├── ecosia/ └── testutil/ ``` diff --git a/go.mod b/go.mod index 5195d6d..fecc7ee 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/karust/openserp -go 1.24 +go 1.24.1 toolchain go1.24.6 @@ -9,19 +9,19 @@ require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 github.com/PuerkitoBio/goquery v1.10.3 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/corpix/uarand v0.2.0 + github.com/bogdanfinn/fhttp v0.6.8 + github.com/bogdanfinn/tls-client v1.15.1 github.com/go-rod/rod v0.116.2 - github.com/gofiber/fiber/v2 v2.52.9 + github.com/gofiber/fiber/v2 v2.52.13 github.com/google/uuid v1.6.0 github.com/markusmobius/go-trafilatura v1.12.2 - github.com/refraction-networking/utls v1.8.0 - github.com/sirupsen/logrus v1.9.3 + github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.9.1 - github.com/spf13/pflag v1.0.7 + github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.20.1 github.com/ysmood/gson v0.7.3 - golang.org/x/net v0.43.0 - golang.org/x/time v0.12.0 + golang.org/x/net v0.50.0 + golang.org/x/time v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -31,6 +31,11 @@ require ( github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect + github.com/bdandy/go-errors v1.2.2 // indirect + github.com/bdandy/go-socks4 v1.2.3 // indirect + github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect + github.com/bogdanfinn/utls v1.7.7-barnius // indirect + github.com/bogdanfinn/websocket v1.5.5-barnius // indirect github.com/elliotchance/pie/v2 v2.9.0 // indirect github.com/forPelevin/gomoji v1.2.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect @@ -42,7 +47,7 @@ require ( github.com/hablullah/go-juliandays v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/markusmobius/go-dateparser v1.2.3 // indirect github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 // indirect github.com/markusmobius/go-htmldate v1.9.1 // indirect @@ -50,14 +55,15 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/rs/zerolog v1.33.0 // indirect github.com/sagikazarmark/locafero v0.10.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/cast v1.9.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect github.com/tetratelabs/wazero v1.8.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.65.0 // indirect @@ -68,9 +74,8 @@ require ( github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/got v0.41.0 // indirect github.com/ysmood/leakless v0.9.0 // indirect - golang.org/x/crypto v0.41.0 // indirect + golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect ) diff --git a/go.sum b/go.sum index 3febe48..96045e8 100644 --- a/go.sum +++ b/go.sum @@ -16,9 +16,21 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhP github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bdandy/go-errors v1.2.2 h1:WdFv/oukjTJCLa79UfkGmwX7ZxONAihKu4V0mLIs11Q= +github.com/bdandy/go-errors v1.2.2/go.mod h1:NkYHl4Fey9oRRdbB1CoC6e84tuqQHiqrOcZpqFEkBxM= +github.com/bdandy/go-socks4 v1.2.3 h1:Q6Y2heY1GRjCtHbmlKfnwrKVU/k81LS8mRGLRlmDlic= +github.com/bdandy/go-socks4 v1.2.3/go.mod h1:98kiVFgpdogR8aIGLWLvjDVZ8XcKPsSI/ypGrO+bqHI= +github.com/bogdanfinn/fhttp v0.6.8 h1:LiQyHOY3i0QoxxNB7nq27/nGNNbtPj0fuBPozhR7Ws4= +github.com/bogdanfinn/fhttp v0.6.8/go.mod h1:A+EKDzMx2hb4IUbMx4TlkoHnaJEiLl8r/1Ss1Y+5e5M= +github.com/bogdanfinn/quic-go-utls v1.0.9-utls h1:tV6eDEiRbRCcepALSzxR94JUVD3N3ACIiRLgyc2Ep8s= +github.com/bogdanfinn/quic-go-utls v1.0.9-utls/go.mod h1:aHph9B9H9yPOt5xnhWKSOum27DJAqpiHzwX+gjvaXcg= +github.com/bogdanfinn/tls-client v1.15.1 h1:KiFAlED55DJ8Fcocn+/1nX6PrDFcttIHAf/GDkV6KN8= +github.com/bogdanfinn/tls-client v1.15.1/go.mod h1:LsU6mXVn8MOFDwTkyRfI7V1BZM1p0wf2ZfZsICW/1fM= +github.com/bogdanfinn/utls v1.7.7-barnius h1:OuJ497cc7F3yKNVHRsYPQdGggmk5x6+V5ZlrCR7fOLU= +github.com/bogdanfinn/utls v1.7.7-barnius/go.mod h1:aAK1VZQlpKZClF1WEQeq6kyclbkPq4hz6xTbB5xSlmg= +github.com/bogdanfinn/websocket v1.5.5-barnius h1:bY+qnxpai1qe7Jmjx+Sds/cmOSpuuLoR8x61rWltjOI= +github.com/bogdanfinn/websocket v1.5.5-barnius/go.mod h1:gvvEw6pTKHb7yOiFvIfAFTStQWyrm25BMVCTj5wRSsI= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= -github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -40,8 +52,8 @@ github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f/go.mod h1 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw= -github.com/gofiber/fiber/v2 v2.52.9/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= +github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk= +github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -56,13 +68,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE= github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho= @@ -90,8 +99,8 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/refraction-networking/utls v1.8.0 h1:L38krhiTAyj9EeiQQa2sg+hYb4qwLCqdMcpZrRfbONE= -github.com/refraction-networking/utls v1.8.0/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -109,8 +118,8 @@ github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5 github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= @@ -120,16 +129,18 @@ github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqe github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= -github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng= github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550= github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -163,14 +174,16 @@ github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -180,6 +193,7 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211104170005-ce137452f963/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -187,8 +201,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -198,9 +212,9 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -210,8 +224,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -223,6 +237,7 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -230,10 +245,10 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=