openserp/core/http_client_test.go

package core

import (
	"context"
	"errors"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
)

type trackingBody struct {
	io.Reader
	closed bool
}

func (b *trackingBody) Close() error {
	b.closed = true
	return nil
}

func TestDrainAndCloseResponseDrainsAndCloses(t *testing.T) {
	body := &trackingBody{Reader: strings.NewReader("unread payload")}
	resp := &http.Response{Body: body}

	DrainAndCloseResponse(resp)

	if !body.closed {
		t.Fatal("expected body to be closed")
	}

	n, err := body.Read(make([]byte, 1))
	if err != io.EOF || n != 0 {
		t.Fatalf("expected body drained to EOF, got n=%d err=%v", n, err)
	}
}

func TestClassifySearchHTTPStatus(t *testing.T) {
	tests := []struct {
		name   string
		status int
		want   error
	}{
		{name: "unknown browser status", status: 0, want: nil},
		{name: "ok", status: http.StatusOK, want: nil},
		{name: "blocked", status: http.StatusForbidden, want: ErrBlocked},
		{name: "rate limited", status: http.StatusTooManyRequests, want: ErrRateLimited},
		{name: "server error", status: http.StatusBadGateway, want: ErrBlocked},
		{name: "unexpected status", status: http.StatusNotFound, want: ErrParser},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			err := ClassifySearchHTTPStatus(tt.status)
			if tt.want == nil {
				if err != nil {
					t.Fatalf("expected nil error, got %v", err)
				}
				return
			}
			if !errors.Is(err, tt.want) {
				t.Fatalf("expected %v, got %v", tt.want, err)
			}
		})
	}
}

func TestDrainAndCloseResponseNilSafe(t *testing.T) {
	DrainAndCloseResponse(nil)
	DrainAndCloseResponse(&http.Response{})
}

func TestRawHTTPClientTracksNetworkBytes(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		_, _ = w.Write([]byte("payload"))
	}))
	defer server.Close()

	client, err := NewRawHTTPClient(Query{})
	if err != nil {
		t.Fatalf("new raw client: %v", err)
	}

	ctx := WithNetworkUsage(context.Background())
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
	if err != nil {
		t.Fatalf("new request: %v", err)
	}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatalf("do request: %v", err)
	}
	defer DrainAndCloseResponse(resp)

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("read body: %v", err)
	}
	if string(body) != "payload" {
		t.Fatalf("unexpected body: %q", string(body))
	}
	if got := NetworkBytesFromContext(ctx); got < int64(len(body)) {
		t.Fatalf("expected tracked bytes >= body length, got %d", got)
	}
}

func TestRawHTTPClientAppliesBrowserHeaderDefaults(t *testing.T) {
	resetRawHTTPClientCache(t)

	var userAgent, acceptLanguage, secCHUA string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		userAgent = r.Header.Get("User-Agent")
		acceptLanguage = r.Header.Get("Accept-Language")
		secCHUA = r.Header.Get("Sec-CH-UA")
		_, _ = w.Write([]byte("ok"))
	}))
	defer server.Close()

	client, err := NewRawHTTPClient(Query{LangCode: "fr", Region: "FR"})
	if err != nil {
		t.Fatalf("new raw client: %v", err)
	}

	ctx := WithBrowserProfileUsage(context.Background())
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
	if err != nil {
		t.Fatalf("new request: %v", err)
	}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatalf("do request: %v", err)
	}
	DrainAndCloseResponse(resp)

	if userAgent == "" || strings.Contains(userAgent, "Go-http-client") {
		t.Fatalf("unexpected User-Agent %q", userAgent)
	}
	if want := BuildAcceptLanguageHeader("fr-FR"); acceptLanguage != want {
		t.Fatalf("Accept-Language = %q, want %q", acceptLanguage, want)
	}
	if secCHUA == "" {
		t.Fatal("expected Sec-CH-UA to be set")
	}
	if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" {
		t.Fatalf("expected one recorded profile id, got %v", ids)
	}
}

func TestRawHTTPClientGuardRejectsInitialPrivateURLWithProxy(t *testing.T) {
	client, err := NewRawHTTPClient(Query{
		ProxyURL:             "http://127.0.0.1:1",
		GuardPrivateNetworks: true,
	})
	if err != nil {
		t.Fatalf("new raw client: %v", err)
	}

	resp, err := client.Get("http://127.0.0.1/")
	DrainAndCloseResponse(resp)
	if !errors.Is(err, ErrTargetNotAllowed) {
		t.Fatalf("expected target guard error, got %v", err)
	}
}

func TestRawHTTPClientProxyAuthError(t *testing.T) {
	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusProxyAuthRequired)
		_, _ = w.Write([]byte("proxy auth required"))
	}))
	defer proxy.Close()

	client, err := NewRawHTTPClient(Query{ProxyURL: proxy.URL})
	if err != nil {
		t.Fatalf("new raw client: %v", err)
	}

	ctx := WithNetworkUsage(context.Background())
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com/", nil)
	if err != nil {
		t.Fatalf("new request: %v", err)
	}
	resp, err := client.Do(req)
	DrainAndCloseResponse(resp)
	if !errors.Is(err, ErrProxyAuth) {
		t.Fatalf("expected proxy auth error, got %v", err)
	}
}

// TestRawSearchRequestReusesPooledClient checks that same-profile calls share
// one connection and a stable Chrome UA. Bytes/headers are covered elsewhere.
func TestRawSearchRequestReusesPooledClient(t *testing.T) {
	resetRawHTTPClientCache(t)

	var connCount atomic.Int32
	var mu sync.Mutex
	var userAgents []string

	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mu.Lock()
		userAgents = append(userAgents, r.Header.Get("User-Agent"))
		mu.Unlock()
		_, _ = w.Write([]byte("ok"))
	}))
	server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
		if state == http.StateNew {
			connCount.Add(1)
		}
	}
	server.Start()
	defer server.Close()

	query := Query{LangCode: "de", Region: "DE"}
	for i := 0; i < 2; i++ {
		ctx := WithEngine(WithBrowserProfileUsage(context.Background()), "google")
		readRawSearchBodyForTest(t, ctx, server.URL, query)
		if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" {
			t.Fatalf("request %d recorded profile ids %v, want exactly one", i, ids)
		}
	}

	if got := connCount.Load(); got != 1 {
		t.Fatalf("expected one reused TCP connection, got %d", got)
	}

	mu.Lock()
	defer mu.Unlock()
	if len(userAgents) != 2 {
		t.Fatalf("expected two captured User-Agents, got %d", len(userAgents))
	}
	if userAgents[0] == "" || strings.Contains(userAgents[0], "Go-http-client") || userAgents[0] != userAgents[1] {
		t.Fatalf("expected stable Chrome User-Agent, got %q then %q", userAgents[0], userAgents[1])
	}
}

// TestRawRequestProfilesRoundRobinCoherently checks rotation hits every Chrome
// major and keeps UA major == Sec-CH-UA major == TLS fingerprint.
func TestRawRequestProfilesRoundRobinCoherently(t *testing.T) {
	tlsByMajor := map[int]string{}
	for _, p := range rawChromeProfiles {
		tlsByMajor[p.major] = p.tls.GetClientHelloStr()
	}

	seen := map[int]bool{}
	for i := 0; i < 200; i++ {
		ctx := WithProxyLaneKey(WithEngine(context.Background(), "google"),
			ProxyLaneKey{Engine: "google", SessionID: "sid-" + strconv.Itoa(i)})

		profile := rawRequestProfileFor(ctx, Query{Region: "US"})
		major, err := strconv.Atoi(chromeMajorVersion(extractChromeVersion(profile.userAgent)))
		if err != nil {
			t.Fatalf("raw User-Agent has no Chrome major: %q", profile.userAgent)
		}
		wantTLS, ok := tlsByMajor[major]
		if !ok {
			t.Fatalf("UA major %d has no configured tls-client profile", major)
		}
		if profile.tlsProfile.GetClientHelloStr() != wantTLS {
			t.Fatalf("major %d: TLS fingerprint does not match UA", major)
		}
		if !strings.Contains(profile.secCHUA, `;v="`+strconv.Itoa(major)+`"`) {
			t.Fatalf("Sec-CH-UA %q does not match UA major %d", profile.secCHUA, major)
		}
		seen[major] = true
	}

	for _, p := range rawChromeProfiles {
		if !seen[p.major] {
			t.Fatalf("configured Chrome major %d never selected; seen=%v", p.major, seen)
		}
	}
}

func readRawSearchBodyForTest(t *testing.T, ctx context.Context, searchURL string, query Query) string {
	t.Helper()

	resp, err := RawSearchRequest(ctx, searchURL, query)
	if err != nil {
		t.Fatalf("raw search request: %v", err)
	}
	defer DrainAndCloseResponse(resp)

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("read raw search body: %v", err)
	}
	return string(body)
}

func resetRawHTTPClientCache(t *testing.T) {
	t.Helper()

	rawHTTPClientCache.Lock()
	entries := rawHTTPClientCache.clients
	rawHTTPClientCache.clients = map[rawHTTPClientKey]*rawHTTPClientEntry{}
	rawHTTPClientCache.Unlock()

	for _, entry := range entries {
		entry.client.CloseIdleConnections()
	}
}