fix integration tests, dedupe engine smoke tests

baidu/search_integration_test.go

@@ -4,52 +4,14 @@
 package baidu
 
 import (
-	"context"
 	"testing"
 
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
 	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestSearchBaidu(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	baid := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := baid.Search(context.Background(), query)
-	ithelper.HandleError(t, "baidu web search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first result title is empty")
-	}
-}
-
-func TestImageSearchBaidu(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	baid := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golden retriever puppy", Limit: 10}
-	results, err := baid.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "baidu image search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty image results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first image result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first image result title is empty")
-	}
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, core.SearchEngineOptions{})
+	})
 }

bing/search_integration_test.go

@@ -4,52 +4,14 @@
 package bing
 
 import (
-	"context"
 	"testing"
 
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
 	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestSearchBing(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	bing := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := bing.Search(context.Background(), query)
-	ithelper.HandleError(t, "bing web search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first result title is empty")
-	}
-}
-
-func TestImageSearchBing(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	bing := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golden retriever puppy", Limit: 10}
-	results, err := bing.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "bing image search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty image results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first image result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first image result title is empty")
-	}
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, core.SearchEngineOptions{})
+	})
 }

core/proxy_per_context_spike_test.go

@@ -1,93 +0,0 @@
-//go:build integration
-// +build integration
-
-package core
-
-import (
-	"context"
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/karust/openserp/testutil"
-)
-
-func TestProxyPerContextAuthIsolationSpike(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	type authHit struct {
-		proxy string
-		auth  string
-	}
-	var (
-		mu   sync.Mutex
-		hits []authHit
-	)
-
-	newAuthProxy := func(name, username, password string) *httptest.Server {
-		want := "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password))
-		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			got := r.Header.Get("Proxy-Authorization")
-			mu.Lock()
-			hits = append(hits, authHit{proxy: name, auth: got})
-			mu.Unlock()
-			if got != want {
-				w.Header().Set("Proxy-Authenticate", `Basic realm="openserp-spike"`)
-				w.WriteHeader(http.StatusProxyAuthRequired)
-				return
-			}
-			w.Header().Set("Content-Type", "text/html; charset=utf-8")
-			_, _ = fmt.Fprintf(w, "<html><body>%s</body></html>", name)
-		}))
-	}
-
-	proxyA := newAuthProxy("proxy-a", "user-a", "pass-a")
-	defer proxyA.Close()
-	proxyB := newAuthProxy("proxy-b", "user-b", "pass-b")
-	defer proxyB.Close()
-
-	browser, err := NewBrowser(BrowserOpts{IsHeadless: true, Timeout: 20 * time.Second})
-	if err != nil {
-		t.Fatalf("create browser: %v", err)
-	}
-	defer closeTestBrowser(t, browser)
-
-	run := func(proxyURL string) error {
-		ctx := WithRequestProxyURL(context.Background(), proxyURL)
-		page, err := browser.Navigate(ctx, "http://proxy-auth-spike.invalid/")
-		if err != nil {
-			return err
-		}
-		defer func() {
-			_ = ClosePageWithTimeout(context.Background(), page, time.Second)
-		}()
-		body, err := page.Timeout(5 * time.Second).Element("body")
-		if err != nil {
-			return err
-		}
-		_, err = body.Text()
-		return err
-	}
-
-	errCh := make(chan error, 2)
-	go func() { errCh <- run(strings.Replace(proxyA.URL, "http://", "http://user-a:pass-a@", 1)) }()
-	go func() { errCh <- run(strings.Replace(proxyB.URL, "http://", "http://user-b:pass-b@", 1)) }()
-
-	for i := 0; i < 2; i++ {
-		if err := <-errCh; err != nil {
-			t.Fatalf("proxied navigation failed: %v", err)
-		}
-	}
-
-	mu.Lock()
-	defer mu.Unlock()
-	t.Logf("per-context auth spike report: %d proxy requests observed; Browser serializes authenticated proxy auth handlers to avoid cross-context credential leakage", len(hits))
-	for _, hit := range hits {
-		t.Logf("proxy=%s auth_prefix=%t", hit.proxy, strings.HasPrefix(hit.auth, "Basic "))
-	}
-}

core/proxy_per_context_test.go

@@ -0,0 +1,82 @@
+//go:build integration
+// +build integration
+
+package core
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/karust/openserp/testutil"
+)
+
+// TestPerContextProxyIsolation verifies the supported per-request proxy path:
+// a shared Chrome (launched without a process-level proxy) routes each request
+// through the unauthenticated proxy from its context, with no cross-context
+// leakage between concurrent navigations.
+//
+// Authenticated per-request proxies are intentionally NOT supported on a
+// shared browser: Chrome's Fetch-based proxy auth is browser-global, so
+// concurrent contexts with different credentials race and fail with
+// ERR_INVALID_AUTH_CREDENTIALS. The server routes authenticated proxies to a
+// dedicated Chrome process per auth identity instead (see browserPool in
+// cmd/serve.go).
+func TestPerContextProxyIsolation(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	newProxy := func(name string) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			_, _ = fmt.Fprintf(w, "<html><body>%s</body></html>", name)
+		}))
+	}
+
+	proxyA := newProxy("proxy-a")
+	defer proxyA.Close()
+	proxyB := newProxy("proxy-b")
+	defer proxyB.Close()
+
+	browser, err := NewBrowser(BrowserOpts{IsHeadless: true, Timeout: 20 * time.Second})
+	if err != nil {
+		t.Fatalf("create browser: %v", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	run := func(proxyURL, want string) error {
+		ctx := WithRequestProxyURL(context.Background(), proxyURL)
+		page, err := browser.Navigate(ctx, "http://per-context-proxy.invalid/")
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = ClosePageWithTimeout(context.Background(), page, time.Second)
+		}()
+		body, err := page.Timeout(5 * time.Second).Element("body")
+		if err != nil {
+			return err
+		}
+		text, err := body.Text()
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(text, want) {
+			return fmt.Errorf("expected response from %s, got %q", want, text)
+		}
+		return nil
+	}
+
+	errCh := make(chan error, 2)
+	go func() { errCh <- run(proxyA.URL, "proxy-a") }()
+	go func() { errCh <- run(proxyB.URL, "proxy-b") }()
+
+	for i := 0; i < 2; i++ {
+		if err := <-errCh; err != nil {
+			t.Fatalf("proxied navigation failed: %v", err)
+		}
+	}
+}

duckduckgo/search_integration_test.go

@@ -4,52 +4,14 @@
 package duckduckgo
 
 import (
-	"context"
 	"testing"
 
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
 	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestSearchDuckDuckGo(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	ddg := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "wikipedia", Limit: 10}
-	results, err := ddg.Search(context.Background(), query)
-	ithelper.HandleError(t, "duckduckgo web search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first result title is empty")
-	}
-}
-
-func TestImageSearchDuckDuckGo(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	ddg := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golden retriever puppy", Limit: 10}
-	results, err := ddg.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "duckduckgo image search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty image results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first image result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first image result title is empty")
-	}
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, core.SearchEngineOptions{})
+	})
 }

ecosia/search_integration_test.go

@@ -4,52 +4,14 @@
 package ecosia
 
 import (
-	"context"
 	"testing"
 
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
 	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestSearchEcosia(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	engine := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := engine.Search(context.Background(), query)
-	ithelper.HandleError(t, "ecosia web search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first result title is empty")
-	}
-}
-
-func TestImageSearchEcosia(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	engine := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golden retriever puppy", Limit: 10}
-	results, err := engine.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "ecosia image search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty image results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first image result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first image result title is empty")
-	}
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, core.SearchEngineOptions{})
+	})
 }

google/search_integration_test.go

@@ -4,52 +4,14 @@
 package google
 
 import (
-	"context"
 	"testing"
 
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
 	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestSearchGoogle(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	gogl := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := gogl.Search(context.Background(), query)
-	ithelper.HandleError(t, "google web search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first result title is empty")
-	}
-}
-
-func TestImageSearchGoogle(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	gogl := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golden retriever puppy", Limit: 10}
-	results, err := gogl.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "google image search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty image results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first image result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first image result title is empty")
-	}
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, core.SearchEngineOptions{})
+	})
 }

testutil/ithelper/ithelper.go

@@ -10,40 +10,46 @@ import (
 	"github.com/karust/openserp/testutil"
 )
 
-// HandleError skips on captcha/timeout (expected in live environments),
-// fatals on other errors. In strict mode, captcha/timeout also fatal.
+// flakyLiveErrors are failure modes expected when hitting live engines from
+// arbitrary IPs (captcha walls, IP blocks, rate limits, slow pages). They do
+// not indicate broken code, so tests skip instead of failing unless
+// OPENSERP_INTEGRATION_STRICT is set.
+var flakyLiveErrors = []error{
+	core.ErrCaptcha,
+	core.ErrBlocked,
+	core.ErrRateLimited,
+	core.ErrSearchTimeout,
+	context.DeadlineExceeded,
+	context.Canceled,
+}
+
+// HandleError skips on captcha/block/rate-limit/timeout (expected against
+// live engines), fatals on other errors. In strict mode everything fatals.
 func HandleError(t *testing.T, operation string, err error) {
 	t.Helper()
 	if err == nil {
 		return
 	}
 
-	if err == core.ErrCaptcha {
-		t.Logf("captcha detected during %s: %v", operation, err)
-		if testutil.IntegrationStrict() {
-			t.Fatalf("%s failed (strict mode): %v", operation, err)
+	flaky := core.IsContextDone(err)
+	for _, sentinel := range flakyLiveErrors {
+		if errors.Is(err, sentinel) {
+			flaky = true
+			break
 		}
-		t.Skipf("skipping flaky live %s due to captcha: %v", operation, err)
 	}
-
-	if err == core.ErrSearchTimeout {
-		if testutil.IntegrationStrict() {
-			t.Fatalf("%s failed (strict mode): %v", operation, err)
-		}
-		t.Skipf("skipping flaky live %s due to timeout: %v", operation, err)
+	if !flaky {
+		t.Fatalf("%s failed: %v", operation, err)
 	}
-	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || core.IsContextDone(err) {
-		if testutil.IntegrationStrict() {
-			t.Fatalf("%s failed (strict mode): %v", operation, err)
-		}
-		t.Skipf("skipping flaky live %s due to context deadline: %v", operation, err)
+	if testutil.IntegrationStrict() {
+		t.Fatalf("%s failed (strict mode): %v", operation, err)
 	}
-
-	t.Fatalf("%s failed: %v", operation, err)
+	t.Skipf("skipping flaky live %s: %v", operation, err)
 }
 
-// CreateBrowser creates a browser configured for integration tests.
-// Respects OPENSERP_INTEGRATION_HEADFUL for debugging.
+// CreateBrowser creates a browser configured for integration tests and closes
+// it when the test finishes. Respects OPENSERP_INTEGRATION_HEADFUL for
+// debugging (browser and page are left open for inspection).
 func CreateBrowser(t *testing.T) *core.Browser {
 	t.Helper()
 	headful := testutil.IntegrationHeadful()
@@ -57,5 +63,48 @@ func CreateBrowser(t *testing.T) *core.Browser {
 	if err != nil {
 		t.Fatalf("failed to create test browser: %v", err)
 	}
+	if !headful {
+		t.Cleanup(func() {
+			if cerr := b.Close(); cerr != nil {
+				t.Logf("close test browser: %v", cerr)
+			}
+		})
+	}
 	return b
 }
+
+// RunEngineTests runs the live web + image search smoke checks shared by every
+// engine's integration test: results come back, and the first one has a URL
+// and a title. newEngine receives a fresh browser per subtest.
+func RunEngineTests(t *testing.T, newEngine func(*core.Browser) core.SearchEngine) {
+	testutil.RequireIntegration(t)
+
+	t.Run("web", func(t *testing.T) {
+		engine := newEngine(CreateBrowser(t))
+		query := core.Query{Text: "golang programming", Limit: 10}
+		results, err := engine.Search(context.Background(), query)
+		HandleError(t, engine.Name()+" web search", err)
+		requireResults(t, results)
+	})
+
+	t.Run("image", func(t *testing.T) {
+		engine := newEngine(CreateBrowser(t))
+		query := core.Query{Text: "golden retriever puppy", Limit: 10}
+		results, err := engine.SearchImage(context.Background(), query)
+		HandleError(t, engine.Name()+" image search", err)
+		requireResults(t, results)
+	})
+}
+
+func requireResults(t *testing.T, results []core.SearchResult) {
+	t.Helper()
+	if len(results) == 0 {
+		t.Fatal("returned empty results")
+	}
+	if results[0].URL == "" {
+		t.Fatal("first result URL is empty")
+	}
+	if results[0].Title == "" {
+		t.Fatal("first result title is empty")
+	}
+}

yandex/search_integration_test.go

@@ -0,0 +1,17 @@
+//go:build integration
+// +build integration
+
+package yandex
+
+import (
+	"testing"
+
+	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/testutil/ithelper"
+)
+
+func TestSearchYandex(t *testing.T) {
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, core.SearchEngineOptions{})
+	})
+}

yandex/search_test.go

@@ -1,43 +0,0 @@
-//go:build integration
-// +build integration
-
-package yandex
-
-import (
-	"context"
-	"testing"
-
-	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
-	"github.com/karust/openserp/testutil/ithelper"
-)
-
-func TestSearchYandex(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	yand := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "HEY", Limit: 10}
-	results, err := yand.Search(context.Background(), query)
-	ithelper.HandleError(t, "yandex web search", err)
-
-	if len(results) == 0 {
-		t.Fatalf("[SearchYandex] returned empty result")
-	}
-}
-
-func TestImageYandex(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	yand := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "furry tiger", Limit: 30}
-	results, err := yand.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "yandex image search", err)
-
-	if len(results) == 0 {
-		t.Fatalf("[ImageYandex] returned empty result")
-	}
-}