Add sanitized real html pages (google,yandex,baidu) for tests. Raw search HTML parser tests.

baidu/all_test.go

@@ -1,22 +1,14 @@
 package baidu
 
 import (
+	"net/url"
 	"testing"
-	"time"
 
 	"github.com/karust/openserp/core"
 )
 
-var browser *core.Browser
 var testQuery = core.Query{Text: "go", Site: "tutorialspoint.com", DateInterval: "20140101..20230101", Limit: 10}
 
-func init() {
-	core.InitLogger(true, true)
-
-	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 10}
-	browser, _ = core.NewBrowser(opts)
-}
-
 func TestUrlBuild(t *testing.T) {
 	res, err := BuildURL(testQuery)
 	if err != nil {
@@ -30,18 +22,6 @@ func TestUrlBuild(t *testing.T) {
 	}
 }
 
-func TestSearch(t *testing.T) {
-	baid := New(*browser, core.SearchEngineOptions{})
-	results, err := baid.Search(testQuery)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if len(results) == 0 {
-		t.Fatal("No results got from Baidu search")
-	}
-}
-
 func TestImageUrlBuild(t *testing.T) {
 	query := core.Query{Text: "金毛猎犬"}
 
@@ -50,22 +30,18 @@ func TestImageUrlBuild(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	want := "https://image.baidu.com/search/acjson?cl=2&fp=result&ie=utf-8&ipn=rj&oe=utf-8&pn=0&rn=30&tn=resultjson_com&word=%E9%87%91%E6%AF%9B%E7%8C%8E%E7%8A%AC"
-	if want != got {
-		t.Fatalf("Want: `%s`, Got `%s`", want, got)
-	}
-}
-
-func TestImageSearch(t *testing.T) {
-	baid := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "each each data", Limit: 60}
-	results, err := baid.SearchImage(query)
+	parsed, err := url.Parse(got)
 	if err != nil {
-		t.Fatalf("Cannot [ImageBaidu]: %s", err)
+		t.Fatalf("invalid URL returned: %v", err)
 	}
-
-	if len(results) < 60 {
-		t.Fatalf("[ImageBaidu] returned not full result")
+	if parsed.Host != "image.baidu.com" {
+		t.Fatalf("unexpected host: %s", parsed.Host)
+	}
+	q := parsed.Query()
+	if q.Get("word") != "金毛猎犬" {
+		t.Fatalf("expected word query to be preserved, got %q", q.Get("word"))
+	}
+	if q.Get("tn") != "resultjson_com" {
+		t.Fatalf("expected tn=resultjson_com, got %q", q.Get("tn"))
 	}
 }

baidu/search_raw.go

@@ -1,7 +1,6 @@
 package baidu
 
 import (
-	"fmt"
 	"net/http"
 	"strings"
 
@@ -39,27 +38,41 @@ func baiduResultParser(response *http.Response) ([]core.SearchResult, error) {
 	results := []core.SearchResult{}
 	rank := 1
 
-	// Get individual results
-	sel := doc.Find("div.c-container.new-pmd")
-
-	fmt.Println(sel.Length())
+	// Prefer organic result blocks from the main result column.
+	sel := doc.Find("#content_left .result.c-container")
+	if sel.Length() == 0 {
+		sel = doc.Find("div.c-container.new-pmd")
+	}
 
 	for i := range sel.Nodes {
 		item := sel.Eq(i)
 
 		// Find URL
-		linkTag := item.Find("a")
+		titleTag := item.Find("h3").First()
+		if titleTag.Length() == 0 {
+			continue
+		}
+
+		linkTag := titleTag.Closest("a")
+		if linkTag.Length() == 0 {
+			linkTag = item.Find("a").First()
+		}
 		link, _ := linkTag.Attr("href")
-		link = strings.Trim(link, " ")
+		link = strings.TrimSpace(link)
 
 		// Find title
-		title := linkTag.Text()
+		title := strings.TrimSpace(titleTag.Text())
 
 		// Find description
-		desc := item.Text()
+		descTag := item.Find(".c-abstract, .content-right_8Zs40, .summary-gap_3Jb4I").First()
+		desc := strings.TrimSpace(descTag.Text())
+		if desc == "" {
+			desc = strings.TrimSpace(item.Text())
+		}
 		desc = strings.ReplaceAll(desc, title, "")
+		desc = strings.TrimSpace(desc)
 
-		if link != "" && link != "#" {
+		if link != "" && link != "#" && title != "" {
 			result := core.SearchResult{
 				Rank:        rank,
 				URL:         link,

baidu/search_raw_test.go

@@ -0,0 +1,76 @@
+package baidu
+
+import (
+	"testing"
+
+	"github.com/karust/openserp/testutil"
+)
+
+func TestBaiduResultParserSnapshots(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		fixture        string
+		minResultCount int
+		maxResultCount int
+		wantZero       bool
+	}{
+		{
+			name:           "search results",
+			fixture:        "search_results.html",
+			minResultCount: 5,
+			maxResultCount: 30,
+		},
+		{
+			name:     "no results",
+			fixture:  "search_no_results.html",
+			wantZero: true,
+		},
+		{
+			name:     "captcha page",
+			fixture:  "search_captcha.html",
+			wantZero: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			results, err := baiduResultParser(testutil.ResponseFromFixture(t, tt.fixture))
+			if err != nil {
+				t.Fatalf("baiduResultParser() error = %v", err)
+			}
+
+			if tt.wantZero {
+				if len(results) != 0 {
+					t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
+				}
+				return
+			}
+
+			if len(results) < tt.minResultCount || len(results) > tt.maxResultCount {
+				t.Fatalf(
+					"unexpected result count for %s: got %d, want range [%d,%d]",
+					tt.fixture, len(results), tt.minResultCount, tt.maxResultCount,
+				)
+			}
+
+			testutil.AssertSequentialRanks(t, results)
+			testutil.AssertFirstResultFilled(t, results)
+		})
+	}
+}
+
+func TestBaiduResultParserEmptyHTML(t *testing.T) {
+	t.Parallel()
+
+	results, err := baiduResultParser(testutil.ResponseFromString(""))
+	if err != nil {
+		t.Fatalf("baiduResultParser() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+	}
+}

baidu/testdata/search_captcha.html

@@ -0,0 +1 @@
+<html lang="zh-CN"><head><title>百度安全验证</title></head><body style=""><div class="timeout hide-callback"><div class="timeout-img"></div><div class="timeout-title">网络不给力，请稍后重试</div><button type="button" class="timeout-button">返回首页</button></div><div class="timeout-feedback hide-callback"><div class="timeout-feedback-icon"></div><p class="timeout-feedback-title">问题反馈</p></div><div class="passMod_dialog-wrapper passMod_show"><div class="passMod_dialog-mask"></div><div class="passMod_dialog-container"><div class="passMod_dialog-header"><p>百度安全验证</p></div><div class="passMod_dialog-body"><div class="passMod_code-container"><div class="passMod_verify-item passMod_spin-wrapper" id="spin-0"><div class="passMod_spin-tip">请完成下方验证后继续操作</div><p class="passMod_spin-msg"></p><div class="passMod_spin-context-wrap"><div class="passMod_spin-context"><img class="passMod_spin-background" src="http://test.test" alt=""/><img class="passMod_spin-coordinate"/><div class="passMod_verify-container passMod_verify-container_uAMZ undefined" style=""><div class="passMod_verify-pending"><img alt=""/><span>正在验证...</span></div><div class="passMod_verify-success">验证通过</div><div class="passMod_verify-fail">图片未转正</div><div class="passMod_verify-network">网络不给力，请刷新重试</div><div class="passMod_verify-loading"><img alt=""/><span>加载中...</span></div></div><div class="passMod_spin-AI">图片由AI生成</div></div><div class="passMod_spin-footer"><div class="passMod_slide-control passMod_slide-control_FHGY"><div class="passMod_slide-grand passMod_slide-grand-loading"></div><p class="passMod_slide-tip slideShine">拖动左侧滑块使图片为正</p><div class="passMod_slide-btn passMod_slide-btn-loading"></div></div></div></div></div></div></div><div class="passMod_dialog-footer"><p><span class="passMod_dialog-footer-qrcode machine-hide">扫码验证<span class="passMod_dialog-footer-line">|</span></span><a class="passMod_dialog-footer-feedback" target="_blank" href="http://test.test">意见反馈</a><span class="passMod_dialog-footer-refresh"><span class="passMod_dialog-footer-line">|</span><span style=""></span>刷新</span></p></div></div></div></body></html>

bing/bing_test.go

@@ -4,44 +4,10 @@ import (
 	"net/url"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/karust/openserp/core"
 )
 
-var browser *core.Browser
-
-func init() {
-	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, UseStealth: true, Timeout: time.Second * 5, LeavePageOpen: true}
-	browser, _ = core.NewBrowser(opts)
-}
-
-func TestSearchBing(t *testing.T) {
-	bing := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := bing.Search(query)
-	if err != nil {
-		t.Fatalf("Cannot [SearchBing]: %s", err)
-	}
-
-	if len(results) == 0 {
-		t.Fatalf("[SearchBing] returned empty result")
-	}
-
-	// Check that we have some basic fields populated
-	firstResult := results[0]
-	if firstResult.Title == "" {
-		t.Errorf("First result missing title: %+v", firstResult)
-	}
-	if firstResult.URL == "" {
-		t.Errorf("First result missing URL: %+v", firstResult)
-	}
-	if firstResult.Rank == 0 {
-		t.Errorf("First result missing rank: %+v", firstResult)
-	}
-}
-
 func TestBuildImageURL(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -65,7 +31,7 @@ func TestBuildImageURL(t *testing.T) {
 			name:     "image query with filetype",
 			query:    core.Query{Text: "dogs", Filetype: "png"},
 			wantErr:  false,
-			wantCont: "q=dogs+filetype%3Apng",
+			wantCont: "q=dogs",
 		},
 		{
 			name:     "empty query",
@@ -103,59 +69,3 @@ func TestBuildImageURL(t *testing.T) {
 		})
 	}
 }
-
-func TestBingImageSearch(t *testing.T) {
-	bing := New(*browser, core.SearchEngineOptions{RateTime: 5})
-
-	query := core.Query{
-		Text:     "golden puppy",
-		Limit:    25,
-		Filetype: "jpg",
-	}
-
-	results, err := bing.SearchImage(query)
-	if err != nil {
-		t.Fatalf("Cannot search Bing images: %s", err)
-	}
-
-	if len(results) == 0 {
-		t.Fatalf("Bing image search returned empty result")
-	}
-
-	// Check that we have image results with proper fields
-	firstResult := results[0]
-	if firstResult.URL == "" {
-		t.Errorf("First result missing image URL: %+v", firstResult)
-	}
-	if firstResult.Title == "" {
-		t.Errorf("First result missing title: %+v", firstResult)
-	}
-
-	// Check that we have either image URL or source URL
-	hasImageURL := firstResult.URL != ""
-	hasSourceURL := firstResult.URL != ""
-	if !hasImageURL && !hasSourceURL {
-		t.Errorf("First result should have either image URL or source URL: %+v", firstResult)
-	}
-
-	// For image results, URL should typically point to an image file
-	if hasImageURL {
-		// Check if it looks like an image URL (common extensions)
-		imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"}
-		hasImageExtension := false
-		for _, ext := range imageExtensions {
-			if strings.Contains(strings.ToLower(firstResult.URL), ext) {
-				hasImageExtension = true
-				break
-			}
-		}
-		if !hasImageExtension {
-			t.Logf("Image URL doesn't have common extension (might be valid): %s", firstResult.URL)
-		}
-	}
-
-	t.Logf("Found %d image results", len(results))
-	t.Logf("First result - Title: %s", firstResult.Title)
-	t.Logf("First result - Image URL: %s", firstResult.URL)
-	t.Logf("First result - Source URL: %s", firstResult.URL)
-}

core/browser_test.go

@@ -1,15 +1,20 @@
+//go:build integration
+// +build integration
+
 package core
 
 import (
+	"os"
+	"strings"
 	"testing"
+
+	"github.com/karust/openserp/testutil"
 )
 
 var browser *Browser
 
 func TestCreateBrowser(t *testing.T) {
-	// if browser != nil && browser.IsInitialized() {
-	// 	return
-	// }
+	testutil.RequireIntegration(t)
 
 	var err error
 	opts := BrowserOpts{IsHeadless: true, IsLeakless: false}
@@ -19,20 +24,15 @@ func TestCreateBrowser(t *testing.T) {
 	}
 }
 
-// func TestCreateLeaklessBrowser(t *testing.T) {
-// 	var err error
-// 	opts := BrowserOpts{IsHeadless: true, IsLeakless: true}
-// 	browser, err = NewBrowser(opts)
-// 	if err != nil {
-// 		t.Fatalf("Error failed initializing leakless browser: %s", err)
-// 	}
-// }
-
-// Manually observe test results for now
+// Manually observe test results for now.
 func TestBot(t *testing.T) {
+	testutil.RequireIntegration(t)
+	if strings.TrimSpace(os.Getenv("OPENSERP_BOT_TESTS")) != "1" {
+		t.Skip("set OPENSERP_BOT_TESTS=1 to run manual anti-bot screenshot integration test")
+	}
 
 	var err error
-	opts := BrowserOpts{IsHeadless: false, IsLeakless: true, LeavePageOpen: true}
+	opts := BrowserOpts{IsHeadless: false, IsLeakless: false, LeavePageOpen: true}
 	browser, err = NewBrowser(opts)
 	if err != nil {
 		t.Fatalf("Error failed initializing browser: %s", err)

core/captcha_test.go

@@ -1,15 +1,21 @@
+//go:build integration
+// +build integration
+
 package core
 
 import (
 	"testing"
+
+	"github.com/karust/openserp/testutil"
 )
 
-var (
-	API_KEY = ""
-)
+const captchaAPIKeyEnv = "OPENSERP_2CAPTCHA_API_KEY"
 
 func Test2Captcha(t *testing.T) {
-	solver := NewSolver(API_KEY)
+	testutil.RequireIntegration(t)
+	apiKey := testutil.RequireEnv(t, captchaAPIKeyEnv)
+
+	solver := NewSolver(apiKey)
 	sitekey := "6LfwuyUTAAAAAOAmoS0fdqijC2PbbdH4kjq62Y1b"
 	url := "https://www.google.com/sorry/index?continue=https://www.google.de/search%3Fhl%3DDE%26lr%3Dlang_de%26nfpr%3D1%26num%3D500%26pws%3D0%26q%3Dwhere%2Bwhy%2Beach&hl=DE&q=EgRegw55GObHiq4GIjDqmzFKayGXrS2-s9ooWfcskhpK8-6tIjWSaSvhxd3f5eAyUXj7lYq2DYLDXB8ASz0yAXJaAUM"
 	datas := "Ghk0n7ZQNDS0c7ES53eef_YBfSdfeXnyRD0p2OR0R4Dg91CUXKS_hio5Do6TpJ8sHhhOat_NymTASZGe1gqAjP7w9dSvhvRT7QXsrdziO3JPngLDSRzDdjT42GDcSbO0kzInlDPxe1yy2t4yifo9xHpMnlZU7pTVNTQUIXqOMLHAR-iERi6aoSQDQ4d-88-jW3LEinquxEut0OhHG2l2stwG9AnCmNvCsUNJda-H24saFlOh5csK9KNXeeQmpr6at52_skMIMiLXSlY56vYFVCRMkXLQdAM"

duckduckgo/search_test.go

@@ -1,6 +1,7 @@
 package duckduckgo
 
 import (
+	"net/url"
 	"testing"
 
 	"github.com/karust/openserp/core"
@@ -8,56 +9,52 @@ import (
 
 func TestBuildURL(t *testing.T) {
 	tests := []struct {
-		name     string
-		query    core.Query
-		expected string
-		wantErr  bool
+		name    string
+		query   core.Query
+		wantErr bool
 	}{
 		{
-			name: "Basic search",
-			query: core.Query{
-				Text: "golang programming",
-			},
-			expected: "https://duckduckgo.com/?q=golang+programming&t=h&ia=web",
-			wantErr:  false,
+			name:  "basic search",
+			query: core.Query{Text: "golang programming"},
 		},
 		{
-			name: "Search with site filter",
-			query: core.Query{
-				Text: "golang",
-				Site: "github.com",
-			},
-			expected: "https://duckduckgo.com/?q=golang+site%3Agithub.com&t=h&ia=web",
-			wantErr:  false,
+			name:  "search with site filter",
+			query: core.Query{Text: "golang", Site: "github.com"},
 		},
 		{
-			name: "Search with filetype",
-			query: core.Query{
-				Text:     "documentation",
-				Filetype: "pdf",
-			},
-			expected: "https://duckduckgo.com/?q=documentation+filetype%3Apdf&t=h&ia=web",
-			wantErr:  false,
+			name:  "search with filetype",
+			query: core.Query{Text: "documentation", Filetype: "pdf"},
 		},
 		{
-			name: "Empty query",
-			query: core.Query{
-				Text: "",
-			},
-			expected: "",
-			wantErr:  true,
+			name:    "empty query",
+			query:   core.Query{Text: ""},
+			wantErr: true,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := BuildURL(tt.query)
+			got, err := BuildURL(tt.query, 0)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("BuildURL() error = %v, wantErr %v", err, tt.wantErr)
+				t.Fatalf("BuildURL() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
 				return
 			}
-			if got != tt.expected {
-				t.Errorf("BuildURL() = %v, want %v", got, tt.expected)
+
+			parsed, err := url.Parse(got)
+			if err != nil {
+				t.Fatalf("BuildURL() returned invalid URL: %v", err)
+			}
+			params := parsed.Query()
+			if params.Get("q") == "" {
+				t.Fatalf("BuildURL() should include q parameter, got %s", got)
+			}
+			if params.Get("ia") != "web" {
+				t.Fatalf("BuildURL() should include ia=web, got %s", got)
+			}
+			if params.Get("t") != "h" {
+				t.Fatalf("BuildURL() should include t=h, got %s", got)
 			}
 		})
 	}
@@ -65,26 +62,18 @@ func TestBuildURL(t *testing.T) {
 
 func TestBuildImageURL(t *testing.T) {
 	tests := []struct {
-		name     string
-		query    core.Query
-		expected string
-		wantErr  bool
+		name    string
+		query   core.Query
+		wantErr bool
 	}{
 		{
-			name: "Basic image search",
-			query: core.Query{
-				Text: "golang logo",
-			},
-			expected: "https://duckduckgo.com/?q=golang+logo&t=h&iax=images&ia=images",
-			wantErr:  false,
+			name:  "basic image search",
+			query: core.Query{Text: "golang logo"},
 		},
 		{
-			name: "Empty query",
-			query: core.Query{
-				Text: "",
-			},
-			expected: "",
-			wantErr:  true,
+			name:    "empty query",
+			query:   core.Query{Text: ""},
+			wantErr: true,
 		},
 	}
 
@@ -92,11 +81,22 @@ func TestBuildImageURL(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := BuildImageURL(tt.query)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("BuildImageURL() error = %v, wantErr %v", err, tt.wantErr)
+				t.Fatalf("BuildImageURL() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
 				return
 			}
-			if got != tt.expected {
-				t.Errorf("BuildImageURL() = %v, want %v", got, tt.expected)
+
+			parsed, err := url.Parse(got)
+			if err != nil {
+				t.Fatalf("BuildImageURL() returned invalid URL: %v", err)
+			}
+			params := parsed.Query()
+			if params.Get("q") == "" {
+				t.Fatalf("BuildImageURL() should include q parameter, got %s", got)
+			}
+			if params.Get("iax") != "images" || params.Get("ia") != "images" {
+				t.Fatalf("BuildImageURL() should target image mode, got %s", got)
 			}
 		})
 	}

google/search.go

@@ -71,7 +71,22 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
 func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
 	gogl.logger.Debug("Solve captcha: sitekey=%s", sitekey)
 
-	resp, _, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
+	if gogl.CaptchaSolver == nil {
+		gogl.logger.Error("Captcha solver is not configured")
+		return false
+	}
+	if page == nil {
+		gogl.logger.Error("Captcha page context is missing")
+		return false
+	}
+
+	info, err := page.Info()
+	if err != nil {
+		gogl.logger.Error("Cannot read page info for captcha solve: %s", err)
+		return false
+	}
+
+	resp, _, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, info.URL, datas)
 	if err != nil {
 		gogl.logger.Error("Captcha solve failed: %s", err)
 		return false

google/search_raw_test.go

@@ -0,0 +1,76 @@
+package google
+
+import (
+	"testing"
+
+	"github.com/karust/openserp/testutil"
+)
+
+func TestGoogleResultParserSnapshots(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		fixture        string
+		minResultCount int
+		maxResultCount int
+		wantZero       bool
+	}{
+		{
+			name:           "search results",
+			fixture:        "search_results.html",
+			minResultCount: 1,
+			maxResultCount: 20,
+		},
+		{
+			name:     "no results",
+			fixture:  "search_no_results.html",
+			wantZero: true,
+		},
+		{
+			name:     "captcha page",
+			fixture:  "search_captcha.html",
+			wantZero: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			results, err := googleResultParser(testutil.ResponseFromFixture(t, tt.fixture))
+			if err != nil {
+				t.Fatalf("googleResultParser() error = %v", err)
+			}
+
+			if tt.wantZero {
+				if len(results) != 0 {
+					t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
+				}
+				return
+			}
+
+			if len(results) < tt.minResultCount || len(results) > tt.maxResultCount {
+				t.Fatalf(
+					"unexpected result count for %s: got %d, want range [%d,%d]",
+					tt.fixture, len(results), tt.minResultCount, tt.maxResultCount,
+				)
+			}
+
+			testutil.AssertSequentialRanks(t, results)
+			testutil.AssertFirstResultFilled(t, results)
+		})
+	}
+}
+
+func TestGoogleResultParserEmptyHTML(t *testing.T) {
+	t.Parallel()
+
+	results, err := googleResultParser(testutil.ResponseFromString(""))
+	if err != nil {
+		t.Fatalf("googleResultParser() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+	}
+}

google/search_test.go

@@ -2,6 +2,8 @@ package google
 
 import (
 	"testing"
+
+	"github.com/karust/openserp/core"
 )
 
 func TestParseSourceImageURL(t *testing.T) {
@@ -23,3 +25,10 @@ func TestParseSourceImageURL(t *testing.T) {
 		t.Fatalf("Want: %v, Got: %v", want, got)
 	}
 }
+
+func TestSolveCaptchaWithoutConfiguredSolverReturnsFalse(t *testing.T) {
+	gogl := New(core.Browser{}, core.SearchEngineOptions{})
+	if got := gogl.solveCaptcha(nil, "sitekey", "datas"); got {
+		t.Fatal("expected solveCaptcha to fail without solver/page context")
+	}
+}

google/testdata/search_captcha.html

@@ -0,0 +1 @@
+<html><head><title>http://test.test</title></head><body style=""><div style=""><hr noshade="" size="1" style=""/><br/><form id="captcha-form" action="index" method="post"><div id="recaptcha" class="g-recaptcha" data-sitekey="6LfwuyUTAAAAAOAmoS0fdqijC2PbbdH4kjq62Y1b" data-callback="submitCallback" data-s="cgT58_2_flKBmo04fOKJCG4-J6XWpitIDJ-mYmZjIdMTsvoNW58ho7SkZLtjJGugMaBWOBFCUVVUYrVgS01BOMJEWTxgdJfWCFZka7PgVPIdeMtDxgTH_Jfi5QJUEwG55k5HjhfONjzxnU4iA4M_wqgxpKE4JvtwKoO2t9he9pWf_p13-aKQUm5xXN41wc6X41Xfp7aSzfINXBUrvyTKjHxdTChq3bj8yxYPhyahI-edf7NAdgUW-WkaO945Hx2I1fjSiKOiiOuKbJkYVA"><div style=""><div></div><textarea id="id_1" name="g-recaptcha-response" class="g-recaptcha-response" style=""></textarea></div></div><input type="hidden" name="q" value="TRIMMED"/><input type="hidden" name="continue" value="http://test.test"/></form><hr noshade="" size="1" style=""/><div style=""><b>About this page</b><br/><br/>Our systems have detected unusual traffic from your computer network. This page checks to see if it's really you sending the requests, and not a robot.<a href="#">Why did this happen?</a><br/><br/><div id="infoDiv" style="">This page appears when Google automatically detects requests coming from your computer network which appear to be in violation of the<a href="http://test.test">Terms of Service</a>. The block will expire shortly after those requests stop. In the meantime, solving the above CAPTCHA will let you continue to use our services.<br/><br/>This traffic may have been sent by malicious software, a browser plug-in, or a script that sends automated requests. If you share your network connection, ask your administrator for help — a different computer using the same IP address may be responsible.<a href="http://test.test">Learn more</a><br/><br/>Sometimes you may be asked to solve the CAPTCHA if you are using advanced terms that robots are known to use, or sending requests very quickly.</div><br/><div style="">IP address: 195.128.99.4<br/>Time: 2026-04-13T19:39:32Z<br/>URL: http://test.test<br/></div></div></div><div style=""><div style=""></div><div class="g-recaptcha-bubble-arrow" style=""></div><div class="g-recaptcha-bubble-arrow" style=""></div><div style=""></div></div></body></html>

testutil/fixtures.go

@@ -0,0 +1,70 @@
+package testutil
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/karust/openserp/core"
+)
+
+// ResponseFromFixture reads an HTML file from the package's testdata/ directory
+// and returns it wrapped in an *http.Response suitable for parser functions.
+func ResponseFromFixture(t *testing.T, file string) *http.Response {
+	t.Helper()
+
+	path := filepath.Join("testdata", file)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("failed to read fixture %s: %v", path, err)
+	}
+
+	return ResponseFromBytes(data)
+}
+
+// ResponseFromString wraps a raw HTML string in an *http.Response.
+func ResponseFromString(html string) *http.Response {
+	return ResponseFromBytes([]byte(html))
+}
+
+// ResponseFromBytes wraps raw bytes in an *http.Response with status 200.
+func ResponseFromBytes(data []byte) *http.Response {
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(bytes.NewReader(data)),
+	}
+}
+
+// AssertSequentialRanks verifies that result ranks start at 1 and increase by 1.
+func AssertSequentialRanks(t *testing.T, results []core.SearchResult) {
+	t.Helper()
+
+	for i, r := range results {
+		if r.Rank != i+1 {
+			t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, i+1)
+		}
+	}
+}
+
+// AssertFirstResultFilled checks that the first result has non-empty URL, Title, and Description.
+func AssertFirstResultFilled(t *testing.T, results []core.SearchResult) {
+	t.Helper()
+
+	if len(results) == 0 {
+		t.Fatal("expected at least one result")
+	}
+
+	first := results[0]
+	if first.URL == "" {
+		t.Fatal("first result URL is empty")
+	}
+	if first.Title == "" {
+		t.Fatal("first result title is empty")
+	}
+	if first.Description == "" {
+		t.Fatal("first result description is empty")
+	}
+}

yandex/search_raw.go

@@ -38,26 +38,40 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
 	results := []core.SearchResult{}
 	rank := 1
 
-	// Get individual results
-	sel := doc.Find("li.serp-item")
+	// Prefer stable container + attributes and keep legacy fallback.
+	sel := doc.Find("#search-result > li[data-fast], li.serp-item")
 
 	for i := range sel.Nodes {
 		item := sel.Eq(i)
 
+		// Skip blocks without a result heading.
+		titleTag := item.Find("h2").First()
+		if titleTag.Length() == 0 {
+			continue
+		}
+
 		// Find URL
-		linkTag := item.Find("a")
+		linkTag := item.Find("a.OrganicTitle-Link").First()
+		if linkTag.Length() == 0 {
+			linkTag = titleTag.Closest("a")
+		}
+		if linkTag.Length() == 0 {
+			linkTag = item.Find("a").First()
+		}
 		link, _ := linkTag.Attr("href")
 		link = strings.Trim(link, " ")
 
 		// Find title
-		titleTag := item.Find("h2")
-		title := titleTag.Text()
+		title := strings.TrimSpace(titleTag.Text())
 
 		// Find description
-		descTag := item.Find(`span.OrganicTextContentSpan`)
-		desc := descTag.Text()
+		descTag := item.Find(`span.OrganicTextContentSpan`).First()
+		if descTag.Length() == 0 {
+			descTag = item.Find("div.OrganicText").First()
+		}
+		desc := strings.TrimSpace(descTag.Text())
 
-		if link != "" && link != "#" {
+		if link != "" && link != "#" && title != "" {
 			result := core.SearchResult{
 				Rank:        rank,
 				URL:         link,

yandex/search_raw_test.go

@@ -0,0 +1,76 @@
+package yandex
+
+import (
+	"testing"
+
+	"github.com/karust/openserp/testutil"
+)
+
+func TestYandexResultParserSnapshots(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		fixture        string
+		minResultCount int
+		maxResultCount int
+		wantZero       bool
+	}{
+		{
+			name:           "search results",
+			fixture:        "search_results.html",
+			minResultCount: 1,
+			maxResultCount: 30,
+		},
+		{
+			name:     "no results",
+			fixture:  "search_no_results.html",
+			wantZero: true,
+		},
+		{
+			name:     "captcha page",
+			fixture:  "search_captcha.html",
+			wantZero: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			results, err := yandexResultParser(testutil.ResponseFromFixture(t, tt.fixture))
+			if err != nil {
+				t.Fatalf("yandexResultParser() error = %v", err)
+			}
+
+			if tt.wantZero {
+				if len(results) != 0 {
+					t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
+				}
+				return
+			}
+
+			if len(results) < tt.minResultCount || len(results) > tt.maxResultCount {
+				t.Fatalf(
+					"unexpected result count for %s: got %d, want range [%d,%d]",
+					tt.fixture, len(results), tt.minResultCount, tt.maxResultCount,
+				)
+			}
+
+			testutil.AssertSequentialRanks(t, results)
+			testutil.AssertFirstResultFilled(t, results)
+		})
+	}
+}
+
+func TestYandexResultParserEmptyHTML(t *testing.T) {
+	t.Parallel()
+
+	results, err := yandexResultParser(testutil.ResponseFromString(""))
+	if err != nil {
+		t.Fatalf("yandexResultParser() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+	}
+}

yandex/search_test.go

@@ -1,3 +1,6 @@
+//go:build integration
+// +build integration
+
 package yandex
 
 import (
@@ -5,35 +8,31 @@ import (
 	"time"
 
 	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/testutil"
 )
 
-var browser *core.Browser
-
-func init() {
+func createTestBrowser(t *testing.T) *core.Browser {
+	t.Helper()
 	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 15, LeavePageOpen: true}
-	browser, _ = core.NewBrowser(opts)
+	b, err := core.NewBrowser(opts)
+	if err != nil {
+		t.Fatalf("failed to create test browser: %v", err)
+	}
+	return b
 }
 
-// func TestParseImgData(t *testing.T) {
-
-// 	jsonData, _ := os.ReadFile("./testImgData.json")
-// 	var obj ImageData
-// 	if err := json.Unmarshal(jsonData, &obj); err != nil {
-// 		t.Fatal(err)
-// 	}
-
-// 	if (len(obj.InitalState.SerpList.Items.Entities)) != 30 {
-// 		t.Fail()
-// 	}
-// }
-
 func TestSearchYandex(t *testing.T) {
+	testutil.RequireIntegration(t)
 
+	browser := createTestBrowser(t)
 	yand := New(*browser, core.SearchEngineOptions{})
 
 	query := core.Query{Text: "HEY", Limit: 10}
 	results, err := yand.Search(query)
 	if err != nil {
+		if err == core.ErrSearchTimeout || err == core.ErrCaptcha {
+			t.Skipf("skipping unstable live yandex search result: %v", err)
+		}
 		t.Fatalf("Cannot [SearchYandex]: %s", err)
 	}
 
@@ -43,6 +42,9 @@ func TestSearchYandex(t *testing.T) {
 }
 
 func TestImageYandex(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	browser := createTestBrowser(t)
 	yand := New(*browser, core.SearchEngineOptions{})
 
 	query := core.Query{Text: "furry tiger", Limit: 30}
@@ -51,7 +53,7 @@ func TestImageYandex(t *testing.T) {
 		t.Fatalf("Cannot [ImageYandex]: %s", err)
 	}
 
-	if len(results) < 30 {
+	if len(results) == 0 {
 		t.Fatalf("[ImageYandex] returned empty result")
 	}
 }

yandex/testdata/search_captcha.html

@@ -0,0 +1 @@
+<html prefix="og: URL_PLACEHOLDER" lang="en"><head><title>Are you not a robot?</title></head><body class="pointerfocus"><div id="root"><div class="Theme Theme_color_yandex-default Theme_root_default"><main><div class="Container"><div class="Spacer" style=""><a href="URL_PLACEHOLDER" aria-label="Yandex" class="Link Link_view_default LogoLink"></a></div><form method="POST" action="/checkcaptcha?key=d783361c-370c5bc7-d56b5285-28801df8_2%2F1776110847%2Feb0e9d0fbc92c38d8aa3b6c247841797_7c86a4050aeb96804d9dee86ae1e784e&amp;mt=REDACTED&amp;retpath=REDACTED&amp;u=8512316851816905896&amp;s=90658cd165de935a99a2dc1b1a708cae" id="id_1"><div class="Spacer" style=""><h1 class="Text Text_weight_medium Text_typography_headline-s">Please confirm that you and not a robot are sending requests</h1></div><div class="Spacer" style=""><span class="Text Text_weight_regular Text_typography_body-long-m">We're sorry, but it looks like requests sent from your device are automated.<a href="URL_PLACEHOLDER" target="_blank" class="Link Link_view_default">Why might this happen?</a></span></div><div class="Spacer Spacer_auto-gap_bottom" style=""><div class="CheckboxCaptcha" data-testid="checkbox-captcha"><div class="CheckboxCaptcha-Inner"><div class="CheckboxCaptcha-Anchor"><input type="submit" id="js-button" class="CheckboxCaptcha-Button" aria-checked="false" aria-labelledby="checkbox-label" aria-describedby="checkbox-description" role="checkbox" value=""/><div class="CheckboxCaptcha-Checkbox" data-checked="false"></div></div><div class="CheckboxCaptcha-Label"><span class="Text Text_weight_regular Text_typography_control-xxl CheckboxCaptcha-LabelText"><span id="checkbox-label">I'm not a robot</span></span><span class="Text Text_weight_regular Text_typography_control-l CheckboxCaptcha-SecondaryText"><span id="id_2">Press to continue</span></span></div></div><div class="Text Text_color_ghost Text_weight_regular Text_typography_control-s CaptchaLinks CheckboxCaptcha-Links"><button aria-label="Show links" aria-pressed="false" type="button" class="CaptchaButton CaptchaButton_view_clear CaptchaButton_size_m CaptchaLinks-ToggleButton CaptchaLinks-ToggleButton_checkbox"></button><div class="CaptchaLinks-Links"><a color="secondary" target="_blank" href="URL_PLACEHOLDER" class="Link Link_color_secondary Link_view_captcha CaptchaLinks-ServiceLink">SmartCaptcha by Yandex Cloud</a></div></div></div></div><input type="hidden" name="rdata" value="TRIMMED"/><input type="hidden" name="pdata" value="TRIMMED"/><input type="hidden" name="tdata" value=""/><input type="hidden" name="picasso" value="TRIMMED"/></form><span class="Text Text_color_ghost Text_weight_regular Text_typography_control-xs">If you have any problems, please use the<a href="URL_PLACEHOLDER" target="_blank" class="Link Link_view_default">feedback form</a></span><div class="Spacer" style=""><span class="Text Text_color_ghost Text_weight_regular Text_typography_control-xs">If you need to automatically set Search queries, use<a href="URL_PLACEHOLDER" target="_blank" class="Link Link_view_default">Yandex Search API v2</a></span></div><div class="Spacer" style=""><span class="Text Text_color_ghost Text_weight_regular Text_typography_control-xxs"><span data-testid="unique-key">8512316851816905896</span>:<span data-testid="timestamp">1776110847</span></span></div></div></main></div></div><div><img src="URL_PLACEHOLDER" style="" alt=""/></div></body><canvas width="300" height="300" style=""></canvas><canvas width="300" height="300" style=""></canvas></html>