fix docs + google: gate organic results by matched selector, harden answer-box parse

cmd/root.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	version               = "0.8.1"
+	version               = "0.8.2"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )

docs/CONTRIBUTING.md

@@ -8,8 +8,8 @@
 4. Run `make fmt`, `make lint`, and `make test` before opening the PR.
 5. Explain what changed, why it changed, and how you tested it.
 
-Good first issues are curated in [`GOOD_FIRST_ISSUES.md`](GOOD_FIRST_ISSUES.md).
-New engine work should start with [`ADDING_AN_ENGINE.md`](ADDING_AN_ENGINE.md).
+Good first issues are curated in [`GOOD_FIRST_ISSUES.md`](https://github.com/karust/openserp/blob/main/docs/GOOD_FIRST_ISSUES.md).
+New engine work should start with [`ADDING_AN_ENGINE.md`](https://github.com/karust/openserp/blob/main/docs/ADDING_AN_ENGINE.md).
 
 ## Development Setup
 
@@ -92,7 +92,7 @@ Update [`config.yaml`](../config.yaml) with your engine section:
 - Parser tests (prefer deterministic fixtures in `testdata/`)
 - Integration tests guarded by `testutil.RequireIntegration(t)`
 
-See [`ADDING_AN_ENGINE.md`](ADDING_AN_ENGINE.md) for the full checklist.
+See [`ADDING_AN_ENGINE.md`](https://github.com/karust/openserp/blob/main/docs/ADDING_AN_ENGINE.md) for the full checklist.
 
 ## Code Style and Quality Checks
 

docs/GOOD_FIRST_ISSUES.md

@@ -14,26 +14,11 @@ with [`ADDING_AN_ENGINE.md`](ADDING_AN_ENGINE.md).
 
 Area: CLI, tests
 
-Add a table test verifying every supported engine name is accepted by the CLI and
-an unknown engine returns a deterministic, user-friendly error listing valid
-engines. Covers `google`, `yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`.
-Start in [`cmd/search.go`](../cmd/search.go).
-
-### [#28 Add Yandex parser fallback micro-fixtures](https://github.com/karust/openserp/issues/28)
-
-Area: parser tests
-
-Add compact Yandex HTML fixtures covering link, title, and snippet fallback
-selectors without depending on full saved result pages. No browser or network.
-Start in `yandex/selectors.go` and `yandex/parse_html.go`.
-
-### [#29 Add Baidu parser fallback micro-fixtures](https://github.com/karust/openserp/issues/29)
-
-Area: parser tests
-
-Add compact Baidu HTML fixtures for title, URL, and description fallback paths.
-Missing optional fields must not panic; result order stays stable. No browser or
-network. Start in `baidu/selectors.go` and `baidu/parse_html.go`.
+Make the unknown-engine error list valid engine names, then add a table test for
+engine dispatch in both modes: browser mode accepts all six engines (`google`,
+`yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`); raw mode accepts
+`google/yandex/baidu/ecosia` and rejects `bing`/`duckduckgo` with a clear message.
+Both dispatch switches live in [`cmd/search.go`](../cmd/search.go).
 
 ### [#30 Document raw-mode support per engine](https://github.com/karust/openserp/issues/30)
 
@@ -47,10 +32,11 @@ search-endpoint section. Start in `cmd/serve.go` and `README.md`.
 
 Area: release tooling
 
-Add a script under `.release/` that builds the binary, starts the server, checks
-`/health`, and exits cleanly. It must fail fast with a useful error when the
-server does not become healthy. Docker and `go install` checks are follow-ups.
-Document it in [`.release/build.md`](../.release/build.md).
+Add a script under `scripts/` (e.g. `scripts/smoke-check.sh`) that builds the
+binary, starts the server, polls `/health` until ready, then shuts down and exits
+cleanly. It must fail fast with a non-zero exit when the server does not become
+healthy. Docker and `go install` checks are follow-ups. Document it in
+[`CONTRIBUTING.md`](CONTRIBUTING.md).
 
 ## Backlog
 

google/search.go

@@ -222,10 +222,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 		gogl.acceptCookies(page)
 	}
 
-	// Wait for result containers (data-hveid + data-ved) to hydrate. WaitLoad in
-	// Navigate fires before Google's right-rail/answers script attaches these
-	// attributes, so a one-shot Search races the DOM and frequently sees nothing.
-	searchResultElems, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, gogl.GetSelectorTimeout())
+	// Wait for the canonical organic wrapper first, then Google's broader
+	// data-hveid/data-ved layout. Headless and headful Chrome can receive
+	// different SERP markup for the same query.
+	searchResultElems, matchedSelector, err := core.WaitForElements(ctx, page, searchResultSelectors(), gogl.GetSelectorTimeout())
 	if err != nil {
 		if gogl.checkCaptcha(page, query.ProxyURL) {
 			gogl.logger.Error("Captcha detected: %s", url)
@@ -240,6 +240,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 		}
 		return nil, core.ErrSearchTimeout
 	}
+	gogl.logger.Debug("Search result selector matched: %s (%d elements)", matchedSelector, len(searchResultElems))
 
 	totalResults, err := gogl.getTotalResults(page)
 	if err != nil {
@@ -250,12 +251,18 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 	rank := query.Start
 	adRank := 1
 	absoluteRank := query.Start + 1
+	// When matched by the canonical organic selector (div.tF2Cxc) every element
+	// is already an organic result, but the wrapper itself often lacks data-ved
+	// (it sits on the outer .g/data-hveid container). Only require data-ved when
+	// we fell back to the broad attribute selector, which also matches non-result
+	// blocks (knowledge panels, nav) that must be filtered out.
+	matchedOrganic := matchedSelector == Selectors.Results
 	for _, resEl := range searchResultElems {
 		srchRes := core.SearchResult{}
 
 		isAd := googleElementHasAdMarker(resEl)
 		isAnswerBox := query.Features && core.HasAttribute(resEl, "data-ulkwtsb") && !core.HasAttribute(resEl, "data-ispaa")
-		isResultCandidate := core.HasAttribute(resEl, "data-ved")
+		isResultCandidate := matchedOrganic || core.HasAttribute(resEl, "data-ved")
 
 		if isAd {
 			// 1. Parse ads
@@ -364,7 +371,11 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 				}
 				srchRes.URL = href.String()
 				srchRes.Title = answerText[0]
-				srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n")
+				// answerText is [title, body..., source, meta]; drop the trailing
+				// two metadata lines, but never slice past the title — a short
+				// answer (len 2) would otherwise produce answerText[1:0] and panic.
+				descEnd := max(len(answerText)-2, 1)
+				srchRes.Description = strings.Join(answerText[1:descEnd], "\n")
 				srchRes.Rank = -1 * (i + 1)
 				srchRes.Type = core.ResultTypePeopleAlsoAsk
 				searchResults = append(searchResults, srchRes)
@@ -379,17 +390,11 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 			}
 			srchRes.Title, _ = titleTag.Text()
 
-			// Get URL from parent link of h3
-			link, err := titleTag.Parent()
-			if err == nil {
-				isLink, matchErr := link.Matches(Selectors.Link)
-				if matchErr != nil {
-					gogl.logger.Debug("Failed to match link selector: %s", matchErr)
-				}
-				if isLink {
-					href, _ := link.Property("href")
-					srchRes.URL = href.String()
-				}
+			// Get URL from the nearest link around the title.
+			link := core.ClosestMatching(titleTag, Selectors.Link, 3)
+			if link != nil {
+				href, _ := link.Property("href")
+				srchRes.URL = href.String()
 			}
 
 			// Skip if URL is empty
@@ -404,18 +409,18 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 			} else if descTag, err := resEl.Element(Selectors.DescFallback); err == nil {
 				desc, _ = descTag.Text()
 			} else {
-				// Structural fallback
-				parent, err := titleTag.Parent()
-				if err == nil {
-					parent, err = parent.Parent()
-					if err == nil {
-						parent, err = parent.Parent()
-						if err == nil {
-							if descTag, err := parent.Next(); err == nil {
-								if descDiv, err := descTag.Element(Selectors.DescAny); err == nil {
-									desc, _ = descDiv.Text()
-								}
-							}
+				// Structural fallback: the description lives in the sibling block
+				// after the title's great-grandparent wrapper.
+				anchor := titleTag
+				for i := 0; i < 3 && anchor != nil; i++ {
+					if anchor, err = anchor.Parent(); err != nil {
+						anchor = nil
+					}
+				}
+				if anchor != nil {
+					if sib, err := anchor.Next(); err == nil {
+						if descDiv, err := sib.Element(Selectors.DescAny); err == nil {
+							desc, _ = descDiv.Text()
 						}
 					}
 				}

google/search_test.go

@@ -61,8 +61,8 @@ func TestBuildSearchURL(t *testing.T) {
 				if got := params.Get("pws"); got != "0" {
 					t.Fatalf("unexpected pws value: %q", got)
 				}
-				if got := params.Get("nfpr"); got != "1" {
-					t.Fatalf("unexpected nfpr value: %q", got)
+				if got := params.Get("nfpr"); got != "" {
+					t.Fatalf("nfpr should be omitted by default, got %q", got)
 				}
 			},
 		},

google/selectors.go

@@ -53,3 +53,7 @@ var Selectors = struct {
 	// ImageTitle selectors are tried in order to recover a human-readable title.
 	ImageTitle: []string{"h3", "a"},
 }
+
+func searchResultSelectors() []string {
+	return []string{Selectors.Results, Selectors.ResultsBroad}
+}

google/url.go

@@ -298,8 +298,7 @@ func BuildURL(q core.Query) (string, error) {
 		params.Add("lr", "lang_"+locale.language)
 	}
 
-	params.Add("pws", "0")  // Do not personalize earch results
-	params.Add("nfpr", "1") // Do not auto correct search queries
+	params.Add("pws", "0") // Do not personalize search results
 	params.Add("sourceid", "chrome")
 	params.Add("ie", "UTF-8")
 
@@ -367,8 +366,7 @@ func BuildImageURL(q core.Query) (string, error) {
 		params.Add("lr", "lang_"+locale.language)
 	}
 
-	params.Add("pws", "0")  // Do not personalize earch results
-	params.Add("nfpr", "1") // Do not auto correct search queries
+	params.Add("pws", "0") // Do not personalize search results
 
 	base.RawQuery = params.Encode()
 	return base.String(), nil