From 45287112dfde5449de1fee451d1d24c9afc45c46 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Wed, 10 Jun 2026 15:15:42 +0300 Subject: [PATCH] fix docs + google: gate organic results by matched selector, harden answer-box parse --- cmd/root.go | 2 +- docs/CONTRIBUTING.md | 6 ++-- docs/GOOD_FIRST_ISSUES.md | 34 +++++++-------------- google/search.go | 63 +++++++++++++++++++++------------------ google/search_test.go | 4 +-- google/selectors.go | 4 +++ google/url.go | 6 ++-- 7 files changed, 56 insertions(+), 63 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 7761562..5f6a4d4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( ) const ( - version = "0.8.1" + version = "0.8.2" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 2052362..2765121 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -8,8 +8,8 @@ 4. Run `make fmt`, `make lint`, and `make test` before opening the PR. 5. Explain what changed, why it changed, and how you tested it. -Good first issues are curated in [`GOOD_FIRST_ISSUES.md`](GOOD_FIRST_ISSUES.md). -New engine work should start with [`ADDING_AN_ENGINE.md`](ADDING_AN_ENGINE.md). +Good first issues are curated in [`GOOD_FIRST_ISSUES.md`](https://github.com/karust/openserp/blob/main/docs/GOOD_FIRST_ISSUES.md). +New engine work should start with [`ADDING_AN_ENGINE.md`](https://github.com/karust/openserp/blob/main/docs/ADDING_AN_ENGINE.md). ## Development Setup @@ -92,7 +92,7 @@ Update [`config.yaml`](../config.yaml) with your engine section: - Parser tests (prefer deterministic fixtures in `testdata/`) - Integration tests guarded by `testutil.RequireIntegration(t)` -See [`ADDING_AN_ENGINE.md`](ADDING_AN_ENGINE.md) for the full checklist. +See [`ADDING_AN_ENGINE.md`](https://github.com/karust/openserp/blob/main/docs/ADDING_AN_ENGINE.md) for the full checklist. ## Code Style and Quality Checks diff --git a/docs/GOOD_FIRST_ISSUES.md b/docs/GOOD_FIRST_ISSUES.md index aa13256..6cfe2d1 100644 --- a/docs/GOOD_FIRST_ISSUES.md +++ b/docs/GOOD_FIRST_ISSUES.md @@ -14,26 +14,11 @@ with [`ADDING_AN_ENGINE.md`](ADDING_AN_ENGINE.md). Area: CLI, tests -Add a table test verifying every supported engine name is accepted by the CLI and -an unknown engine returns a deterministic, user-friendly error listing valid -engines. Covers `google`, `yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`. -Start in [`cmd/search.go`](../cmd/search.go). - -### [#28 Add Yandex parser fallback micro-fixtures](https://github.com/karust/openserp/issues/28) - -Area: parser tests - -Add compact Yandex HTML fixtures covering link, title, and snippet fallback -selectors without depending on full saved result pages. No browser or network. -Start in `yandex/selectors.go` and `yandex/parse_html.go`. - -### [#29 Add Baidu parser fallback micro-fixtures](https://github.com/karust/openserp/issues/29) - -Area: parser tests - -Add compact Baidu HTML fixtures for title, URL, and description fallback paths. -Missing optional fields must not panic; result order stays stable. No browser or -network. Start in `baidu/selectors.go` and `baidu/parse_html.go`. +Make the unknown-engine error list valid engine names, then add a table test for +engine dispatch in both modes: browser mode accepts all six engines (`google`, +`yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`); raw mode accepts +`google/yandex/baidu/ecosia` and rejects `bing`/`duckduckgo` with a clear message. +Both dispatch switches live in [`cmd/search.go`](../cmd/search.go). ### [#30 Document raw-mode support per engine](https://github.com/karust/openserp/issues/30) @@ -47,10 +32,11 @@ search-endpoint section. Start in `cmd/serve.go` and `README.md`. Area: release tooling -Add a script under `.release/` that builds the binary, starts the server, checks -`/health`, and exits cleanly. It must fail fast with a useful error when the -server does not become healthy. Docker and `go install` checks are follow-ups. -Document it in [`.release/build.md`](../.release/build.md). +Add a script under `scripts/` (e.g. `scripts/smoke-check.sh`) that builds the +binary, starts the server, polls `/health` until ready, then shuts down and exits +cleanly. It must fail fast with a non-zero exit when the server does not become +healthy. Docker and `go install` checks are follow-ups. Document it in +[`CONTRIBUTING.md`](CONTRIBUTING.md). ## Backlog diff --git a/google/search.go b/google/search.go index 82a65bc..3733393 100644 --- a/google/search.go +++ b/google/search.go @@ -222,10 +222,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor gogl.acceptCookies(page) } - // Wait for result containers (data-hveid + data-ved) to hydrate. WaitLoad in - // Navigate fires before Google's right-rail/answers script attaches these - // attributes, so a one-shot Search races the DOM and frequently sees nothing. - searchResultElems, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, gogl.GetSelectorTimeout()) + // Wait for the canonical organic wrapper first, then Google's broader + // data-hveid/data-ved layout. Headless and headful Chrome can receive + // different SERP markup for the same query. + searchResultElems, matchedSelector, err := core.WaitForElements(ctx, page, searchResultSelectors(), gogl.GetSelectorTimeout()) if err != nil { if gogl.checkCaptcha(page, query.ProxyURL) { gogl.logger.Error("Captcha detected: %s", url) @@ -240,6 +240,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } return nil, core.ErrSearchTimeout } + gogl.logger.Debug("Search result selector matched: %s (%d elements)", matchedSelector, len(searchResultElems)) totalResults, err := gogl.getTotalResults(page) if err != nil { @@ -250,12 +251,18 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor rank := query.Start adRank := 1 absoluteRank := query.Start + 1 + // When matched by the canonical organic selector (div.tF2Cxc) every element + // is already an organic result, but the wrapper itself often lacks data-ved + // (it sits on the outer .g/data-hveid container). Only require data-ved when + // we fell back to the broad attribute selector, which also matches non-result + // blocks (knowledge panels, nav) that must be filtered out. + matchedOrganic := matchedSelector == Selectors.Results for _, resEl := range searchResultElems { srchRes := core.SearchResult{} isAd := googleElementHasAdMarker(resEl) isAnswerBox := query.Features && core.HasAttribute(resEl, "data-ulkwtsb") && !core.HasAttribute(resEl, "data-ispaa") - isResultCandidate := core.HasAttribute(resEl, "data-ved") + isResultCandidate := matchedOrganic || core.HasAttribute(resEl, "data-ved") if isAd { // 1. Parse ads @@ -364,7 +371,11 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } srchRes.URL = href.String() srchRes.Title = answerText[0] - srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n") + // answerText is [title, body..., source, meta]; drop the trailing + // two metadata lines, but never slice past the title — a short + // answer (len 2) would otherwise produce answerText[1:0] and panic. + descEnd := max(len(answerText)-2, 1) + srchRes.Description = strings.Join(answerText[1:descEnd], "\n") srchRes.Rank = -1 * (i + 1) srchRes.Type = core.ResultTypePeopleAlsoAsk searchResults = append(searchResults, srchRes) @@ -379,17 +390,11 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } srchRes.Title, _ = titleTag.Text() - // Get URL from parent link of h3 - link, err := titleTag.Parent() - if err == nil { - isLink, matchErr := link.Matches(Selectors.Link) - if matchErr != nil { - gogl.logger.Debug("Failed to match link selector: %s", matchErr) - } - if isLink { - href, _ := link.Property("href") - srchRes.URL = href.String() - } + // Get URL from the nearest link around the title. + link := core.ClosestMatching(titleTag, Selectors.Link, 3) + if link != nil { + href, _ := link.Property("href") + srchRes.URL = href.String() } // Skip if URL is empty @@ -404,18 +409,18 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } else if descTag, err := resEl.Element(Selectors.DescFallback); err == nil { desc, _ = descTag.Text() } else { - // Structural fallback - parent, err := titleTag.Parent() - if err == nil { - parent, err = parent.Parent() - if err == nil { - parent, err = parent.Parent() - if err == nil { - if descTag, err := parent.Next(); err == nil { - if descDiv, err := descTag.Element(Selectors.DescAny); err == nil { - desc, _ = descDiv.Text() - } - } + // Structural fallback: the description lives in the sibling block + // after the title's great-grandparent wrapper. + anchor := titleTag + for i := 0; i < 3 && anchor != nil; i++ { + if anchor, err = anchor.Parent(); err != nil { + anchor = nil + } + } + if anchor != nil { + if sib, err := anchor.Next(); err == nil { + if descDiv, err := sib.Element(Selectors.DescAny); err == nil { + desc, _ = descDiv.Text() } } } diff --git a/google/search_test.go b/google/search_test.go index c7ea644..d860d4d 100644 --- a/google/search_test.go +++ b/google/search_test.go @@ -61,8 +61,8 @@ func TestBuildSearchURL(t *testing.T) { if got := params.Get("pws"); got != "0" { t.Fatalf("unexpected pws value: %q", got) } - if got := params.Get("nfpr"); got != "1" { - t.Fatalf("unexpected nfpr value: %q", got) + if got := params.Get("nfpr"); got != "" { + t.Fatalf("nfpr should be omitted by default, got %q", got) } }, }, diff --git a/google/selectors.go b/google/selectors.go index dbc2a01..7ed19d5 100644 --- a/google/selectors.go +++ b/google/selectors.go @@ -53,3 +53,7 @@ var Selectors = struct { // ImageTitle selectors are tried in order to recover a human-readable title. ImageTitle: []string{"h3", "a"}, } + +func searchResultSelectors() []string { + return []string{Selectors.Results, Selectors.ResultsBroad} +} diff --git a/google/url.go b/google/url.go index 6bbbc2a..87a0e22 100644 --- a/google/url.go +++ b/google/url.go @@ -298,8 +298,7 @@ func BuildURL(q core.Query) (string, error) { params.Add("lr", "lang_"+locale.language) } - params.Add("pws", "0") // Do not personalize earch results - params.Add("nfpr", "1") // Do not auto correct search queries + params.Add("pws", "0") // Do not personalize search results params.Add("sourceid", "chrome") params.Add("ie", "UTF-8") @@ -367,8 +366,7 @@ func BuildImageURL(q core.Query) (string, error) { params.Add("lr", "lang_"+locale.language) } - params.Add("pws", "0") // Do not personalize earch results - params.Add("nfpr", "1") // Do not auto correct search queries + params.Add("pws", "0") // Do not personalize search results base.RawQuery = params.Encode() return base.String(), nil