zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-16 21:36:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): clean search, add extract/format flags; harden engines & proxy rotation; fix bugs; update docs

Browse Source 
- Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution)
- Unify engines behind a single engineSpec registry (CLI + serve share it)
- Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers
- Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks
- Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable
This commit is contained in:
Rustem Kamalov
2026-06-16 03:51:37 +03:00
parent ca3143ccd1
commit 9cc69da758
33 changed files with 1613 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
bing/parse_html.go
View File

@@ -47,7 +47,10 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
return
}
title := titleTag.Text()
title := firstNonEmptyAttr(titleTag, "aria-label", "title")
if title == "" {
title = normalizeWhitespace(titleTag.Text())
}
if title == "" {
title = extractFirstText(item, Selectors.TitleFallbacks)
}
@@ -82,19 +85,30 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
func extractFirstText(item *goquery.Selection, selectors []string) string {
for _, selector := range selectors {
if tag := item.Find(selector).First(); tag.Length() > 0 {
if text := strings.TrimSpace(tag.Text()); text != "" {
if text := normalizeWhitespace(tag.Text()); text != "" {
return text
}
if label, exists := tag.Attr("aria-label"); exists {
if label = strings.TrimSpace(label); label != "" {
return label
}
if label := firstNonEmptyAttr(tag, "aria-label", "title"); label != "" {
return label
}
}
}
return ""
}
func firstNonEmptyAttr(item *goquery.Selection, attrs ...string) string {
for _, attr := range attrs {
value, exists := item.Attr(attr)
if !exists {
continue
}
if value = normalizeWhitespace(value); value != "" {
return value
}
}
return ""
}
// descriptionFromItem extracts a description using the same 4-step fallback
// chain as the rod-based browser parser. Bing renders snippet text with heavy
// source-indentation whitespace, so each candidate is whitespace-collapsed.
@@ -118,8 +132,8 @@ func descriptionFromItem(item *goquery.Selection, title string) string {
return normalizeWhitespace(strings.Replace(item.Text(), title, "", 1))
}
// normalizeWhitespace collapses runs of whitespace (including the newlines and
// indentation Bing leaves in snippet markup) into single spaces.
// normalizeWhitespace collapses Bing's snippet-markup whitespace into single
// spaces (see core.NormalizeWhitespace).
func normalizeWhitespace(s string) string {
return strings.Join(strings.Fields(s), " ")
return core.NormalizeWhitespace(s)
}