zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-05 16:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(engines): better classify captcha/blocked/no-results pages instead of returning empty results, also enable it to <engine>/parse endpoints. Enable allow_request_proxy_url by default

Browse Source
This commit is contained in:
Rustem Kamalov
2026-07-01 01:38:03 +03:00
parent e7af939599
commit 3d5eddfeed
35 changed files with 605 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
core/server_parse_test.go
View File

@@ -87,6 +87,29 @@ func TestParseEndpointParserErrorReturns400(t *testing.T) {
}
}
func TestParseEndpointSearchErrorReturnsTypedError(t *testing.T) {
engine := &parserMock{
engineMock: engineMock{name: "google", initialized: true},
parseHTMLFn: func(_ io.Reader) ([]SearchResult, error) {
return nil, ErrCaptcha
},
}
srv := NewServerWithOptions("127.0.0.1", 7126, DefaultServerOptions(), engine)
resp := postHTML(t, srv, "/google/parse", "<html>captcha</html>")
if resp.StatusCode != http.StatusTooManyRequests {
t.Fatalf("expected 429, got %d", resp.StatusCode)
}
var body map[string]any
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
t.Fatalf("decode error body: %v", err)
}
if body["error"] != "captcha_detected" {
t.Fatalf("expected captcha_detected, got %#v", body["error"])
}
}
func TestParseEndpointNotRegisteredForNonParserEngine(t *testing.T) {
// engineMock does NOT implement HTMLParser so no /mock/parse route is registered.
engine := &engineMock{name: "mock", initialized: true}