Multi
Page with sufficient content for batch extraction test that verifies concurrent processing works correctly.
`))
}))
defer target.Close()
+ s := batchExtractTestServer(true)
+ resp := postBatchExtract(t, s, fmt.Sprintf(`{"urls":["%s/1","%s/2","%s/3"]}`, target.URL, target.URL, target.URL))
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
+ }
+ if items := decodeBatchItems(t, resp); len(items) != 3 {
+ t.Fatalf("items count = %d, want 3", len(items))
+ }
+}
+
+func batchExtractTestServer(allowPrivate bool) *Server {
opts := DefaultServerOptions()
opts.Extract = extractpkg.Config{
Enabled: true,
@@ -177,51 +164,163 @@ func TestBatchExtractHandlesMultipleURLs(t *testing.T) {
Timeout: time.Second,
MaxBytes: 256 * 1024,
MaxConcurrent: 2,
- AllowPrivateNetworks: true,
+ AllowPrivateNetworks: allowPrivate,
}
- s := NewServerWithOptions("127.0.0.1", 0, opts)
+ return NewServerWithOptions("127.0.0.1", 0, opts)
+}
- body := fmt.Sprintf(`{"urls":["%s/1","%s/2","%s/3"]}`, target.URL, target.URL, target.URL)
+func postBatchExtract(t *testing.T, s *Server, body string, header ...[2]string) *http.Response {
+ t.Helper()
req, err := http.NewRequest(http.MethodPost, "/extract/batch", strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
req.Header.Set("Content-Type", "application/json")
-
+ for _, h := range header {
+ req.Header.Set(h[0], h[1])
+ }
resp, err := s.app.Test(req)
if err != nil {
t.Fatal(err)
}
+ return resp
+}
+
+func decodeBatchItems(t *testing.T, resp *http.Response) []batchExtractItem {
+ t.Helper()
+ var items []batchExtractItem
+ if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ return items
+}
+
+func TestBatchExtractReturnsPerURLErrors(t *testing.T) {
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte(`