diff --git a/README.md b/README.md index e9d67b2..a8c3827 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OpenSERP (Search Engine Results) -![OpenSERP](/logo.svg) +![OpenSERP](./logo.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp) [![Go Reference](https://pkg.go.dev/badge/github/karust/openserp?style=for-the-badge)](https://pkg.go.dev/github.com/karust/openserp) @@ -8,12 +8,15 @@ [![Docker Pulls](https://img.shields.io/docker/v/karust/openserp)](https://hub.docker.com/repository/docker/karust/openserp) [![CI](https://github.com/karust/openserp/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/karust/openserp/actions/workflows/ci.yml) -**OpenSERP** is an API and CLI for accessing search engine results from **Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia**. -A developer-friendly alternative to paid SERP API services! +**OpenSERP** is a free, open-source API and CLI for accessing normalized search engine results from **Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia**. + +Run it locally, self-host it, or use the optional hosted API when you do not want to manage infrastructure. **Official website:** [openserp.org](https://openserp.org) -> 💡 OpenSerp is free and open-source. Only links listed in this repository and on the official website are associated with the project. +**Feedback:** [GitHub Issues](https://github.com/karust/openserp/issues) · [Telegram](https://t.me/+RJEKspw3mUlhZDMy) · [feedback@openserp.org](mailto:feedback@openserp.org) + +> 💡 OpenSERP is free and open-source. Only links listed in this repository and on the official website are associated with the project. ## Features @@ -25,7 +28,7 @@ A developer-friendly alternative to paid SERP API services! - 🐳 **Docker-ready** - local and container deployment - 📝 **Data Formats** - JSON, Markdown, Text, NdJSON response formats -## Quick Start⚡️ +## ⚡ Quick Start ### Docker @@ -46,11 +49,22 @@ go build -o openserp . ./openserp serve ``` +## Deployment Options + +- **Self-hosted (this repo)** - free, MIT-licensed, with full control over runtime, proxies, cache, and scaling. +- **[Hosted API](https://openserp.org/cloud)** - optional managed version from the project maintainers, with the same API shape. + +The hosted API helps fund continued development of the open-source project. Same endpoints, same response schema, and client code can migrate either direction. + ## API Docs +Once the server is running, the interactive docs are available locally: + - Swagger UI: `http://127.0.0.1:7000/docs` - OpenAPI YAML: `http://127.0.0.1:7000/openapi.yaml` +To browse the spec without running the server, see [docs/openapi.yaml](./docs/openapi.yaml). For a higher-level overview of how OpenSERP works internally, see the [architecture docs](https://openserp.org/docs/architecture/). + ## Search Endpoints Available engine names: `google`, `yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`. @@ -76,7 +90,7 @@ curl "http://127.0.0.1:7000/mega/search?text=golang&limit=10" # Fast mode: only one fastest engine is queried curl "http://127.0.0.1:7000/mega/search?text=golang&mode=fast&engines=google,bing,yandex" -# Any mode: sequential fallback in provided order (default orded if none provided) +# Any mode: sequential fallback in provided order (default order if none provided) curl "http://127.0.0.1:7000/mega/search?text=golang&mode=any&engines=google,yandex,bing" # Balanced mode (default): parallel all engines with aggregation controls @@ -99,17 +113,17 @@ curl "http://127.0.0.1:7000/mega/engines" Common parameters: -| Parameter | Description | Example | -| --------- | -------------------------------------------------------------------- | ------------------------------------ | -| `text` | Search query | `golang programming` | -| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | -| `region` | Market/location hint. Yandex accepts numeric `lr`; others use country hints. | `213`, `RU`, `en-US` | -| `date` | Date range | `20250101..20251231` | -| `file` | File extension | `pdf`, `doc`, `xls` | -| `site` | Site-specific search | `github.com` | -| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` | -| `start` | Pagination offset | `0`, `10`, `20` | -| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | +| Parameter | Description | Example | +| --------- | ---------------------------------------------------------------------------- | ------------------------------------ | +| `text` | Search query | `golang programming` | +| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | +| `region` | Market/location hint. Yandex accepts numeric `lr`; others use country hints. | `213`, `RU`, `en-US` | +| `date` | Date range | `20250101..20251231` | +| `file` | File extension | `pdf`, `doc`, `xls` | +| `site` | Site-specific search | `github.com` | +| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` | +| `start` | Pagination offset | `0`, `10`, `20` | +| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | Engine-specific parameters: @@ -120,6 +134,9 @@ Engine-specific parameters: ## Search Response Example +
+Search response example + ```json { "query": { @@ -163,9 +180,14 @@ Engine-specific parameters: } ``` +
+ ## Mega Response Notes -`/mega/search` returns the same envelope plus `clusters`. Results are deduplicated by normalized URL; clusters keep the per-engine occurrences: +`/mega/search` returns the same envelope plus `clusters`. Results are deduplicated by normalized URL; clusters keep the per-engine occurrences. + +
+Cluster example ```json { @@ -183,8 +205,13 @@ Engine-specific parameters: } ``` +
+ ## Image Response Example +
+Image result example + ```json { "id": "i_a1b2c3d4e5f6a1b2", @@ -205,6 +232,8 @@ Engine-specific parameters: } ``` +
+ ## Error Responses `400 Bad Request`: @@ -241,6 +270,8 @@ Simple global proxy: Advanced proxy configuration is available in [config.yaml](./config.yaml). You can enable tagged proxy pools and per-request override via `X-Use-Proxy: ` or `X-Use-Proxy: direct`. +A [managed API](https://openserp.org/cloud) is also available for teams that do not want to operate infrastructure. + ## Health & Stats ```bash @@ -260,8 +291,10 @@ This project is licensed under the MIT License. See [LICENSE](LICENSE). Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md). -## Updates +## Feedback & Updates -If you want to follow updates to the hosted version — status, changes, and occasional notes on the OSS project — join the [Telegram channel](https://t.me/+RJEKspw3mUlhZDMy). +- [GitHub Issues](https://github.com/karust/openserp/issues) - bugs, feature ideas, and reproducible issues. +- [Telegram channel](https://t.me/openserp_cloud) - OpenSERP news, release notes, and project updates. Direct messages are open for quick feedback and hosted API questions. +- [feedback@openserp.org](mailto:feedback@openserp.org) - private notes, longer feedback, or anything that does not fit GitHub Issues. -###### _"OpenSERP" is the name of this open-source project. The official [website](https://openserp.org). Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._ +###### _"OpenSERP" is the name of this open-source project. The official website is [openserp.org](https://openserp.org). Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._ diff --git a/cmd/root.go b/cmd/root.go index 0441ce1..fbf3149 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( ) const ( - version = "0.7.12" + version = "0.7.13" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/cache.go b/core/cache.go index 5d939f6..9f3e693 100644 --- a/core/cache.go +++ b/core/cache.go @@ -39,14 +39,14 @@ func BuildCacheKey(engine string, action string, q Query) string { country, class, provider := cacheProxyMarket(q) raw := fmt.Sprintf( "%s|%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s|%s|%s", - engine, - action, - q.Text, - q.LangCode, - q.Region, - q.DateInterval, - q.Filetype, - q.Site, + cacheToken(engine), + cacheToken(action), + strings.TrimSpace(q.Text), + cacheToken(q.LangCode), + cacheToken(q.Region), + strings.TrimSpace(q.DateInterval), + cacheToken(q.Filetype), + cacheToken(q.Site), q.Limit, q.Start, q.Filter, @@ -59,20 +59,24 @@ func BuildCacheKey(engine string, action string, q Query) string { return hex.EncodeToString(hash[:]) } +func cacheToken(value string) string { + return strings.ToLower(strings.TrimSpace(value)) +} + func cacheProxyMarket(q Query) (country string, class string, provider string) { - country = strings.ToLower(strings.TrimSpace(q.ProxyCountry)) + country = cacheToken(q.ProxyCountry) if country == "" { // Region is a stronger market signal than LangCode; LangCode is the last // fallback. TODO: Use explicit balancer market metadata everywhere. if region := CountryFromRegion(q.Region); region != "" { country = strings.ToLower(region) } else { - country = strings.ToLower(strings.TrimSpace(q.LangCode)) + country = cacheToken(q.LangCode) } } return country, - strings.ToLower(strings.TrimSpace(q.ProxyClass)), - strings.ToLower(strings.TrimSpace(q.ProxyProvider)) + cacheToken(q.ProxyClass), + cacheToken(q.ProxyProvider) } func ShouldBypassCacheForProxyMarket(q Query) bool { diff --git a/core/cache_test.go b/core/cache_test.go index 32d8c87..6f6ed3b 100644 --- a/core/cache_test.go +++ b/core/cache_test.go @@ -187,6 +187,32 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) { } } +func TestBuildCacheKeyNormalizesStableFields(t *testing.T) { + base := BuildCacheKey("google", "search", Query{ + Text: " golang ", + LangCode: "EN", + Region: " us ", + Filetype: "PDF", + Site: "EXAMPLE.COM", + Limit: 10, + ProxyCountry: " US ", + ProxyClass: " Residential ", + }) + same := BuildCacheKey("Google", "Search", Query{ + Text: "golang", + LangCode: "en", + Region: "US", + Filetype: "pdf", + Site: "example.com", + Limit: 10, + ProxyCountry: "us", + ProxyClass: "residential", + }) + if same != base { + t.Fatal("expected cache key to normalize engine/action, locale, filters, and proxy market fields") + } +} + func TestBuildCacheKeyUsesProxyMarketNotSessionOrURL(t *testing.T) { base := Query{ Text: "golang", diff --git a/core/common.go b/core/common.go index f69844a..1dca59d 100644 --- a/core/common.go +++ b/core/common.go @@ -308,12 +308,12 @@ const MaxQueryLimit = 100 // headers. It validates numeric/boolean inputs and returns an *APIError for // invalid client input (400) or a plain error for internal failures. func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { - searchQuery.Text = reqCtx.Query("text") - searchQuery.LangCode = reqCtx.Query("lang") - searchQuery.Region = reqCtx.Query("region") - searchQuery.DateInterval = reqCtx.Query("date") - searchQuery.Filetype = reqCtx.Query("file") - searchQuery.Site = reqCtx.Query("site") + searchQuery.Text = strings.TrimSpace(reqCtx.Query("text")) + searchQuery.LangCode = strings.TrimSpace(reqCtx.Query("lang")) + searchQuery.Region = strings.TrimSpace(reqCtx.Query("region")) + searchQuery.DateInterval = strings.TrimSpace(reqCtx.Query("date")) + searchQuery.Filetype = strings.TrimSpace(reqCtx.Query("file")) + searchQuery.Site = strings.TrimSpace(reqCtx.Query("site")) limitRaw := reqCtx.Query("limit", "25") limit, err := strconv.Atoi(limitRaw) diff --git a/core/server_test.go b/core/server_test.go index 6cd64fb..29ed304 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -291,6 +291,12 @@ func TestInvalidQueryParametersReturnJSONError(t *testing.T) { message: "query cannot be empty", reason: ReasonEmptyQuery, }, + { + name: "whitespace text query", + path: "/google/search?text=%20%20%20", + message: "query cannot be empty", + reason: ReasonEmptyQuery, + }, { name: "limit too high", path: "/google/search?text=golang&limit=999",