Files
openserp/core/logger.go
Rustem Kamalov 9cc69da758 feat(cli): clean search, add extract/format flags; harden engines & proxy rotation; fix bugs; update docs
- Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution)
- Unify engines behind a single engineSpec registry (CLI + serve share it)
- Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers
- Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks
- Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable
2026-06-16 03:51:37 +03:00

297 lines
7.8 KiB
Go

package core
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/sirupsen/logrus"
)
type loggerContextKey string
const (
requestIDContextKey loggerContextKey = "request_id"
tenantContextKey loggerContextKey = "tenant"
engineContextKey loggerContextKey = "engine"
queryHashContextKey loggerContextKey = "query_hash"
LogFormatJSON = "json"
LogFormatText = "text"
)
func NormalizeLogFormat(raw string) (string, error) {
format := strings.ToLower(strings.TrimSpace(raw))
if format == "" {
return LogFormatText, nil
}
switch format {
case LogFormatJSON, LogFormatText:
return format, nil
default:
return "", fmt.Errorf("invalid logging.format %q: expected json or text", raw)
}
}
func WithRequestID(ctx context.Context, requestID string) context.Context {
requestID = strings.TrimSpace(requestID)
if requestID == "" {
return EnsureContext(ctx)
}
return context.WithValue(EnsureContext(ctx), requestIDContextKey, requestID)
}
func WithTenant(ctx context.Context, tenant string) context.Context {
tenant = strings.TrimSpace(tenant)
if tenant == "" {
return EnsureContext(ctx)
}
return context.WithValue(EnsureContext(ctx), tenantContextKey, tenant)
}
func WithEngine(ctx context.Context, engine string) context.Context {
engine = strings.TrimSpace(engine)
if engine == "" {
return EnsureContext(ctx)
}
return context.WithValue(EnsureContext(ctx), engineContextKey, engine)
}
func WithQueryHash(ctx context.Context, queryHash string) context.Context {
queryHash = strings.TrimSpace(queryHash)
if queryHash == "" {
return EnsureContext(ctx)
}
return context.WithValue(EnsureContext(ctx), queryHashContextKey, queryHash)
}
func RequestIDFromContext(ctx context.Context) string {
value, _ := EnsureContext(ctx).Value(requestIDContextKey).(string)
return strings.TrimSpace(value)
}
func TenantFromContext(ctx context.Context) string {
value, _ := EnsureContext(ctx).Value(tenantContextKey).(string)
return strings.TrimSpace(value)
}
func WithRequest(ctx context.Context) *logrus.Entry {
ctx = EnsureContext(ctx)
fields := logrus.Fields{}
if requestID, ok := ctx.Value(requestIDContextKey).(string); ok && strings.TrimSpace(requestID) != "" {
fields["request_id"] = strings.TrimSpace(requestID)
}
if tenant, ok := ctx.Value(tenantContextKey).(string); ok && strings.TrimSpace(tenant) != "" {
fields["tenant"] = strings.TrimSpace(tenant)
}
if engine, ok := ctx.Value(engineContextKey).(string); ok && strings.TrimSpace(engine) != "" {
fields["engine"] = strings.TrimSpace(engine)
}
if queryHash, ok := ctx.Value(queryHashContextKey).(string); ok && strings.TrimSpace(queryHash) != "" {
fields["query_hash"] = strings.TrimSpace(queryHash)
}
profileIDs := BrowserProfileIDsFromContext(ctx)
if len(profileIDs) == 1 {
fields["browser_profile_id"] = profileIDs[0]
} else if len(profileIDs) > 1 {
fields["browser_profile_ids"] = strings.Join(profileIDs, ",")
}
return logrus.WithFields(fields)
}
func WithRequestEngine(ctx context.Context, engine string) *logrus.Entry {
return WithRequest(WithEngine(ctx, engine))
}
func QueryHash(raw string) string {
normalized := strings.TrimSpace(strings.ToLower(raw))
if normalized == "" {
return ""
}
hash := md5.Sum([]byte(normalized))
return hex.EncodeToString(hash[:])
}
func QueryHashFromQuery(q Query) string {
raw := strings.Join([]string{
q.Text,
q.Site,
q.Filetype,
q.LangCode,
q.Region,
q.DateInterval,
}, "|")
return QueryHash(raw)
}
func formatMessage(message string, args ...any) string {
if len(args) == 0 {
return message
}
return fmt.Sprintf(message, args...)
}
// EngineLogger provides structured logging for search engines with a fixed engine field.
type EngineLogger struct {
engine string
entry *logrus.Entry
}
func NewEngineLogger(engine string) *EngineLogger {
engine = strings.ToLower(strings.TrimSpace(engine))
return &EngineLogger{engine: engine, entry: logrus.WithField("engine", engine)}
}
func (el *EngineLogger) WithRequest(ctx context.Context) *EngineLogger {
return &EngineLogger{engine: el.engine, entry: WithRequestEngine(ctx, el.engine)}
}
// Fields returns a new EngineLogger with additional structured fields merged in.
func (el *EngineLogger) Fields(fields logrus.Fields) *EngineLogger {
return &EngineLogger{engine: el.engine, entry: el.entry.WithFields(fields)}
}
func (el *EngineLogger) Debug(message string, args ...any) {
el.entry.Debug(formatMessage(message, args...))
}
func (el *EngineLogger) Info(message string, args ...any) {
el.entry.Info(formatMessage(message, args...))
}
func (el *EngineLogger) Warn(message string, args ...any) {
el.entry.Warn(formatMessage(message, args...))
}
func (el *EngineLogger) Error(message string, args ...any) {
el.entry.Error(formatMessage(message, args...))
}
func (el *EngineLogger) Fatal(message string, args ...any) {
el.entry.Fatal(formatMessage(message, args...))
}
func (el *EngineLogger) Panic(message string, args ...any) {
el.entry.Panic(formatMessage(message, args...))
}
// bracketFormatter emits bracket-delimited fields:
//
// [time][level][engine=..][request_id=..][query_hash=..][extra fields sorted][msg]
//
// request_id is truncated to last 8 chars; query_hash to first 12.
type bracketFormatter struct {
TimestampFormat string
}
func (f *bracketFormatter) Format(entry *logrus.Entry) ([]byte, error) {
ts := entry.Time.Format(f.TimestampFormat)
var buf bytes.Buffer
fmt.Fprintf(&buf, "[%s][%s]", ts, entry.Level.String())
// Context identity fields in fixed order, then remaining fields sorted, then msg last.
priority := []string{"engine", "tenant", "request_id", "query_hash"}
written := make(map[string]bool, len(entry.Data))
for _, key := range priority {
val, ok := entry.Data[key]
if !ok {
continue
}
s := fmt.Sprintf("%v", val)
switch key {
case "request_id":
if len(s) > 8 {
s = s[len(s)-8:]
}
case "query_hash":
if len(s) > 12 {
s = s[:12]
}
}
fmt.Fprintf(&buf, "[%s=%s]", key, quoteIfNeeded(s))
written[key] = true
}
rest := make([]string, 0, len(entry.Data))
for k := range entry.Data {
if !written[k] {
rest = append(rest, k)
}
}
sort.Strings(rest)
for _, k := range rest {
fmt.Fprintf(&buf, "[%s=%s]", k, quoteIfNeeded(fmt.Sprintf("%v", entry.Data[k])))
}
// Message last so context fields are scannable without scrolling past a long msg.
if entry.Message != "" {
fmt.Fprintf(&buf, "[%s]", quoteIfNeeded(entry.Message))
}
buf.WriteByte('\n')
return buf.Bytes(), nil
}
// quoteIfNeeded wraps s in double-quotes if it contains spaces.
func quoteIfNeeded(s string) string {
if strings.ContainsAny(s, " \t") {
return `"` + strings.ReplaceAll(s, `"`, `\"`) + `"`
}
return s
}
func InitLogger(isVerbose, isDebug, isQuiet bool, format string) {
switch format {
case LogFormatText:
logrus.SetFormatter(&bracketFormatter{TimestampFormat: "2006-01-02 15:04:05"})
case LogFormatJSON:
logrus.SetFormatter(&logrus.JSONFormatter{
TimestampFormat: time.RFC3339Nano,
})
}
// Logs go to stderr (+ optional file) so stdout carries only the payload.
switch {
case isDebug:
logrus.SetOutput(io.MultiWriter(os.Stderr))
logrus.SetReportCaller(true)
case isQuiet:
// One-shot CLI default: stderr only, no ./logs.txt in the user's CWD.
logrus.SetOutput(os.Stderr)
logrus.SetReportCaller(false)
default:
f, err := os.OpenFile("./logs.txt", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to open logs file ./logs.txt: %v\n", err)
logrus.SetOutput(io.MultiWriter(os.Stderr))
} else {
logrus.SetOutput(io.MultiWriter(f, os.Stderr))
}
logrus.SetReportCaller(false)
}
level := logrus.InfoLevel
if isVerbose {
level = logrus.DebugLevel
}
if isDebug {
level = logrus.TraceLevel
}
if isQuiet && !isVerbose && !isDebug {
// Quiet keeps only warnings/errors on stderr.
level = logrus.WarnLevel
}
logrus.SetLevel(level)
}