chore: refactor the imports (#2406)

* feat: mysql and redis metric support

* feat: mysql and redis metric support

* feat: mysql and redis metric support

Co-authored-by: dawn.zhou <dawn.zhou@yijinin.com>

.github/FUNDING.yml

@@ -9,4 +9,5 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
 liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
-custom: https://gitee.com/kevwan/static/raw/master/images/sponsor.jpg
+custom: # https://gitee.com/kevwan/static/raw/master/images/sponsor.jpg
+ethereum: 0x5052b7f6B937B02563996D23feb69b38D06Ca150 | kevwan

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"

.github/workflows/codeql-analysis.yml

@@ -35,11 +35,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/go.yml

@@ -7,32 +7,50 @@ on:
     branches: [ master ]
 
 jobs:
-  build:
-    name: Build
+  test-linux:
+    name: Linux
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+        id: go
 
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.15
-      id: go
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
 
-    - name: Get dependencies
-      run: |
-        go get -v -t -d ./...
+      - name: Lint
+        run: |
+          go vet -stdmethods=false $(go list ./...)
+          go install mvdan.cc/gofumpt@latest
+          test -z "$(gofumpt -l -extra .)" || echo "Please run 'gofumpt -l -w -extra .'"
 
-    - name: Lint
-      run: |
-        go vet -stdmethods=false $(go list ./...)
-        go install mvdan.cc/gofumpt@latest
-        test -z "$(gofumpt -s -l -extra .)" || echo "Please run 'gofumpt -l -w -extra .'"
+      - name: Test
+        run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
 
-    - name: Test
-      run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
+      - name: Codecov
+        uses: codecov/codecov-action@v2
 
-    - name: Codecov
-      uses: codecov/codecov-action@v2
+  test-win:
+    name: Windows
+    runs-on: windows-latest
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+
+      - name: Checkout codebase
+        uses: actions/checkout@v3
+
+      - name: Test
+        run: |
+          go mod verify
+          go mod download
+          go test -v -race ./...
+          cd tools/goctl && go build -v goctl.go

.github/workflows/issues.yml

@@ -9,8 +9,8 @@ jobs:
     steps:
       - uses: actions/stale@v3
         with:
-          days-before-issue-stale: 30
-          days-before-issue-close: 14
+          days-before-issue-stale: 365
+          days-before-issue-close: 90
           stale-issue-label: "stale"
           stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
           close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."

.github/workflows/release.yaml

@@ -0,0 +1,28 @@
+on:
+  push:
+    tags:
+      - "tools/goctl/*"
+jobs:
+  releases-matrix:
+    name: Release goctl binary
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # build and publish in parallel: linux/386, linux/amd64, linux/arm64,
+        # windows/386, windows/amd64, windows/arm64, darwin/amd64, darwin/arm64
+        goos: [ linux, windows, darwin ]
+        goarch: [ "386", amd64, arm64 ]
+        exclude:
+          - goarch: "386"
+            goos: darwin
+    steps:
+      - uses: actions/checkout@v3
+      - uses: zeromicro/go-zero-release-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          goversion: "https://dl.google.com/go/go1.17.5.linux-amd64.tar.gz"
+          project_path: "tools/goctl"
+          binary_name: "goctl"
+          extra_files: tools/goctl/readme.md tools/goctl/readme-cn.md

.github/workflows/reviewdog.yml

@@ -5,7 +5,7 @@ jobs:
     name: runner / staticcheck
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: reviewdog/action-staticcheck@v1
         with:
           github_token: ${{ secrets.github_token }}

.gitignore

@@ -16,7 +16,9 @@
 **/logs
 
 # for test purpose
-adhoc
+**/adhoc
+go.work
+go.work.sum
 
 # gitlab ci
 .cache

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 xiaoheiban_server_go
+Copyright (c) 2022 zeromicro
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

ROADMAP.md

@@ -20,9 +20,9 @@ We hope that the items listed below will inspire further engagement from the com
 - [x] Support `goctl bug` to report bugs conveniently
 
 ## 2022
-- [ ] Support `goctl mock` command to start a mocking server with given `.api` file
-- [ ] Add `httpx.Client` with governance, like circuit breaker etc.
+- [x] Support `context` in redis related methods for timeout and tracing
+- [x] Support `context` in sql related methods for timeout and tracing
+- [x] Support `context` in mongodb related methods for timeout and tracing
+- [x] Add `httpc.Do` with HTTP call governance, like circuit breaker etc.
 - [ ] Support `goctl doctor` command to report potential issues for given service
-- [ ] Support `context` in redis related methods for timeout and tracing
-- [ ] Support `context` in sql related methods for timeout and tracing
-- [ ] Support `context` in mongodb related methods for timeout and tracing
+- [ ] Support `goctl mock` command to start a mocking server with given `.api` file

core/bloom/bloom.go

@@ -69,11 +69,8 @@ func (f *Filter) Exists(data []byte) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	if !isSet {
-		return false, nil
-	}
 
-	return true, nil
+	return isSet, nil
 }
 
 func (f *Filter) getLocations(data []byte) []uint {

core/breaker/breaker.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/zeromicro/go-zero/core/mathx"
 	"github.com/zeromicro/go-zero/core/proc"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/stringx"
-	"github.com/zeromicro/go-zero/core/timex"
 )
 
 const (
@@ -171,7 +171,7 @@ func (lt loggedThrottle) allow() (Promise, error) {
 func (lt loggedThrottle) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
 	return lt.logError(lt.internalThrottle.doReq(req, fallback, func(err error) bool {
 		accept := acceptable(err)
-		if !accept {
+		if !accept && err != nil {
 			lt.errWin.add(err.Error())
 		}
 		return accept
@@ -198,7 +198,7 @@ type errorWindow struct {
 
 func (ew *errorWindow) add(reason string) {
 	ew.lock.Lock()
-	ew.reasons[ew.index] = fmt.Sprintf("%s %s", timex.Time().Format(timeFormat), reason)
+	ew.reasons[ew.index] = fmt.Sprintf("%s %s", time.Now().Format(timeFormat), reason)
 	ew.index = (ew.index + 1) % numHistoryReasons
 	ew.count = mathx.MinInt(ew.count+1, numHistoryReasons)
 	ew.lock.Unlock()

core/codec/rsa.go

@@ -7,7 +7,7 @@ import (
 	"encoding/base64"
 	"encoding/pem"
 	"errors"
-	"io/ioutil"
+	"os"
 )
 
 var (
@@ -48,7 +48,7 @@ type (
 
 // NewRsaDecrypter returns a RsaDecrypter with the given file.
 func NewRsaDecrypter(file string) (RsaDecrypter, error) {
-	content, err := ioutil.ReadFile(file)
+	content, err := os.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}

core/collection/cache.go

@@ -26,7 +26,7 @@ type (
 	// CacheOption defines the method to customize a Cache.
 	CacheOption func(cache *Cache)
 
-	// A Cache object is a in-memory cache.
+	// A Cache object is an in-memory cache.
 	Cache struct {
 		name           string
 		lock           sync.Mutex
@@ -98,13 +98,18 @@ func (c *Cache) Get(key string) (interface{}, bool) {
 
 // Set sets value into c with key.
 func (c *Cache) Set(key string, value interface{}) {
+	c.SetWithExpire(key, value, c.expire)
+}
+
+// SetWithExpire sets value into c with key and expire with the given value.
+func (c *Cache) SetWithExpire(key string, value interface{}, expire time.Duration) {
 	c.lock.Lock()
 	_, ok := c.data[key]
 	c.data[key] = value
 	c.lruCache.add(key)
 	c.lock.Unlock()
 
-	expiry := c.unstableExpiry.AroundDuration(c.expire)
+	expiry := c.unstableExpiry.AroundDuration(expire)
 	if ok {
 		c.timingWheel.MoveTimer(key, expiry)
 	} else {

core/collection/cache_test.go

@@ -18,7 +18,7 @@ func TestCacheSet(t *testing.T) {
 	assert.Nil(t, err)
 
 	cache.Set("first", "first element")
-	cache.Set("second", "second element")
+	cache.SetWithExpire("second", "second element", time.Second*3)
 
 	value, ok := cache.Get("first")
 	assert.True(t, ok)

core/collection/fifo_test.go

@@ -61,3 +61,41 @@ func TestPutMore(t *testing.T) {
 		assert.Equal(t, string(element), string(body.([]byte)))
 	}
 }
+
+func TestPutMoreWithHeaderNotZero(t *testing.T) {
+	elements := [][]byte{
+		[]byte("hello"),
+		[]byte("world"),
+		[]byte("again"),
+	}
+	queue := NewQueue(4)
+	for i := range elements {
+		queue.Put(elements[i])
+	}
+
+	// take 1
+	body, ok := queue.Take()
+	assert.True(t, ok)
+	element, ok := body.([]byte)
+	assert.True(t, ok)
+	assert.Equal(t, element, []byte("hello"))
+
+	// put more
+	queue.Put([]byte("b4"))
+	queue.Put([]byte("b5")) // will store in elements[0]
+	queue.Put([]byte("b6")) // cause expansion
+
+	results := [][]byte{
+		[]byte("world"),
+		[]byte("again"),
+		[]byte("b4"),
+		[]byte("b5"),
+		[]byte("b6"),
+	}
+
+	for _, element := range results {
+		body, ok := queue.Take()
+		assert.True(t, ok)
+		assert.Equal(t, string(element), string(body.([]byte)))
+	}
+}

core/collection/safemap.go

@@ -68,6 +68,24 @@ func (m *SafeMap) Get(key interface{}) (interface{}, bool) {
 	return val, ok
 }
 
+// Range calls f sequentially for each key and value present in the map.
+// If f returns false, range stops the iteration.
+func (m *SafeMap) Range(f func(key, val interface{}) bool) {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	for k, v := range m.dirtyOld {
+		if !f(k, v) {
+			return
+		}
+	}
+	for k, v := range m.dirtyNew {
+		if !f(k, v) {
+			return
+		}
+	}
+}
+
 // Set sets the value into m with the given key.
 func (m *SafeMap) Set(key, value interface{}) {
 	m.lock.Lock()

core/collection/safemap_test.go

@@ -1,6 +1,7 @@
 package collection
 
 import (
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -107,3 +108,42 @@ func testSafeMapWithParameters(t *testing.T, size, exception int) {
 		}
 	}
 }
+
+func TestSafeMap_Range(t *testing.T) {
+	const (
+		size       = 100000
+		exception1 = 5
+		exception2 = 500
+	)
+
+	m := NewSafeMap()
+	newMap := NewSafeMap()
+
+	for i := 0; i < size; i++ {
+		m.Set(i, i)
+	}
+	for i := 0; i < size; i++ {
+		if i%exception1 == 0 {
+			m.Del(i)
+		}
+	}
+
+	for i := size; i < size<<1; i++ {
+		m.Set(i, i)
+	}
+	for i := size; i < size<<1; i++ {
+		if i%exception2 != 0 {
+			m.Del(i)
+		}
+	}
+
+	var count int32
+	m.Range(func(k, v interface{}) bool {
+		atomic.AddInt32(&count, 1)
+		newMap.Set(k, v)
+		return true
+	})
+	assert.Equal(t, int(atomic.LoadInt32(&count)), m.Size())
+	assert.Equal(t, m.dirtyNew, newMap.dirtyNew)
+	assert.Equal(t, m.dirtyOld, newMap.dirtyOld)
+}

core/collection/timingwheel.go

@@ -2,6 +2,7 @@ package collection
 
 import (
 	"container/list"
+	"errors"
 	"fmt"
 	"time"
 
@@ -12,6 +13,11 @@ import (
 
 const drainWorkers = 8
 
+var (
+	ErrClosed   = errors.New("TimingWheel is closed already")
+	ErrArgument = errors.New("incorrect task argument")
+)
+
 type (
 	// Execute defines the method to execute the task.
 	Execute func(key, value interface{})
@@ -59,14 +65,15 @@ type (
 // NewTimingWheel returns a TimingWheel.
 func NewTimingWheel(interval time.Duration, numSlots int, execute Execute) (*TimingWheel, error) {
 	if interval <= 0 || numSlots <= 0 || execute == nil {
-		return nil, fmt.Errorf("interval: %v, slots: %d, execute: %p", interval, numSlots, execute)
+		return nil, fmt.Errorf("interval: %v, slots: %d, execute: %p",
+			interval, numSlots, execute)
 	}
 
 	return newTimingWheelWithClock(interval, numSlots, execute, timex.NewTicker(interval))
 }
 
-func newTimingWheelWithClock(interval time.Duration, numSlots int, execute Execute, ticker timex.Ticker) (
-	*TimingWheel, error) {
+func newTimingWheelWithClock(interval time.Duration, numSlots int, execute Execute,
+	ticker timex.Ticker) (*TimingWheel, error) {
 	tw := &TimingWheel{
 		interval:      interval,
 		ticker:        ticker,
@@ -89,47 +96,67 @@ func newTimingWheelWithClock(interval time.Duration, numSlots int, execute Execu
 }
 
 // Drain drains all items and executes them.
-func (tw *TimingWheel) Drain(fn func(key, value interface{})) {
-	tw.drainChannel <- fn
+func (tw *TimingWheel) Drain(fn func(key, value interface{})) error {
+	select {
+	case tw.drainChannel <- fn:
+		return nil
+	case <-tw.stopChannel:
+		return ErrClosed
+	}
 }
 
 // MoveTimer moves the task with the given key to the given delay.
-func (tw *TimingWheel) MoveTimer(key interface{}, delay time.Duration) {
+func (tw *TimingWheel) MoveTimer(key interface{}, delay time.Duration) error {
 	if delay <= 0 || key == nil {
-		return
+		return ErrArgument
 	}
 
-	tw.moveChannel <- baseEntry{
+	select {
+	case tw.moveChannel <- baseEntry{
 		delay: delay,
 		key:   key,
+	}:
+		return nil
+	case <-tw.stopChannel:
+		return ErrClosed
 	}
 }
 
 // RemoveTimer removes the task with the given key.
-func (tw *TimingWheel) RemoveTimer(key interface{}) {
+func (tw *TimingWheel) RemoveTimer(key interface{}) error {
 	if key == nil {
-		return
+		return ErrArgument
 	}
 
-	tw.removeChannel <- key
+	select {
+	case tw.removeChannel <- key:
+		return nil
+	case <-tw.stopChannel:
+		return ErrClosed
+	}
 }
 
 // SetTimer sets the task value with the given key to the delay.
-func (tw *TimingWheel) SetTimer(key, value interface{}, delay time.Duration) {
+func (tw *TimingWheel) SetTimer(key, value interface{}, delay time.Duration) error {
 	if delay <= 0 || key == nil {
-		return
+		return ErrArgument
 	}
 
-	tw.setChannel <- timingEntry{
+	select {
+	case tw.setChannel <- timingEntry{
 		baseEntry: baseEntry{
 			delay: delay,
 			key:   key,
 		},
 		value: value,
+	}:
+		return nil
+	case <-tw.stopChannel:
+		return ErrClosed
 	}
 }
 
-// Stop stops tw.
+// Stop stops tw. No more actions after stopping a TimingWheel.
 func (tw *TimingWheel) Stop() {
 	close(tw.stopChannel)
 }

core/collection/timingwheel_test.go

@@ -28,7 +28,6 @@ func TestTimingWheel_Drain(t *testing.T) {
 	ticker := timex.NewFakeTicker()
 	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
 	}, ticker)
-	defer tw.Stop()
 	tw.SetTimer("first", 3, testStep*4)
 	tw.SetTimer("second", 5, testStep*7)
 	tw.SetTimer("third", 7, testStep*7)
@@ -56,6 +55,8 @@ func TestTimingWheel_Drain(t *testing.T) {
 	})
 	time.Sleep(time.Millisecond * 100)
 	assert.Equal(t, 0, count)
+	tw.Stop()
+	assert.Equal(t, ErrClosed, tw.Drain(func(key, value interface{}) {}))
 }
 
 func TestTimingWheel_SetTimerSoon(t *testing.T) {
@@ -102,6 +103,13 @@ func TestTimingWheel_SetTimerWrongDelay(t *testing.T) {
 	})
 }
 
+func TestTimingWheel_SetTimerAfterClose(t *testing.T) {
+	ticker := timex.NewFakeTicker()
+	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {}, ticker)
+	tw.Stop()
+	assert.Equal(t, ErrClosed, tw.SetTimer("any", 3, testStep))
+}
+
 func TestTimingWheel_MoveTimer(t *testing.T) {
 	run := syncx.NewAtomicBool()
 	ticker := timex.NewFakeTicker()
@@ -111,7 +119,6 @@ func TestTimingWheel_MoveTimer(t *testing.T) {
 		assert.Equal(t, 3, v.(int))
 		ticker.Done()
 	}, ticker)
-	defer tw.Stop()
 	tw.SetTimer("any", 3, testStep*4)
 	tw.MoveTimer("any", testStep*7)
 	tw.MoveTimer("any", -testStep)
@@ -125,6 +132,8 @@ func TestTimingWheel_MoveTimer(t *testing.T) {
 	}
 	assert.Nil(t, ticker.Wait(waitTime))
 	assert.True(t, run.True())
+	tw.Stop()
+	assert.Equal(t, ErrClosed, tw.MoveTimer("any", time.Millisecond))
 }
 
 func TestTimingWheel_MoveTimerSoon(t *testing.T) {
@@ -175,6 +184,7 @@ func TestTimingWheel_RemoveTimer(t *testing.T) {
 		ticker.Tick()
 	}
 	tw.Stop()
+	assert.Equal(t, ErrClosed, tw.RemoveTimer("any"))
 }
 
 func TestTimingWheel_SetTimer(t *testing.T) {

core/color/color.go

@@ -0,0 +1,73 @@
+package color
+
+import "github.com/fatih/color"
+
+const (
+	// NoColor is no color for both foreground and background.
+	NoColor Color = iota
+	// FgBlack is the foreground color black.
+	FgBlack
+	// FgRed is the foreground color red.
+	FgRed
+	// FgGreen is the foreground color green.
+	FgGreen
+	// FgYellow is the foreground color yellow.
+	FgYellow
+	// FgBlue is the foreground color blue.
+	FgBlue
+	// FgMagenta is the foreground color magenta.
+	FgMagenta
+	// FgCyan is the foreground color cyan.
+	FgCyan
+	// FgWhite is the foreground color white.
+	FgWhite
+
+	// BgBlack is the background color black.
+	BgBlack
+	// BgRed is the background color red.
+	BgRed
+	// BgGreen is the background color green.
+	BgGreen
+	// BgYellow is the background color yellow.
+	BgYellow
+	// BgBlue is the background color blue.
+	BgBlue
+	// BgMagenta is the background color magenta.
+	BgMagenta
+	// BgCyan is the background color cyan.
+	BgCyan
+	// BgWhite is the background color white.
+	BgWhite
+)
+
+var colors = map[Color][]color.Attribute{
+	FgBlack:   {color.FgBlack, color.Bold},
+	FgRed:     {color.FgRed, color.Bold},
+	FgGreen:   {color.FgGreen, color.Bold},
+	FgYellow:  {color.FgYellow, color.Bold},
+	FgBlue:    {color.FgBlue, color.Bold},
+	FgMagenta: {color.FgMagenta, color.Bold},
+	FgCyan:    {color.FgCyan, color.Bold},
+	FgWhite:   {color.FgWhite, color.Bold},
+	BgBlack:   {color.BgBlack, color.FgHiWhite, color.Bold},
+	BgRed:     {color.BgRed, color.FgHiWhite, color.Bold},
+	BgGreen:   {color.BgGreen, color.FgHiWhite, color.Bold},
+	BgYellow:  {color.BgHiYellow, color.FgHiBlack, color.Bold},
+	BgBlue:    {color.BgBlue, color.FgHiWhite, color.Bold},
+	BgMagenta: {color.BgMagenta, color.FgHiWhite, color.Bold},
+	BgCyan:    {color.BgCyan, color.FgHiWhite, color.Bold},
+	BgWhite:   {color.BgHiWhite, color.FgHiBlack, color.Bold},
+}
+
+type Color uint32
+
+// WithColor returns a string with the given color applied.
+func WithColor(text string, colour Color) string {
+	c := color.New(colors[colour]...)
+	return c.Sprint(text)
+}
+
+// WithColorPadding returns a string with the given color applied with leading and trailing spaces.
+func WithColorPadding(text string, colour Color) string {
+	return WithColor(" "+text+" ", colour)
+}

core/color/color_test.go

@@ -0,0 +1,17 @@
+package color
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWithColor(t *testing.T) {
+	output := WithColor("Hello", BgRed)
+	assert.Equal(t, "Hello", output)
+}
+
+func TestWithColorPadding(t *testing.T) {
+	output := WithColorPadding("Hello", BgRed)
+	assert.Equal(t, " Hello ", output)
+}

core/conf/config.go

@@ -2,28 +2,29 @@ package conf
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
+	"strings"
 
 	"github.com/zeromicro/go-zero/core/mapping"
 )
 
 var loaders = map[string]func([]byte, interface{}) error{
-	".json": LoadConfigFromJsonBytes,
-	".yaml": LoadConfigFromYamlBytes,
-	".yml":  LoadConfigFromYamlBytes,
+	".json": LoadFromJsonBytes,
+	".toml": LoadFromTomlBytes,
+	".yaml": LoadFromYamlBytes,
+	".yml":  LoadFromYamlBytes,
 }
 
-// LoadConfig loads config into v from file, .json, .yaml and .yml are acceptable.
-func LoadConfig(file string, v interface{}, opts ...Option) error {
-	content, err := ioutil.ReadFile(file)
+// Load loads config into v from file, .json, .yaml and .yml are acceptable.
+func Load(file string, v interface{}, opts ...Option) error {
+	content, err := os.ReadFile(file)
 	if err != nil {
 		return err
 	}
 
-	loader, ok := loaders[path.Ext(file)]
+	loader, ok := loaders[strings.ToLower(path.Ext(file))]
 	if !ok {
 		return fmt.Errorf("unrecognized file type: %s", file)
 	}
@@ -40,19 +41,42 @@ func LoadConfig(file string, v interface{}, opts ...Option) error {
 	return loader(content, v)
 }
 
-// LoadConfigFromJsonBytes loads config into v from content json bytes.
-func LoadConfigFromJsonBytes(content []byte, v interface{}) error {
+// LoadConfig loads config into v from file, .json, .yaml and .yml are acceptable.
+// Deprecated: use Load instead.
+func LoadConfig(file string, v interface{}, opts ...Option) error {
+	return Load(file, v, opts...)
+}
+
+// LoadFromJsonBytes loads config into v from content json bytes.
+func LoadFromJsonBytes(content []byte, v interface{}) error {
 	return mapping.UnmarshalJsonBytes(content, v)
 }
 
-// LoadConfigFromYamlBytes loads config into v from content yaml bytes.
-func LoadConfigFromYamlBytes(content []byte, v interface{}) error {
+// LoadConfigFromJsonBytes loads config into v from content json bytes.
+// Deprecated: use LoadFromJsonBytes instead.
+func LoadConfigFromJsonBytes(content []byte, v interface{}) error {
+	return LoadFromJsonBytes(content, v)
+}
+
+// LoadFromTomlBytes loads config into v from content toml bytes.
+func LoadFromTomlBytes(content []byte, v interface{}) error {
+	return mapping.UnmarshalTomlBytes(content, v)
+}
+
+// LoadFromYamlBytes loads config into v from content yaml bytes.
+func LoadFromYamlBytes(content []byte, v interface{}) error {
 	return mapping.UnmarshalYamlBytes(content, v)
 }
 
+// LoadConfigFromYamlBytes loads config into v from content yaml bytes.
+// Deprecated: use LoadFromYamlBytes instead.
+func LoadConfigFromYamlBytes(content []byte, v interface{}) error {
+	return LoadFromYamlBytes(content, v)
+}
+
 // MustLoad loads config into v from path, exits on error.
 func MustLoad(path string, v interface{}, opts ...Option) {
-	if err := LoadConfig(path, v, opts...); err != nil {
+	if err := Load(path, v, opts...); err != nil {
 		log.Fatalf("error: config file %s, %s", path, err.Error())
 	}
 }

core/conf/config_test.go

@@ -1,7 +1,6 @@
 package conf
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -11,14 +10,14 @@ import (
 )
 
 func TestLoadConfig_notExists(t *testing.T) {
-	assert.NotNil(t, LoadConfig("not_a_file", nil))
+	assert.NotNil(t, Load("not_a_file", nil))
 }
 
 func TestLoadConfig_notRecogFile(t *testing.T) {
 	filename, err := fs.TempFilenameWithText("hello")
 	assert.Nil(t, err)
 	defer os.Remove(filename)
-	assert.NotNil(t, LoadConfig(filename, nil))
+	assert.NotNil(t, Load(filename, nil))
 }
 
 func TestConfigJson(t *testing.T) {
@@ -57,6 +56,58 @@ func TestConfigJson(t *testing.T) {
 	}
 }
 
+func TestConfigToml(t *testing.T) {
+	text := `a = "foo"
+b = 1
+c = "${FOO}"
+d = "abcd!@#$112"
+`
+	os.Setenv("FOO", "2")
+	defer os.Unsetenv("FOO")
+	tmpfile, err := createTempFile(".toml", text)
+	assert.Nil(t, err)
+	defer os.Remove(tmpfile)
+
+	var val struct {
+		A string `json:"a"`
+		B int    `json:"b"`
+		C string `json:"c"`
+		D string `json:"d"`
+	}
+	MustLoad(tmpfile, &val)
+	assert.Equal(t, "foo", val.A)
+	assert.Equal(t, 1, val.B)
+	assert.Equal(t, "${FOO}", val.C)
+	assert.Equal(t, "abcd!@#$112", val.D)
+}
+
+func TestConfigTomlEnv(t *testing.T) {
+	text := `a = "foo"
+b = 1
+c = "${FOO}"
+d = "abcd!@#112"
+`
+	os.Setenv("FOO", "2")
+	defer os.Unsetenv("FOO")
+	tmpfile, err := createTempFile(".toml", text)
+	assert.Nil(t, err)
+	defer os.Remove(tmpfile)
+
+	var val struct {
+		A string `json:"a"`
+		B int    `json:"b"`
+		C string `json:"c"`
+		D string `json:"d"`
+	}
+
+	MustLoad(tmpfile, &val, UseEnv())
+	assert.Equal(t, "foo", val.A)
+	assert.Equal(t, 1, val.B)
+	assert.Equal(t, "2", val.C)
+	assert.Equal(t, "abcd!@#112", val.D)
+
+}
+
 func TestConfigJsonEnv(t *testing.T) {
 	tests := []string{
 		".json",
@@ -94,12 +145,12 @@ func TestConfigJsonEnv(t *testing.T) {
 }
 
 func createTempFile(ext, text string) (string, error) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
+	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
 	if err != nil {
 		return "", err
 	}
 
-	if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+	if err := os.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
 		return "", err
 	}
 

core/conf/readme.md

@@ -0,0 +1,57 @@
+## How to use
+
+1. Define a config structure, like below:
+
+```go
+type RestfulConf struct {
+	Host         string        `json:",default=0.0.0.0"`
+	Port         int
+	LogMode      string        `json:",options=[file,console]"`
+	Verbose      bool          `json:",optional"`
+	MaxConns     int           `json:",default=10000"`
+	MaxBytes     int64         `json:",default=1048576"`
+	Timeout      time.Duration `json:",default=3s"`
+	CpuThreshold int64         `json:",default=900,range=[0:1000]"`
+}
+```
+
+2. Write the yaml, toml or json config file:
+
+- yaml example
+
+```yaml
+# most fields are optional or have default values
+Port: 8080
+LogMode: console
+# you can use env settings
+MaxBytes: ${MAX_BYTES}
+```
+
+- toml example
+
+```toml
+# most fields are optional or have default values
+Port = 8_080
+LogMode = "console"
+# you can use env settings
+MaxBytes = "${MAX_BYTES}"
+```
+
+3. Load the config from a file:
+
+```go
+// exit on error
+var config RestfulConf
+conf.MustLoad(configFile, &config)
+
+// or handle the error on your own
+var config RestfulConf
+if err := conf.Load(configFile, &config); err != nil {
+  log.Fatal(err)
+}
+
+// enable reading from environments
+var config RestfulConf
+conf.MustLoad(configFile, &config, conf.UseEnv())
+```
+

core/discov/config.go

@@ -2,6 +2,13 @@ package discov
 
 import "errors"
 
+var (
+	// errEmptyEtcdHosts indicates that etcd hosts are empty.
+	errEmptyEtcdHosts = errors.New("empty etcd hosts")
+	// errEmptyEtcdKey indicates that etcd key is empty.
+	errEmptyEtcdKey = errors.New("empty etcd key")
+)
+
 // EtcdConf is the config item with the given key on etcd.
 type EtcdConf struct {
 	Hosts              []string
@@ -27,9 +34,9 @@ func (c EtcdConf) HasTLS() bool {
 // Validate validates c.
 func (c EtcdConf) Validate() error {
 	if len(c.Hosts) == 0 {
-		return errors.New("empty etcd hosts")
+		return errEmptyEtcdHosts
 	} else if len(c.Key) == 0 {
-		return errors.New("empty etcd key")
+		return errEmptyEtcdKey
 	} else {
 		return nil
 	}

core/discov/internal/accountmanager.go

@@ -3,7 +3,7 @@ package internal
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"io/ioutil"
+	"os"
 	"sync"
 )
 
@@ -37,7 +37,7 @@ func AddTLS(endpoints []string, certFile, certKeyFile, caFile string, insecureSk
 		return err
 	}
 
-	caData, err := ioutil.ReadFile(caFile)
+	caData, err := os.ReadFile(caFile)
 	if err != nil {
 		return err
 	}

core/discov/internal/registry.go

@@ -191,9 +191,11 @@ func (c *cluster) handleWatchEvents(key string, events []*clientv3.Event) {
 				})
 			}
 		case clientv3.EventTypeDelete:
+			c.lock.Lock()
 			if vals, ok := c.values[key]; ok {
 				delete(vals, string(ev.Kv.Key))
 			}
+			c.lock.Unlock()
 			for _, l := range listeners {
 				l.OnDelete(KV{
 					Key: string(ev.Kv.Key),
@@ -206,7 +208,7 @@ func (c *cluster) handleWatchEvents(key string, events []*clientv3.Event) {
 	}
 }
 
-func (c *cluster) load(cli EtcdClient, key string) {
+func (c *cluster) load(cli EtcdClient, key string) int64 {
 	var resp *clientv3.GetResponse
 	for {
 		var err error
@@ -230,6 +232,8 @@ func (c *cluster) load(cli EtcdClient, key string) {
 	}
 
 	c.handleChanges(key, kvs)
+
+	return resp.Header.Revision
 }
 
 func (c *cluster) monitor(key string, l UpdateListener) error {
@@ -242,9 +246,9 @@ func (c *cluster) monitor(key string, l UpdateListener) error {
 		return err
 	}
 
-	c.load(cli, key)
+	rev := c.load(cli, key)
 	c.watchGroup.Run(func() {
-		c.watch(cli, key)
+		c.watch(cli, key, rev)
 	})
 
 	return nil
@@ -276,22 +280,28 @@ func (c *cluster) reload(cli EtcdClient) {
 	for _, key := range keys {
 		k := key
 		c.watchGroup.Run(func() {
-			c.load(cli, k)
-			c.watch(cli, k)
+			rev := c.load(cli, k)
+			c.watch(cli, k, rev)
 		})
 	}
 }
 
-func (c *cluster) watch(cli EtcdClient, key string) {
+func (c *cluster) watch(cli EtcdClient, key string, rev int64) {
 	for {
-		if c.watchStream(cli, key) {
+		if c.watchStream(cli, key, rev) {
 			return
 		}
 	}
 }
 
-func (c *cluster) watchStream(cli EtcdClient, key string) bool {
-	rch := cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
+func (c *cluster) watchStream(cli EtcdClient, key string, rev int64) bool {
+	var rch clientv3.WatchChan
+	if rev != 0 {
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix(), clientv3.WithRev(rev+1))
+	} else {
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
+	}
+
 	for {
 		select {
 		case wresp, ok := <-rch:

core/discov/internal/registry_test.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"go.etcd.io/etcd/api/v3/etcdserverpb"
 	"sync"
 	"testing"
 
@@ -112,6 +113,7 @@ func TestCluster_Load(t *testing.T) {
 	restore := setMockClient(cli)
 	defer restore()
 	cli.EXPECT().Get(gomock.Any(), "any/", gomock.Any()).Return(&clientv3.GetResponse{
+		Header: &etcdserverpb.ResponseHeader{},
 		Kvs: []*mvccpb.KeyValue{
 			{
 				Key:   []byte("hello"),
@@ -168,7 +170,7 @@ func TestCluster_Watch(t *testing.T) {
 			listener.EXPECT().OnDelete(gomock.Any()).Do(func(_ interface{}) {
 				wg.Done()
 			}).MaxTimes(1)
-			go c.watch(cli, "any")
+			go c.watch(cli, "any", 0)
 			ch <- clientv3.WatchResponse{
 				Events: []*clientv3.Event{
 					{
@@ -212,7 +214,7 @@ func TestClusterWatch_RespFailures(t *testing.T) {
 				ch <- resp
 				close(c.done)
 			}()
-			c.watch(cli, "any")
+			c.watch(cli, "any", 0)
 		})
 	}
 }
@@ -232,7 +234,7 @@ func TestClusterWatch_CloseChan(t *testing.T) {
 		close(ch)
 		close(c.done)
 	}()
-	c.watch(cli, "any")
+	c.watch(cli, "any", 0)
 }
 
 func TestValueOnlyContext(t *testing.T) {

core/errorx/batcherror.go

@@ -11,10 +11,12 @@ type (
 	errorArray []error
 )
 
-// Add adds err to be.
-func (be *BatchError) Add(err error) {
-	if err != nil {
-		be.errs = append(be.errs, err)
+// Add adds errs to be, nil errors are ignored.
+func (be *BatchError) Add(errs ...error) {
+	for _, err := range errs {
+		if err != nil {
+			be.errs = append(be.errs, err)
+		}
 	}
 }
 

core/errorx/wrap.go

@@ -0,0 +1,21 @@
+package errorx
+
+import "fmt"
+
+// Wrap returns an error that wraps err with given message.
+func Wrap(err error, message string) error {
+	if err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("%s: %w", message, err)
+}
+
+// Wrapf returns an error that wraps err with given format and args.
+func Wrapf(err error, format string, args ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), err)
+}

core/errorx/wrap_test.go

@@ -0,0 +1,24 @@
+package errorx
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWrap(t *testing.T) {
+	assert.Nil(t, Wrap(nil, "test"))
+	assert.Equal(t, "foo: bar", Wrap(errors.New("bar"), "foo").Error())
+
+	err := errors.New("foo")
+	assert.True(t, errors.Is(Wrap(err, "bar"), err))
+}
+
+func TestWrapf(t *testing.T) {
+	assert.Nil(t, Wrapf(nil, "%s", "test"))
+	assert.Equal(t, "foo bar: quz", Wrapf(errors.New("quz"), "foo %s", "bar").Error())
+
+	err := errors.New("foo")
+	assert.True(t, errors.Is(Wrapf(err, "foo %s", "bar"), err))
+}

core/filex/rangereader.go

@@ -5,6 +5,9 @@ import (
 	"os"
 )
 
+// errExceedFileSize indicates that the file size is exceeded.
+var errExceedFileSize = errors.New("exceed file size")
+
 // A RangeReader is used to read a range of content from a file.
 type RangeReader struct {
 	file  *os.File
@@ -29,7 +32,7 @@ func (rr *RangeReader) Read(p []byte) (n int, err error) {
 	}
 
 	if rr.stop < rr.start || rr.start >= stat.Size() {
-		return 0, errors.New("exceed file size")
+		return 0, errExceedFileSize
 	}
 
 	if rr.stop-rr.start < int64(len(p)) {

core/fs/temps.go

@@ -1,7 +1,6 @@
 package fs
 
 import (
-	"io/ioutil"
 	"os"
 
 	"github.com/zeromicro/go-zero/core/hash"
@@ -12,12 +11,12 @@ import (
 // The file is kept as open, the caller should close the file handle,
 // and remove the file by name.
 func TempFileWithText(text string) (*os.File, error) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text)))
+	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text)))
 	if err != nil {
 		return nil, err
 	}
 
-	if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+	if err := os.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
 		return nil, err
 	}
 

core/fs/temps_test.go

@@ -0,0 +1,49 @@
+package fs
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTempFileWithText(t *testing.T) {
+	f, err := TempFileWithText("test")
+	if err != nil {
+		t.Error(err)
+	}
+	if f == nil {
+		t.Error("TempFileWithText returned nil")
+	}
+	if f.Name() == "" {
+		t.Error("TempFileWithText returned empty file name")
+	}
+	defer os.Remove(f.Name())
+
+	bs, err := io.ReadAll(f)
+	assert.Nil(t, err)
+	if len(bs) != 4 {
+		t.Error("TempFileWithText returned wrong file size")
+	}
+	if f.Close() != nil {
+		t.Error("TempFileWithText returned error on close")
+	}
+}
+
+func TestTempFilenameWithText(t *testing.T) {
+	f, err := TempFilenameWithText("test")
+	if err != nil {
+		t.Error(err)
+	}
+	if f == "" {
+		t.Error("TempFilenameWithText returned empty file name")
+	}
+	defer os.Remove(f)
+
+	bs, err := os.ReadFile(f)
+	assert.Nil(t, err)
+	if len(bs) != 4 {
+		t.Error("TempFilenameWithText returned wrong file size")
+	}
+}

core/fx/stream_test.go

@@ -1,7 +1,7 @@
 package fx
 
 import (
-	"io/ioutil"
+	"io"
 	"log"
 	"math/rand"
 	"reflect"
@@ -238,7 +238,7 @@ func TestLast(t *testing.T) {
 
 func TestMap(t *testing.T) {
 	runCheckedTest(t, func(t *testing.T) {
-		log.SetOutput(ioutil.Discard)
+		log.SetOutput(io.Discard)
 
 		tests := []struct {
 			mapper MapFunc

core/iox/read.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 )
@@ -26,7 +25,7 @@ type (
 func DupReadCloser(reader io.ReadCloser) (io.ReadCloser, io.ReadCloser) {
 	var buf bytes.Buffer
 	tee := io.TeeReader(reader, &buf)
-	return ioutil.NopCloser(tee), ioutil.NopCloser(&buf)
+	return io.NopCloser(tee), io.NopCloser(&buf)
 }
 
 // KeepSpace customizes the reading functions to keep leading and tailing spaces.
@@ -54,7 +53,7 @@ func ReadBytes(reader io.Reader, buf []byte) error {
 
 // ReadText reads content from the given file with leading and tailing spaces trimmed.
 func ReadText(filename string) (string, error) {
-	content, err := ioutil.ReadFile(filename)
+	content, err := os.ReadFile(filename)
 	if err != nil {
 		return "", err
 	}

core/iox/read_test.go

@@ -3,7 +3,6 @@ package iox
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -97,10 +96,10 @@ func TestReadTextLines(t *testing.T) {
 
 func TestDupReadCloser(t *testing.T) {
 	input := "hello"
-	reader := ioutil.NopCloser(bytes.NewBufferString(input))
+	reader := io.NopCloser(bytes.NewBufferString(input))
 	r1, r2 := DupReadCloser(reader)
 	verify := func(r io.Reader) {
-		output, err := ioutil.ReadAll(r)
+		output, err := io.ReadAll(r)
 		assert.Nil(t, err)
 		assert.Equal(t, input, string(output))
 	}
@@ -110,7 +109,7 @@ func TestDupReadCloser(t *testing.T) {
 }
 
 func TestReadBytes(t *testing.T) {
-	reader := ioutil.NopCloser(bytes.NewBufferString("helloworld"))
+	reader := io.NopCloser(bytes.NewBufferString("helloworld"))
 	buf := make([]byte, 5)
 	err := ReadBytes(reader, buf)
 	assert.Nil(t, err)
@@ -118,7 +117,7 @@ func TestReadBytes(t *testing.T) {
 }
 
 func TestReadBytesNotEnough(t *testing.T) {
-	reader := ioutil.NopCloser(bytes.NewBufferString("hell"))
+	reader := io.NopCloser(bytes.NewBufferString("hell"))
 	buf := make([]byte, 5)
 	err := ReadBytes(reader, buf)
 	assert.Equal(t, io.EOF, err)

core/iox/textfile_test.go

@@ -1,7 +1,6 @@
 package iox
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -13,7 +12,7 @@ func TestCountLines(t *testing.T) {
 2
 3
 4`
-	file, err := ioutil.TempFile(os.TempDir(), "test-")
+	file, err := os.CreateTemp(os.TempDir(), "test-")
 	if err != nil {
 		t.Fatal(err)
 	}

core/jsonx/json.go

@@ -13,6 +13,16 @@ func Marshal(v interface{}) ([]byte, error) {
 	return json.Marshal(v)
 }
 
+// MarshalToString marshals v into a string.
+func MarshalToString(v interface{}) (string, error) {
+	data, err := Marshal(v)
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
+}
+
 // Unmarshal unmarshals data bytes into v.
 func Unmarshal(data []byte, v interface{}) error {
 	decoder := json.NewDecoder(bytes.NewReader(data))
@@ -51,5 +61,5 @@ func unmarshalUseNumber(decoder *json.Decoder, v interface{}) error {
 }
 
 func formatError(v string, err error) error {
-	return fmt.Errorf("string: `%s`, error: `%s`", v, err.Error())
+	return fmt.Errorf("string: `%s`, error: `%w`", v, err)
 }

core/jsonx/json_test.go

@@ -0,0 +1,103 @@
+package jsonx
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarshal(t *testing.T) {
+	var v = struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}{
+		Name: "John",
+		Age:  30,
+	}
+	bs, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, `{"name":"John","age":30}`, string(bs))
+}
+
+func TestMarshalToString(t *testing.T) {
+	var v = struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}{
+		Name: "John",
+		Age:  30,
+	}
+	toString, err := MarshalToString(v)
+	assert.Nil(t, err)
+	assert.Equal(t, `{"name":"John","age":30}`, toString)
+
+	_, err = MarshalToString(make(chan int))
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshal(t *testing.T) {
+	const s = `{"name":"John","age":30}`
+	var v struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+	err := Unmarshal([]byte(s), &v)
+	assert.Nil(t, err)
+	assert.Equal(t, "John", v.Name)
+	assert.Equal(t, 30, v.Age)
+}
+
+func TestUnmarshalError(t *testing.T) {
+	const s = `{"name":"John","age":30`
+	var v struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+	err := Unmarshal([]byte(s), &v)
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshalFromString(t *testing.T) {
+	const s = `{"name":"John","age":30}`
+	var v struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+	err := UnmarshalFromString(s, &v)
+	assert.Nil(t, err)
+	assert.Equal(t, "John", v.Name)
+	assert.Equal(t, 30, v.Age)
+}
+
+func TestUnmarshalFromStringError(t *testing.T) {
+	const s = `{"name":"John","age":30`
+	var v struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+	err := UnmarshalFromString(s, &v)
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshalFromRead(t *testing.T) {
+	const s = `{"name":"John","age":30}`
+	var v struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+	err := UnmarshalFromReader(strings.NewReader(s), &v)
+	assert.Nil(t, err)
+	assert.Equal(t, "John", v.Name)
+	assert.Equal(t, 30, v.Age)
+}
+
+func TestUnmarshalFromReaderError(t *testing.T) {
+	const s = `{"name":"John","age":30`
+	var v struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+	err := UnmarshalFromReader(strings.NewReader(s), &v)
+	assert.NotNil(t, err)
+}

core/limit/periodlimit.go

@@ -1,6 +1,7 @@
 package limit
 
 import (
+	"context"
 	"errors"
 	"strconv"
 	"time"
@@ -14,8 +15,8 @@ local window = tonumber(ARGV[2])
 local current = redis.call("INCRBY", KEYS[1], 1)
 if current == 1 then
     redis.call("expire", KEYS[1], window)
-    return 1
-elseif current < limit then
+end
+if current < limit then
     return 1
 elseif current == limit then
     return 2
@@ -74,7 +75,12 @@ func NewPeriodLimit(period, quota int, limitStore *redis.Redis, keyPrefix string
 
 // Take requests a permit, it returns the permit state.
 func (h *PeriodLimit) Take(key string) (int, error) {
-	resp, err := h.limitStore.Eval(periodScript, []string{h.keyPrefix + key}, []string{
+	return h.TakeCtx(context.Background(), key)
+}
+
+// TakeCtx requests a permit with context, it returns the permit state.
+func (h *PeriodLimit) TakeCtx(ctx context.Context, key string) (int, error) {
+	resp, err := h.limitStore.EvalCtx(ctx, periodScript, []string{h.keyPrefix + key}, []string{
 		strconv.Itoa(h.quota),
 		strconv.Itoa(h.calcExpireSeconds()),
 	})

core/limit/periodlimit_test.go

@@ -23,10 +23,9 @@ func TestPeriodLimit_RedisUnavailable(t *testing.T) {
 
 	const (
 		seconds = 1
-		total   = 100
 		quota   = 5
 	)
-	l := NewPeriodLimit(seconds, quota, redis.NewRedis(s.Addr(), redis.NodeType), "periodlimit")
+	l := NewPeriodLimit(seconds, quota, redis.New(s.Addr()), "periodlimit")
 	s.Close()
 	val, err := l.Take("first")
 	assert.NotNil(t, err)
@@ -66,3 +65,13 @@ func testPeriodLimit(t *testing.T, opts ...PeriodOption) {
 	assert.Equal(t, 1, hitQuota)
 	assert.Equal(t, total-quota, overQuota)
 }
+
+func TestQuotaFull(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+
+	l := NewPeriodLimit(1, 1, redis.New(s.Addr()), "periodlimit")
+	val, err := l.Take("first")
+	assert.Nil(t, err)
+	assert.Equal(t, HitQuota, val)
+}

core/logx/color.go

@@ -0,0 +1,26 @@
+package logx
+
+import (
+	"sync/atomic"
+
+	"github.com/zeromicro/go-zero/core/color"
+)
+
+// WithColor is a helper function to add color to a string, only in plain encoding.
+func WithColor(text string, colour color.Color) string {
+	if atomic.LoadUint32(&encoding) == plainEncodingType {
+		return color.WithColor(text, colour)
+	}
+
+	return text
+}
+
+// WithColorPadding is a helper function to add color to a string with leading and trailing spaces,
+// only in plain encoding.
+func WithColorPadding(text string, colour color.Color) string {
+	if atomic.LoadUint32(&encoding) == plainEncodingType {
+		return color.WithColorPadding(text, colour)
+	}
+
+	return text
+}

core/logx/color_test.go

@@ -0,0 +1,33 @@
+package logx
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/color"
+)
+
+func TestWithColor(t *testing.T) {
+	old := atomic.SwapUint32(&encoding, plainEncodingType)
+	defer atomic.StoreUint32(&encoding, old)
+
+	output := WithColor("hello", color.BgBlue)
+	assert.Equal(t, "hello", output)
+
+	atomic.StoreUint32(&encoding, jsonEncodingType)
+	output = WithColor("hello", color.BgBlue)
+	assert.Equal(t, "hello", output)
+}
+
+func TestWithColorPadding(t *testing.T) {
+	old := atomic.SwapUint32(&encoding, plainEncodingType)
+	defer atomic.StoreUint32(&encoding, old)
+
+	output := WithColorPadding("hello", color.BgBlue)
+	assert.Equal(t, " hello ", output)
+
+	atomic.StoreUint32(&encoding, jsonEncodingType)
+	output = WithColorPadding("hello", color.BgBlue)
+	assert.Equal(t, "hello", output)
+}

core/logx/config.go

@@ -11,4 +11,16 @@ type LogConf struct {
 	Compress            bool   `json:",optional"`
 	KeepDays            int    `json:",optional"`
 	StackCooldownMillis int    `json:",default=100"`
+	// MaxBackups represents how many backup log files will be kept. 0 means all files will be kept forever.
+	// Only take effect when RotationRuleType is `size`.
+	// Even thougth `MaxBackups` sets 0, log files will still be removed
+	// if the `KeepDays` limitation is reached.
+	MaxBackups int `json:",default=0"`
+	// MaxSize represents how much space the writing log file takes up. 0 means no limit. The unit is `MB`.
+	// Only take effect when RotationRuleType is `size`
+	MaxSize int `json:",default=0"`
+	// RotationRuleType represents the type of log rotation rule. Default is `daily`.
+	// daily: daily rotation.
+	// size: size limited rotation.
+	Rotation string `json:",default=daily,options=[daily,size]"`
 }

core/logx/contextlogger.go

@@ -0,0 +1,145 @@
+package logx
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/zeromicro/go-zero/core/timex"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// WithContext sets ctx to log, for keeping tracing information.
+func WithContext(ctx context.Context) Logger {
+	return &contextLogger{
+		ctx: ctx,
+	}
+}
+
+type contextLogger struct {
+	logEntry
+	ctx context.Context
+}
+
+func (l *contextLogger) Error(v ...interface{}) {
+	l.err(fmt.Sprint(v...))
+}
+
+func (l *contextLogger) Errorf(format string, v ...interface{}) {
+	l.err(fmt.Sprintf(format, v...))
+}
+
+func (l *contextLogger) Errorv(v interface{}) {
+	l.err(fmt.Sprint(v))
+}
+
+func (l *contextLogger) Errorw(msg string, fields ...LogField) {
+	l.err(msg, fields...)
+}
+
+func (l *contextLogger) Info(v ...interface{}) {
+	l.info(fmt.Sprint(v...))
+}
+
+func (l *contextLogger) Infof(format string, v ...interface{}) {
+	l.info(fmt.Sprintf(format, v...))
+}
+
+func (l *contextLogger) Infov(v interface{}) {
+	l.info(v)
+}
+
+func (l *contextLogger) Infow(msg string, fields ...LogField) {
+	l.info(msg, fields...)
+}
+
+func (l *contextLogger) Slow(v ...interface{}) {
+	l.slow(fmt.Sprint(v...))
+}
+
+func (l *contextLogger) Slowf(format string, v ...interface{}) {
+	l.slow(fmt.Sprintf(format, v...))
+}
+
+func (l *contextLogger) Slowv(v interface{}) {
+	l.slow(v)
+}
+
+func (l *contextLogger) Sloww(msg string, fields ...LogField) {
+	l.slow(msg, fields...)
+}
+
+func (l *contextLogger) WithContext(ctx context.Context) Logger {
+	if ctx == nil {
+		return l
+	}
+
+	l.ctx = ctx
+	return l
+}
+
+func (l *contextLogger) WithDuration(duration time.Duration) Logger {
+	l.Duration = timex.ReprOfDuration(duration)
+	return l
+}
+
+func (l *contextLogger) buildFields(fields ...LogField) []LogField {
+	if len(l.Duration) > 0 {
+		fields = append(fields, Field(durationKey, l.Duration))
+	}
+
+	traceID := traceIdFromContext(l.ctx)
+	if len(traceID) > 0 {
+		fields = append(fields, Field(traceKey, traceID))
+	}
+
+	spanID := spanIdFromContext(l.ctx)
+	if len(spanID) > 0 {
+		fields = append(fields, Field(spanKey, spanID))
+	}
+
+	val := l.ctx.Value(fieldsContextKey)
+	if val != nil {
+		if arr, ok := val.([]LogField); ok {
+			fields = append(fields, arr...)
+		}
+	}
+
+	return fields
+}
+
+func (l *contextLogger) err(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Error(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *contextLogger) info(v interface{}, fields ...LogField) {
+	if shallLog(InfoLevel) {
+		getWriter().Info(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *contextLogger) slow(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Slow(v, l.buildFields(fields...)...)
+	}
+}
+
+func spanIdFromContext(ctx context.Context) string {
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasSpanID() {
+		return spanCtx.SpanID().String()
+	}
+
+	return ""
+}
+
+func traceIdFromContext(ctx context.Context) string {
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasTraceID() {
+		return spanCtx.TraceID().String()
+	}
+
+	return ""
+}

core/logx/contextlogger_test.go

@@ -0,0 +1,240 @@
+package logx
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func TestTraceLog(t *testing.T) {
+	SetLevel(InfoLevel)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
+	defer span.End()
+
+	WithContext(ctx).Info(testlog)
+	validate(t, w.String(), true, true)
+}
+
+func TestTraceError(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
+	defer span.End()
+
+	var nilCtx context.Context
+	l := WithContext(context.Background())
+	l = l.WithContext(nilCtx)
+	l = l.WithContext(ctx)
+	SetLevel(ErrorLevel)
+	l.WithDuration(time.Second).Error(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Errorf(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Errorv(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Errorw(testlog, Field("basket", "ball"))
+	validate(t, w.String(), true, true)
+	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
+	assert.True(t, strings.Contains(w.String(), "ball"), w.String())
+}
+
+func TestTraceInfo(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
+	defer span.End()
+
+	SetLevel(InfoLevel)
+	l := WithContext(ctx)
+	l.WithDuration(time.Second).Info(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Infof(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Infov(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Infow(testlog, Field("basket", "ball"))
+	validate(t, w.String(), true, true)
+	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
+	assert.True(t, strings.Contains(w.String(), "ball"), w.String())
+}
+
+func TestTraceInfoConsole(t *testing.T) {
+	old := atomic.SwapUint32(&encoding, jsonEncodingType)
+	defer atomic.StoreUint32(&encoding, old)
+
+	w := new(mockWriter)
+	o := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(o)
+	}()
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
+	defer span.End()
+
+	l := WithContext(ctx)
+	SetLevel(InfoLevel)
+	l.WithDuration(time.Second).Info(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Infof(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Infov(testlog)
+	validate(t, w.String(), true, true)
+}
+
+func TestTraceSlow(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
+	defer span.End()
+
+	l := WithContext(ctx)
+	SetLevel(InfoLevel)
+	l.WithDuration(time.Second).Slow(testlog)
+	assert.True(t, strings.Contains(w.String(), traceKey))
+	assert.True(t, strings.Contains(w.String(), spanKey))
+	w.Reset()
+	l.WithDuration(time.Second).Slowf(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Slowv(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Sloww(testlog, Field("basket", "ball"))
+	validate(t, w.String(), true, true)
+	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
+	assert.True(t, strings.Contains(w.String(), "ball"), w.String())
+}
+
+func TestTraceWithoutContext(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	l := WithContext(context.Background())
+	SetLevel(InfoLevel)
+	l.WithDuration(time.Second).Info(testlog)
+	validate(t, w.String(), false, false)
+	w.Reset()
+	l.WithDuration(time.Second).Infof(testlog)
+	validate(t, w.String(), false, false)
+}
+
+func TestLogWithFields(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	ctx := WithFields(context.Background(), Field("foo", "bar"))
+	l := WithContext(ctx)
+	SetLevel(InfoLevel)
+	l.Info(testlog)
+
+	var val mockValue
+	assert.Nil(t, json.Unmarshal([]byte(w.String()), &val))
+	assert.Equal(t, "bar", val.Foo)
+}
+
+func validate(t *testing.T, body string, expectedTrace, expectedSpan bool) {
+	var val mockValue
+	dec := json.NewDecoder(strings.NewReader(body))
+
+	for {
+		var doc mockValue
+		err := dec.Decode(&doc)
+		if err == io.EOF {
+			// all done
+			break
+		}
+		if err != nil {
+			continue
+		}
+
+		val = doc
+	}
+
+	assert.Equal(t, expectedTrace, len(val.Trace) > 0, body)
+	assert.Equal(t, expectedSpan, len(val.Span) > 0, body)
+}
+
+type mockValue struct {
+	Trace string `json:"trace"`
+	Span  string `json:"span"`
+	Foo   string `json:"foo"`
+}

core/logx/durationlogger.go

@@ -1,17 +1,13 @@
 package logx
 
 import (
+	"context"
 	"fmt"
-	"io"
 	"time"
 
 	"github.com/zeromicro/go-zero/core/timex"
 )
 
-const durationCallerDepth = 3
-
-type durationLogger logEntry
-
 // WithDuration returns a Logger which logs the given duration.
 func WithDuration(d time.Duration) Logger {
 	return &durationLogger{
@@ -19,57 +15,62 @@ func WithDuration(d time.Duration) Logger {
 	}
 }
 
+type durationLogger logEntry
+
 func (l *durationLogger) Error(v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
-	}
+	l.err(fmt.Sprint(v...))
 }
 
 func (l *durationLogger) Errorf(format string, v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
-	}
+	l.err(fmt.Sprintf(format, v...))
 }
 
 func (l *durationLogger) Errorv(v interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(errorLog, levelError, v)
-	}
+	l.err(v)
+}
+
+func (l *durationLogger) Errorw(msg string, fields ...LogField) {
+	l.err(msg, fields...)
 }
 
 func (l *durationLogger) Info(v ...interface{}) {
-	if shallLog(InfoLevel) {
-		l.write(infoLog, levelInfo, fmt.Sprint(v...))
-	}
+	l.info(fmt.Sprint(v...))
 }
 
 func (l *durationLogger) Infof(format string, v ...interface{}) {
-	if shallLog(InfoLevel) {
-		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
-	}
+	l.info(fmt.Sprintf(format, v...))
 }
 
 func (l *durationLogger) Infov(v interface{}) {
-	if shallLog(InfoLevel) {
-		l.write(infoLog, levelInfo, v)
-	}
+	l.info(v)
+}
+
+func (l *durationLogger) Infow(msg string, fields ...LogField) {
+	l.info(msg, fields...)
 }
 
 func (l *durationLogger) Slow(v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(slowLog, levelSlow, fmt.Sprint(v...))
-	}
+	l.slow(fmt.Sprint(v...))
 }
 
 func (l *durationLogger) Slowf(format string, v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
-	}
+	l.slow(fmt.Sprintf(format, v...))
 }
 
 func (l *durationLogger) Slowv(v interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(slowLog, levelSlow, v)
+	l.slow(v)
+}
+
+func (l *durationLogger) Sloww(msg string, fields ...LogField) {
+	l.slow(msg, fields...)
+}
+
+func (l *durationLogger) WithContext(ctx context.Context) Logger {
+	return &contextLogger{
+		ctx: ctx,
+		logEntry: logEntry{
+			Duration: l.Duration,
+		},
 	}
 }
 
@@ -78,16 +79,23 @@ func (l *durationLogger) WithDuration(duration time.Duration) Logger {
 	return l
 }
 
-func (l *durationLogger) write(writer io.Writer, level string, val interface{}) {
-	switch encoding {
-	case plainEncodingType:
-		writePlainAny(writer, level, val, l.Duration)
-	default:
-		outputJson(writer, &durationLogger{
-			Timestamp: getTimestamp(),
-			Level:     level,
-			Content:   val,
-			Duration:  l.Duration,
-		})
+func (l *durationLogger) err(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		fields = append(fields, Field(durationKey, l.Duration))
+		getWriter().Error(v, fields...)
+	}
+}
+
+func (l *durationLogger) info(v interface{}, fields ...LogField) {
+	if shallLog(InfoLevel) {
+		fields = append(fields, Field(durationKey, l.Duration))
+		getWriter().Info(v, fields...)
+	}
+}
+
+func (l *durationLogger) slow(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		fields = append(fields, Field(durationKey, l.Duration))
+		getWriter().Slow(v, fields...)
 	}
 }

core/logx/durationlogger_test.go

@@ -1,86 +1,161 @@
 package logx
 
 import (
-	"log"
+	"context"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 )
 
 func TestWithDurationError(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Error("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
 }
 
 func TestWithDurationErrorf(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Errorf("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
 }
 
 func TestWithDurationErrorv(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Errorv("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
+}
+
+func TestWithDurationErrorw(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	WithDuration(time.Second).Errorw("foo", Field("foo", "bar"))
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
+	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
+	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
 }
 
 func TestWithDurationInfo(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Info("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
 }
 
 func TestWithDurationInfoConsole(t *testing.T) {
-	old := encoding
-	encoding = plainEncodingType
+	old := atomic.LoadUint32(&encoding)
+	atomic.StoreUint32(&encoding, plainEncodingType)
 	defer func() {
-		encoding = old
+		atomic.StoreUint32(&encoding, old)
 	}()
 
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	o := writer.Swap(w)
+	defer writer.Store(o)
+
 	WithDuration(time.Second).Info("foo")
-	assert.True(t, strings.Contains(builder.String(), "ms"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "ms"), w.String())
 }
 
 func TestWithDurationInfof(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Infof("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
 }
 
 func TestWithDurationInfov(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Infov("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
+}
+
+func TestWithDurationInfow(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	WithDuration(time.Second).Infow("foo", Field("foo", "bar"))
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
+	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
+	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
+}
+
+func TestWithDurationWithContextInfow(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
+	WithDuration(time.Second).WithContext(ctx).Infow("foo", Field("foo", "bar"))
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
+	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
+	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
+	assert.True(t, strings.Contains(w.String(), "trace"), w.String())
+	assert.True(t, strings.Contains(w.String(), "span"), w.String())
 }
 
 func TestWithDurationSlow(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Slow("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
 }
 
 func TestWithDurationSlowf(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).WithDuration(time.Hour).Slowf("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
 }
 
 func TestWithDurationSlowv(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).WithDuration(time.Hour).Slowv("foo")
-	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
+}
+
+func TestWithDurationSloww(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	WithDuration(time.Second).WithDuration(time.Hour).Sloww("foo", Field("foo", "bar"))
+	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
+	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
+	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
 }

core/logx/fields.go

@@ -0,0 +1,18 @@
+package logx
+
+import "context"
+
+var fieldsContextKey contextKey
+
+type contextKey struct{}
+
+// WithFields returns a new context with the given fields.
+func WithFields(ctx context.Context, fields ...LogField) context.Context {
+	if val := ctx.Value(fieldsContextKey); val != nil {
+		if arr, ok := val.([]LogField); ok {
+			return context.WithValue(ctx, fieldsContextKey, append(arr, fields...))
+		}
+	}
+
+	return context.WithValue(ctx, fieldsContextKey, fields)
+}

core/logx/fields_test.go

@@ -0,0 +1,35 @@
+package logx
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWithFields(t *testing.T) {
+	ctx := WithFields(context.Background(), Field("a", 1), Field("b", 2))
+	vals := ctx.Value(fieldsContextKey)
+	assert.NotNil(t, vals)
+	fields, ok := vals.([]LogField)
+	assert.True(t, ok)
+	assert.EqualValues(t, []LogField{Field("a", 1), Field("b", 2)}, fields)
+}
+
+func TestWithFieldsAppend(t *testing.T) {
+	var dummyKey struct{}
+	ctx := context.WithValue(context.Background(), dummyKey, "dummy")
+	ctx = WithFields(ctx, Field("a", 1), Field("b", 2))
+	ctx = WithFields(ctx, Field("c", 3), Field("d", 4))
+	vals := ctx.Value(fieldsContextKey)
+	assert.NotNil(t, vals)
+	fields, ok := vals.([]LogField)
+	assert.True(t, ok)
+	assert.Equal(t, "dummy", ctx.Value(dummyKey))
+	assert.EqualValues(t, []LogField{
+		Field("a", 1),
+		Field("b", 2),
+		Field("c", 3),
+		Field("d", 4),
+	}, fields)
+}

core/logx/lesslogger_test.go

@@ -1,7 +1,6 @@
 package logx
 
 import (
-	"log"
 	"strings"
 	"testing"
 
@@ -9,23 +8,27 @@ import (
 )
 
 func TestLessLogger_Error(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	l := NewLessLogger(500)
 	for i := 0; i < 100; i++ {
 		l.Error("hello")
 	}
 
-	assert.Equal(t, 1, strings.Count(builder.String(), "\n"))
+	assert.Equal(t, 1, strings.Count(w.String(), "\n"))
 }
 
 func TestLessLogger_Errorf(t *testing.T) {
-	var builder strings.Builder
-	log.SetOutput(&builder)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	l := NewLessLogger(500)
 	for i := 0; i < 100; i++ {
 		l.Errorf("hello")
 	}
 
-	assert.Equal(t, 1, strings.Count(builder.String(), "\n"))
+	assert.Equal(t, 1, strings.Count(w.String(), "\n"))
 }

core/logx/logger.go

@@ -0,0 +1,38 @@
+package logx
+
+import (
+	"context"
+	"time"
+)
+
+// A Logger represents a logger.
+type Logger interface {
+	// Error logs a message at error level.
+	Error(...interface{})
+	// Errorf logs a message at error level.
+	Errorf(string, ...interface{})
+	// Errorv logs a message at error level.
+	Errorv(interface{})
+	// Errorw logs a message at error level.
+	Errorw(string, ...LogField)
+	// Info logs a message at info level.
+	Info(...interface{})
+	// Infof logs a message at info level.
+	Infof(string, ...interface{})
+	// Infov logs a message at info level.
+	Infov(interface{})
+	// Infow logs a message at info level.
+	Infow(string, ...LogField)
+	// Slow logs a message at slow level.
+	Slow(...interface{})
+	// Slowf logs a message at slow level.
+	Slowf(string, ...interface{})
+	// Slowv logs a message at slow level.
+	Slowv(interface{})
+	// Sloww logs a message at slow level.
+	Sloww(string, ...LogField)
+	// WithContext returns a new logger with the given context.
+	WithContext(context.Context) Logger
+	// WithDuration returns a new logger with the given duration.
+	WithDuration(time.Duration) Logger
+}

core/logx/logs.go

@@ -1,93 +1,31 @@
 package logx
 
 import (
-	"bytes"
-	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
-	"runtime"
 	"runtime/debug"
-	"strconv"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/zeromicro/go-zero/core/iox"
 	"github.com/zeromicro/go-zero/core/sysx"
-	"github.com/zeromicro/go-zero/core/timex"
 )
 
-const (
-	// InfoLevel logs everything
-	InfoLevel = iota
-	// ErrorLevel includes errors, slows, stacks
-	ErrorLevel
-	// SevereLevel only log severe messages
-	SevereLevel
-)
-
-const (
-	jsonEncodingType = iota
-	plainEncodingType
-
-	jsonEncoding     = "json"
-	plainEncoding    = "plain"
-	plainEncodingSep = '\t'
-)
-
-const (
-	accessFilename = "access.log"
-	errorFilename  = "error.log"
-	severeFilename = "severe.log"
-	slowFilename   = "slow.log"
-	statFilename   = "stat.log"
-
-	consoleMode = "console"
-	volumeMode  = "volume"
-
-	levelAlert  = "alert"
-	levelInfo   = "info"
-	levelError  = "error"
-	levelSevere = "severe"
-	levelFatal  = "fatal"
-	levelSlow   = "slow"
-	levelStat   = "stat"
-
-	backupFileDelimiter = "-"
-	callerInnerDepth    = 5
-	flags               = 0x0
-)
+const callerDepth = 5
 
 var (
-	// ErrLogPathNotSet is an error that indicates the log path is not set.
-	ErrLogPathNotSet = errors.New("log path must be set")
-	// ErrLogNotInitialized is an error that log is not initialized.
-	ErrLogNotInitialized = errors.New("log not initialized")
-	// ErrLogServiceNameNotSet is an error that indicates that the service name is not set.
-	ErrLogServiceNameNotSet = errors.New("log service name must be set")
-
-	timeFormat   = "2006-01-02T15:04:05.000Z07"
-	writeConsole bool
-	logLevel     uint32
-	encoding     = jsonEncodingType
+	timeFormat = "2006-01-02T15:04:05.000Z07:00"
+	logLevel   uint32
+	encoding   uint32 = jsonEncodingType
 	// use uint32 for atomic operations
+	disableLog  uint32
 	disableStat uint32
-	infoLog     io.WriteCloser
-	errorLog    io.WriteCloser
-	severeLog   io.WriteCloser
-	slowLog     io.WriteCloser
-	statLog     io.WriteCloser
-	stackLog    io.Writer
-
-	once        sync.Once
-	initialized uint32
 	options     logOptions
+	writer      = new(atomicWriter)
+	setupOnce   sync.Once
 )
 
 type (
@@ -95,109 +33,40 @@ type (
 		Timestamp string      `json:"@timestamp"`
 		Level     string      `json:"level"`
 		Duration  string      `json:"duration,omitempty"`
+		Caller    string      `json:"caller,omitempty"`
 		Content   interface{} `json:"content"`
 	}
 
+	logEntryWithFields map[string]interface{}
+
 	logOptions struct {
 		gzipEnabled           bool
 		logStackCooldownMills int
 		keepDays              int
+		maxBackups            int
+		maxSize               int
+		rotationRule          string
+	}
+
+	// LogField is a key-value pair that will be added to the log entry.
+	LogField struct {
+		Key   string
+		Value interface{}
 	}
 
 	// LogOption defines the method to customize the logging.
 	LogOption func(options *logOptions)
-
-	// A Logger represents a logger.
-	Logger interface {
-		Error(...interface{})
-		Errorf(string, ...interface{})
-		Errorv(interface{})
-		Info(...interface{})
-		Infof(string, ...interface{})
-		Infov(interface{})
-		Slow(...interface{})
-		Slowf(string, ...interface{})
-		Slowv(interface{})
-		WithDuration(time.Duration) Logger
-	}
 )
 
-// MustSetup sets up logging with given config c. It exits on error.
-func MustSetup(c LogConf) {
-	Must(SetUp(c))
-}
-
-// SetUp sets up the logx. If already set up, just return nil.
-// we allow SetUp to be called multiple times, because for example
-// we need to allow different service frameworks to initialize logx respectively.
-// the same logic for SetUp
-func SetUp(c LogConf) error {
-	if len(c.TimeFormat) > 0 {
-		timeFormat = c.TimeFormat
-	}
-	switch c.Encoding {
-	case plainEncoding:
-		encoding = plainEncodingType
-	default:
-		encoding = jsonEncodingType
-	}
-
-	switch c.Mode {
-	case consoleMode:
-		setupWithConsole(c)
-		return nil
-	case volumeMode:
-		return setupWithVolume(c)
-	default:
-		return setupWithFiles(c)
-	}
-}
-
 // Alert alerts v in alert level, and the message is written to error log.
 func Alert(v string) {
-	outputText(errorLog, levelAlert, v)
+	getWriter().Alert(v)
 }
 
 // Close closes the logging.
 func Close() error {
-	if writeConsole {
-		return nil
-	}
-
-	if atomic.LoadUint32(&initialized) == 0 {
-		return ErrLogNotInitialized
-	}
-
-	atomic.StoreUint32(&initialized, 0)
-
-	if infoLog != nil {
-		if err := infoLog.Close(); err != nil {
-			return err
-		}
-	}
-
-	if errorLog != nil {
-		if err := errorLog.Close(); err != nil {
-			return err
-		}
-	}
-
-	if severeLog != nil {
-		if err := severeLog.Close(); err != nil {
-			return err
-		}
-	}
-
-	if slowLog != nil {
-		if err := slowLog.Close(); err != nil {
-			return err
-		}
-	}
-
-	if statLog != nil {
-		if err := statLog.Close(); err != nil {
-			return err
-		}
+	if w := writer.Swap(nil); w != nil {
+		return w.(io.Closer).Close()
 	}
 
 	return nil
@@ -205,16 +74,8 @@ func Close() error {
 
 // Disable disables the logging.
 func Disable() {
-	once.Do(func() {
-		atomic.StoreUint32(&initialized, 1)
-
-		infoLog = iox.NopCloser(ioutil.Discard)
-		errorLog = iox.NopCloser(ioutil.Discard)
-		severeLog = iox.NopCloser(ioutil.Discard)
-		slowLog = iox.NopCloser(ioutil.Discard)
-		statLog = iox.NopCloser(ioutil.Discard)
-		stackLog = ioutil.Discard
-	})
+	atomic.StoreUint32(&disableLog, 1)
+	writer.Store(nopWriter{})
 }
 
 // DisableStat disables the stat logs.
@@ -224,22 +85,12 @@ func DisableStat() {
 
 // Error writes v into error log.
 func Error(v ...interface{}) {
-	ErrorCaller(1, v...)
-}
-
-// ErrorCaller writes v with context into error log.
-func ErrorCaller(callDepth int, v ...interface{}) {
-	errorTextSync(fmt.Sprint(v...), callDepth+callerInnerDepth)
-}
-
-// ErrorCallerf writes v with context in format into error log.
-func ErrorCallerf(callDepth int, format string, v ...interface{}) {
-	errorTextSync(fmt.Errorf(format, v...).Error(), callDepth+callerInnerDepth)
+	errorTextSync(fmt.Sprint(v...))
 }
 
 // Errorf writes v with format into error log.
 func Errorf(format string, v ...interface{}) {
-	ErrorCallerf(1, format, v...)
+	errorTextSync(fmt.Errorf(format, v...).Error())
 }
 
 // ErrorStack writes v along with call stack into error log.
@@ -260,6 +111,49 @@ func Errorv(v interface{}) {
 	errorAnySync(v)
 }
 
+// Errorw writes msg along with fields into error log.
+func Errorw(msg string, fields ...LogField) {
+	errorFieldsSync(msg, fields...)
+}
+
+// Field returns a LogField for the given key and value.
+func Field(key string, value interface{}) LogField {
+	switch val := value.(type) {
+	case error:
+		return LogField{Key: key, Value: val.Error()}
+	case []error:
+		var errs []string
+		for _, err := range val {
+			errs = append(errs, err.Error())
+		}
+		return LogField{Key: key, Value: errs}
+	case time.Duration:
+		return LogField{Key: key, Value: fmt.Sprint(val)}
+	case []time.Duration:
+		var durs []string
+		for _, dur := range val {
+			durs = append(durs, fmt.Sprint(dur))
+		}
+		return LogField{Key: key, Value: durs}
+	case []time.Time:
+		var times []string
+		for _, t := range val {
+			times = append(times, fmt.Sprint(t))
+		}
+		return LogField{Key: key, Value: times}
+	case fmt.Stringer:
+		return LogField{Key: key, Value: val.String()}
+	case []fmt.Stringer:
+		var strs []string
+		for _, str := range val {
+			strs = append(strs, str.String())
+		}
+		return LogField{Key: key, Value: strs}
+	default:
+		return LogField{Key: key, Value: val}
+	}
+}
+
 // Info writes v into access log.
 func Info(v ...interface{}) {
 	infoTextSync(fmt.Sprint(v...))
@@ -275,14 +169,31 @@ func Infov(v interface{}) {
 	infoAnySync(v)
 }
 
-// Must checks if err is nil, otherwise logs the err and exits.
+// Infow writes msg along with fields into access log.
+func Infow(msg string, fields ...LogField) {
+	infoFieldsSync(msg, fields...)
+}
+
+// Must checks if err is nil, otherwise logs the error and exits.
 func Must(err error) {
-	if err != nil {
-		msg := formatWithCaller(err.Error(), 3)
-		log.Print(msg)
-		outputText(severeLog, levelFatal, msg)
-		os.Exit(1)
+	if err == nil {
+		return
 	}
+
+	msg := err.Error()
+	log.Print(msg)
+	getWriter().Severe(msg)
+	os.Exit(1)
+}
+
+// MustSetup sets up logging with given config c. It exits on error.
+func MustSetup(c LogConf) {
+	Must(SetUp(c))
+}
+
+// Reset clears the writer and resets the log level.
+func Reset() Writer {
+	return writer.Swap(nil)
 }
 
 // SetLevel sets the logging level. It can be used to suppress some logs.
@@ -290,6 +201,47 @@ func SetLevel(level uint32) {
 	atomic.StoreUint32(&logLevel, level)
 }
 
+// SetWriter sets the logging writer. It can be used to customize the logging.
+func SetWriter(w Writer) {
+	if atomic.LoadUint32(&disableLog) == 0 {
+		writer.Store(w)
+	}
+}
+
+// SetUp sets up the logx. If already set up, just return nil.
+// we allow SetUp to be called multiple times, because for example
+// we need to allow different service frameworks to initialize logx respectively.
+func SetUp(c LogConf) (err error) {
+	// Just ignore the subsequent SetUp calls.
+	// Because multiple services in one process might call SetUp respectively.
+	// Need to wait for the first caller to complete the execution.
+	setupOnce.Do(func() {
+		setupLogLevel(c)
+
+		if len(c.TimeFormat) > 0 {
+			timeFormat = c.TimeFormat
+		}
+
+		switch c.Encoding {
+		case plainEncoding:
+			atomic.StoreUint32(&encoding, plainEncodingType)
+		default:
+			atomic.StoreUint32(&encoding, jsonEncodingType)
+		}
+
+		switch c.Mode {
+		case fileMode:
+			err = setupWithFiles(c)
+		case volumeMode:
+			err = setupWithVolume(c)
+		default:
+			setupWithConsole()
+		}
+	})
+
+	return
+}
+
 // Severe writes v into severe log.
 func Severe(v ...interface{}) {
 	severeSync(fmt.Sprint(v...))
@@ -315,6 +267,11 @@ func Slowv(v interface{}) {
 	slowAnySync(v)
 }
 
+// Sloww writes msg along with fields into slow log.
+func Sloww(msg string, fields ...LogField) {
+	slowFieldsSync(msg, fields...)
+}
+
 // Stat writes v into stat log.
 func Stat(v ...interface{}) {
 	statSync(fmt.Sprint(v...))
@@ -346,63 +303,67 @@ func WithGzip() LogOption {
 	}
 }
 
+// WithMaxBackups customizes how many log files backups will be kept.
+func WithMaxBackups(count int) LogOption {
+	return func(opts *logOptions) {
+		opts.maxBackups = count
+	}
+}
+
+// WithMaxSize customizes how much space the writing log file can take up.
+func WithMaxSize(size int) LogOption {
+	return func(opts *logOptions) {
+		opts.maxSize = size
+	}
+}
+
+// WithRotation customizes which log rotation rule to use.
+func WithRotation(r string) LogOption {
+	return func(opts *logOptions) {
+		opts.rotationRule = r
+	}
+}
+
 func createOutput(path string) (io.WriteCloser, error) {
 	if len(path) == 0 {
 		return nil, ErrLogPathNotSet
 	}
 
-	return NewLogger(path, DefaultRotateRule(path, backupFileDelimiter, options.keepDays,
-		options.gzipEnabled), options.gzipEnabled)
+	switch options.rotationRule {
+	case sizeRotationRule:
+		return NewLogger(path, NewSizeLimitRotateRule(path, backupFileDelimiter, options.keepDays,
+			options.maxSize, options.maxBackups, options.gzipEnabled), options.gzipEnabled)
+	default:
+		return NewLogger(path, DefaultRotateRule(path, backupFileDelimiter, options.keepDays,
+			options.gzipEnabled), options.gzipEnabled)
+	}
 }
 
 func errorAnySync(v interface{}) {
 	if shallLog(ErrorLevel) {
-		outputAny(errorLog, levelError, v)
+		getWriter().Error(v)
 	}
 }
 
-func errorTextSync(msg string, callDepth int) {
+func errorFieldsSync(content string, fields ...LogField) {
 	if shallLog(ErrorLevel) {
-		outputError(errorLog, msg, callDepth)
+		getWriter().Error(content, fields...)
 	}
 }
 
-func formatWithCaller(msg string, callDepth int) string {
-	var buf strings.Builder
-
-	caller := getCaller(callDepth)
-	if len(caller) > 0 {
-		buf.WriteString(caller)
-		buf.WriteByte(' ')
+func errorTextSync(msg string) {
+	if shallLog(ErrorLevel) {
+		getWriter().Error(msg)
 	}
-
-	buf.WriteString(msg)
-
-	return buf.String()
 }
 
-func getCaller(callDepth int) string {
-	var buf strings.Builder
-
-	_, file, line, ok := runtime.Caller(callDepth)
-	if ok {
-		short := file
-		for i := len(file) - 1; i > 0; i-- {
-			if file[i] == '/' {
-				short = file[i+1:]
-				break
-			}
-		}
-		buf.WriteString(short)
-		buf.WriteByte(':')
-		buf.WriteString(strconv.Itoa(line))
+func getWriter() Writer {
+	w := writer.Load()
+	if w == nil {
+		w = writer.StoreIfNil(newConsoleWriter())
 	}
 
-	return buf.String()
-}
-
-func getTimestamp() string {
-	return timex.Time().Format(timeFormat)
+	return w
 }
 
 func handleOptions(opts []LogOption) {
@@ -413,56 +374,19 @@ func handleOptions(opts []LogOption) {
 
 func infoAnySync(val interface{}) {
 	if shallLog(InfoLevel) {
-		outputAny(infoLog, levelInfo, val)
+		getWriter().Info(val)
+	}
+}
+
+func infoFieldsSync(content string, fields ...LogField) {
+	if shallLog(InfoLevel) {
+		getWriter().Info(content, fields...)
 	}
 }
 
 func infoTextSync(msg string) {
 	if shallLog(InfoLevel) {
-		outputText(infoLog, levelInfo, msg)
-	}
-}
-
-func outputAny(writer io.Writer, level string, val interface{}) {
-	switch encoding {
-	case plainEncodingType:
-		writePlainAny(writer, level, val)
-	default:
-		info := logEntry{
-			Timestamp: getTimestamp(),
-			Level:     level,
-			Content:   val,
-		}
-		outputJson(writer, info)
-	}
-}
-
-func outputText(writer io.Writer, level, msg string) {
-	switch encoding {
-	case plainEncodingType:
-		writePlainText(writer, level, msg)
-	default:
-		info := logEntry{
-			Timestamp: getTimestamp(),
-			Level:     level,
-			Content:   msg,
-		}
-		outputJson(writer, info)
-	}
-}
-
-func outputError(writer io.Writer, msg string, callDepth int) {
-	content := formatWithCaller(msg, callDepth)
-	outputText(writer, levelError, content)
-}
-
-func outputJson(writer io.Writer, info interface{}) {
-	if content, err := json.Marshal(info); err != nil {
-		log.Println(err.Error())
-	} else if atomic.LoadUint32(&initialized) == 0 || writer == nil {
-		log.Println(string(content))
-	} else {
-		writer.Write(append(content, '\n'))
+		getWriter().Info(msg)
 	}
 }
 
@@ -477,72 +401,18 @@ func setupLogLevel(c LogConf) {
 	}
 }
 
-func setupWithConsole(c LogConf) {
-	once.Do(func() {
-		atomic.StoreUint32(&initialized, 1)
-		writeConsole = true
-		setupLogLevel(c)
-
-		infoLog = newLogWriter(log.New(os.Stdout, "", flags))
-		errorLog = newLogWriter(log.New(os.Stderr, "", flags))
-		severeLog = newLogWriter(log.New(os.Stderr, "", flags))
-		slowLog = newLogWriter(log.New(os.Stderr, "", flags))
-		stackLog = newLessWriter(errorLog, options.logStackCooldownMills)
-		statLog = infoLog
-	})
+func setupWithConsole() {
+	SetWriter(newConsoleWriter())
 }
 
 func setupWithFiles(c LogConf) error {
-	var opts []LogOption
-	var err error
-
-	if len(c.Path) == 0 {
-		return ErrLogPathNotSet
+	w, err := newFileWriter(c)
+	if err != nil {
+		return err
 	}
 
-	opts = append(opts, WithCooldownMillis(c.StackCooldownMillis))
-	if c.Compress {
-		opts = append(opts, WithGzip())
-	}
-	if c.KeepDays > 0 {
-		opts = append(opts, WithKeepDays(c.KeepDays))
-	}
-
-	accessFile := path.Join(c.Path, accessFilename)
-	errorFile := path.Join(c.Path, errorFilename)
-	severeFile := path.Join(c.Path, severeFilename)
-	slowFile := path.Join(c.Path, slowFilename)
-	statFile := path.Join(c.Path, statFilename)
-
-	once.Do(func() {
-		atomic.StoreUint32(&initialized, 1)
-		handleOptions(opts)
-		setupLogLevel(c)
-
-		if infoLog, err = createOutput(accessFile); err != nil {
-			return
-		}
-
-		if errorLog, err = createOutput(errorFile); err != nil {
-			return
-		}
-
-		if severeLog, err = createOutput(severeFile); err != nil {
-			return
-		}
-
-		if slowLog, err = createOutput(slowFile); err != nil {
-			return
-		}
-
-		if statLog, err = createOutput(statFile); err != nil {
-			return
-		}
-
-		stackLog = newLessWriter(errorLog, options.logStackCooldownMills)
-	})
-
-	return err
+	SetWriter(w)
+	return nil
 }
 
 func setupWithVolume(c LogConf) error {
@@ -556,7 +426,7 @@ func setupWithVolume(c LogConf) error {
 
 func severeSync(msg string) {
 	if shallLog(SevereLevel) {
-		outputText(severeLog, levelSevere, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+		getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
@@ -570,99 +440,30 @@ func shallLogStat() bool {
 
 func slowAnySync(v interface{}) {
 	if shallLog(ErrorLevel) {
-		outputAny(slowLog, levelSlow, v)
+		getWriter().Slow(v)
+	}
+}
+
+func slowFieldsSync(content string, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Slow(content, fields...)
 	}
 }
 
 func slowTextSync(msg string) {
 	if shallLog(ErrorLevel) {
-		outputText(slowLog, levelSlow, msg)
+		getWriter().Slow(msg)
 	}
 }
 
 func stackSync(msg string) {
 	if shallLog(ErrorLevel) {
-		outputText(stackLog, levelError, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+		getWriter().Stack(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
 func statSync(msg string) {
 	if shallLogStat() && shallLog(InfoLevel) {
-		outputText(statLog, levelStat, msg)
+		getWriter().Stat(msg)
 	}
 }
-
-func writePlainAny(writer io.Writer, level string, val interface{}, fields ...string) {
-	switch v := val.(type) {
-	case string:
-		writePlainText(writer, level, v, fields...)
-	case error:
-		writePlainText(writer, level, v.Error(), fields...)
-	case fmt.Stringer:
-		writePlainText(writer, level, v.String(), fields...)
-	default:
-		var buf bytes.Buffer
-		buf.WriteString(getTimestamp())
-		buf.WriteByte(plainEncodingSep)
-		buf.WriteString(level)
-		for _, item := range fields {
-			buf.WriteByte(plainEncodingSep)
-			buf.WriteString(item)
-		}
-		buf.WriteByte(plainEncodingSep)
-		if err := json.NewEncoder(&buf).Encode(val); err != nil {
-			log.Println(err.Error())
-			return
-		}
-		buf.WriteByte('\n')
-		if atomic.LoadUint32(&initialized) == 0 || writer == nil {
-			log.Println(buf.String())
-			return
-		}
-
-		if _, err := writer.Write(buf.Bytes()); err != nil {
-			log.Println(err.Error())
-		}
-	}
-}
-
-func writePlainText(writer io.Writer, level, msg string, fields ...string) {
-	var buf bytes.Buffer
-	buf.WriteString(getTimestamp())
-	buf.WriteByte(plainEncodingSep)
-	buf.WriteString(level)
-	for _, item := range fields {
-		buf.WriteByte(plainEncodingSep)
-		buf.WriteString(item)
-	}
-	buf.WriteByte(plainEncodingSep)
-	buf.WriteString(msg)
-	buf.WriteByte('\n')
-	if atomic.LoadUint32(&initialized) == 0 || writer == nil {
-		log.Println(buf.String())
-		return
-	}
-
-	if _, err := writer.Write(buf.Bytes()); err != nil {
-		log.Println(err.Error())
-	}
-}
-
-type logWriter struct {
-	logger *log.Logger
-}
-
-func newLogWriter(logger *log.Logger) logWriter {
-	return logWriter{
-		logger: logger,
-	}
-}
-
-func (lw logWriter) Close() error {
-	return nil
-}
-
-func (lw logWriter) Write(data []byte) (int, error) {
-	lw.logger.Print(string(data))
-	return len(data), nil
-}

core/logx/logs_test.go

@@ -5,9 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
+	"reflect"
 	"runtime"
 	"strings"
 	"sync"
@@ -19,8 +19,9 @@ import (
 )
 
 var (
-	s    = []byte("Sending #11 notification (id: 1451875113812010473) in #1 connection")
-	pool = make(chan []byte, 1)
+	s           = []byte("Sending #11 notification (id: 1451875113812010473) in #1 connection")
+	pool        = make(chan []byte, 1)
+	_    Writer = (*mockWriter)(nil)
 )
 
 type mockWriter struct {
@@ -28,10 +29,46 @@ type mockWriter struct {
 	builder strings.Builder
 }
 
-func (mw *mockWriter) Write(data []byte) (int, error) {
+func (mw *mockWriter) Alert(v interface{}) {
 	mw.lock.Lock()
 	defer mw.lock.Unlock()
-	return mw.builder.Write(data)
+	output(&mw.builder, levelAlert, v)
+}
+
+func (mw *mockWriter) Error(v interface{}, fields ...LogField) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelError, v, fields...)
+}
+
+func (mw *mockWriter) Info(v interface{}, fields ...LogField) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelInfo, v, fields...)
+}
+
+func (mw *mockWriter) Severe(v interface{}) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelSevere, v)
+}
+
+func (mw *mockWriter) Slow(v interface{}, fields ...LogField) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelSlow, v, fields...)
+}
+
+func (mw *mockWriter) Stack(v interface{}) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelError, v)
+}
+
+func (mw *mockWriter) Stat(v interface{}, fields ...LogField) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelStat, v, fields...)
 }
 
 func (mw *mockWriter) Close() error {
@@ -56,99 +93,215 @@ func (mw *mockWriter) String() string {
 	return mw.builder.String()
 }
 
+func TestField(t *testing.T) {
+	tests := []struct {
+		name string
+		f    LogField
+		want map[string]interface{}
+	}{
+		{
+			name: "error",
+			f:    Field("foo", errors.New("bar")),
+			want: map[string]interface{}{
+				"foo": "bar",
+			},
+		},
+		{
+			name: "errors",
+			f:    Field("foo", []error{errors.New("bar"), errors.New("baz")}),
+			want: map[string]interface{}{
+				"foo": []interface{}{"bar", "baz"},
+			},
+		},
+		{
+			name: "strings",
+			f:    Field("foo", []string{"bar", "baz"}),
+			want: map[string]interface{}{
+				"foo": []interface{}{"bar", "baz"},
+			},
+		},
+		{
+			name: "duration",
+			f:    Field("foo", time.Second),
+			want: map[string]interface{}{
+				"foo": "1s",
+			},
+		},
+		{
+			name: "durations",
+			f:    Field("foo", []time.Duration{time.Second, 2 * time.Second}),
+			want: map[string]interface{}{
+				"foo": []interface{}{"1s", "2s"},
+			},
+		},
+		{
+			name: "times",
+			f: Field("foo", []time.Time{
+				time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC),
+				time.Date(2020, time.January, 2, 0, 0, 0, 0, time.UTC),
+			}),
+			want: map[string]interface{}{
+				"foo": []interface{}{"2020-01-01 00:00:00 +0000 UTC", "2020-01-02 00:00:00 +0000 UTC"},
+			},
+		},
+		{
+			name: "stringer",
+			f:    Field("foo", ValStringer{val: "bar"}),
+			want: map[string]interface{}{
+				"foo": "bar",
+			},
+		},
+		{
+			name: "stringers",
+			f:    Field("foo", []fmt.Stringer{ValStringer{val: "bar"}, ValStringer{val: "baz"}}),
+			want: map[string]interface{}{
+				"foo": []interface{}{"bar", "baz"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			w := new(mockWriter)
+			old := writer.Swap(w)
+			defer writer.Store(old)
+
+			Infow("foo", test.f)
+			validateFields(t, w.String(), test.want)
+		})
+	}
+}
+
 func TestFileLineFileMode(t *testing.T) {
-	writer := new(mockWriter)
-	errorLog = writer
-	atomic.StoreUint32(&initialized, 1)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	file, line := getFileLine()
 	Error("anything")
-	assert.True(t, writer.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
 
-	writer.Reset()
 	file, line = getFileLine()
 	Errorf("anything %s", "format")
-	assert.True(t, writer.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestFileLineConsoleMode(t *testing.T) {
-	writer := new(mockWriter)
-	writeConsole = true
-	errorLog = newLogWriter(log.New(writer, "[ERROR] ", flags))
-	atomic.StoreUint32(&initialized, 1)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	file, line := getFileLine()
 	Error("anything")
-	assert.True(t, writer.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
 
-	writer.Reset()
+	w.Reset()
 	file, line = getFileLine()
 	Errorf("anything %s", "format")
-	assert.True(t, writer.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestStructedLogAlert(t *testing.T) {
-	doTestStructedLog(t, levelAlert, func(writer io.WriteCloser) {
-		errorLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelAlert, w, func(v ...interface{}) {
 		Alert(fmt.Sprint(v...))
 	})
 }
 
 func TestStructedLogError(t *testing.T) {
-	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
-		errorLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelError, w, func(v ...interface{}) {
 		Error(v...)
 	})
 }
 
 func TestStructedLogErrorf(t *testing.T) {
-	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
-		errorLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelError, w, func(v ...interface{}) {
 		Errorf("%s", fmt.Sprint(v...))
 	})
 }
 
 func TestStructedLogErrorv(t *testing.T) {
-	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
-		errorLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelError, w, func(v ...interface{}) {
 		Errorv(fmt.Sprint(v...))
 	})
 }
 
+func TestStructedLogErrorw(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelError, w, func(v ...interface{}) {
+		Errorw(fmt.Sprint(v...), Field("foo", "bar"))
+	})
+}
+
 func TestStructedLogInfo(t *testing.T) {
-	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelInfo, w, func(v ...interface{}) {
 		Info(v...)
 	})
 }
 
 func TestStructedLogInfof(t *testing.T) {
-	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelInfo, w, func(v ...interface{}) {
 		Infof("%s", fmt.Sprint(v...))
 	})
 }
 
 func TestStructedLogInfov(t *testing.T) {
-	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelInfo, w, func(v ...interface{}) {
 		Infov(fmt.Sprint(v...))
 	})
 }
 
+func TestStructedLogInfow(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelInfo, w, func(v ...interface{}) {
+		Infow(fmt.Sprint(v...), Field("foo", "bar"))
+	})
+}
+
 func TestStructedLogInfoConsoleAny(t *testing.T) {
-	doTestStructedLogConsole(t, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
-		old := encoding
-		encoding = plainEncodingType
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLogConsole(t, w, func(v ...interface{}) {
+		old := atomic.LoadUint32(&encoding)
+		atomic.StoreUint32(&encoding, plainEncodingType)
 		defer func() {
-			encoding = old
+			atomic.StoreUint32(&encoding, old)
 		}()
 
 		Infov(v)
@@ -156,13 +309,15 @@ func TestStructedLogInfoConsoleAny(t *testing.T) {
 }
 
 func TestStructedLogInfoConsoleAnyString(t *testing.T) {
-	doTestStructedLogConsole(t, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
-		old := encoding
-		encoding = plainEncodingType
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLogConsole(t, w, func(v ...interface{}) {
+		old := atomic.LoadUint32(&encoding)
+		atomic.StoreUint32(&encoding, plainEncodingType)
 		defer func() {
-			encoding = old
+			atomic.StoreUint32(&encoding, old)
 		}()
 
 		Infov(fmt.Sprint(v...))
@@ -170,13 +325,15 @@ func TestStructedLogInfoConsoleAnyString(t *testing.T) {
 }
 
 func TestStructedLogInfoConsoleAnyError(t *testing.T) {
-	doTestStructedLogConsole(t, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
-		old := encoding
-		encoding = plainEncodingType
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLogConsole(t, w, func(v ...interface{}) {
+		old := atomic.LoadUint32(&encoding)
+		atomic.StoreUint32(&encoding, plainEncodingType)
 		defer func() {
-			encoding = old
+			atomic.StoreUint32(&encoding, old)
 		}()
 
 		Infov(errors.New(fmt.Sprint(v...)))
@@ -184,13 +341,15 @@ func TestStructedLogInfoConsoleAnyError(t *testing.T) {
 }
 
 func TestStructedLogInfoConsoleAnyStringer(t *testing.T) {
-	doTestStructedLogConsole(t, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
-		old := encoding
-		encoding = plainEncodingType
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLogConsole(t, w, func(v ...interface{}) {
+		old := atomic.LoadUint32(&encoding)
+		atomic.StoreUint32(&encoding, plainEncodingType)
 		defer func() {
-			encoding = old
+			atomic.StoreUint32(&encoding, old)
 		}()
 
 		Infov(ValStringer{
@@ -200,13 +359,15 @@ func TestStructedLogInfoConsoleAnyStringer(t *testing.T) {
 }
 
 func TestStructedLogInfoConsoleText(t *testing.T) {
-	doTestStructedLogConsole(t, func(writer io.WriteCloser) {
-		infoLog = writer
-	}, func(v ...interface{}) {
-		old := encoding
-		encoding = plainEncodingType
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLogConsole(t, w, func(v ...interface{}) {
+		old := atomic.LoadUint32(&encoding)
+		atomic.StoreUint32(&encoding, plainEncodingType)
 		defer func() {
-			encoding = old
+			atomic.StoreUint32(&encoding, old)
 		}()
 
 		Info(fmt.Sprint(v...))
@@ -214,69 +375,94 @@ func TestStructedLogInfoConsoleText(t *testing.T) {
 }
 
 func TestStructedLogSlow(t *testing.T) {
-	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
-		slowLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelSlow, w, func(v ...interface{}) {
 		Slow(v...)
 	})
 }
 
 func TestStructedLogSlowf(t *testing.T) {
-	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
-		slowLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelSlow, w, func(v ...interface{}) {
 		Slowf(fmt.Sprint(v...))
 	})
 }
 
 func TestStructedLogSlowv(t *testing.T) {
-	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
-		slowLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelSlow, w, func(v ...interface{}) {
 		Slowv(fmt.Sprint(v...))
 	})
 }
 
+func TestStructedLogSloww(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelSlow, w, func(v ...interface{}) {
+		Sloww(fmt.Sprint(v...), Field("foo", time.Second))
+	})
+}
+
 func TestStructedLogStat(t *testing.T) {
-	doTestStructedLog(t, levelStat, func(writer io.WriteCloser) {
-		statLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelStat, w, func(v ...interface{}) {
 		Stat(v...)
 	})
 }
 
 func TestStructedLogStatf(t *testing.T) {
-	doTestStructedLog(t, levelStat, func(writer io.WriteCloser) {
-		statLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelStat, w, func(v ...interface{}) {
 		Statf(fmt.Sprint(v...))
 	})
 }
 
 func TestStructedLogSevere(t *testing.T) {
-	doTestStructedLog(t, levelSevere, func(writer io.WriteCloser) {
-		severeLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelSevere, w, func(v ...interface{}) {
 		Severe(v...)
 	})
 }
 
 func TestStructedLogSeveref(t *testing.T) {
-	doTestStructedLog(t, levelSevere, func(writer io.WriteCloser) {
-		severeLog = writer
-	}, func(v ...interface{}) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelSevere, w, func(v ...interface{}) {
 		Severef(fmt.Sprint(v...))
 	})
 }
 
 func TestStructedLogWithDuration(t *testing.T) {
 	const message = "hello there"
-	writer := new(mockWriter)
-	infoLog = writer
-	atomic.StoreUint32(&initialized, 1)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Info(message)
 	var entry logEntry
-	if err := json.Unmarshal([]byte(writer.builder.String()), &entry); err != nil {
+	if err := json.Unmarshal([]byte(w.String()), &entry); err != nil {
 		t.Error(err)
 	}
 	assert.Equal(t, levelInfo, entry.Level)
@@ -287,11 +473,12 @@ func TestStructedLogWithDuration(t *testing.T) {
 func TestSetLevel(t *testing.T) {
 	SetLevel(ErrorLevel)
 	const message = "hello there"
-	writer := new(mockWriter)
-	infoLog = writer
-	atomic.StoreUint32(&initialized, 1)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	Info(message)
-	assert.Equal(t, 0, writer.builder.Len())
+	assert.Equal(t, 0, w.builder.Len())
 }
 
 func TestSetLevelTwiceWithMode(t *testing.T) {
@@ -300,29 +487,35 @@ func TestSetLevelTwiceWithMode(t *testing.T) {
 		"console",
 		"volumn",
 	}
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	for _, mode := range testModes {
-		testSetLevelTwiceWithMode(t, mode)
+		testSetLevelTwiceWithMode(t, mode, w)
 	}
 }
 
 func TestSetLevelWithDuration(t *testing.T) {
 	SetLevel(ErrorLevel)
 	const message = "hello there"
-	writer := new(mockWriter)
-	infoLog = writer
-	atomic.StoreUint32(&initialized, 1)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	WithDuration(time.Second).Info(message)
-	assert.Equal(t, 0, writer.builder.Len())
+	assert.Equal(t, 0, w.builder.Len())
 }
 
 func TestErrorfWithWrappedError(t *testing.T) {
 	SetLevel(ErrorLevel)
 	const message = "there"
-	writer := new(mockWriter)
-	errorLog = writer
-	atomic.StoreUint32(&initialized, 1)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
 	Errorf("hello %w", errors.New(message))
-	assert.True(t, strings.Contains(writer.builder.String(), "hello there"))
+	assert.True(t, strings.Contains(w.String(), "hello there"))
 }
 
 func TestMustNil(t *testing.T) {
@@ -330,6 +523,11 @@ func TestMustNil(t *testing.T) {
 }
 
 func TestSetup(t *testing.T) {
+	defer func() {
+		SetLevel(InfoLevel)
+		atomic.StoreUint32(&encoding, jsonEncodingType)
+	}()
+
 	MustSetup(LogConf{
 		ServiceName: "any",
 		Mode:        "console",
@@ -344,6 +542,17 @@ func TestSetup(t *testing.T) {
 		Mode:        "volume",
 		Path:        os.TempDir(),
 	})
+	MustSetup(LogConf{
+		ServiceName: "any",
+		Mode:        "console",
+		TimeFormat:  timeFormat,
+	})
+	MustSetup(LogConf{
+		ServiceName: "any",
+		Mode:        "console",
+		Encoding:    plainEncoding,
+	})
+
 	assert.NotNil(t, setupWithVolume(LogConf{}))
 	assert.NotNil(t, setupWithFiles(LogConf{}))
 	assert.Nil(t, setupWithFiles(LogConf{
@@ -364,6 +573,8 @@ func TestSetup(t *testing.T) {
 	_, err := createOutput("")
 	assert.NotNil(t, err)
 	Disable()
+	SetLevel(InfoLevel)
+	atomic.StoreUint32(&encoding, jsonEncodingType)
 }
 
 func TestDisable(t *testing.T) {
@@ -373,7 +584,6 @@ func TestDisable(t *testing.T) {
 	WithKeepDays(1)(&opt)
 	WithGzip()(&opt)
 	assert.Nil(t, Close())
-	writeConsole = false
 	assert.Nil(t, Close())
 }
 
@@ -381,11 +591,22 @@ func TestDisableStat(t *testing.T) {
 	DisableStat()
 
 	const message = "hello there"
-	writer := new(mockWriter)
-	statLog = writer
-	atomic.StoreUint32(&initialized, 1)
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
 	Stat(message)
-	assert.Equal(t, 0, writer.builder.Len())
+	assert.Equal(t, 0, w.builder.Len())
+}
+
+func TestSetWriter(t *testing.T) {
+	atomic.StoreUint32(&disableLog, 0)
+	Reset()
+	SetWriter(nopWriter{})
+	assert.NotNil(t, writer.Load())
+	assert.True(t, writer.Load() == nopWriter{})
+	mocked := new(mockWriter)
+	SetWriter(mocked)
+	assert.Equal(t, mocked, writer.Load())
 }
 
 func TestWithGzip(t *testing.T) {
@@ -428,7 +649,7 @@ func BenchmarkCopyByteSlice(b *testing.B) {
 		buf = make([]byte, len(s))
 		copy(buf, s)
 	}
-	fmt.Fprint(ioutil.Discard, buf)
+	fmt.Fprint(io.Discard, buf)
 }
 
 func BenchmarkCopyOnWriteByteSlice(b *testing.B) {
@@ -437,7 +658,7 @@ func BenchmarkCopyOnWriteByteSlice(b *testing.B) {
 		size := len(s)
 		buf = s[:size:size]
 	}
-	fmt.Fprint(ioutil.Discard, buf)
+	fmt.Fprint(io.Discard, buf)
 }
 
 func BenchmarkCacheByteSlice(b *testing.B) {
@@ -451,7 +672,7 @@ func BenchmarkCacheByteSlice(b *testing.B) {
 func BenchmarkLogs(b *testing.B) {
 	b.ReportAllocs()
 
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 	for i := 0; i < b.N; i++ {
 		Info(i)
 	}
@@ -487,15 +708,11 @@ func put(b []byte) {
 	}
 }
 
-func doTestStructedLog(t *testing.T, level string, setup func(writer io.WriteCloser),
-	write func(...interface{})) {
+func doTestStructedLog(t *testing.T, level string, w *mockWriter, write func(...interface{})) {
 	const message = "hello there"
-	writer := new(mockWriter)
-	setup(writer)
-	atomic.StoreUint32(&initialized, 1)
 	write(message)
 	var entry logEntry
-	if err := json.Unmarshal([]byte(writer.builder.String()), &entry); err != nil {
+	if err := json.Unmarshal([]byte(w.String()), &entry); err != nil {
 		t.Error(err)
 	}
 	assert.Equal(t, level, entry.Level)
@@ -504,18 +721,14 @@ func doTestStructedLog(t *testing.T, level string, setup func(writer io.WriteClo
 	assert.True(t, strings.Contains(val, message))
 }
 
-func doTestStructedLogConsole(t *testing.T, setup func(writer io.WriteCloser),
-	write func(...interface{})) {
+func doTestStructedLogConsole(t *testing.T, w *mockWriter, write func(...interface{})) {
 	const message = "hello there"
-	writer := new(mockWriter)
-	setup(writer)
-	atomic.StoreUint32(&initialized, 1)
 	write(message)
-	println(writer.String())
-	assert.True(t, strings.Contains(writer.String(), message))
+	assert.True(t, strings.Contains(w.String(), message))
 }
 
-func testSetLevelTwiceWithMode(t *testing.T, mode string) {
+func testSetLevelTwiceWithMode(t *testing.T, mode string, w *mockWriter) {
+	writer.Store(nil)
 	SetUp(LogConf{
 		Mode:  mode,
 		Level: "error",
@@ -527,17 +740,14 @@ func testSetLevelTwiceWithMode(t *testing.T, mode string) {
 		Path:  "/dev/null",
 	})
 	const message = "hello there"
-	writer := new(mockWriter)
-	infoLog = writer
-	atomic.StoreUint32(&initialized, 1)
 	Info(message)
-	assert.Equal(t, 0, writer.builder.Len())
+	assert.Equal(t, 0, w.builder.Len())
 	Infof(message)
-	assert.Equal(t, 0, writer.builder.Len())
+	assert.Equal(t, 0, w.builder.Len())
 	ErrorStack(message)
-	assert.Equal(t, 0, writer.builder.Len())
+	assert.Equal(t, 0, w.builder.Len())
 	ErrorStackf(message)
-	assert.Equal(t, 0, writer.builder.Len())
+	assert.Equal(t, 0, w.builder.Len())
 }
 
 type ValStringer struct {
@@ -547,3 +757,18 @@ type ValStringer struct {
 func (v ValStringer) String() string {
 	return v.val
 }
+
+func validateFields(t *testing.T, content string, fields map[string]interface{}) {
+	var m map[string]interface{}
+	if err := json.Unmarshal([]byte(content), &m); err != nil {
+		t.Error(err)
+	}
+
+	for k, v := range fields {
+		if reflect.TypeOf(v).Kind() == reflect.Slice {
+			assert.EqualValues(t, v, m[k])
+		} else {
+			assert.Equal(t, v, m[k], content)
+		}
+	}
+}

core/logx/logwriter.go

@@ -0,0 +1,22 @@
+package logx
+
+import "log"
+
+type logWriter struct {
+	logger *log.Logger
+}
+
+func newLogWriter(logger *log.Logger) logWriter {
+	return logWriter{
+		logger: logger,
+	}
+}
+
+func (lw logWriter) Close() error {
+	return nil
+}
+
+func (lw logWriter) Write(data []byte) (int, error) {
+	lw.logger.Print(string(data))
+	return len(data), nil
+}

core/logx/readme-cn.md

@@ -0,0 +1,206 @@
+<IMG align="right" width="150px" src="https://raw.githubusercontent.com/zeromicro/zero-doc/main/doc/images/go-zero.png">
+
+# logx
+
+[English](readme.md) | 简体中文
+
+## logx 配置
+
+```go
+type LogConf struct {
+	ServiceName         string              `json:",optional"`
+	Mode                string              `json:",default=console,options=[console,file,volume]"`
+	Encoding            string              `json:",default=json,options=[json,plain]"`
+	TimeFormat          string              `json:",optional"`
+	Path                string              `json:",default=logs"`
+	Level               string              `json:",default=info,options=[info,error,severe]"`
+	Compress            bool                `json:",optional"`
+	KeepDays            int                 `json:",optional"`
+	StackCooldownMillis int                 `json:",default=100"`
+	MaxBackups          int                 `json:",default=0"`
+	MaxSize             int                 `json:",default=0"`
+	Rotation            string              `json:",default=daily,options=[daily,size]"`
+}
+```
+
+- `ServiceName`：设置服务名称，可选。在 `volume` 模式下，该名称用于生成日志文件。在 `rest/zrpc` 服务中，名称将被自动设置为 `rest`或`zrpc` 的名称。
+- `Mode`：输出日志的模式，默认是 `console`
+  - `console` 模式将日志写到 `stdout/stderr`
+  - `file` 模式将日志写到 `Path` 指定目录的文件中
+  - `volume` 模式在 docker 中使用，将日志写入挂载的卷中
+- `Encoding`: 指示如何对日志进行编码，默认是 `json`
+  - `json`模式以 json 格式写日志
+  - `plain`模式用纯文本写日志，并带有终端颜色显示
+- `TimeFormat`：自定义时间格式，可选。默认是 `2006-01-02T15:04:05.000Z07:00`
+- `Path`：设置日志路径，默认为 `logs`
+- `Level`: 用于过滤日志的日志级别。默认为 `info`
+  - `info`，所有日志都被写入
+  - `error`, `info` 的日志被丢弃
+  - `severe`, `info` 和 `error` 日志被丢弃，只有 `severe` 日志被写入
+- `Compress`: 是否压缩日志文件，只在 `file` 模式下工作
+- `KeepDays`：日志文件被保留多少天，在给定的天数之后，过期的文件将被自动删除。对 `console` 模式没有影响
+- `StackCooldownMillis`：多少毫秒后再次写入堆栈跟踪。用来避免堆栈跟踪日志过多
+- `MaxBackups`: 多少个日志文件备份将被保存。0代表所有备份都被保存。当`Rotation`被设置为`size`时才会起作用。注意：`KeepDays`选项的优先级会比`MaxBackups`高，即使`MaxBackups`被设置为0，当达到`KeepDays`上限时备份文件同样会被删除。
+- `MaxSize`: 当前被写入的日志文件最大可占用多少空间。0代表没有上限。单位为`MB`。当`Rotation`被设置为`size`时才会起作用。
+- `Rotation`: 日志轮转策略类型。默认为`daily`（按天轮转）。
+  - `daily` 按天轮转。
+  - `size` 按日志大小轮转。
+
+
+## 打印日志方法
+
+```go
+type Logger interface {
+	// Error logs a message at error level.
+	Error(...interface{})
+	// Errorf logs a message at error level.
+	Errorf(string, ...interface{})
+	// Errorv logs a message at error level.
+	Errorv(interface{})
+	// Errorw logs a message at error level.
+	Errorw(string, ...LogField)
+	// Info logs a message at info level.
+	Info(...interface{})
+	// Infof logs a message at info level.
+	Infof(string, ...interface{})
+	// Infov logs a message at info level.
+	Infov(interface{})
+	// Infow logs a message at info level.
+	Infow(string, ...LogField)
+	// Slow logs a message at slow level.
+	Slow(...interface{})
+	// Slowf logs a message at slow level.
+	Slowf(string, ...interface{})
+	// Slowv logs a message at slow level.
+	Slowv(interface{})
+	// Sloww logs a message at slow level.
+	Sloww(string, ...LogField)
+	// WithContext returns a new logger with the given context.
+	WithContext(context.Context) Logger
+	// WithDuration returns a new logger with the given duration.
+	WithDuration(time.Duration) Logger
+}
+```
+
+- `Error`, `Info`, `Slow`: 将任何类型的信息写进日志，使用 `fmt.Sprint(...)` 来转换为 `string`
+- `Errorf`, `Infof`, `Slowf`: 将指定格式的信息写入日志
+- `Errorv`, `Infov`, `Slowv`: 将任何类型的信息写入日志，用 `json marshal` 编码
+- `Errorw`, `Infow`, `Sloww`: 写日志，并带上给定的 `key:value` 字段
+- `WithContext`：将给定的 ctx 注入日志信息，例如用于记录 `trace-id`和`span-id`
+- `WithDuration`: 将指定的时间写入日志信息中，字段名为 `duration`
+
+## 与第三方日志库集成
+
+- zap
+  - 实现：[https://github.com/zeromicro/zero-contrib/blob/main/logx/zapx/zap.go](https://github.com/zeromicro/zero-contrib/blob/main/logx/zapx/zap.go)
+  - 使用示例：[https://github.com/zeromicro/zero-examples/blob/main/logx/zaplog/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/zaplog/main.go)
+- logrus
+  - 实现：[https://github.com/zeromicro/zero-contrib/blob/main/logx/logrusx/logrus.go](https://github.com/zeromicro/zero-contrib/blob/main/logx/logrusx/logrus.go)
+  - 使用示例：[https://github.com/zeromicro/zero-examples/blob/main/logx/logrus/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/logrus/main.go)
+
+对于其它的日志库，请参考上面示例实现，并欢迎提交 `PR` 到 [https://github.com/zeromicro/zero-contrib](https://github.com/zeromicro/zero-contrib)
+
+## 将日志写到指定的存储
+
+`logx`定义了两个接口，方便自定义 `logx`，将日志写入任何存储。
+
+- `logx.NewWriter(w io.Writer)`
+- `logx.SetWriter(write logx.Writer)`
+
+例如，如果我们想把日志写进kafka，而不是控制台或文件，我们可以像下面这样做。
+
+```go
+type KafkaWriter struct {
+	Pusher *kq.Pusher
+}
+
+func NewKafkaWriter(pusher *kq.Pusher) *KafkaWriter {
+	return &KafkaWriter{
+		Pusher: pusher,
+	}
+}
+
+func (w *KafkaWriter) Write(p []byte) (n int, err error) {
+	// writing log with newlines, trim them.
+	if err := w.Pusher.Push(strings.TrimSpace(string(p))); err != nil {
+		return 0, err
+	}
+
+	return len(p), nil
+}
+
+func main() {
+	pusher := kq.NewPusher([]string{"localhost:9092"}, "go-zero")
+	defer pusher.Close()
+
+	writer := logx.NewWriter(NewKafkaWriter(pusher))
+	logx.SetWriter(writer)
+  
+	// more code
+}
+```
+
+完整代码：[https://github.com/zeromicro/zero-examples/blob/main/logx/tokafka/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/tokafka/main.go)
+
+## 过滤敏感字段
+
+如果我们需要防止  `password` 字段被记录下来，我们可以像下面这样实现。
+
+```go
+type (
+	Message struct {
+		Name     string
+		Password string
+		Message  string
+	}
+
+	SensitiveLogger struct {
+		logx.Writer
+	}
+)
+
+func NewSensitiveLogger(writer logx.Writer) *SensitiveLogger {
+	return &SensitiveLogger{
+		Writer: writer,
+	}
+}
+
+func (l *SensitiveLogger) Info(msg interface{}, fields ...logx.LogField) {
+	if m, ok := msg.(Message); ok {
+		l.Writer.Info(Message{
+			Name:     m.Name,
+			Password: "******",
+			Message:  m.Message,
+		}, fields...)
+	} else {
+		l.Writer.Info(msg, fields...)
+	}
+}
+
+func main() {
+	// setup logx to make sure originalWriter not nil,
+	// the injected writer is only for filtering, like a middleware.
+
+	originalWriter := logx.Reset()
+	writer := NewSensitiveLogger(originalWriter)
+	logx.SetWriter(writer)
+
+	logx.Infov(Message{
+		Name:     "foo",
+		Password: "shouldNotAppear",
+		Message:  "bar",
+	})
+  
+	// more code
+}
+```
+
+完整代码：[https://github.com/zeromicro/zero-examples/blob/main/logx/filterfields/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/filterfields/main.go)
+
+## 更多示例
+
+[https://github.com/zeromicro/zero-examples/tree/main/logx](https://github.com/zeromicro/zero-examples/tree/main/logx)
+
+## Give a Star! ⭐
+
+如果你正在使用或者觉得这个项目对你有帮助，请 **star** 支持，感谢！

core/logx/readme.md

@@ -0,0 +1,205 @@
+<img align="right" width="150px" src="https://raw.githubusercontent.com/zeromicro/zero-doc/main/doc/images/go-zero.png">
+
+# logx
+
+English | [简体中文](readme-cn.md)
+
+## logx configurations
+
+```go
+type LogConf struct {
+	ServiceName         string              `json:",optional"`
+	Mode                string              `json:",default=console,options=[console,file,volume]"`
+	Encoding            string              `json:",default=json,options=[json,plain]"`
+	TimeFormat          string              `json:",optional"`
+	Path                string              `json:",default=logs"`
+	Level               string              `json:",default=info,options=[info,error,severe]"`
+	Compress            bool                `json:",optional"`
+	KeepDays            int                 `json:",optional"`
+	StackCooldownMillis int                 `json:",default=100"`
+	MaxBackups          int                 `json:",default=0"`
+	MaxSize             int                 `json:",default=0"`
+	Rotation            string              `json:",default=daily,options=[daily,size]"`
+}
+```
+
+- `ServiceName`: set the service name, optional. on `volume` mode, the name is used to generate the log files. Within `rest/zrpc` services, the name will be set to the name of `rest` or `zrpc` automatically.
+- `Mode`: the mode to output the logs, default is `console`.
+  -  `console` mode writes the logs to `stdout/stderr`.
+  - `file` mode writes the logs to the files specified by `Path`.
+  - `volume` mode is used in docker, to write logs into mounted volumes.
+- `Encoding`: indicates how to encode the logs, default is `json`.
+  - `json` mode writes the logs in json format.
+  - `plain` mode writes the logs with plain text, with terminal color enabled.
+- `TimeFormat`: customize the time format, optional. Default is `2006-01-02T15:04:05.000Z07:00`.
+- `Path`: set the log path, default to `logs`.
+- `Level`: the logging level to filter logs. Default is `info`.
+  - `info`, all logs are written.
+  - `error`, `info` logs are suppressed.
+  - `severe`, `info` and `error` logs are suppressed, only `severe` logs are written.
+- `Compress`: whether or not to compress log files, only works with `file` mode.
+- `KeepDays`: how many days that the log files are kept, after the given days, the outdated files will be deleted automatically. It has no effect on `console` mode.
+- `StackCooldownMillis`: how many milliseconds to rewrite stacktrace again. It’s used to avoid stacktrace flooding.
+- `MaxBackups`: represents how many backup log files will be kept. 0 means all files will be kept forever. Only take effect when `Rotation` is `size`. NOTE: the level of option `KeepDays` will be higher. Even thougth `MaxBackups` sets 0, log files will still be removed if the `KeepDays` limitation is reached.
+- `MaxSize`: represents how much space the writing log file takes up. 0 means no limit. The unit is `MB`. Only take effect when `Rotation` is `size`.
+- `Rotation`: represents the type of log rotation rule. Default is `daily`.
+  - `daily` rotate the logs by day.
+  - `size` rotate the logs by size of logs.
+
+## Logging methods
+
+```go
+type Logger interface {
+	// Error logs a message at error level.
+	Error(...interface{})
+	// Errorf logs a message at error level.
+	Errorf(string, ...interface{})
+	// Errorv logs a message at error level.
+	Errorv(interface{})
+	// Errorw logs a message at error level.
+	Errorw(string, ...LogField)
+	// Info logs a message at info level.
+	Info(...interface{})
+	// Infof logs a message at info level.
+	Infof(string, ...interface{})
+	// Infov logs a message at info level.
+	Infov(interface{})
+	// Infow logs a message at info level.
+	Infow(string, ...LogField)
+	// Slow logs a message at slow level.
+	Slow(...interface{})
+	// Slowf logs a message at slow level.
+	Slowf(string, ...interface{})
+	// Slowv logs a message at slow level.
+	Slowv(interface{})
+	// Sloww logs a message at slow level.
+	Sloww(string, ...LogField)
+	// WithContext returns a new logger with the given context.
+	WithContext(context.Context) Logger
+	// WithDuration returns a new logger with the given duration.
+	WithDuration(time.Duration) Logger
+}
+```
+
+- `Error`, `Info`, `Slow`: write any kind of messages into logs, with like `fmt.Sprint(…)`.
+- `Errorf`, `Infof`, `Slowf`: write messages with given format into logs.
+- `Errorv`, `Infov`, `Slowv`: write any kind of messages into logs, with json marshalling to encode them.
+- `Errorw`, `Infow`, `Sloww`: write the string message with given `key:value` fields.
+- `WithContext`: inject the given ctx into the log messages, typically used to log `trace-id` and `span-id`.
+- `WithDuration`: write elapsed duration into the log messages, with key `duration`.
+
+## Integrating with third-party logging libs
+
+- zap
+  - implementation: [https://github.com/zeromicro/zero-contrib/blob/main/logx/zapx/zap.go](https://github.com/zeromicro/zero-contrib/blob/main/logx/zapx/zap.go)
+  - usage example: [https://github.com/zeromicro/zero-examples/blob/main/logx/zaplog/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/zaplog/main.go)
+- logrus
+  - implementation: [https://github.com/zeromicro/zero-contrib/blob/main/logx/logrusx/logrus.go](https://github.com/zeromicro/zero-contrib/blob/main/logx/logrusx/logrus.go)
+  - usage example: [https://github.com/zeromicro/zero-examples/blob/main/logx/logrus/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/logrus/main.go)
+
+For more libs, please implement and PR to [https://github.com/zeromicro/zero-contrib](https://github.com/zeromicro/zero-contrib)
+
+## Write the logs to specific stores
+
+`logx` defined two interfaces to let you customize `logx` to write logs into any stores.
+
+- `logx.NewWriter(w io.Writer)`
+- `logx.SetWriter(writer logx.Writer)`
+
+For example, if we want to write the logs into kafka instead of console or files, we can do it like below:
+
+```go
+type KafkaWriter struct {
+	Pusher *kq.Pusher
+}
+
+func NewKafkaWriter(pusher *kq.Pusher) *KafkaWriter {
+	return &KafkaWriter{
+		Pusher: pusher,
+	}
+}
+
+func (w *KafkaWriter) Write(p []byte) (n int, err error) {
+	// writing log with newlines, trim them.
+	if err := w.Pusher.Push(strings.TrimSpace(string(p))); err != nil {
+		return 0, err
+	}
+
+	return len(p), nil
+}
+
+func main() {
+	pusher := kq.NewPusher([]string{"localhost:9092"}, "go-zero")
+	defer pusher.Close()
+
+	writer := logx.NewWriter(NewKafkaWriter(pusher))
+	logx.SetWriter(writer)
+  
+	// more code
+}
+```
+
+Complete code: [https://github.com/zeromicro/zero-examples/blob/main/logx/tokafka/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/tokafka/main.go)
+
+## Filtering sensitive fields
+
+If we need to prevent the `password` fields from logging, we can do it like below:
+
+```go
+type (
+	Message struct {
+		Name     string
+		Password string
+		Message  string
+	}
+
+	SensitiveLogger struct {
+		logx.Writer
+	}
+)
+
+func NewSensitiveLogger(writer logx.Writer) *SensitiveLogger {
+	return &SensitiveLogger{
+		Writer: writer,
+	}
+}
+
+func (l *SensitiveLogger) Info(msg interface{}, fields ...logx.LogField) {
+	if m, ok := msg.(Message); ok {
+		l.Writer.Info(Message{
+			Name:     m.Name,
+			Password: "******",
+			Message:  m.Message,
+		}, fields...)
+	} else {
+		l.Writer.Info(msg, fields...)
+	}
+}
+
+func main() {
+	// setup logx to make sure originalWriter not nil,
+	// the injected writer is only for filtering, like a middleware.
+
+	originalWriter := logx.Reset()
+	writer := NewSensitiveLogger(originalWriter)
+	logx.SetWriter(writer)
+
+	logx.Infov(Message{
+		Name:     "foo",
+		Password: "shouldNotAppear",
+		Message:  "bar",
+	})
+  
+	// more code
+}
+```
+
+Complete code: [https://github.com/zeromicro/zero-examples/blob/main/logx/filterfields/main.go](https://github.com/zeromicro/zero-examples/blob/main/logx/filterfields/main.go)
+
+## More examples
+
+[https://github.com/zeromicro/zero-examples/tree/main/logx](https://github.com/zeromicro/zero-examples/tree/main/logx)
+
+## Give a Star! ⭐
+
+If you like or are using this project to learn or start your solution, please give it a star. Thanks!

core/logx/rotatelogger.go

@@ -9,21 +9,24 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/zeromicro/go-zero/core/fs"
 	"github.com/zeromicro/go-zero/core/lang"
-	"github.com/zeromicro/go-zero/core/timex"
 )
 
 const (
 	dateFormat      = "2006-01-02"
+	fileTimeFormat  = time.RFC3339
 	hoursPerDay     = 24
 	bufferSize      = 100
 	defaultDirMode  = 0o755
 	defaultFileMode = 0o600
+	gzipExt         = ".gz"
+	megaBytes       = 1 << 20
 )
 
 // ErrLogFileClosed is an error that indicates the log file is already closed.
@@ -35,7 +38,7 @@ type (
 		BackupFileName() string
 		MarkRotated()
 		OutdatedFiles() []string
-		ShallRotate() bool
+		ShallRotate(size int64) bool
 	}
 
 	// A RotateLogger is a Logger that can rotate log files with given rules.
@@ -48,8 +51,9 @@ type (
 		rule     RotateRule
 		compress bool
 		// can't use threading.RoutineGroup because of cycle import
-		waitGroup sync.WaitGroup
-		closeOnce sync.Once
+		waitGroup   sync.WaitGroup
+		closeOnce   sync.Once
+		currentSize int64
 	}
 
 	// A DailyRotateRule is a rule to daily rotate the log files.
@@ -60,6 +64,13 @@ type (
 		days        int
 		gzip        bool
 	}
+
+	// SizeLimitRotateRule a rotation rule that make the log file rotated base on size
+	SizeLimitRotateRule struct {
+		DailyRotateRule
+		maxSize    int64
+		maxBackups int
+	}
 )
 
 // DefaultRotateRule is a default log rotating rule, currently DailyRotateRule.
@@ -91,7 +102,7 @@ func (r *DailyRotateRule) OutdatedFiles() []string {
 
 	var pattern string
 	if r.gzip {
-		pattern = fmt.Sprintf("%s%s*.gz", r.filename, r.delimiter)
+		pattern = fmt.Sprintf("%s%s*%s", r.filename, r.delimiter, gzipExt)
 	} else {
 		pattern = fmt.Sprintf("%s%s*", r.filename, r.delimiter)
 	}
@@ -106,7 +117,7 @@ func (r *DailyRotateRule) OutdatedFiles() []string {
 	boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay*r.days)).Format(dateFormat)
 	fmt.Fprintf(&buf, "%s%s%s", r.filename, r.delimiter, boundary)
 	if r.gzip {
-		buf.WriteString(".gz")
+		buf.WriteString(gzipExt)
 	}
 	boundaryFile := buf.String()
 
@@ -121,10 +132,100 @@ func (r *DailyRotateRule) OutdatedFiles() []string {
 }
 
 // ShallRotate checks if the file should be rotated.
-func (r *DailyRotateRule) ShallRotate() bool {
+func (r *DailyRotateRule) ShallRotate(_ int64) bool {
 	return len(r.rotatedTime) > 0 && getNowDate() != r.rotatedTime
 }
 
+// NewSizeLimitRotateRule returns the rotation rule with size limit
+func NewSizeLimitRotateRule(filename, delimiter string, days, maxSize, maxBackups int, gzip bool) RotateRule {
+	return &SizeLimitRotateRule{
+		DailyRotateRule: DailyRotateRule{
+			rotatedTime: getNowDateInRFC3339Format(),
+			filename:    filename,
+			delimiter:   delimiter,
+			days:        days,
+			gzip:        gzip,
+		},
+		maxSize:    int64(maxSize) * megaBytes,
+		maxBackups: maxBackups,
+	}
+}
+
+func (r *SizeLimitRotateRule) BackupFileName() string {
+	dir := filepath.Dir(r.filename)
+	prefix, ext := r.parseFilename()
+	timestamp := getNowDateInRFC3339Format()
+	return filepath.Join(dir, fmt.Sprintf("%s%s%s%s", prefix, r.delimiter, timestamp, ext))
+}
+
+func (r *SizeLimitRotateRule) MarkRotated() {
+	r.rotatedTime = getNowDateInRFC3339Format()
+}
+
+func (r *SizeLimitRotateRule) OutdatedFiles() []string {
+	dir := filepath.Dir(r.filename)
+	prefix, ext := r.parseFilename()
+
+	var pattern string
+	if r.gzip {
+		pattern = fmt.Sprintf("%s%s%s%s*%s%s", dir, string(filepath.Separator),
+			prefix, r.delimiter, ext, gzipExt)
+	} else {
+		pattern = fmt.Sprintf("%s%s%s%s*%s", dir, string(filepath.Separator),
+			prefix, r.delimiter, ext)
+	}
+
+	files, err := filepath.Glob(pattern)
+	if err != nil {
+		Errorf("failed to delete outdated log files, error: %s", err)
+		return nil
+	}
+
+	sort.Strings(files)
+
+	outdated := make(map[string]lang.PlaceholderType)
+
+	// test if too many backups
+	if r.maxBackups > 0 && len(files) > r.maxBackups {
+		for _, f := range files[:len(files)-r.maxBackups] {
+			outdated[f] = lang.Placeholder
+		}
+		files = files[len(files)-r.maxBackups:]
+	}
+
+	// test if any too old backups
+	if r.days > 0 {
+		boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay*r.days)).Format(fileTimeFormat)
+		boundaryFile := filepath.Join(dir, fmt.Sprintf("%s%s%s%s", prefix, r.delimiter, boundary, ext))
+		if r.gzip {
+			boundaryFile += gzipExt
+		}
+		for _, f := range files {
+			if f >= boundaryFile {
+				break
+			}
+			outdated[f] = lang.Placeholder
+		}
+	}
+
+	var result []string
+	for k := range outdated {
+		result = append(result, k)
+	}
+	return result
+}
+
+func (r *SizeLimitRotateRule) ShallRotate(size int64) bool {
+	return r.maxSize > 0 && r.maxSize < size
+}
+
+func (r *SizeLimitRotateRule) parseFilename() (prefix, ext string) {
+	logName := filepath.Base(r.filename)
+	ext = filepath.Ext(r.filename)
+	prefix = logName[:len(logName)-len(ext)]
+	return
+}
+
 // NewLogger returns a RotateLogger with given filename and rule, etc.
 func NewLogger(filename string, rule RotateRule, compress bool) (*RotateLogger, error) {
 	l := &RotateLogger{
@@ -211,6 +312,12 @@ func (l *RotateLogger) maybeCompressFile(file string) {
 			ErrorStack(r)
 		}
 	}()
+
+	if _, err := os.Stat(file); err != nil {
+		// file not exists or other error, ignore compression
+		return
+	}
+
 	compressLogFile(file)
 }
 
@@ -277,25 +384,27 @@ func (l *RotateLogger) startWorker() {
 }
 
 func (l *RotateLogger) write(v []byte) {
-	if l.rule.ShallRotate() {
+	if l.rule.ShallRotate(l.currentSize + int64(len(v))) {
 		if err := l.rotate(); err != nil {
 			log.Println(err)
 		} else {
 			l.rule.MarkRotated()
+			l.currentSize = 0
 		}
 	}
 	if l.fp != nil {
 		l.fp.Write(v)
+		l.currentSize += int64(len(v))
 	}
 }
 
 func compressLogFile(file string) {
-	start := timex.Now()
+	start := time.Now()
 	Infof("compressing log file: %s", file)
 	if err := gzipFile(file); err != nil {
 		Errorf("compress error: %s", err)
 	} else {
-		Infof("compressed log file: %s, took %s", file, timex.Since(start))
+		Infof("compressed log file: %s, took %s", file, time.Since(start))
 	}
 }
 
@@ -303,6 +412,10 @@ func getNowDate() string {
 	return time.Now().Format(dateFormat)
 }
 
+func getNowDateInRFC3339Format() string {
+	return time.Now().Format(fileTimeFormat)
+}
+
 func gzipFile(file string) error {
 	in, err := os.Open(file)
 	if err != nil {
@@ -310,7 +423,7 @@ func gzipFile(file string) error {
 	}
 	defer in.Close()
 
-	out, err := os.Create(fmt.Sprintf("%s.gz", file))
+	out, err := os.Create(fmt.Sprintf("%s%s", file, gzipExt))
 	if err != nil {
 		return err
 	}

core/logx/rotatelogger_test.go

@@ -29,7 +29,34 @@ func TestDailyRotateRuleOutdatedFiles(t *testing.T) {
 func TestDailyRotateRuleShallRotate(t *testing.T) {
 	var rule DailyRotateRule
 	rule.rotatedTime = time.Now().Add(time.Hour * 24).Format(dateFormat)
-	assert.True(t, rule.ShallRotate())
+	assert.True(t, rule.ShallRotate(0))
+}
+
+func TestSizeLimitRotateRuleMarkRotated(t *testing.T) {
+	var rule SizeLimitRotateRule
+	rule.MarkRotated()
+	assert.Equal(t, getNowDateInRFC3339Format(), rule.rotatedTime)
+}
+
+func TestSizeLimitRotateRuleOutdatedFiles(t *testing.T) {
+	var rule SizeLimitRotateRule
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.days = 1
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.gzip = true
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.maxBackups = 0
+	assert.Empty(t, rule.OutdatedFiles())
+}
+
+func TestSizeLimitRotateRuleShallRotate(t *testing.T) {
+	var rule SizeLimitRotateRule
+	rule.rotatedTime = time.Now().Add(time.Hour * 24).Format(fileTimeFormat)
+	rule.maxSize = 0
+	assert.False(t, rule.ShallRotate(0))
+	rule.maxSize = 100
+	assert.False(t, rule.ShallRotate(0))
+	assert.True(t, rule.ShallRotate(101*megaBytes))
 }
 
 func TestRotateLoggerClose(t *testing.T) {
@@ -57,6 +84,12 @@ func TestRotateLoggerGetBackupFilename(t *testing.T) {
 }
 
 func TestRotateLoggerMayCompressFile(t *testing.T) {
+	old := os.Stdout
+	os.Stdout = os.NewFile(0, os.DevNull)
+	defer func() {
+		os.Stdout = old
+	}()
+
 	filename, err := fs.TempFilenameWithText("foo")
 	assert.Nil(t, err)
 	if len(filename) > 0 {
@@ -70,15 +103,18 @@ func TestRotateLoggerMayCompressFile(t *testing.T) {
 }
 
 func TestRotateLoggerMayCompressFileTrue(t *testing.T) {
+	old := os.Stdout
+	os.Stdout = os.NewFile(0, os.DevNull)
+	defer func() {
+		os.Stdout = old
+	}()
+
 	filename, err := fs.TempFilenameWithText("foo")
 	assert.Nil(t, err)
 	logger, err := NewLogger(filename, new(DailyRotateRule), true)
 	assert.Nil(t, err)
 	if len(filename) > 0 {
-		defer func() {
-			os.Remove(filename)
-			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
-		}()
+		defer os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
 	}
 	logger.maybeCompressFile(filename)
 	_, err = os.Stat(filename)
@@ -92,7 +128,6 @@ func TestRotateLoggerRotate(t *testing.T) {
 	assert.Nil(t, err)
 	if len(filename) > 0 {
 		defer func() {
-			os.Remove(filename)
 			os.Remove(logger.getBackupFilename())
 			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
 		}()
@@ -102,6 +137,10 @@ func TestRotateLoggerRotate(t *testing.T) {
 	case *os.LinkError:
 		// avoid rename error on docker container
 		assert.Equal(t, syscall.EXDEV, v.Err)
+	case *os.PathError:
+		// ignore remove error for tests,
+		// files are cleaned in GitHub actions.
+		assert.Equal(t, "remove", v.Op)
 	default:
 		assert.Nil(t, err)
 	}
@@ -115,12 +154,177 @@ func TestRotateLoggerWrite(t *testing.T) {
 	assert.Nil(t, err)
 	if len(filename) > 0 {
 		defer func() {
-			os.Remove(filename)
 			os.Remove(logger.getBackupFilename())
 			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
 		}()
 	}
+	// the following write calls cannot be changed to Write, because of DATA RACE.
 	logger.write([]byte(`foo`))
 	rule.rotatedTime = time.Now().Add(-time.Hour * 24).Format(dateFormat)
 	logger.write([]byte(`bar`))
+	logger.Close()
+	logger.write([]byte(`baz`))
+}
+
+func TestLogWriterClose(t *testing.T) {
+	assert.Nil(t, newLogWriter(nil).Close())
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleClose(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), false)
+	assert.Nil(t, err)
+	assert.Nil(t, logger.Close())
+}
+
+func TestRotateLoggerGetBackupWithSizeLimitRotateRuleFilename(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), false)
+	assert.Nil(t, err)
+	assert.True(t, len(logger.getBackupFilename()) > 0)
+	logger.backup = ""
+	assert.True(t, len(logger.getBackupFilename()) > 0)
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleMayCompressFile(t *testing.T) {
+	old := os.Stdout
+	os.Stdout = os.NewFile(0, os.DevNull)
+	defer func() {
+		os.Stdout = old
+	}()
+
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), false)
+	assert.Nil(t, err)
+	logger.maybeCompressFile(filename)
+	_, err = os.Stat(filename)
+	assert.Nil(t, err)
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleMayCompressFileTrue(t *testing.T) {
+	old := os.Stdout
+	os.Stdout = os.NewFile(0, os.DevNull)
+	defer func() {
+		os.Stdout = old
+	}()
+
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+	}
+	logger.maybeCompressFile(filename)
+	_, err = os.Stat(filename)
+	assert.NotNil(t, err)
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleRotate(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer func() {
+			os.Remove(logger.getBackupFilename())
+			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+		}()
+	}
+	err = logger.rotate()
+	switch v := err.(type) {
+	case *os.LinkError:
+		// avoid rename error on docker container
+		assert.Equal(t, syscall.EXDEV, v.Err)
+	case *os.PathError:
+		// ignore remove error for tests,
+		// files are cleaned in GitHub actions.
+		assert.Equal(t, "remove", v.Op)
+	default:
+		assert.Nil(t, err)
+	}
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleWrite(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	rule := new(SizeLimitRotateRule)
+	logger, err := NewLogger(filename, rule, true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer func() {
+			os.Remove(logger.getBackupFilename())
+			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+		}()
+	}
+	// the following write calls cannot be changed to Write, because of DATA RACE.
+	logger.write([]byte(`foo`))
+	rule.rotatedTime = time.Now().Add(-time.Hour * 24).Format(dateFormat)
+	logger.write([]byte(`bar`))
+	logger.Close()
+	logger.write([]byte(`baz`))
+}
+
+func BenchmarkRotateLogger(b *testing.B) {
+	filename := "./test.log"
+	filename2 := "./test2.log"
+	dailyRotateRuleLogger, err1 := NewLogger(
+		filename,
+		DefaultRotateRule(
+			filename,
+			backupFileDelimiter,
+			1,
+			true,
+		),
+		true,
+	)
+	if err1 != nil {
+		b.Logf("Failed to new daily rotate rule logger: %v", err1)
+		b.FailNow()
+	}
+	sizeLimitRotateRuleLogger, err2 := NewLogger(
+		filename2,
+		NewSizeLimitRotateRule(
+			filename,
+			backupFileDelimiter,
+			1,
+			100,
+			10,
+			true,
+		),
+		true,
+	)
+	if err2 != nil {
+		b.Logf("Failed to new size limit rotate rule logger: %v", err1)
+		b.FailNow()
+	}
+	defer func() {
+		dailyRotateRuleLogger.Close()
+		sizeLimitRotateRuleLogger.Close()
+		os.Remove(filename)
+		os.Remove(filename2)
+	}()
+
+	b.Run("daily rotate rule", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			dailyRotateRuleLogger.write([]byte("testing\ntesting\n"))
+		}
+	})
+	b.Run("size limit rotate rule", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			sizeLimitRotateRuleLogger.write([]byte("testing\ntesting\n"))
+		}
+	})
 }

core/logx/syslog_test.go

@@ -29,20 +29,24 @@ func TestRedirector(t *testing.T) {
 }
 
 func captureOutput(f func()) string {
-	atomic.StoreUint32(&initialized, 1)
-	writer := new(mockWriter)
-	infoLog = writer
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
 
 	prevLevel := atomic.LoadUint32(&logLevel)
 	SetLevel(InfoLevel)
 	f()
 	SetLevel(prevLevel)
 
-	return writer.builder.String()
+	return w.String()
 }
 
 func getContent(jsonStr string) string {
 	var entry logEntry
 	json.Unmarshal([]byte(jsonStr), &entry)
-	return entry.Content.(string)
+	val, ok := entry.Content.(string)
+	if ok {
+		return val
+	}
+	return ""
 }

core/logx/tracelogger.go

@@ -1,123 +0,0 @@
-package logx
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"time"
-
-	"github.com/zeromicro/go-zero/core/timex"
-	"go.opentelemetry.io/otel/trace"
-)
-
-type traceLogger struct {
-	logEntry
-	Trace string `json:"trace,omitempty"`
-	Span  string `json:"span,omitempty"`
-	ctx   context.Context
-}
-
-func (l *traceLogger) Error(v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
-	}
-}
-
-func (l *traceLogger) Errorf(format string, v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
-	}
-}
-
-func (l *traceLogger) Errorv(v interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(errorLog, levelError, v)
-	}
-}
-
-func (l *traceLogger) Info(v ...interface{}) {
-	if shallLog(InfoLevel) {
-		l.write(infoLog, levelInfo, fmt.Sprint(v...))
-	}
-}
-
-func (l *traceLogger) Infof(format string, v ...interface{}) {
-	if shallLog(InfoLevel) {
-		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
-	}
-}
-
-func (l *traceLogger) Infov(v interface{}) {
-	if shallLog(InfoLevel) {
-		l.write(infoLog, levelInfo, v)
-	}
-}
-
-func (l *traceLogger) Slow(v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(slowLog, levelSlow, fmt.Sprint(v...))
-	}
-}
-
-func (l *traceLogger) Slowf(format string, v ...interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
-	}
-}
-
-func (l *traceLogger) Slowv(v interface{}) {
-	if shallLog(ErrorLevel) {
-		l.write(slowLog, levelSlow, v)
-	}
-}
-
-func (l *traceLogger) WithDuration(duration time.Duration) Logger {
-	l.Duration = timex.ReprOfDuration(duration)
-	return l
-}
-
-func (l *traceLogger) write(writer io.Writer, level string, val interface{}) {
-	traceID := traceIdFromContext(l.ctx)
-	spanID := spanIdFromContext(l.ctx)
-
-	switch encoding {
-	case plainEncodingType:
-		writePlainAny(writer, level, val, l.Duration, traceID, spanID)
-	default:
-		outputJson(writer, &traceLogger{
-			logEntry: logEntry{
-				Timestamp: getTimestamp(),
-				Level:     level,
-				Duration:  l.Duration,
-				Content:   val,
-			},
-			Trace: traceID,
-			Span:  spanID,
-		})
-	}
-}
-
-// WithContext sets ctx to log, for keeping tracing information.
-func WithContext(ctx context.Context) Logger {
-	return &traceLogger{
-		ctx: ctx,
-	}
-}
-
-func spanIdFromContext(ctx context.Context) string {
-	spanCtx := trace.SpanContextFromContext(ctx)
-	if spanCtx.HasSpanID() {
-		return spanCtx.SpanID().String()
-	}
-
-	return ""
-}
-
-func traceIdFromContext(ctx context.Context) string {
-	spanCtx := trace.SpanContextFromContext(ctx)
-	if spanCtx.HasTraceID() {
-		return spanCtx.TraceID().String()
-	}
-
-	return ""
-}

core/logx/tracelogger_test.go

@@ -1,154 +0,0 @@
-package logx
-
-import (
-	"context"
-	"log"
-	"strings"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"go.opentelemetry.io/otel"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-)
-
-const (
-	traceKey = "trace"
-	spanKey  = "span"
-)
-
-func TestTraceLog(t *testing.T) {
-	var buf mockWriter
-	atomic.StoreUint32(&initialized, 1)
-	otp := otel.GetTracerProvider()
-	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
-	otel.SetTracerProvider(tp)
-	defer otel.SetTracerProvider(otp)
-
-	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
-	WithContext(ctx).(*traceLogger).write(&buf, levelInfo, testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-}
-
-func TestTraceError(t *testing.T) {
-	var buf mockWriter
-	atomic.StoreUint32(&initialized, 1)
-	errorLog = newLogWriter(log.New(&buf, "", flags))
-	otp := otel.GetTracerProvider()
-	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
-	otel.SetTracerProvider(tp)
-	defer otel.SetTracerProvider(otp)
-
-	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
-	l := WithContext(ctx).(*traceLogger)
-	SetLevel(InfoLevel)
-	l.WithDuration(time.Second).Error(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-	buf.Reset()
-	l.WithDuration(time.Second).Errorf(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-	buf.Reset()
-	l.WithDuration(time.Second).Errorv(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-}
-
-func TestTraceInfo(t *testing.T) {
-	var buf mockWriter
-	atomic.StoreUint32(&initialized, 1)
-	infoLog = newLogWriter(log.New(&buf, "", flags))
-	otp := otel.GetTracerProvider()
-	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
-	otel.SetTracerProvider(tp)
-	defer otel.SetTracerProvider(otp)
-
-	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
-	l := WithContext(ctx).(*traceLogger)
-	SetLevel(InfoLevel)
-	l.WithDuration(time.Second).Info(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-	buf.Reset()
-	l.WithDuration(time.Second).Infof(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-	buf.Reset()
-	l.WithDuration(time.Second).Infov(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-}
-
-func TestTraceInfoConsole(t *testing.T) {
-	old := encoding
-	encoding = plainEncodingType
-	defer func() {
-		encoding = old
-	}()
-
-	var buf mockWriter
-	atomic.StoreUint32(&initialized, 1)
-	infoLog = newLogWriter(log.New(&buf, "", flags))
-	otp := otel.GetTracerProvider()
-	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
-	otel.SetTracerProvider(tp)
-	defer otel.SetTracerProvider(otp)
-
-	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
-	l := WithContext(ctx).(*traceLogger)
-	SetLevel(InfoLevel)
-	l.WithDuration(time.Second).Info(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceIdFromContext(ctx)))
-	assert.True(t, strings.Contains(buf.String(), spanIdFromContext(ctx)))
-	buf.Reset()
-	l.WithDuration(time.Second).Infof(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceIdFromContext(ctx)))
-	assert.True(t, strings.Contains(buf.String(), spanIdFromContext(ctx)))
-	buf.Reset()
-	l.WithDuration(time.Second).Infov(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceIdFromContext(ctx)))
-	assert.True(t, strings.Contains(buf.String(), spanIdFromContext(ctx)))
-}
-
-func TestTraceSlow(t *testing.T) {
-	var buf mockWriter
-	atomic.StoreUint32(&initialized, 1)
-	slowLog = newLogWriter(log.New(&buf, "", flags))
-	otp := otel.GetTracerProvider()
-	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
-	otel.SetTracerProvider(tp)
-	defer otel.SetTracerProvider(otp)
-
-	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
-	l := WithContext(ctx).(*traceLogger)
-	SetLevel(InfoLevel)
-	l.WithDuration(time.Second).Slow(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-	buf.Reset()
-	l.WithDuration(time.Second).Slowf(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-	buf.Reset()
-	l.WithDuration(time.Second).Slowv(testlog)
-	assert.True(t, strings.Contains(buf.String(), traceKey))
-	assert.True(t, strings.Contains(buf.String(), spanKey))
-}
-
-func TestTraceWithoutContext(t *testing.T) {
-	var buf mockWriter
-	atomic.StoreUint32(&initialized, 1)
-	infoLog = newLogWriter(log.New(&buf, "", flags))
-	l := WithContext(context.Background()).(*traceLogger)
-	SetLevel(InfoLevel)
-	l.WithDuration(time.Second).Info(testlog)
-	assert.False(t, strings.Contains(buf.String(), traceKey))
-	assert.False(t, strings.Contains(buf.String(), spanKey))
-	buf.Reset()
-	l.WithDuration(time.Second).Infof(testlog)
-	assert.False(t, strings.Contains(buf.String(), traceKey))
-	assert.False(t, strings.Contains(buf.String(), spanKey))
-}

core/logx/util.go

@@ -0,0 +1,35 @@
+package logx
+
+import (
+	"fmt"
+	"runtime"
+	"strings"
+	"time"
+)
+
+func getCaller(callDepth int) string {
+	_, file, line, ok := runtime.Caller(callDepth)
+	if !ok {
+		return ""
+	}
+
+	return prettyCaller(file, line)
+}
+
+func getTimestamp() string {
+	return time.Now().Format(timeFormat)
+}
+
+func prettyCaller(file string, line int) string {
+	idx := strings.LastIndexByte(file, '/')
+	if idx < 0 {
+		return fmt.Sprintf("%s:%d", file, line)
+	}
+
+	idx = strings.LastIndexByte(file[:idx], '/')
+	if idx < 0 {
+		return fmt.Sprintf("%s:%d", file, line)
+	}
+
+	return fmt.Sprintf("%s:%d", file[idx+1:], line)
+}

core/logx/util_test.go

@@ -0,0 +1,72 @@
+package logx
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetCaller(t *testing.T) {
+	_, file, _, _ := runtime.Caller(0)
+	assert.Contains(t, getCaller(1), filepath.Base(file))
+	assert.True(t, len(getCaller(1<<10)) == 0)
+}
+
+func TestGetTimestamp(t *testing.T) {
+	ts := getTimestamp()
+	tm, err := time.Parse(timeFormat, ts)
+	assert.Nil(t, err)
+	assert.True(t, time.Since(tm) < time.Minute)
+}
+
+func TestPrettyCaller(t *testing.T) {
+	tests := []struct {
+		name string
+		file string
+		line int
+		want string
+	}{
+		{
+			name: "regular",
+			file: "logx_test.go",
+			line: 123,
+			want: "logx_test.go:123",
+		},
+		{
+			name: "relative",
+			file: "adhoc/logx_test.go",
+			line: 123,
+			want: "adhoc/logx_test.go:123",
+		},
+		{
+			name: "long path",
+			file: "github.com/zeromicro/go-zero/core/logx/util_test.go",
+			line: 12,
+			want: "logx/util_test.go:12",
+		},
+		{
+			name: "local path",
+			file: "/Users/kevin/go-zero/core/logx/util_test.go",
+			line: 1234,
+			want: "logx/util_test.go:1234",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.want, prettyCaller(test.file, test.line))
+		})
+	}
+}
+
+func BenchmarkGetCaller(b *testing.B) {
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		getCaller(1)
+	}
+}

core/logx/vars.go

@@ -0,0 +1,60 @@
+package logx
+
+import "errors"
+
+const (
+	// InfoLevel logs everything
+	InfoLevel uint32 = iota
+	// ErrorLevel includes errors, slows, stacks
+	ErrorLevel
+	// SevereLevel only log severe messages
+	SevereLevel
+)
+
+const (
+	jsonEncodingType = iota
+	plainEncodingType
+
+	plainEncoding    = "plain"
+	plainEncodingSep = '\t'
+	sizeRotationRule = "size"
+)
+
+const (
+	accessFilename = "access.log"
+	errorFilename  = "error.log"
+	severeFilename = "severe.log"
+	slowFilename   = "slow.log"
+	statFilename   = "stat.log"
+
+	fileMode   = "file"
+	volumeMode = "volume"
+
+	levelAlert  = "alert"
+	levelInfo   = "info"
+	levelError  = "error"
+	levelSevere = "severe"
+	levelFatal  = "fatal"
+	levelSlow   = "slow"
+	levelStat   = "stat"
+
+	backupFileDelimiter = "-"
+	flags               = 0x0
+)
+
+const (
+	callerKey    = "caller"
+	contentKey   = "content"
+	durationKey  = "duration"
+	levelKey     = "level"
+	spanKey      = "span"
+	timestampKey = "@timestamp"
+	traceKey     = "trace"
+)
+
+var (
+	// ErrLogPathNotSet is an error that indicates the log path is not set.
+	ErrLogPathNotSet = errors.New("log path must be set")
+	// ErrLogServiceNameNotSet is an error that indicates that the service name is not set.
+	ErrLogServiceNameNotSet = errors.New("log service name must be set")
+)

core/logx/writer.go

@@ -0,0 +1,371 @@
+package logx
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"path"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	fatihcolor "github.com/fatih/color"
+	"github.com/zeromicro/go-zero/core/color"
+)
+
+type (
+	Writer interface {
+		Alert(v interface{})
+		Close() error
+		Error(v interface{}, fields ...LogField)
+		Info(v interface{}, fields ...LogField)
+		Severe(v interface{})
+		Slow(v interface{}, fields ...LogField)
+		Stack(v interface{})
+		Stat(v interface{}, fields ...LogField)
+	}
+
+	atomicWriter struct {
+		writer Writer
+		lock   sync.RWMutex
+	}
+
+	concreteWriter struct {
+		infoLog   io.WriteCloser
+		errorLog  io.WriteCloser
+		severeLog io.WriteCloser
+		slowLog   io.WriteCloser
+		statLog   io.WriteCloser
+		stackLog  io.Writer
+	}
+)
+
+// NewWriter creates a new Writer with the given io.Writer.
+func NewWriter(w io.Writer) Writer {
+	lw := newLogWriter(log.New(w, "", flags))
+
+	return &concreteWriter{
+		infoLog:   lw,
+		errorLog:  lw,
+		severeLog: lw,
+		slowLog:   lw,
+		statLog:   lw,
+		stackLog:  lw,
+	}
+}
+
+func (w *atomicWriter) Load() Writer {
+	w.lock.RLock()
+	defer w.lock.RUnlock()
+	return w.writer
+}
+
+func (w *atomicWriter) Store(v Writer) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+	w.writer = v
+}
+
+func (w *atomicWriter) StoreIfNil(v Writer) Writer {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if w.writer == nil {
+		w.writer = v
+	}
+
+	return w.writer
+}
+
+func (w *atomicWriter) Swap(v Writer) Writer {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+	old := w.writer
+	w.writer = v
+	return old
+}
+
+func newConsoleWriter() Writer {
+	outLog := newLogWriter(log.New(fatihcolor.Output, "", flags))
+	errLog := newLogWriter(log.New(fatihcolor.Error, "", flags))
+	return &concreteWriter{
+		infoLog:   outLog,
+		errorLog:  errLog,
+		severeLog: errLog,
+		slowLog:   errLog,
+		stackLog:  newLessWriter(errLog, options.logStackCooldownMills),
+		statLog:   outLog,
+	}
+}
+
+func newFileWriter(c LogConf) (Writer, error) {
+	var err error
+	var opts []LogOption
+	var infoLog io.WriteCloser
+	var errorLog io.WriteCloser
+	var severeLog io.WriteCloser
+	var slowLog io.WriteCloser
+	var statLog io.WriteCloser
+	var stackLog io.Writer
+
+	if len(c.Path) == 0 {
+		return nil, ErrLogPathNotSet
+	}
+
+	opts = append(opts, WithCooldownMillis(c.StackCooldownMillis))
+	if c.Compress {
+		opts = append(opts, WithGzip())
+	}
+	if c.KeepDays > 0 {
+		opts = append(opts, WithKeepDays(c.KeepDays))
+	}
+	if c.MaxBackups > 0 {
+		opts = append(opts, WithMaxBackups(c.MaxBackups))
+	}
+	if c.MaxSize > 0 {
+		opts = append(opts, WithMaxSize(c.MaxSize))
+	}
+
+	opts = append(opts, WithRotation(c.Rotation))
+
+	accessFile := path.Join(c.Path, accessFilename)
+	errorFile := path.Join(c.Path, errorFilename)
+	severeFile := path.Join(c.Path, severeFilename)
+	slowFile := path.Join(c.Path, slowFilename)
+	statFile := path.Join(c.Path, statFilename)
+
+	handleOptions(opts)
+	setupLogLevel(c)
+
+	if infoLog, err = createOutput(accessFile); err != nil {
+		return nil, err
+	}
+
+	if errorLog, err = createOutput(errorFile); err != nil {
+		return nil, err
+	}
+
+	if severeLog, err = createOutput(severeFile); err != nil {
+		return nil, err
+	}
+
+	if slowLog, err = createOutput(slowFile); err != nil {
+		return nil, err
+	}
+
+	if statLog, err = createOutput(statFile); err != nil {
+		return nil, err
+	}
+
+	stackLog = newLessWriter(errorLog, options.logStackCooldownMills)
+
+	return &concreteWriter{
+		infoLog:   infoLog,
+		errorLog:  errorLog,
+		severeLog: severeLog,
+		slowLog:   slowLog,
+		statLog:   statLog,
+		stackLog:  stackLog,
+	}, nil
+}
+
+func (w *concreteWriter) Alert(v interface{}) {
+	output(w.errorLog, levelAlert, v)
+}
+
+func (w *concreteWriter) Close() error {
+	if err := w.infoLog.Close(); err != nil {
+		return err
+	}
+
+	if err := w.errorLog.Close(); err != nil {
+		return err
+	}
+
+	if err := w.severeLog.Close(); err != nil {
+		return err
+	}
+
+	if err := w.slowLog.Close(); err != nil {
+		return err
+	}
+
+	return w.statLog.Close()
+}
+
+func (w *concreteWriter) Error(v interface{}, fields ...LogField) {
+	output(w.errorLog, levelError, v, fields...)
+}
+
+func (w *concreteWriter) Info(v interface{}, fields ...LogField) {
+	output(w.infoLog, levelInfo, v, fields...)
+}
+
+func (w *concreteWriter) Severe(v interface{}) {
+	output(w.severeLog, levelFatal, v)
+}
+
+func (w *concreteWriter) Slow(v interface{}, fields ...LogField) {
+	output(w.slowLog, levelSlow, v, fields...)
+}
+
+func (w *concreteWriter) Stack(v interface{}) {
+	output(w.stackLog, levelError, v)
+}
+
+func (w *concreteWriter) Stat(v interface{}, fields ...LogField) {
+	output(w.statLog, levelStat, v, fields...)
+}
+
+type nopWriter struct{}
+
+func (n nopWriter) Alert(_ interface{}) {
+}
+
+func (n nopWriter) Close() error {
+	return nil
+}
+
+func (n nopWriter) Error(_ interface{}, _ ...LogField) {
+}
+
+func (n nopWriter) Info(_ interface{}, _ ...LogField) {
+}
+
+func (n nopWriter) Severe(_ interface{}) {
+}
+
+func (n nopWriter) Slow(_ interface{}, _ ...LogField) {
+}
+
+func (n nopWriter) Stack(_ interface{}) {
+}
+
+func (n nopWriter) Stat(_ interface{}, _ ...LogField) {
+}
+
+func buildFields(fields ...LogField) []string {
+	var items []string
+
+	for _, field := range fields {
+		items = append(items, fmt.Sprintf("%s=%v", field.Key, field.Value))
+	}
+
+	return items
+}
+
+func output(writer io.Writer, level string, val interface{}, fields ...LogField) {
+	fields = append(fields, Field(callerKey, getCaller(callerDepth)))
+
+	switch atomic.LoadUint32(&encoding) {
+	case plainEncodingType:
+		writePlainAny(writer, level, val, buildFields(fields...)...)
+	default:
+		entry := make(logEntryWithFields)
+		for _, field := range fields {
+			entry[field.Key] = field.Value
+		}
+		entry[timestampKey] = getTimestamp()
+		entry[levelKey] = level
+		entry[contentKey] = val
+		writeJson(writer, entry)
+	}
+}
+
+func wrapLevelWithColor(level string) string {
+	var colour color.Color
+	switch level {
+	case levelAlert:
+		colour = color.FgRed
+	case levelError:
+		colour = color.FgRed
+	case levelFatal:
+		colour = color.FgRed
+	case levelInfo:
+		colour = color.FgBlue
+	case levelSlow:
+		colour = color.FgYellow
+	case levelStat:
+		colour = color.FgGreen
+	}
+
+	if colour == color.NoColor {
+		return level
+	}
+
+	return color.WithColorPadding(level, colour)
+}
+
+func writeJson(writer io.Writer, info interface{}) {
+	if content, err := json.Marshal(info); err != nil {
+		log.Println(err.Error())
+	} else if writer == nil {
+		log.Println(string(content))
+	} else {
+		writer.Write(append(content, '\n'))
+	}
+}
+
+func writePlainAny(writer io.Writer, level string, val interface{}, fields ...string) {
+	level = wrapLevelWithColor(level)
+
+	switch v := val.(type) {
+	case string:
+		writePlainText(writer, level, v, fields...)
+	case error:
+		writePlainText(writer, level, v.Error(), fields...)
+	case fmt.Stringer:
+		writePlainText(writer, level, v.String(), fields...)
+	default:
+		writePlainValue(writer, level, v, fields...)
+	}
+}
+
+func writePlainText(writer io.Writer, level, msg string, fields ...string) {
+	var buf strings.Builder
+	buf.WriteString(getTimestamp())
+	buf.WriteByte(plainEncodingSep)
+	buf.WriteString(level)
+	buf.WriteByte(plainEncodingSep)
+	buf.WriteString(msg)
+	for _, item := range fields {
+		buf.WriteByte(plainEncodingSep)
+		buf.WriteString(item)
+	}
+	buf.WriteByte('\n')
+	if writer == nil {
+		log.Println(buf.String())
+		return
+	}
+
+	if _, err := fmt.Fprint(writer, buf.String()); err != nil {
+		log.Println(err.Error())
+	}
+}
+
+func writePlainValue(writer io.Writer, level string, val interface{}, fields ...string) {
+	var buf strings.Builder
+	buf.WriteString(getTimestamp())
+	buf.WriteByte(plainEncodingSep)
+	buf.WriteString(level)
+	buf.WriteByte(plainEncodingSep)
+	if err := json.NewEncoder(&buf).Encode(val); err != nil {
+		log.Println(err.Error())
+		return
+	}
+
+	for _, item := range fields {
+		buf.WriteByte(plainEncodingSep)
+		buf.WriteString(item)
+	}
+	buf.WriteByte('\n')
+	if writer == nil {
+		log.Println(buf.String())
+		return
+	}
+
+	if _, err := fmt.Fprint(writer, buf.String()); err != nil {
+		log.Println(err.Error())
+	}
+}

core/logx/writer_test.go

@@ -0,0 +1,179 @@
+package logx
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"log"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewWriter(t *testing.T) {
+	const literal = "foo bar"
+	var buf bytes.Buffer
+	w := NewWriter(&buf)
+	w.Info(literal)
+	assert.Contains(t, buf.String(), literal)
+}
+
+func TestConsoleWriter(t *testing.T) {
+	var buf bytes.Buffer
+	w := newConsoleWriter()
+	lw := newLogWriter(log.New(&buf, "", 0))
+	w.(*concreteWriter).errorLog = lw
+	w.Alert("foo bar 1")
+	var val mockedEntry
+	if err := json.Unmarshal(buf.Bytes(), &val); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, levelAlert, val.Level)
+	assert.Equal(t, "foo bar 1", val.Content)
+
+	buf.Reset()
+	w.(*concreteWriter).errorLog = lw
+	w.Error("foo bar 2")
+	if err := json.Unmarshal(buf.Bytes(), &val); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, levelError, val.Level)
+	assert.Equal(t, "foo bar 2", val.Content)
+
+	buf.Reset()
+	w.(*concreteWriter).infoLog = lw
+	w.Info("foo bar 3")
+	if err := json.Unmarshal(buf.Bytes(), &val); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, levelInfo, val.Level)
+	assert.Equal(t, "foo bar 3", val.Content)
+
+	buf.Reset()
+	w.(*concreteWriter).severeLog = lw
+	w.Severe("foo bar 4")
+	if err := json.Unmarshal(buf.Bytes(), &val); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, levelFatal, val.Level)
+	assert.Equal(t, "foo bar 4", val.Content)
+
+	buf.Reset()
+	w.(*concreteWriter).slowLog = lw
+	w.Slow("foo bar 5")
+	if err := json.Unmarshal(buf.Bytes(), &val); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, levelSlow, val.Level)
+	assert.Equal(t, "foo bar 5", val.Content)
+
+	buf.Reset()
+	w.(*concreteWriter).statLog = lw
+	w.Stat("foo bar 6")
+	if err := json.Unmarshal(buf.Bytes(), &val); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, levelStat, val.Level)
+	assert.Equal(t, "foo bar 6", val.Content)
+
+	w.(*concreteWriter).infoLog = hardToCloseWriter{}
+	assert.NotNil(t, w.Close())
+	w.(*concreteWriter).infoLog = easyToCloseWriter{}
+	w.(*concreteWriter).errorLog = hardToCloseWriter{}
+	assert.NotNil(t, w.Close())
+	w.(*concreteWriter).errorLog = easyToCloseWriter{}
+	w.(*concreteWriter).severeLog = hardToCloseWriter{}
+	assert.NotNil(t, w.Close())
+	w.(*concreteWriter).severeLog = easyToCloseWriter{}
+	w.(*concreteWriter).slowLog = hardToCloseWriter{}
+	assert.NotNil(t, w.Close())
+	w.(*concreteWriter).slowLog = easyToCloseWriter{}
+	w.(*concreteWriter).statLog = hardToCloseWriter{}
+	assert.NotNil(t, w.Close())
+	w.(*concreteWriter).statLog = easyToCloseWriter{}
+}
+
+func TestNopWriter(t *testing.T) {
+	assert.NotPanics(t, func() {
+		var w nopWriter
+		w.Alert("foo")
+		w.Error("foo")
+		w.Info("foo")
+		w.Severe("foo")
+		w.Stack("foo")
+		w.Stat("foo")
+		w.Slow("foo")
+		w.Close()
+	})
+}
+
+func TestWriteJson(t *testing.T) {
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+	writeJson(nil, "foo")
+	assert.Contains(t, buf.String(), "foo")
+	buf.Reset()
+	writeJson(nil, make(chan int))
+	assert.Contains(t, buf.String(), "unsupported type")
+}
+
+func TestWritePlainAny(t *testing.T) {
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+	writePlainAny(nil, levelInfo, "foo")
+	assert.Contains(t, buf.String(), "foo")
+
+	buf.Reset()
+	writePlainAny(nil, levelError, make(chan int))
+	assert.Contains(t, buf.String(), "unsupported type")
+	writePlainAny(nil, levelSlow, 100)
+	assert.Contains(t, buf.String(), "100")
+
+	buf.Reset()
+	writePlainAny(hardToWriteWriter{}, levelStat, 100)
+	assert.Contains(t, buf.String(), "write error")
+
+	buf.Reset()
+	writePlainAny(hardToWriteWriter{}, levelSevere, "foo")
+	assert.Contains(t, buf.String(), "write error")
+
+	buf.Reset()
+	writePlainAny(hardToWriteWriter{}, levelAlert, "foo")
+	assert.Contains(t, buf.String(), "write error")
+
+	buf.Reset()
+	writePlainAny(hardToWriteWriter{}, levelFatal, "foo")
+	assert.Contains(t, buf.String(), "write error")
+
+}
+
+type mockedEntry struct {
+	Level   string `json:"level"`
+	Content string `json:"content"`
+}
+
+type easyToCloseWriter struct{}
+
+func (h easyToCloseWriter) Write(_ []byte) (_ int, _ error) {
+	return
+}
+
+func (h easyToCloseWriter) Close() error {
+	return nil
+}
+
+type hardToCloseWriter struct{}
+
+func (h hardToCloseWriter) Write(_ []byte) (_ int, _ error) {
+	return
+}
+
+func (h hardToCloseWriter) Close() error {
+	return errors.New("close error")
+}
+
+type hardToWriteWriter struct{}
+
+func (h hardToWriteWriter) Write(_ []byte) (_ int, _ error) {
+	return 0, errors.New("write error")
+}

core/mapping/marshaler.go

@@ -0,0 +1,186 @@
+package mapping
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+)
+
+const (
+	emptyTag       = ""
+	tagKVSeparator = ":"
+)
+
+// Marshal marshals the given val and returns the map that contains the fields.
+// optional=another is not implemented, and it's hard to implement and not common used.
+func Marshal(val interface{}) (map[string]map[string]interface{}, error) {
+	ret := make(map[string]map[string]interface{})
+	tp := reflect.TypeOf(val)
+	if tp.Kind() == reflect.Ptr {
+		tp = tp.Elem()
+	}
+	rv := reflect.ValueOf(val)
+	if rv.Kind() == reflect.Ptr {
+		rv = rv.Elem()
+	}
+
+	for i := 0; i < tp.NumField(); i++ {
+		field := tp.Field(i)
+		value := rv.Field(i)
+		if err := processMember(field, value, ret); err != nil {
+			return nil, err
+		}
+	}
+
+	return ret, nil
+}
+
+func getTag(field reflect.StructField) (string, bool) {
+	tag := string(field.Tag)
+	if i := strings.Index(tag, tagKVSeparator); i >= 0 {
+		return strings.TrimSpace(tag[:i]), true
+	}
+
+	return strings.TrimSpace(tag), false
+}
+
+func processMember(field reflect.StructField, value reflect.Value,
+	collector map[string]map[string]interface{}) error {
+	var key string
+	var opt *fieldOptions
+	var err error
+	tag, ok := getTag(field)
+	if !ok {
+		tag = emptyTag
+		key = field.Name
+	} else {
+		key, opt, err = parseKeyAndOptions(tag, field)
+		if err != nil {
+			return err
+		}
+
+		if err = validate(field, value, opt); err != nil {
+			return err
+		}
+	}
+
+	val := value.Interface()
+	if opt != nil && opt.FromString {
+		val = fmt.Sprint(val)
+	}
+
+	m, ok := collector[tag]
+	if ok {
+		m[key] = val
+	} else {
+		m = map[string]interface{}{
+			key: val,
+		}
+	}
+	collector[tag] = m
+
+	return nil
+}
+
+func validate(field reflect.StructField, value reflect.Value, opt *fieldOptions) error {
+	if opt == nil || !opt.Optional {
+		if err := validateOptional(field, value); err != nil {
+			return err
+		}
+	}
+
+	if opt == nil {
+		return nil
+	}
+
+	if opt.Optional && value.IsZero() {
+		return nil
+	}
+
+	if len(opt.Options) > 0 {
+		if err := validateOptions(value, opt); err != nil {
+			return err
+		}
+	}
+
+	if opt.Range != nil {
+		if err := validateRange(value, opt); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validateOptional(field reflect.StructField, value reflect.Value) error {
+	switch field.Type.Kind() {
+	case reflect.Ptr:
+		if value.IsNil() {
+			return fmt.Errorf("field %q is nil", field.Name)
+		}
+	case reflect.Array, reflect.Slice, reflect.Map:
+		if value.IsNil() || value.Len() == 0 {
+			return fmt.Errorf("field %q is empty", field.Name)
+		}
+	}
+
+	return nil
+}
+
+func validateOptions(value reflect.Value, opt *fieldOptions) error {
+	var found bool
+	val := fmt.Sprint(value.Interface())
+	for i := range opt.Options {
+		if opt.Options[i] == val {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return fmt.Errorf("field %q not in options", val)
+	}
+
+	return nil
+}
+
+func validateRange(value reflect.Value, opt *fieldOptions) error {
+	var val float64
+	switch v := value.Interface().(type) {
+	case int:
+		val = float64(v)
+	case int8:
+		val = float64(v)
+	case int16:
+		val = float64(v)
+	case int32:
+		val = float64(v)
+	case int64:
+		val = float64(v)
+	case uint:
+		val = float64(v)
+	case uint8:
+		val = float64(v)
+	case uint16:
+		val = float64(v)
+	case uint32:
+		val = float64(v)
+	case uint64:
+		val = float64(v)
+	case float32:
+		val = float64(v)
+	case float64:
+		val = v
+	default:
+		return fmt.Errorf("unknown support type for range %q", value.Type().String())
+	}
+
+	// validates [left, right], [left, right), (left, right], (left, right)
+	if val < opt.Range.left ||
+		(!opt.Range.leftInclude && val == opt.Range.left) ||
+		val > opt.Range.right ||
+		(!opt.Range.rightInclude && val == opt.Range.right) {
+		return fmt.Errorf("%v out of range", value.Interface())
+	}
+
+	return nil
+}

core/mapping/marshaler_test.go

@@ -0,0 +1,346 @@
+package mapping
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarshal(t *testing.T) {
+	v := struct {
+		Name      string `path:"name"`
+		Address   string `json:"address,options=[beijing,shanghai]"`
+		Age       int    `json:"age"`
+		Anonymous bool
+	}{
+		Name:      "kevin",
+		Address:   "shanghai",
+		Age:       20,
+		Anonymous: true,
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, "kevin", m["path"]["name"])
+	assert.Equal(t, "shanghai", m["json"]["address"])
+	assert.Equal(t, 20, m["json"]["age"].(int))
+	assert.True(t, m[emptyTag]["Anonymous"].(bool))
+}
+
+func TestMarshal_Ptr(t *testing.T) {
+	v := &struct {
+		Name      string `path:"name"`
+		Address   string `json:"address,options=[beijing,shanghai]"`
+		Age       int    `json:"age"`
+		Anonymous bool
+	}{
+		Name:      "kevin",
+		Address:   "shanghai",
+		Age:       20,
+		Anonymous: true,
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, "kevin", m["path"]["name"])
+	assert.Equal(t, "shanghai", m["json"]["address"])
+	assert.Equal(t, 20, m["json"]["age"].(int))
+	assert.True(t, m[emptyTag]["Anonymous"].(bool))
+}
+
+func TestMarshal_OptionalPtr(t *testing.T) {
+	var val = 1
+	v := struct {
+		Age *int `json:"age"`
+	}{
+		Age: &val,
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, *m["json"]["age"].(*int))
+}
+
+func TestMarshal_OptionalPtrNil(t *testing.T) {
+	v := struct {
+		Age *int `json:"age"`
+	}{}
+
+	_, err := Marshal(v)
+	assert.NotNil(t, err)
+}
+
+func TestMarshal_BadOptions(t *testing.T) {
+	v := struct {
+		Name string `json:"name,options"`
+	}{
+		Name: "kevin",
+	}
+
+	_, err := Marshal(v)
+	assert.NotNil(t, err)
+}
+
+func TestMarshal_NotInOptions(t *testing.T) {
+	v := struct {
+		Name string `json:"name,options=[a,b]"`
+	}{
+		Name: "kevin",
+	}
+
+	_, err := Marshal(v)
+	assert.NotNil(t, err)
+}
+
+func TestMarshal_NotInOptionsOptional(t *testing.T) {
+	v := struct {
+		Name string `json:"name,options=[a,b],optional"`
+	}{}
+
+	_, err := Marshal(v)
+	assert.Nil(t, err)
+}
+
+func TestMarshal_NotInOptionsOptionalWrongValue(t *testing.T) {
+	v := struct {
+		Name string `json:"name,options=[a,b],optional"`
+	}{
+		Name: "kevin",
+	}
+
+	_, err := Marshal(v)
+	assert.NotNil(t, err)
+}
+
+func TestMarshal_Nested(t *testing.T) {
+	type address struct {
+		Country string `json:"country"`
+		City    string `json:"city"`
+	}
+	v := struct {
+		Name    string  `json:"name,options=[kevin,wan]"`
+		Address address `json:"address"`
+	}{
+		Name: "kevin",
+		Address: address{
+			Country: "China",
+			City:    "Shanghai",
+		},
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, "kevin", m["json"]["name"])
+	assert.Equal(t, "China", m["json"]["address"].(address).Country)
+	assert.Equal(t, "Shanghai", m["json"]["address"].(address).City)
+}
+
+func TestMarshal_NestedPtr(t *testing.T) {
+	type address struct {
+		Country string `json:"country"`
+		City    string `json:"city"`
+	}
+	v := struct {
+		Name    string   `json:"name,options=[kevin,wan]"`
+		Address *address `json:"address"`
+	}{
+		Name: "kevin",
+		Address: &address{
+			Country: "China",
+			City:    "Shanghai",
+		},
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, "kevin", m["json"]["name"])
+	assert.Equal(t, "China", m["json"]["address"].(*address).Country)
+	assert.Equal(t, "Shanghai", m["json"]["address"].(*address).City)
+}
+
+func TestMarshal_Slice(t *testing.T) {
+	v := struct {
+		Name []string `json:"name"`
+	}{
+		Name: []string{"kevin", "wan"},
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []string{"kevin", "wan"}, m["json"]["name"].([]string))
+}
+
+func TestMarshal_SliceNil(t *testing.T) {
+	v := struct {
+		Name []string `json:"name"`
+	}{
+		Name: nil,
+	}
+
+	_, err := Marshal(v)
+	assert.NotNil(t, err)
+}
+
+func TestMarshal_Range(t *testing.T) {
+	v := struct {
+		Int     int     `json:"int,range=[1:3]"`
+		Int8    int8    `json:"int8,range=[1:3)"`
+		Int16   int16   `json:"int16,range=(1:3]"`
+		Int32   int32   `json:"int32,range=(1:3)"`
+		Int64   int64   `json:"int64,range=(1:3)"`
+		Uint    uint    `json:"uint,range=[1:3]"`
+		Uint8   uint8   `json:"uint8,range=[1:3)"`
+		Uint16  uint16  `json:"uint16,range=(1:3]"`
+		Uint32  uint32  `json:"uint32,range=(1:3)"`
+		Uint64  uint64  `json:"uint64,range=(1:3)"`
+		Float32 float32 `json:"float32,range=(1:3)"`
+		Float64 float64 `json:"float64,range=(1:3)"`
+	}{
+		Int:     1,
+		Int8:    1,
+		Int16:   2,
+		Int32:   2,
+		Int64:   2,
+		Uint:    1,
+		Uint8:   1,
+		Uint16:  2,
+		Uint32:  2,
+		Uint64:  2,
+		Float32: 2,
+		Float64: 2,
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, m["json"]["int"].(int))
+	assert.Equal(t, int8(1), m["json"]["int8"].(int8))
+	assert.Equal(t, int16(2), m["json"]["int16"].(int16))
+	assert.Equal(t, int32(2), m["json"]["int32"].(int32))
+	assert.Equal(t, int64(2), m["json"]["int64"].(int64))
+	assert.Equal(t, uint(1), m["json"]["uint"].(uint))
+	assert.Equal(t, uint8(1), m["json"]["uint8"].(uint8))
+	assert.Equal(t, uint16(2), m["json"]["uint16"].(uint16))
+	assert.Equal(t, uint32(2), m["json"]["uint32"].(uint32))
+	assert.Equal(t, uint64(2), m["json"]["uint64"].(uint64))
+	assert.Equal(t, float32(2), m["json"]["float32"].(float32))
+	assert.Equal(t, float64(2), m["json"]["float64"].(float64))
+}
+
+func TestMarshal_RangeOut(t *testing.T) {
+	tests := []interface{}{
+		struct {
+			Int int `json:"int,range=[1:3]"`
+		}{
+			Int: 4,
+		},
+		struct {
+			Int int `json:"int,range=(1:3]"`
+		}{
+			Int: 1,
+		},
+		struct {
+			Int int `json:"int,range=[1:3)"`
+		}{
+			Int: 3,
+		},
+		struct {
+			Int int `json:"int,range=(1:3)"`
+		}{
+			Int: 3,
+		},
+		struct {
+			Bool bool `json:"bool,range=(1:3)"`
+		}{
+			Bool: true,
+		},
+	}
+
+	for _, test := range tests {
+		_, err := Marshal(test)
+		assert.NotNil(t, err)
+	}
+}
+
+func TestMarshal_RangeIllegal(t *testing.T) {
+	tests := []interface{}{
+		struct {
+			Int int `json:"int,range=[3:1]"`
+		}{
+			Int: 2,
+		},
+		struct {
+			Int int `json:"int,range=(3:1]"`
+		}{
+			Int: 2,
+		},
+	}
+
+	for _, test := range tests {
+		_, err := Marshal(test)
+		assert.Equal(t, err, errNumberRange)
+	}
+}
+
+func TestMarshal_RangeLeftEqualsToRight(t *testing.T) {
+	tests := []struct {
+		name  string
+		value interface{}
+		err   error
+	}{
+		{
+			name: "left inclusive, right inclusive",
+			value: struct {
+				Int int `json:"int,range=[2:2]"`
+			}{
+				Int: 2,
+			},
+		},
+		{
+			name: "left inclusive, right exclusive",
+			value: struct {
+				Int int `json:"int,range=[2:2)"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+		{
+			name: "left exclusive, right inclusive",
+			value: struct {
+				Int int `json:"int,range=(2:2]"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+		{
+			name: "left exclusive, right exclusive",
+			value: struct {
+				Int int `json:"int,range=(2:2)"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			_, err := Marshal(test.value)
+			assert.Equal(t, test.err, err)
+		})
+	}
+}
+
+func TestMarshal_FromString(t *testing.T) {
+	v := struct {
+		Age int `json:"age,string"`
+	}{
+		Age: 10,
+	}
+
+	m, err := Marshal(v)
+	assert.Nil(t, err)
+	assert.Equal(t, "10", m["json"]["age"].(string))
+}

core/mapping/tomlunmarshaler.go

@@ -0,0 +1,29 @@
+package mapping
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+// UnmarshalTomlBytes unmarshals TOML bytes into the given v.
+func UnmarshalTomlBytes(content []byte, v interface{}) error {
+	return UnmarshalTomlReader(bytes.NewReader(content), v)
+}
+
+// UnmarshalTomlReader unmarshals TOML from the given io.Reader into the given v.
+func UnmarshalTomlReader(r io.Reader, v interface{}) error {
+	var val interface{}
+	if err := toml.NewDecoder(r).Decode(&val); err != nil {
+		return err
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(val); err != nil {
+		return err
+	}
+
+	return UnmarshalJsonReader(&buf, v)
+}

core/mapping/tomlunmarshaler_test.go

@@ -0,0 +1,41 @@
+package mapping
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnmarshalToml(t *testing.T) {
+	const input = `a = "foo"
+b = 1
+c = "${FOO}"
+d = "abcd!@#$112"
+`
+	var val struct {
+		A string `json:"a"`
+		B int    `json:"b"`
+		C string `json:"c"`
+		D string `json:"d"`
+	}
+	assert.Nil(t, UnmarshalTomlBytes([]byte(input), &val))
+	assert.Equal(t, "foo", val.A)
+	assert.Equal(t, 1, val.B)
+	assert.Equal(t, "${FOO}", val.C)
+	assert.Equal(t, "abcd!@#$112", val.D)
+}
+
+func TestUnmarshalTomlErrorToml(t *testing.T) {
+	const input = `foo"
+b = 1
+c = "${FOO}"
+d = "abcd!@#$112"
+`
+	var val struct {
+		A string `json:"a"`
+		B int    `json:"b"`
+		C string `json:"c"`
+		D string `json:"d"`
+	}
+	assert.NotNil(t, UnmarshalTomlBytes([]byte(input), &val))
+}

core/mapping/unmarshaler.go

@@ -97,10 +97,6 @@ func (u *Unmarshaler) unmarshalWithFullName(m Valuer, v interface{}, fullName st
 	numFields := rte.NumField()
 	for i := 0; i < numFields; i++ {
 		field := rte.Field(i)
-		if usingDifferentKeys(u.key, field) {
-			continue
-		}
-
 		if err := u.processField(field, rve.Field(i), m, fullName); err != nil {
 			return err
 		}
@@ -235,7 +231,7 @@ func (u *Unmarshaler) processFieldPrimitive(field reflect.StructField, value ref
 			return u.processFieldPrimitiveWithJSONNumber(field, value, v, opts, fullName)
 		default:
 			if typeKind == valueKind {
-				if err := validateValueInOptions(opts.options(), mapValue); err != nil {
+				if err := validateValueInOptions(mapValue, opts.options()); err != nil {
 					return err
 				}
 
@@ -257,6 +253,10 @@ func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(field reflect.StructFi
 		return err
 	}
 
+	if err := validateValueInOptions(v, opts.options()); err != nil {
+		return err
+	}
+
 	if fieldKind == reflect.Ptr {
 		value = value.Elem()
 	}
@@ -275,6 +275,10 @@ func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(field reflect.StructFi
 			return err
 		}
 
+		if iValue < 0 {
+			return fmt.Errorf("unmarshal %q with bad value %q", fullName, v.String())
+		}
+
 		value.SetUint(uint64(iValue))
 	case reflect.Float32, reflect.Float64:
 		fValue, err := v.Float64()
@@ -448,7 +452,15 @@ func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, map
 	dereffedBaseType := Deref(baseType)
 	dereffedBaseKind := dereffedBaseType.Kind()
 	refValue := reflect.ValueOf(mapValue)
+	if refValue.IsNil() {
+		return nil
+	}
+
 	conv := reflect.MakeSlice(reflect.SliceOf(baseType), refValue.Len(), refValue.Cap())
+	if refValue.Len() == 0 {
+		value.Set(conv)
+		return nil
+	}
 
 	var valid bool
 	for i := 0; i < refValue.Len(); i++ {
@@ -488,10 +500,20 @@ func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, map
 	return nil
 }
 
-func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.Value, mapValue interface{}) error {
+func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.Value,
+	mapValue interface{}) error {
 	var slice []interface{}
-	if err := jsonx.UnmarshalFromString(mapValue.(string), &slice); err != nil {
-		return err
+	switch v := mapValue.(type) {
+	case fmt.Stringer:
+		if err := jsonx.UnmarshalFromString(v.String(), &slice); err != nil {
+			return err
+		}
+	case string:
+		if err := jsonx.UnmarshalFromString(v, &slice); err != nil {
+			return err
+		}
+	default:
+		return errUnsupportedType
 	}
 
 	baseFieldType := Deref(fieldType.Elem())
@@ -512,8 +534,10 @@ func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int,
 	baseKind reflect.Kind, value interface{}) error {
 	ithVal := slice.Index(index)
 	switch v := value.(type) {
-	case json.Number:
+	case fmt.Stringer:
 		return setValue(baseKind, ithVal, v.String())
+	case string:
+		return setValue(baseKind, ithVal, v)
 	default:
 		// don't need to consider the difference between int, int8, int16, int32, int64,
 		// uint, uint8, uint16, uint32, uint64, because they're handled as json.Number.
@@ -723,10 +747,10 @@ func fillWithSameType(field reflect.StructField, value reflect.Value, mapValue i
 	if field.Type.Kind() == reflect.Ptr {
 		baseType := Deref(field.Type)
 		target := reflect.New(baseType).Elem()
-		target.Set(reflect.ValueOf(mapValue))
+		setSameKindValue(baseType, target, mapValue)
 		value.Set(target.Addr())
 	} else {
-		value.Set(reflect.ValueOf(mapValue))
+		setSameKindValue(field.Type, value, mapValue)
 	}
 
 	return nil
@@ -801,3 +825,11 @@ func readKeys(key string) []string {
 
 	return keys
 }
+
+func setSameKindValue(targetType reflect.Type, target reflect.Value, value interface{}) {
+	if reflect.ValueOf(value).Type().AssignableTo(targetType) {
+		target.Set(reflect.ValueOf(value))
+	} else {
+		target.Set(reflect.ValueOf(value).Convert(targetType))
+	}
+}

core/mapping/unmarshaler_test.go

@@ -198,6 +198,49 @@ func TestUnmarshalIntWithDefault(t *testing.T) {
 	assert.Equal(t, 1, in.Int)
 }
 
+func TestUnmarshalBoolSliceRequired(t *testing.T) {
+	type inner struct {
+		Bools []bool `key:"bools"`
+	}
+
+	var in inner
+	assert.NotNil(t, UnmarshalKey(map[string]interface{}{}, &in))
+}
+
+func TestUnmarshalBoolSliceNil(t *testing.T) {
+	type inner struct {
+		Bools []bool `key:"bools,optional"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(map[string]interface{}{}, &in))
+	assert.Nil(t, in.Bools)
+}
+
+func TestUnmarshalBoolSliceNilExplicit(t *testing.T) {
+	type inner struct {
+		Bools []bool `key:"bools,optional"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(map[string]interface{}{
+		"bools": nil,
+	}, &in))
+	assert.Nil(t, in.Bools)
+}
+
+func TestUnmarshalBoolSliceEmpty(t *testing.T) {
+	type inner struct {
+		Bools []bool `key:"bools,optional"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(map[string]interface{}{
+		"bools": []bool{},
+	}, &in))
+	assert.Empty(t, in.Bools)
+}
+
 func TestUnmarshalBoolSliceWithDefault(t *testing.T) {
 	type inner struct {
 		Bools []bool `key:"bools,default=[true,false]"`
@@ -330,28 +373,34 @@ func TestUnmarshalFloat(t *testing.T) {
 
 func TestUnmarshalInt64Slice(t *testing.T) {
 	var v struct {
-		Ages []int64 `key:"ages"`
+		Ages  []int64 `key:"ages"`
+		Slice []int64 `key:"slice"`
 	}
 	m := map[string]interface{}{
-		"ages": []int64{1, 2},
+		"ages":  []int64{1, 2},
+		"slice": []interface{}{},
 	}
 
 	ast := assert.New(t)
 	ast.Nil(UnmarshalKey(m, &v))
 	ast.ElementsMatch([]int64{1, 2}, v.Ages)
+	ast.Equal([]int64{}, v.Slice)
 }
 
 func TestUnmarshalIntSlice(t *testing.T) {
 	var v struct {
-		Ages []int `key:"ages"`
+		Ages  []int `key:"ages"`
+		Slice []int `key:"slice"`
 	}
 	m := map[string]interface{}{
-		"ages": []int{1, 2},
+		"ages":  []int{1, 2},
+		"slice": []interface{}{},
 	}
 
 	ast := assert.New(t)
 	ast.Nil(UnmarshalKey(m, &v))
 	ast.ElementsMatch([]int{1, 2}, v.Ages)
+	ast.Equal([]int{}, v.Slice)
 }
 
 func TestUnmarshalString(t *testing.T) {
@@ -938,6 +987,43 @@ func TestUnmarshalWithStringOptionsCorrect(t *testing.T) {
 	ast.Equal("2", in.Correct)
 }
 
+func TestUnmarshalOptionsOptional(t *testing.T) {
+	type inner struct {
+		Value         string `key:"value,options=first|second,optional"`
+		OptionalValue string `key:"optional_value,options=first|second,optional"`
+		Foo           string `key:"foo,options=[bar,baz]"`
+		Correct       string `key:"correct,options=1|2"`
+	}
+	m := map[string]interface{}{
+		"value":   "first",
+		"foo":     "bar",
+		"correct": "2",
+	}
+
+	var in inner
+	ast := assert.New(t)
+	ast.Nil(UnmarshalKey(m, &in))
+	ast.Equal("first", in.Value)
+	ast.Equal("", in.OptionalValue)
+	ast.Equal("bar", in.Foo)
+	ast.Equal("2", in.Correct)
+}
+
+func TestUnmarshalOptionsOptionalWrongValue(t *testing.T) {
+	type inner struct {
+		Value         string `key:"value,options=first|second,optional"`
+		OptionalValue string `key:"optional_value,options=first|second,optional"`
+		WrongValue    string `key:"wrong_value,options=first|second,optional"`
+	}
+	m := map[string]interface{}{
+		"value":       "first",
+		"wrong_value": "third",
+	}
+
+	var in inner
+	assert.NotNil(t, UnmarshalKey(m, &in))
+}
+
 func TestUnmarshalStringOptionsWithStringOptionsNotString(t *testing.T) {
 	type inner struct {
 		Value   string `key:"value,options=first|second"`
@@ -1084,6 +1170,28 @@ func TestUnmarshalWithIntOptionsIncorrect(t *testing.T) {
 	assert.NotNil(t, UnmarshalKey(m, &in))
 }
 
+func TestUnmarshalWithJsonNumberOptionsIncorrect(t *testing.T) {
+	type inner struct {
+		Value     string `key:"value,options=first|second"`
+		Incorrect int    `key:"incorrect,options=1|2"`
+	}
+	m := map[string]interface{}{
+		"value":     "first",
+		"incorrect": json.Number("3"),
+	}
+
+	var in inner
+	assert.NotNil(t, UnmarshalKey(m, &in))
+}
+
+func TestUnmarshaler_UnmarshalIntOptions(t *testing.T) {
+	var val struct {
+		Sex int `json:"sex,options=0|1"`
+	}
+	input := []byte(`{"sex": 2}`)
+	assert.NotNil(t, UnmarshalJsonBytes(input, &val))
+}
+
 func TestUnmarshalWithUintOptionsCorrect(t *testing.T) {
 	type inner struct {
 		Value  string `key:"value,options=first|second"`
@@ -2573,7 +2681,7 @@ func TestUnmarshalJsonReaderMultiArray(t *testing.T) {
 	assert.Equal(t, 2, len(res.B))
 }
 
-func TestUnmarshalJsonReaderPtrMultiArray(t *testing.T) {
+func TestUnmarshalJsonReaderPtrMultiArrayString(t *testing.T) {
 	payload := `{"a": "133", "b": [["add", "cccd"], ["eeee"]]}`
 	var res struct {
 		A string      `json:"a"`
@@ -2586,6 +2694,32 @@ func TestUnmarshalJsonReaderPtrMultiArray(t *testing.T) {
 	assert.Equal(t, 2, len(res.B[0]))
 }
 
+func TestUnmarshalJsonReaderPtrMultiArrayString_Int(t *testing.T) {
+	payload := `{"a": "133", "b": [[11, 22], [33]]}`
+	var res struct {
+		A string      `json:"a"`
+		B [][]*string `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(res.B))
+	assert.Equal(t, 2, len(res.B[0]))
+}
+
+func TestUnmarshalJsonReaderPtrMultiArrayInt(t *testing.T) {
+	payload := `{"a": "133", "b": [[11, 22], [33]]}`
+	var res struct {
+		A string   `json:"a"`
+		B [][]*int `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(res.B))
+	assert.Equal(t, 2, len(res.B[0]))
+}
+
 func TestUnmarshalJsonReaderPtrArray(t *testing.T) {
 	payload := `{"a": "133", "b": ["add", "cccd", "eeee"]}`
 	var res struct {
@@ -2598,6 +2732,30 @@ func TestUnmarshalJsonReaderPtrArray(t *testing.T) {
 	assert.Equal(t, 3, len(res.B))
 }
 
+func TestUnmarshalJsonReaderPtrArray_Int(t *testing.T) {
+	payload := `{"a": "133", "b": [11, 22, 33]}`
+	var res struct {
+		A string    `json:"a"`
+		B []*string `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(res.B))
+}
+
+func TestUnmarshalJsonReaderPtrInt(t *testing.T) {
+	payload := `{"a": "133", "b": [11, 22, 33]}`
+	var res struct {
+		A string    `json:"a"`
+		B []*string `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(res.B))
+}
+
 func TestUnmarshalJsonWithoutKey(t *testing.T) {
 	payload := `{"A": "1", "B": "2"}`
 	var res struct {
@@ -2611,6 +2769,116 @@ func TestUnmarshalJsonWithoutKey(t *testing.T) {
 	assert.Equal(t, "2", res.B)
 }
 
+func TestUnmarshalJsonUintNegative(t *testing.T) {
+	payload := `{"a": -1}`
+	var res struct {
+		A uint `json:"a"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshalJsonDefinedInt(t *testing.T) {
+	type Int int
+	var res struct {
+		A Int `json:"a"`
+	}
+	payload := `{"a": -1}`
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, Int(-1), res.A)
+}
+
+func TestUnmarshalJsonDefinedString(t *testing.T) {
+	type String string
+	var res struct {
+		A String `json:"a"`
+	}
+	payload := `{"a": "foo"}`
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, String("foo"), res.A)
+}
+
+func TestUnmarshalJsonDefinedStringPtr(t *testing.T) {
+	type String string
+	var res struct {
+		A *String `json:"a"`
+	}
+	payload := `{"a": "foo"}`
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, String("foo"), *res.A)
+}
+
+func TestUnmarshalJsonReaderComplex(t *testing.T) {
+	type (
+		MyInt      int
+		MyTxt      string
+		MyTxtArray []string
+
+		Req struct {
+			MyInt      MyInt      `json:"my_int"` // int.. ok
+			MyTxtArray MyTxtArray `json:"my_txt_array"`
+			MyTxt      MyTxt      `json:"my_txt"` // but string is not assignable
+			Int        int        `json:"int"`
+			Txt        string     `json:"txt"`
+		}
+	)
+	body := `{
+  "my_int": 100,
+  "my_txt_array": [
+    "a",
+    "b"
+  ],
+  "my_txt": "my_txt",
+  "int": 200,
+  "txt": "txt"
+}`
+	var req Req
+	err := UnmarshalJsonReader(strings.NewReader(body), &req)
+	assert.Nil(t, err)
+	assert.Equal(t, MyInt(100), req.MyInt)
+	assert.Equal(t, MyTxt("my_txt"), req.MyTxt)
+	assert.EqualValues(t, MyTxtArray([]string{"a", "b"}), req.MyTxtArray)
+	assert.Equal(t, 200, req.Int)
+	assert.Equal(t, "txt", req.Txt)
+}
+
+func TestUnmarshalJsonReaderArrayBool(t *testing.T) {
+	payload := `{"id": false}`
+	var res struct {
+		ID []string `json:"id"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshalJsonReaderArrayInt(t *testing.T) {
+	payload := `{"id": 123}`
+	var res struct {
+		ID []string `json:"id"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshalJsonReaderArrayString(t *testing.T) {
+	payload := `{"id": "123"}`
+	var res struct {
+		ID []string `json:"id"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.NotNil(t, err)
+}
+
 func BenchmarkDefaultValue(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		var a struct {

core/mapping/utils.go

@@ -143,6 +143,23 @@ func doParseKeyAndOptions(field reflect.StructField, value string) (string, *fie
 	return key, &fieldOpts, nil
 }
 
+// ensureValue ensures nested members not to be nil.
+// If pointer value is nil, set to a new value.
+func ensureValue(v reflect.Value) reflect.Value {
+	for {
+		if v.Kind() != reflect.Ptr {
+			break
+		}
+
+		if v.IsNil() {
+			v.Set(reflect.New(v.Type().Elem()))
+		}
+		v = v.Elem()
+	}
+
+	return v
+}
+
 func implicitValueRequiredStruct(tag string, tp reflect.Type) (bool, error) {
 	numFields := tp.NumField()
 	for i := 0; i < numFields; i++ {
@@ -294,6 +311,20 @@ func parseNumberRange(str string) (*numberRange, error) {
 		right = math.MaxFloat64
 	}
 
+	if left > right {
+		return nil, errNumberRange
+	}
+
+	// [2:2] valid
+	// [2:2) invalid
+	// (2:2] invalid
+	// (2:2) invalid
+	if left == right {
+		if !leftInclude || !rightInclude {
+			return nil, errNumberRange
+		}
+	}
+
 	return &numberRange{
 		left:         left,
 		leftInclude:  leftInclude,
@@ -478,6 +509,7 @@ func setValue(kind reflect.Kind, value reflect.Value, str string) error {
 		return errValueNotSettable
 	}
 
+	value = ensureValue(value)
 	v, err := convertType(kind, str)
 	if err != nil {
 		return err
@@ -592,16 +624,16 @@ func validateNumberRange(fv float64, nr *numberRange) error {
 	return nil
 }
 
-func validateValueInOptions(options []string, value interface{}) error {
+func validateValueInOptions(val interface{}, options []string) error {
 	if len(options) > 0 {
-		switch v := value.(type) {
+		switch v := val.(type) {
 		case string:
 			if !stringx.Contains(options, v) {
 				return fmt.Errorf(`error: value "%s" is not defined in options "%v"`, v, options)
 			}
 		default:
 			if !stringx.Contains(options, Repr(v)) {
-				return fmt.Errorf(`error: value "%v" is not defined in options "%v"`, value, options)
+				return fmt.Errorf(`error: value "%v" is not defined in options "%v"`, val, options)
 			}
 		}
 	}

core/mapping/utils_test.go

@@ -15,7 +15,7 @@ type Foo struct {
 	StrWithTagAndOption string `key:"stringwithtag,string"`
 }
 
-func TestDeferInt(t *testing.T) {
+func TestDerefInt(t *testing.T) {
 	i := 1
 	s := "hello"
 	number := struct {
@@ -60,6 +60,51 @@ func TestDeferInt(t *testing.T) {
 	}
 }
 
+func TestDerefValInt(t *testing.T) {
+	i := 1
+	s := "hello"
+	number := struct {
+		f float64
+	}{
+		f: 6.4,
+	}
+	cases := []struct {
+		t      reflect.Value
+		expect reflect.Kind
+	}{
+		{
+			t:      reflect.ValueOf(i),
+			expect: reflect.Int,
+		},
+		{
+			t:      reflect.ValueOf(&i),
+			expect: reflect.Int,
+		},
+		{
+			t:      reflect.ValueOf(s),
+			expect: reflect.String,
+		},
+		{
+			t:      reflect.ValueOf(&s),
+			expect: reflect.String,
+		},
+		{
+			t:      reflect.ValueOf(number.f),
+			expect: reflect.Float64,
+		},
+		{
+			t:      reflect.ValueOf(&number.f),
+			expect: reflect.Float64,
+		},
+	}
+
+	for _, each := range cases {
+		t.Run(each.t.String(), func(t *testing.T) {
+			assert.Equal(t, each.expect, ensureValue(each.t).Kind())
+		})
+	}
+}
+
 func TestParseKeyAndOptionWithoutTag(t *testing.T) {
 	var foo Foo
 	rte := reflect.TypeOf(&foo).Elem()

core/mapping/yamlunmarshaler.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
-	"io/ioutil"
 
 	"gopkg.in/yaml.v2"
 )
@@ -14,7 +13,7 @@ const yamlTagKey = "json"
 
 var (
 	// ErrUnsupportedType is an error that indicates the config format is not supported.
-	ErrUnsupportedType = errors.New("only map-like configs are suported")
+	ErrUnsupportedType = errors.New("only map-like configs are supported")
 
 	yamlUnmarshaler = NewUnmarshaler(yamlTagKey)
 )
@@ -29,39 +28,6 @@ func UnmarshalYamlReader(reader io.Reader, v interface{}) error {
 	return unmarshalYamlReader(reader, v, yamlUnmarshaler)
 }
 
-func unmarshalYamlBytes(content []byte, v interface{}, unmarshaler *Unmarshaler) error {
-	var o interface{}
-	if err := yamlUnmarshal(content, &o); err != nil {
-		return err
-	}
-
-	if m, ok := o.(map[string]interface{}); ok {
-		return unmarshaler.Unmarshal(m, v)
-	}
-
-	return ErrUnsupportedType
-}
-
-func unmarshalYamlReader(reader io.Reader, v interface{}, unmarshaler *Unmarshaler) error {
-	content, err := ioutil.ReadAll(reader)
-	if err != nil {
-		return err
-	}
-
-	return unmarshalYamlBytes(content, v, unmarshaler)
-}
-
-// yamlUnmarshal YAML to map[string]interface{} instead of map[interface{}]interface{}.
-func yamlUnmarshal(in []byte, out interface{}) error {
-	var res interface{}
-	if err := yaml.Unmarshal(in, &res); err != nil {
-		return err
-	}
-
-	*out.(*interface{}) = cleanupMapValue(res)
-	return nil
-}
-
 func cleanupInterfaceMap(in map[interface{}]interface{}) map[string]interface{} {
 	res := make(map[string]interface{})
 	for k, v := range in {
@@ -96,3 +62,40 @@ func cleanupMapValue(v interface{}) interface{} {
 		return Repr(v)
 	}
 }
+
+func unmarshal(unmarshaler *Unmarshaler, o interface{}, v interface{}) error {
+	if m, ok := o.(map[string]interface{}); ok {
+		return unmarshaler.Unmarshal(m, v)
+	}
+
+	return ErrUnsupportedType
+}
+
+func unmarshalYamlBytes(content []byte, v interface{}, unmarshaler *Unmarshaler) error {
+	var o interface{}
+	if err := yamlUnmarshal(content, &o); err != nil {
+		return err
+	}
+
+	return unmarshal(unmarshaler, o, v)
+}
+
+func unmarshalYamlReader(reader io.Reader, v interface{}, unmarshaler *Unmarshaler) error {
+	var res interface{}
+	if err := yaml.NewDecoder(reader).Decode(&res); err != nil {
+		return err
+	}
+
+	return unmarshal(unmarshaler, cleanupMapValue(res), v)
+}
+
+// yamlUnmarshal YAML to map[string]interface{} instead of map[interface{}]interface{}.
+func yamlUnmarshal(in []byte, out interface{}) error {
+	var res interface{}
+	if err := yaml.Unmarshal(in, &res); err != nil {
+		return err
+	}
+
+	*out.(*interface{}) = cleanupMapValue(res)
+	return nil
+}

core/mapping/yamlunmarshaler_test.go

@@ -926,14 +926,17 @@ func TestUnmarshalYamlBytesError(t *testing.T) {
 }
 
 func TestUnmarshalYamlReaderError(t *testing.T) {
-	payload := `abcd: cdef`
-	reader := strings.NewReader(payload)
 	var v struct {
 		Any string
 	}
 
+	reader := strings.NewReader(`abcd: cdef`)
 	err := UnmarshalYamlReader(reader, &v)
 	assert.NotNil(t, err)
+
+	reader = strings.NewReader("chenquan")
+	err = UnmarshalYamlReader(reader, &v)
+	assert.ErrorIs(t, err, ErrUnsupportedType)
 }
 
 func TestUnmarshalYamlBadReader(t *testing.T) {
@@ -1011,6 +1014,6 @@ func TestUnmarshalYamlMapRune(t *testing.T) {
 
 type badReader struct{}
 
-func (b *badReader) Read(p []byte) (n int, err error) {
+func (b *badReader) Read(_ []byte) (n int, err error) {
 	return 0, io.ErrLimitReached
 }

core/metric/counter.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -47,10 +48,18 @@ func NewCounterVec(cfg *CounterVecOpts) CounterVec {
 }
 
 func (cv *promCounterVec) Inc(labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	cv.counter.WithLabelValues(labels...).Inc()
 }
 
 func (cv *promCounterVec) Add(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	cv.counter.WithLabelValues(labels...).Add(v)
 }
 

core/metric/counter_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 func TestNewCounterVec(t *testing.T) {
@@ -21,6 +22,7 @@ func TestNewCounterVec(t *testing.T) {
 }
 
 func TestCounterIncr(t *testing.T) {
+	startAgent()
 	counterVec := NewCounterVec(&CounterVecOpts{
 		Namespace: "http_client",
 		Subsystem: "call",
@@ -37,6 +39,7 @@ func TestCounterIncr(t *testing.T) {
 }
 
 func TestCounterAdd(t *testing.T) {
+	startAgent()
 	counterVec := NewCounterVec(&CounterVecOpts{
 		Namespace: "rpc_server",
 		Subsystem: "requests",
@@ -51,3 +54,11 @@ func TestCounterAdd(t *testing.T) {
 	r := testutil.ToFloat64(cv.counter)
 	assert.Equal(t, float64(33), r)
 }
+
+func startAgent() {
+	prometheus.StartAgent(prometheus.Config{
+		Host: "127.0.0.1",
+		Port: 9101,
+		Path: "/metrics",
+	})
+}

core/metric/gauge.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -50,14 +51,26 @@ func NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {
 }
 
 func (gv *promGaugeVec) Inc(labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Inc()
 }
 
 func (gv *promGaugeVec) Add(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Add(v)
 }
 
 func (gv *promGaugeVec) Set(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Set(v)
 }
 

core/metric/gauge_test.go

@@ -21,6 +21,7 @@ func TestNewGaugeVec(t *testing.T) {
 }
 
 func TestGaugeInc(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "rpc_client2",
 		Subsystem: "requests",
@@ -37,6 +38,7 @@ func TestGaugeInc(t *testing.T) {
 }
 
 func TestGaugeAdd(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "rpc_client",
 		Subsystem: "request",
@@ -53,6 +55,7 @@ func TestGaugeAdd(t *testing.T) {
 }
 
 func TestGaugeSet(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "http_client",
 		Subsystem: "request",

core/metric/histogram.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -53,6 +54,10 @@ func NewHistogramVec(cfg *HistogramVecOpts) HistogramVec {
 }
 
 func (hv *promHistogramVec) Observe(v int64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	hv.histogram.WithLabelValues(labels...).Observe(float64(v))
 }
 

core/metric/histogram_test.go

@@ -21,6 +21,7 @@ func TestNewHistogramVec(t *testing.T) {
 }
 
 func TestHistogramObserve(t *testing.T) {
+	startAgent()
 	histogramVec := NewHistogramVec(&HistogramVecOpts{
 		Name:    "counts",
 		Help:    "rpc server requests duration(ms).",

core/mr/mapreduce.go

@@ -3,12 +3,11 @@ package mr
 import (
 	"context"
 	"errors"
-	"fmt"
 	"sync"
+	"sync/atomic"
 
 	"github.com/zeromicro/go-zero/core/errorx"
 	"github.com/zeromicro/go-zero/core/lang"
-	"github.com/zeromicro/go-zero/core/threading"
 )
 
 const (
@@ -42,6 +41,16 @@ type (
 	// Option defines the method to customize the mapreduce.
 	Option func(opts *mapReduceOptions)
 
+	mapperContext struct {
+		ctx       context.Context
+		mapper    MapFunc
+		source    <-chan interface{}
+		panicChan *onceChan
+		collector chan<- interface{}
+		doneChan  <-chan lang.PlaceholderType
+		workers   int
+	}
+
 	mapReduceOptions struct {
 		ctx     context.Context
 		workers int
@@ -90,46 +99,72 @@ func FinishVoid(fns ...func()) {
 
 // ForEach maps all elements from given generate but no output.
 func ForEach(generate GenerateFunc, mapper ForEachFunc, opts ...Option) {
-	drain(Map(generate, func(item interface{}, writer Writer) {
-		mapper(item)
-	}, opts...))
-}
-
-// Map maps all elements generated from given generate func, and returns an output channel.
-func Map(generate GenerateFunc, mapper MapFunc, opts ...Option) chan interface{} {
 	options := buildOptions(opts...)
-	source := buildSource(generate)
-	collector := make(chan interface{}, options.workers)
+	panicChan := &onceChan{channel: make(chan interface{})}
+	source := buildSource(generate, panicChan)
+	collector := make(chan interface{})
 	done := make(chan lang.PlaceholderType)
 
-	go executeMappers(options.ctx, mapper, source, collector, done, options.workers)
+	go executeMappers(mapperContext{
+		ctx: options.ctx,
+		mapper: func(item interface{}, _ Writer) {
+			mapper(item)
+		},
+		source:    source,
+		panicChan: panicChan,
+		collector: collector,
+		doneChan:  done,
+		workers:   options.workers,
+	})
 
-	return collector
+	for {
+		select {
+		case v := <-panicChan.channel:
+			panic(v)
+		case _, ok := <-collector:
+			if !ok {
+				return
+			}
+		}
+	}
 }
 
 // MapReduce maps all elements generated from given generate func,
 // and reduces the output elements with given reducer.
 func MapReduce(generate GenerateFunc, mapper MapperFunc, reducer ReducerFunc,
 	opts ...Option) (interface{}, error) {
-	source := buildSource(generate)
-	return MapReduceChan(source, mapper, reducer, opts...)
+	panicChan := &onceChan{channel: make(chan interface{})}
+	source := buildSource(generate, panicChan)
+	return mapReduceWithPanicChan(source, panicChan, mapper, reducer, opts...)
 }
 
 // MapReduceChan maps all elements from source, and reduce the output elements with given reducer.
 func MapReduceChan(source <-chan interface{}, mapper MapperFunc, reducer ReducerFunc,
 	opts ...Option) (interface{}, error) {
+	panicChan := &onceChan{channel: make(chan interface{})}
+	return mapReduceWithPanicChan(source, panicChan, mapper, reducer, opts...)
+}
+
+// MapReduceChan maps all elements from source, and reduce the output elements with given reducer.
+func mapReduceWithPanicChan(source <-chan interface{}, panicChan *onceChan, mapper MapperFunc,
+	reducer ReducerFunc, opts ...Option) (interface{}, error) {
 	options := buildOptions(opts...)
+	// output is used to write the final result
 	output := make(chan interface{})
 	defer func() {
+		// reducer can only write once, if more, panic
 		for range output {
 			panic("more than one element written in reducer")
 		}
 	}()
 
+	// collector is used to collect data from mapper, and consume in reducer
 	collector := make(chan interface{}, options.workers)
+	// if done is closed, all mappers and reducer should stop processing
 	done := make(chan lang.PlaceholderType)
 	writer := newGuardedWriter(options.ctx, output, done)
 	var closeOnce sync.Once
+	// use atomic.Value to avoid data race
 	var retErr errorx.AtomicError
 	finish := func() {
 		closeOnce.Do(func() {
@@ -151,30 +186,40 @@ func MapReduceChan(source <-chan interface{}, mapper MapperFunc, reducer Reducer
 	go func() {
 		defer func() {
 			drain(collector)
-
 			if r := recover(); r != nil {
-				cancel(fmt.Errorf("%v", r))
-			} else {
-				finish()
+				panicChan.write(r)
 			}
+			finish()
 		}()
 
 		reducer(collector, writer, cancel)
 	}()
 
-	go executeMappers(options.ctx, func(item interface{}, w Writer) {
-		mapper(item, w, cancel)
-	}, source, collector, done, options.workers)
+	go executeMappers(mapperContext{
+		ctx: options.ctx,
+		mapper: func(item interface{}, w Writer) {
+			mapper(item, w, cancel)
+		},
+		source:    source,
+		panicChan: panicChan,
+		collector: collector,
+		doneChan:  done,
+		workers:   options.workers,
+	})
 
 	select {
 	case <-options.ctx.Done():
 		cancel(context.DeadlineExceeded)
 		return nil, context.DeadlineExceeded
-	case value, ok := <-output:
+	case v := <-panicChan.channel:
+		// drain output here, otherwise for loop panic in defer
+		drain(output)
+		panic(v)
+	case v, ok := <-output:
 		if err := retErr.Load(); err != nil {
 			return nil, err
 		} else if ok {
-			return value, nil
+			return v, nil
 		} else {
 			return nil, ErrReduceNoOutput
 		}
@@ -221,12 +266,18 @@ func buildOptions(opts ...Option) *mapReduceOptions {
 	return options
 }
 
-func buildSource(generate GenerateFunc) chan interface{} {
+func buildSource(generate GenerateFunc, panicChan *onceChan) chan interface{} {
 	source := make(chan interface{})
-	threading.GoSafe(func() {
-		defer close(source)
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				panicChan.write(r)
+			}
+			close(source)
+		}()
+
 		generate(source)
-	})
+	}()
 
 	return source
 }
@@ -238,39 +289,43 @@ func drain(channel <-chan interface{}) {
 	}
 }
 
-func executeMappers(ctx context.Context, mapper MapFunc, input <-chan interface{},
-	collector chan<- interface{}, done <-chan lang.PlaceholderType, workers int) {
+func executeMappers(mCtx mapperContext) {
 	var wg sync.WaitGroup
 	defer func() {
 		wg.Wait()
-		close(collector)
+		close(mCtx.collector)
+		drain(mCtx.source)
 	}()
 
-	pool := make(chan lang.PlaceholderType, workers)
-	writer := newGuardedWriter(ctx, collector, done)
-	for {
+	var failed int32
+	pool := make(chan lang.PlaceholderType, mCtx.workers)
+	writer := newGuardedWriter(mCtx.ctx, mCtx.collector, mCtx.doneChan)
+	for atomic.LoadInt32(&failed) == 0 {
 		select {
-		case <-ctx.Done():
+		case <-mCtx.ctx.Done():
 			return
-		case <-done:
+		case <-mCtx.doneChan:
 			return
 		case pool <- lang.Placeholder:
-			item, ok := <-input
+			item, ok := <-mCtx.source
 			if !ok {
 				<-pool
 				return
 			}
 
 			wg.Add(1)
-			// better to safely run caller defined method
-			threading.GoSafe(func() {
+			go func() {
 				defer func() {
+					if r := recover(); r != nil {
+						atomic.AddInt32(&failed, 1)
+						mCtx.panicChan.write(r)
+					}
 					wg.Done()
 					<-pool
 				}()
 
-				mapper(item, writer)
-			})
+				mCtx.mapper(item, writer)
+			}()
 		}
 	}
 }
@@ -316,3 +371,14 @@ func (gw guardedWriter) Write(v interface{}) {
 		gw.channel <- v
 	}
 }
+
+type onceChan struct {
+	channel chan interface{}
+	wrote   int32
+}
+
+func (oc *onceChan) write(val interface{}) {
+	if atomic.CompareAndSwapInt32(&oc.wrote, 0, 1) {
+		oc.channel <- val
+	}
+}

core/mr/mapreduce_fuzz_test.go

@@ -0,0 +1,78 @@
+//go:build go1.18
+// +build go1.18
+
+package mr
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/goleak"
+)
+
+func FuzzMapReduce(f *testing.F) {
+	rand.Seed(time.Now().UnixNano())
+
+	f.Add(uint(10), uint(runtime.NumCPU()))
+	f.Fuzz(func(t *testing.T, num uint, workers uint) {
+		n := int64(num)%5000 + 5000
+		genPanic := rand.Intn(100) == 0
+		mapperPanic := rand.Intn(100) == 0
+		reducerPanic := rand.Intn(100) == 0
+		genIdx := rand.Int63n(n)
+		mapperIdx := rand.Int63n(n)
+		reducerIdx := rand.Int63n(n)
+		squareSum := (n - 1) * n * (2*n - 1) / 6
+
+		fn := func() (interface{}, error) {
+			defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+
+			return MapReduce(func(source chan<- interface{}) {
+				for i := int64(0); i < n; i++ {
+					source <- i
+					if genPanic && i == genIdx {
+						panic("foo")
+					}
+				}
+			}, func(item interface{}, writer Writer, cancel func(error)) {
+				v := item.(int64)
+				if mapperPanic && v == mapperIdx {
+					panic("bar")
+				}
+				writer.Write(v * v)
+			}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+				var idx int64
+				var total int64
+				for v := range pipe {
+					if reducerPanic && idx == reducerIdx {
+						panic("baz")
+					}
+					total += v.(int64)
+					idx++
+				}
+				writer.Write(total)
+			}, WithWorkers(int(workers)%50+runtime.NumCPU()/2))
+		}
+
+		if genPanic || mapperPanic || reducerPanic {
+			var buf strings.Builder
+			buf.WriteString(fmt.Sprintf("n: %d", n))
+			buf.WriteString(fmt.Sprintf(", genPanic: %t", genPanic))
+			buf.WriteString(fmt.Sprintf(", mapperPanic: %t", mapperPanic))
+			buf.WriteString(fmt.Sprintf(", reducerPanic: %t", reducerPanic))
+			buf.WriteString(fmt.Sprintf(", genIdx: %d", genIdx))
+			buf.WriteString(fmt.Sprintf(", mapperIdx: %d", mapperIdx))
+			buf.WriteString(fmt.Sprintf(", reducerIdx: %d", reducerIdx))
+			assert.Panicsf(t, func() { fn() }, buf.String())
+		} else {
+			val, err := fn()
+			assert.Nil(t, err)
+			assert.Equal(t, squareSum, val.(int64))
+		}
+	})
+}

core/mr/mapreduce_fuzzcase_test.go

@@ -0,0 +1,107 @@
+//go:build fuzz
+// +build fuzz
+
+package mr
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/threading"
+	"gopkg.in/cheggaaa/pb.v1"
+)
+
+// If Fuzz stuck, we don't know why, because it only returns hung or unexpected,
+// so we need to simulate the fuzz test in test mode.
+func TestMapReduceRandom(t *testing.T) {
+	rand.Seed(time.Now().UnixNano())
+
+	const (
+		times  = 10000
+		nRange = 500
+		mega   = 1024 * 1024
+	)
+
+	bar := pb.New(times).Start()
+	runner := threading.NewTaskRunner(runtime.NumCPU())
+	var wg sync.WaitGroup
+	wg.Add(times)
+	for i := 0; i < times; i++ {
+		runner.Schedule(func() {
+			start := time.Now()
+			defer func() {
+				if time.Since(start) > time.Minute {
+					t.Fatal("timeout")
+				}
+				wg.Done()
+			}()
+
+			t.Run(strconv.Itoa(i), func(t *testing.T) {
+				n := rand.Int63n(nRange)%nRange + nRange
+				workers := rand.Int()%50 + runtime.NumCPU()/2
+				genPanic := rand.Intn(100) == 0
+				mapperPanic := rand.Intn(100) == 0
+				reducerPanic := rand.Intn(100) == 0
+				genIdx := rand.Int63n(n)
+				mapperIdx := rand.Int63n(n)
+				reducerIdx := rand.Int63n(n)
+				squareSum := (n - 1) * n * (2*n - 1) / 6
+
+				fn := func() (interface{}, error) {
+					return MapReduce(func(source chan<- interface{}) {
+						for i := int64(0); i < n; i++ {
+							source <- i
+							if genPanic && i == genIdx {
+								panic("foo")
+							}
+						}
+					}, func(item interface{}, writer Writer, cancel func(error)) {
+						v := item.(int64)
+						if mapperPanic && v == mapperIdx {
+							panic("bar")
+						}
+						writer.Write(v * v)
+					}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+						var idx int64
+						var total int64
+						for v := range pipe {
+							if reducerPanic && idx == reducerIdx {
+								panic("baz")
+							}
+							total += v.(int64)
+							idx++
+						}
+						writer.Write(total)
+					}, WithWorkers(int(workers)%50+runtime.NumCPU()/2))
+				}
+
+				if genPanic || mapperPanic || reducerPanic {
+					var buf strings.Builder
+					buf.WriteString(fmt.Sprintf("n: %d", n))
+					buf.WriteString(fmt.Sprintf(", genPanic: %t", genPanic))
+					buf.WriteString(fmt.Sprintf(", mapperPanic: %t", mapperPanic))
+					buf.WriteString(fmt.Sprintf(", reducerPanic: %t", reducerPanic))
+					buf.WriteString(fmt.Sprintf(", genIdx: %d", genIdx))
+					buf.WriteString(fmt.Sprintf(", mapperIdx: %d", mapperIdx))
+					buf.WriteString(fmt.Sprintf(", reducerIdx: %d", reducerIdx))
+					assert.Panicsf(t, func() { fn() }, buf.String())
+				} else {
+					val, err := fn()
+					assert.Nil(t, err)
+					assert.Equal(t, squareSum, val.(int64))
+				}
+				bar.Increment()
+			})
+		})
+	}
+
+	wg.Wait()
+	bar.Finish()
+}

core/mr/mapreduce_test.go

@@ -3,7 +3,7 @@ package mr
 import (
 	"context"
 	"errors"
-	"io/ioutil"
+	"io"
 	"log"
 	"runtime"
 	"sync/atomic"
@@ -11,15 +11,13 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/zeromicro/go-zero/core/stringx"
-	"github.com/zeromicro/go-zero/core/syncx"
 	"go.uber.org/goleak"
 )
 
 var errDummy = errors.New("dummy")
 
 func init() {
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 }
 
 func TestFinish(t *testing.T) {
@@ -124,84 +122,69 @@ func TestForEach(t *testing.T) {
 	t.Run("all", func(t *testing.T) {
 		defer goleak.VerifyNone(t)
 
-		ForEach(func(source chan<- interface{}) {
-			for i := 0; i < tasks; i++ {
-				source <- i
-			}
-		}, func(item interface{}) {
-			panic("foo")
+		assert.PanicsWithValue(t, "foo", func() {
+			ForEach(func(source chan<- interface{}) {
+				for i := 0; i < tasks; i++ {
+					source <- i
+				}
+			}, func(item interface{}) {
+				panic("foo")
+			})
 		})
 	})
 }
 
-func TestMap(t *testing.T) {
+func TestGeneratePanic(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
-	tests := []struct {
-		mapper MapFunc
-		expect int
-	}{
-		{
-			mapper: func(item interface{}, writer Writer) {
-				v := item.(int)
-				writer.Write(v * v)
-			},
-			expect: 30,
-		},
-		{
-			mapper: func(item interface{}, writer Writer) {
-				v := item.(int)
-				if v%2 == 0 {
-					return
-				}
-				writer.Write(v * v)
-			},
-			expect: 10,
-		},
-		{
-			mapper: func(item interface{}, writer Writer) {
-				v := item.(int)
-				if v%2 == 0 {
-					panic(v)
-				}
-				writer.Write(v * v)
-			},
-			expect: 10,
-		},
-	}
+	t.Run("all", func(t *testing.T) {
+		assert.PanicsWithValue(t, "foo", func() {
+			ForEach(func(source chan<- interface{}) {
+				panic("foo")
+			}, func(item interface{}) {
+			})
+		})
+	})
+}
 
-	for _, test := range tests {
-		t.Run(stringx.Rand(), func(t *testing.T) {
-			channel := Map(func(source chan<- interface{}) {
-				for i := 1; i < 5; i++ {
+func TestMapperPanic(t *testing.T) {
+	defer goleak.VerifyNone(t)
+
+	const tasks = 1000
+	var run int32
+	t.Run("all", func(t *testing.T) {
+		assert.PanicsWithValue(t, "foo", func() {
+			_, _ = MapReduce(func(source chan<- interface{}) {
+				for i := 0; i < tasks; i++ {
 					source <- i
 				}
-			}, test.mapper, WithWorkers(-1))
-
-			var result int
-			for v := range channel {
-				result += v.(int)
-			}
-
-			assert.Equal(t, test.expect, result)
+			}, func(item interface{}, writer Writer, cancel func(error)) {
+				atomic.AddInt32(&run, 1)
+				panic("foo")
+			}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+			})
 		})
-	}
+		assert.True(t, atomic.LoadInt32(&run) < tasks/2)
+	})
 }
 
 func TestMapReduce(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
 	tests := []struct {
+		name        string
 		mapper      MapperFunc
 		reducer     ReducerFunc
 		expectErr   error
 		expectValue interface{}
 	}{
 		{
+			name:        "simple",
 			expectErr:   nil,
 			expectValue: 30,
 		},
 		{
+			name: "cancel with error",
 			mapper: func(item interface{}, writer Writer, cancel func(error)) {
 				v := item.(int)
 				if v%3 == 0 {
@@ -212,6 +195,7 @@ func TestMapReduce(t *testing.T) {
 			expectErr: errDummy,
 		},
 		{
+			name: "cancel with nil",
 			mapper: func(item interface{}, writer Writer, cancel func(error)) {
 				v := item.(int)
 				if v%3 == 0 {
@@ -223,6 +207,7 @@ func TestMapReduce(t *testing.T) {
 			expectValue: nil,
 		},
 		{
+			name: "cancel with more",
 			reducer: func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
 				var result int
 				for item := range pipe {
@@ -237,45 +222,68 @@ func TestMapReduce(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
-		t.Run(stringx.Rand(), func(t *testing.T) {
-			if test.mapper == nil {
-				test.mapper = func(item interface{}, writer Writer, cancel func(error)) {
-					v := item.(int)
-					writer.Write(v * v)
-				}
-			}
-			if test.reducer == nil {
-				test.reducer = func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
-					var result int
-					for item := range pipe {
-						result += item.(int)
+	t.Run("MapReduce", func(t *testing.T) {
+		for _, test := range tests {
+			t.Run(test.name, func(t *testing.T) {
+				if test.mapper == nil {
+					test.mapper = func(item interface{}, writer Writer, cancel func(error)) {
+						v := item.(int)
+						writer.Write(v * v)
 					}
-					writer.Write(result)
 				}
-			}
-			value, err := MapReduce(func(source chan<- interface{}) {
-				for i := 1; i < 5; i++ {
-					source <- i
+				if test.reducer == nil {
+					test.reducer = func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+						var result int
+						for item := range pipe {
+							result += item.(int)
+						}
+						writer.Write(result)
+					}
 				}
-			}, test.mapper, test.reducer, WithWorkers(runtime.NumCPU()))
+				value, err := MapReduce(func(source chan<- interface{}) {
+					for i := 1; i < 5; i++ {
+						source <- i
+					}
+				}, test.mapper, test.reducer, WithWorkers(runtime.NumCPU()))
 
-			assert.Equal(t, test.expectErr, err)
-			assert.Equal(t, test.expectValue, value)
-		})
-	}
-}
+				assert.Equal(t, test.expectErr, err)
+				assert.Equal(t, test.expectValue, value)
+			})
+		}
+	})
 
-func TestMapReducePanicBothMapperAndReducer(t *testing.T) {
-	defer goleak.VerifyNone(t)
+	t.Run("MapReduce", func(t *testing.T) {
+		for _, test := range tests {
+			t.Run(test.name, func(t *testing.T) {
+				if test.mapper == nil {
+					test.mapper = func(item interface{}, writer Writer, cancel func(error)) {
+						v := item.(int)
+						writer.Write(v * v)
+					}
+				}
+				if test.reducer == nil {
+					test.reducer = func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+						var result int
+						for item := range pipe {
+							result += item.(int)
+						}
+						writer.Write(result)
+					}
+				}
 
-	_, _ = MapReduce(func(source chan<- interface{}) {
-		source <- 0
-		source <- 1
-	}, func(item interface{}, writer Writer, cancel func(error)) {
-		panic("foo")
-	}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
-		panic("bar")
+				source := make(chan interface{})
+				go func() {
+					for i := 1; i < 5; i++ {
+						source <- i
+					}
+					close(source)
+				}()
+
+				value, err := MapReduceChan(source, test.mapper, test.reducer, WithWorkers(-1))
+				assert.Equal(t, test.expectErr, err)
+				assert.Equal(t, test.expectValue, value)
+			})
+		}
 	})
 }
 
@@ -302,16 +310,19 @@ func TestMapReduceVoid(t *testing.T) {
 
 	var value uint32
 	tests := []struct {
+		name        string
 		mapper      MapperFunc
 		reducer     VoidReducerFunc
 		expectValue uint32
 		expectErr   error
 	}{
 		{
+			name:        "simple",
 			expectValue: 30,
 			expectErr:   nil,
 		},
 		{
+			name: "cancel with error",
 			mapper: func(item interface{}, writer Writer, cancel func(error)) {
 				v := item.(int)
 				if v%3 == 0 {
@@ -322,6 +333,7 @@ func TestMapReduceVoid(t *testing.T) {
 			expectErr: errDummy,
 		},
 		{
+			name: "cancel with nil",
 			mapper: func(item interface{}, writer Writer, cancel func(error)) {
 				v := item.(int)
 				if v%3 == 0 {
@@ -332,6 +344,7 @@ func TestMapReduceVoid(t *testing.T) {
 			expectErr: ErrCancelWithNil,
 		},
 		{
+			name: "cancel with more",
 			reducer: func(pipe <-chan interface{}, cancel func(error)) {
 				for item := range pipe {
 					result := atomic.AddUint32(&value, uint32(item.(int)))
@@ -345,7 +358,7 @@ func TestMapReduceVoid(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		t.Run(stringx.Rand(), func(t *testing.T) {
+		t.Run(test.name, func(t *testing.T) {
 			atomic.StoreUint32(&value, 0)
 
 			if test.mapper == nil {
@@ -400,39 +413,59 @@ func TestMapReduceVoidWithDelay(t *testing.T) {
 	assert.Equal(t, 0, result[1])
 }
 
-func TestMapVoid(t *testing.T) {
-	defer goleak.VerifyNone(t)
-
-	const tasks = 1000
-	var count uint32
-	ForEach(func(source chan<- interface{}) {
-		for i := 0; i < tasks; i++ {
-			source <- i
-		}
-	}, func(item interface{}) {
-		atomic.AddUint32(&count, 1)
-	})
-
-	assert.Equal(t, tasks, int(count))
-}
-
 func TestMapReducePanic(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
-	v, err := MapReduce(func(source chan<- interface{}) {
-		source <- 0
-		source <- 1
-	}, func(item interface{}, writer Writer, cancel func(error)) {
-		i := item.(int)
-		writer.Write(i)
-	}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
-		for range pipe {
-			panic("panic")
-		}
+	assert.Panics(t, func() {
+		_, _ = MapReduce(func(source chan<- interface{}) {
+			source <- 0
+			source <- 1
+		}, func(item interface{}, writer Writer, cancel func(error)) {
+			i := item.(int)
+			writer.Write(i)
+		}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+			for range pipe {
+				panic("panic")
+			}
+		})
+	})
+}
+
+func TestMapReducePanicOnce(t *testing.T) {
+	defer goleak.VerifyNone(t)
+
+	assert.Panics(t, func() {
+		_, _ = MapReduce(func(source chan<- interface{}) {
+			for i := 0; i < 100; i++ {
+				source <- i
+			}
+		}, func(item interface{}, writer Writer, cancel func(error)) {
+			i := item.(int)
+			if i == 0 {
+				panic("foo")
+			}
+			writer.Write(i)
+		}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+			for range pipe {
+				panic("bar")
+			}
+		})
+	})
+}
+
+func TestMapReducePanicBothMapperAndReducer(t *testing.T) {
+	defer goleak.VerifyNone(t)
+
+	assert.Panics(t, func() {
+		_, _ = MapReduce(func(source chan<- interface{}) {
+			source <- 0
+			source <- 1
+		}, func(item interface{}, writer Writer, cancel func(error)) {
+			panic("foo")
+		}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+			panic("bar")
+		})
 	})
-	assert.Nil(t, v)
-	assert.NotNil(t, err)
-	assert.Equal(t, "panic", err.Error())
 }
 
 func TestMapReduceVoidCancel(t *testing.T) {
@@ -461,13 +494,13 @@ func TestMapReduceVoidCancel(t *testing.T) {
 func TestMapReduceVoidCancelWithRemains(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
-	var done syncx.AtomicBool
+	var done int32
 	var result []int
 	err := MapReduceVoid(func(source chan<- interface{}) {
 		for i := 0; i < defaultWorkers*2; i++ {
 			source <- i
 		}
-		done.Set(true)
+		atomic.AddInt32(&done, 1)
 	}, func(item interface{}, writer Writer, cancel func(error)) {
 		i := item.(int)
 		if i == defaultWorkers/2 {
@@ -482,7 +515,7 @@ func TestMapReduceVoidCancelWithRemains(t *testing.T) {
 	})
 	assert.NotNil(t, err)
 	assert.Equal(t, "anything", err.Error())
-	assert.True(t, done.True())
+	assert.Equal(t, int32(1), done)
 }
 
 func TestMapReduceWithoutReducerWrite(t *testing.T) {
@@ -507,34 +540,51 @@ func TestMapReduceVoidPanicInReducer(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
 	const message = "foo"
-	var done syncx.AtomicBool
-	err := MapReduceVoid(func(source chan<- interface{}) {
+	assert.Panics(t, func() {
+		var done int32
+		_ = MapReduceVoid(func(source chan<- interface{}) {
+			for i := 0; i < defaultWorkers*2; i++ {
+				source <- i
+			}
+			atomic.AddInt32(&done, 1)
+		}, func(item interface{}, writer Writer, cancel func(error)) {
+			i := item.(int)
+			writer.Write(i)
+		}, func(pipe <-chan interface{}, cancel func(error)) {
+			panic(message)
+		}, WithWorkers(1))
+	})
+}
+
+func TestForEachWithContext(t *testing.T) {
+	defer goleak.VerifyNone(t)
+
+	var done int32
+	ctx, cancel := context.WithCancel(context.Background())
+	ForEach(func(source chan<- interface{}) {
 		for i := 0; i < defaultWorkers*2; i++ {
 			source <- i
 		}
-		done.Set(true)
-	}, func(item interface{}, writer Writer, cancel func(error)) {
+		atomic.AddInt32(&done, 1)
+	}, func(item interface{}) {
 		i := item.(int)
-		writer.Write(i)
-	}, func(pipe <-chan interface{}, cancel func(error)) {
-		panic(message)
-	}, WithWorkers(1))
-	assert.NotNil(t, err)
-	assert.Equal(t, message, err.Error())
-	assert.True(t, done.True())
+		if i == defaultWorkers/2 {
+			cancel()
+		}
+	}, WithContext(ctx))
 }
 
 func TestMapReduceWithContext(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
-	var done syncx.AtomicBool
+	var done int32
 	var result []int
 	ctx, cancel := context.WithCancel(context.Background())
 	err := MapReduceVoid(func(source chan<- interface{}) {
 		for i := 0; i < defaultWorkers*2; i++ {
 			source <- i
 		}
-		done.Set(true)
+		atomic.AddInt32(&done, 1)
 	}, func(item interface{}, writer Writer, c func(error)) {
 		i := item.(int)
 		if i == defaultWorkers/2 {

core/mr/readme-cn.md

@@ -54,7 +54,7 @@ import (
     "fmt"
     "log"
 
-    "github.com/tal-tech/go-zero/core/mr"
+    "github.com/zeromicro/go-zero/core/mr"
 )
 
 func main() {

core/mr/readme.md

@@ -55,7 +55,7 @@ import (
     "fmt"
     "log"
 
-    "github.com/tal-tech/go-zero/core/mr"
+    "github.com/zeromicro/go-zero/core/mr"
 )
 
 func main() {
@@ -87,4 +87,4 @@ More examples: [https://github.com/zeromicro/zero-examples/tree/main/mapreduce](
 
 ## Give a Star! ⭐
 
-If you like or are using this project to learn or start your solution, please give it a star. Thanks!
+If you like or are using this project to learn or start your solution, please give it a star. Thanks!

core/proc/goroutines_test.go

@@ -1,16 +1,23 @@
 package proc
 
 import (
-	"log"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/logx"
 )
 
 func TestDumpGoroutines(t *testing.T) {
 	var buf strings.Builder
-	log.SetOutput(&buf)
+	w := logx.NewWriter(&buf)
+	o := logx.Reset()
+	logx.SetWriter(w)
+	defer func() {
+		logx.Reset()
+		logx.SetWriter(o)
+	}()
+
 	dumpGoroutines()
 	assert.True(t, strings.Contains(buf.String(), ".dump"))
 }

core/proc/profile_test.go

@@ -1,16 +1,24 @@
 package proc
 
 import (
-	"log"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/logx"
 )
 
 func TestProfile(t *testing.T) {
 	var buf strings.Builder
-	log.SetOutput(&buf)
+	w := logx.NewWriter(&buf)
+	o := logx.Reset()
+	logx.SetWriter(w)
+
+	defer func() {
+		logx.Reset()
+		logx.SetWriter(o)
+	}()
+
 	profiler := StartProfile()
 	// start again should not work
 	assert.NotNil(t, StartProfile())

core/proc/shutdown_test.go

@@ -1,3 +1,6 @@
+//go:build linux || darwin
+// +build linux darwin
+
 package proc
 
 import (

core/prof/runtime.go

@@ -0,0 +1,31 @@
+package prof
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+)
+
+const (
+	defaultInterval = time.Second * 5
+	mega            = 1024 * 1024
+)
+
+// DisplayStats prints the goroutine, memory, GC stats with given interval, default to 5 seconds.
+func DisplayStats(interval ...time.Duration) {
+	duration := defaultInterval
+	for _, val := range interval {
+		duration = val
+	}
+
+	go func() {
+		ticker := time.NewTicker(duration)
+		defer ticker.Stop()
+		for range ticker.C {
+			var m runtime.MemStats
+			runtime.ReadMemStats(&m)
+			fmt.Printf("Goroutines: %d, Alloc: %vm, TotalAlloc: %vm, Sys: %vm, NumGC: %v\n",
+				runtime.NumGoroutine(), m.Alloc/mega, m.TotalAlloc/mega, m.Sys/mega, m.NumGC)
+		}
+	}()
+}

core/service/serviceconf.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/zeromicro/go-zero/core/load"
 	"github.com/zeromicro/go-zero/core/logx"
+	"github.com/zeromicro/go-zero/core/proc"
 	"github.com/zeromicro/go-zero/core/prometheus"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/trace"
@@ -56,6 +57,9 @@ func (sc ServiceConf) SetUp() error {
 		sc.Telemetry.Name = sc.Name
 	}
 	trace.StartAgent(sc.Telemetry)
+	proc.AddShutdownListener(func() {
+		trace.StopAgent()
+	})
 
 	if len(sc.MetricsUrl) > 0 {
 		stat.SetReportWriter(stat.NewRemoteWriter(sc.MetricsUrl))

core/stat/alert.go

@@ -15,7 +15,6 @@ import (
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/proc"
 	"github.com/zeromicro/go-zero/core/sysx"
-	"github.com/zeromicro/go-zero/core/timex"
 )
 
 const (
@@ -47,7 +46,7 @@ func Report(msg string) {
 	if fn != nil {
 		reported := lessExecutor.DoOrDiscard(func() {
 			var builder strings.Builder
-			fmt.Fprintf(&builder, "%s\n", timex.Time().Format(timeFormat))
+			fmt.Fprintf(&builder, "%s\n", time.Now().Format(timeFormat))
 			if len(clusterName) > 0 {
 				fmt.Fprintf(&builder, "cluster: %s\n", clusterName)
 			}

core/stat/internal/cgroup_linux.go

@@ -1,78 +1,129 @@
 package internal
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"path"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/zeromicro/go-zero/core/iox"
 	"github.com/zeromicro/go-zero/core/lang"
+	"golang.org/x/sys/unix"
 )
 
-const cgroupDir = "/sys/fs/cgroup"
+const (
+	cgroupDir   = "/sys/fs/cgroup"
+	cpuStatFile = cgroupDir + "/cpu.stat"
+	cpusetFile  = cgroupDir + "/cpuset.cpus.effective"
+)
 
-type cgroup struct {
+var (
+	isUnifiedOnce sync.Once
+	isUnified     bool
+	inUserNS      bool
+	nsOnce        sync.Once
+)
+
+type cgroup interface {
+	cpuQuotaUs() (int64, error)
+	cpuPeriodUs() (uint64, error)
+	cpus() ([]uint64, error)
+	usageAllCpus() (uint64, error)
+}
+
+func currentCgroup() (cgroup, error) {
+	if isCgroup2UnifiedMode() {
+		return currentCgroupV2()
+	}
+
+	return currentCgroupV1()
+}
+
+type cgroupV1 struct {
 	cgroups map[string]string
 }
 
-func (c *cgroup) acctUsageAllCpus() (uint64, error) {
-	data, err := iox.ReadText(path.Join(c.cgroups["cpuacct"], "cpuacct.usage"))
-	if err != nil {
-		return 0, err
-	}
-
-	return parseUint(string(data))
-}
-
-func (c *cgroup) acctUsagePerCpu() ([]uint64, error) {
-	data, err := iox.ReadText(path.Join(c.cgroups["cpuacct"], "cpuacct.usage_percpu"))
-	if err != nil {
-		return nil, err
-	}
-
-	var usage []uint64
-	for _, v := range strings.Fields(string(data)) {
-		u, err := parseUint(v)
-		if err != nil {
-			return nil, err
-		}
-
-		usage = append(usage, u)
-	}
-
-	return usage, nil
-}
-
-func (c *cgroup) cpuQuotaUs() (int64, error) {
+func (c *cgroupV1) cpuQuotaUs() (int64, error) {
 	data, err := iox.ReadText(path.Join(c.cgroups["cpu"], "cpu.cfs_quota_us"))
 	if err != nil {
 		return 0, err
 	}
 
-	return strconv.ParseInt(string(data), 10, 64)
+	return strconv.ParseInt(data, 10, 64)
 }
 
-func (c *cgroup) cpuPeriodUs() (uint64, error) {
+func (c *cgroupV1) cpuPeriodUs() (uint64, error) {
 	data, err := iox.ReadText(path.Join(c.cgroups["cpu"], "cpu.cfs_period_us"))
 	if err != nil {
 		return 0, err
 	}
 
-	return parseUint(string(data))
+	return parseUint(data)
 }
 
-func (c *cgroup) cpus() ([]uint64, error) {
+func (c *cgroupV1) cpus() ([]uint64, error) {
 	data, err := iox.ReadText(path.Join(c.cgroups["cpuset"], "cpuset.cpus"))
 	if err != nil {
 		return nil, err
 	}
 
-	return parseUints(string(data))
+	return parseUints(data)
 }
 
-func currentCgroup() (*cgroup, error) {
+func (c *cgroupV1) usageAllCpus() (uint64, error) {
+	data, err := iox.ReadText(path.Join(c.cgroups["cpuacct"], "cpuacct.usage"))
+	if err != nil {
+		return 0, err
+	}
+
+	return parseUint(data)
+}
+
+type cgroupV2 struct {
+	cgroups map[string]string
+}
+
+func (c *cgroupV2) cpuQuotaUs() (int64, error) {
+	data, err := iox.ReadText(path.Join(cgroupDir, "cpu.cfs_quota_us"))
+	if err != nil {
+		return 0, err
+	}
+
+	return strconv.ParseInt(data, 10, 64)
+}
+
+func (c *cgroupV2) cpuPeriodUs() (uint64, error) {
+	data, err := iox.ReadText(path.Join(cgroupDir, "cpu.cfs_period_us"))
+	if err != nil {
+		return 0, err
+	}
+
+	return parseUint(data)
+}
+
+func (c *cgroupV2) cpus() ([]uint64, error) {
+	data, err := iox.ReadText(cpusetFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return parseUints(data)
+}
+
+func (c *cgroupV2) usageAllCpus() (uint64, error) {
+	usec, err := parseUint(c.cgroups["usage_usec"])
+	if err != nil {
+		return 0, err
+	}
+
+	return usec * uint64(time.Microsecond), nil
+}
+
+func currentCgroupV1() (cgroup, error) {
 	cgroupFile := fmt.Sprintf("/proc/%d/cgroup", os.Getpid())
 	lines, err := iox.ReadTextLines(cgroupFile, iox.WithoutBlank())
 	if err != nil {
@@ -100,11 +151,51 @@ func currentCgroup() (*cgroup, error) {
 		}
 	}
 
-	return &cgroup{
+	return &cgroupV1{
 		cgroups: cgroups,
 	}, nil
 }
 
+func currentCgroupV2() (cgroup, error) {
+	lines, err := iox.ReadTextLines(cpuStatFile, iox.WithoutBlank())
+	if err != nil {
+		return nil, err
+	}
+
+	cgroups := make(map[string]string)
+	for _, line := range lines {
+		cols := strings.Fields(line)
+		if len(cols) != 2 {
+			return nil, fmt.Errorf("invalid cgroupV2 line: %s", line)
+		}
+
+		cgroups[cols[0]] = cols[1]
+	}
+
+	return &cgroupV2{
+		cgroups: cgroups,
+	}, nil
+}
+
+// isCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode.
+func isCgroup2UnifiedMode() bool {
+	isUnifiedOnce.Do(func() {
+		var st unix.Statfs_t
+		err := unix.Statfs(cgroupDir, &st)
+		if err != nil {
+			if os.IsNotExist(err) && runningInUserNS() {
+				// ignore the "not found" error if running in userns
+				isUnified = false
+				return
+			}
+			panic(fmt.Sprintf("cannot statfs cgroup root: %s", err))
+		}
+		isUnified = st.Type == unix.CGROUP2_SUPER_MAGIC
+	})
+
+	return isUnified
+}
+
 func parseUint(s string) (uint64, error) {
 	v, err := strconv.ParseInt(s, 10, 64)
 	if err != nil {
@@ -166,3 +257,36 @@ func parseUints(val string) ([]uint64, error) {
 
 	return sets, nil
 }
+
+// runningInUserNS detects whether we are currently running in a user namespace.
+func runningInUserNS() bool {
+	nsOnce.Do(func() {
+		file, err := os.Open("/proc/self/uid_map")
+		if err != nil {
+			// This kernel-provided file only exists if user namespaces are supported
+			return
+		}
+		defer file.Close()
+
+		buf := bufio.NewReader(file)
+		l, _, err := buf.ReadLine()
+		if err != nil {
+			return
+		}
+
+		line := string(l)
+		var a, b, c int64
+		fmt.Sscanf(line, "%d %d %d", &a, &b, &c)
+
+		/*
+		 * We assume we are in the initial user namespace if we have a full
+		 * range - 4294967295 uids starting at uid 0.
+		 */
+		if a == 0 && b == 0 && c == 4294967295 {
+			return
+		}
+		inUserNS = true
+	})
+
+	return inUserNS
+}

core/stat/internal/cpu_linux.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/zeromicro/go-zero/core/iox"
@@ -20,11 +21,12 @@ var (
 	preTotal  uint64
 	quota     float64
 	cores     uint64
+	initOnce  sync.Once
 )
 
 // if /proc not present, ignore the cpu calculation, like wsl linux
-func init() {
-	cpus, err := perCpuUsage()
+func initialize() {
+	cpus, err := cpuSets()
 	if err != nil {
 		logx.Error(err)
 		return
@@ -69,10 +71,13 @@ func init() {
 
 // RefreshCpu refreshes cpu usage and returns.
 func RefreshCpu() uint64 {
+	initOnce.Do(initialize)
+
 	total, err := totalCpuUsage()
 	if err != nil {
 		return 0
 	}
+
 	system, err := systemCpuUsage()
 	if err != nil {
 		return 0
@@ -117,15 +122,6 @@ func cpuSets() ([]uint64, error) {
 	return cg.cpus()
 }
 
-func perCpuUsage() ([]uint64, error) {
-	cg, err := currentCgroup()
-	if err != nil {
-		return nil, err
-	}
-
-	return cg.acctUsagePerCpu()
-}
-
 func systemCpuUsage() (uint64, error) {
 	lines, err := iox.ReadTextLines("/proc/stat", iox.WithoutBlank())
 	if err != nil {
@@ -157,10 +153,10 @@ func systemCpuUsage() (uint64, error) {
 }
 
 func totalCpuUsage() (usage uint64, err error) {
-	var cg *cgroup
+	var cg cgroup
 	if cg, err = currentCgroup(); err != nil {
 		return
 	}
 
-	return cg.acctUsageAllCpus()
+	return cg.usageAllCpus()
 }

core/stat/remotewriter.go

@@ -10,7 +10,10 @@ import (
 	"github.com/zeromicro/go-zero/core/logx"
 )
 
-const httpTimeout = time.Second * 5
+const (
+	httpTimeout     = time.Second * 5
+	jsonContentType = "application/json; charset=utf-8"
+)
 
 // ErrWriteFailed is an error that indicates failed to submit a StatReport.
 var ErrWriteFailed = errors.New("submit failed")
@@ -36,7 +39,7 @@ func (rw *RemoteWriter) Write(report *StatReport) error {
 	client := &http.Client{
 		Timeout: httpTimeout,
 	}
-	resp, err := client.Post(rw.endpoint, "application/json", bytes.NewBuffer(bs))
+	resp, err := client.Post(rw.endpoint, jsonContentType, bytes.NewReader(bs))
 	if err != nil {
 		return err
 	}