feat: add logger.WithFields (#2546)

* chore: refactor

* chore: refactor

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"

.github/workflows/codeql-analysis.yml

@@ -35,11 +35,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/go.yml

@@ -12,13 +12,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: ^1.15
+          go-version: ^1.16
         id: go
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Get dependencies
         run: |
@@ -34,19 +34,19 @@ jobs:
         run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
 
       - name: Codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
 
   test-win:
     name: Windows
     runs-on: windows-latest
     steps:
       - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: ^1.15
+          go-version: ^1.16
 
       - name: Checkout codebase
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Test
         run: |

.github/workflows/issue-translator.yml

@@ -9,7 +9,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: tomsun28/issues-translate-action@v2.6
+      - uses: usthe/issues-translate-action@v2.7
         with:
           IS_MODIFY_TITLE: true
           # not require, default false, . Decide whether to modify the issue title

.github/workflows/issues.yml

@@ -7,10 +7,10 @@ jobs:
   close-issues:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v6
         with:
-          days-before-issue-stale: 30
-          days-before-issue-close: 14
+          days-before-issue-stale: 365
+          days-before-issue-close: 90
           stale-issue-label: "stale"
           stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
           close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."

.github/workflows/release.yaml

@@ -16,7 +16,7 @@ jobs:
           - goarch: "386"
             goos: darwin
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: zeromicro/go-zero-release-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
@@ -25,4 +25,4 @@ jobs:
           goversion: "https://dl.google.com/go/go1.17.5.linux-amd64.tar.gz"
           project_path: "tools/goctl"
           binary_name: "goctl"
-          extra_files: tools/goctl/goctl.md
+          extra_files: tools/goctl/readme.md tools/goctl/readme-cn.md

.github/workflows/reviewdog.yml

@@ -5,7 +5,7 @@ jobs:
     name: runner / staticcheck
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: reviewdog/action-staticcheck@v1
         with:
           github_token: ${{ secrets.github_token }}

.gitignore

@@ -22,6 +22,7 @@ go.work.sum
 
 # gitlab ci
 .cache
+.golangci.yml
 
 # vim auto backup file
 *~

core/codec/rsa.go

@@ -7,7 +7,7 @@ import (
 	"encoding/base64"
 	"encoding/pem"
 	"errors"
-	"io/ioutil"
+	"os"
 )
 
 var (
@@ -48,7 +48,7 @@ type (
 
 // NewRsaDecrypter returns a RsaDecrypter with the given file.
 func NewRsaDecrypter(file string) (RsaDecrypter, error) {
-	content, err := ioutil.ReadFile(file)
+	content, err := os.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}

core/collection/cache.go

@@ -26,7 +26,7 @@ type (
 	// CacheOption defines the method to customize a Cache.
 	CacheOption func(cache *Cache)
 
-	// A Cache object is a in-memory cache.
+	// A Cache object is an in-memory cache.
 	Cache struct {
 		name           string
 		lock           sync.Mutex

core/collection/safemap.go

@@ -68,6 +68,24 @@ func (m *SafeMap) Get(key interface{}) (interface{}, bool) {
 	return val, ok
 }
 
+// Range calls f sequentially for each key and value present in the map.
+// If f returns false, range stops the iteration.
+func (m *SafeMap) Range(f func(key, val interface{}) bool) {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	for k, v := range m.dirtyOld {
+		if !f(k, v) {
+			return
+		}
+	}
+	for k, v := range m.dirtyNew {
+		if !f(k, v) {
+			return
+		}
+	}
+}
+
 // Set sets the value into m with the given key.
 func (m *SafeMap) Set(key, value interface{}) {
 	m.lock.Lock()

core/collection/safemap_test.go

@@ -1,6 +1,7 @@
 package collection
 
 import (
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -107,3 +108,42 @@ func testSafeMapWithParameters(t *testing.T, size, exception int) {
 		}
 	}
 }
+
+func TestSafeMap_Range(t *testing.T) {
+	const (
+		size       = 100000
+		exception1 = 5
+		exception2 = 500
+	)
+
+	m := NewSafeMap()
+	newMap := NewSafeMap()
+
+	for i := 0; i < size; i++ {
+		m.Set(i, i)
+	}
+	for i := 0; i < size; i++ {
+		if i%exception1 == 0 {
+			m.Del(i)
+		}
+	}
+
+	for i := size; i < size<<1; i++ {
+		m.Set(i, i)
+	}
+	for i := size; i < size<<1; i++ {
+		if i%exception2 != 0 {
+			m.Del(i)
+		}
+	}
+
+	var count int32
+	m.Range(func(k, v interface{}) bool {
+		atomic.AddInt32(&count, 1)
+		newMap.Set(k, v)
+		return true
+	})
+	assert.Equal(t, int(atomic.LoadInt32(&count)), m.Size())
+	assert.Equal(t, m.dirtyNew, newMap.dirtyNew)
+	assert.Equal(t, m.dirtyOld, newMap.dirtyOld)
+}

core/conf/config.go

@@ -2,7 +2,6 @@ package conf
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -20,7 +19,7 @@ var loaders = map[string]func([]byte, interface{}) error{
 
 // Load loads config into v from file, .json, .yaml and .yml are acceptable.
 func Load(file string, v interface{}, opts ...Option) error {
-	content, err := ioutil.ReadFile(file)
+	content, err := os.ReadFile(file)
 	if err != nil {
 		return err
 	}

core/conf/config_test.go

@@ -1,7 +1,6 @@
 package conf
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -106,7 +105,6 @@ d = "abcd!@#112"
 	assert.Equal(t, 1, val.B)
 	assert.Equal(t, "2", val.C)
 	assert.Equal(t, "abcd!@#112", val.D)
-
 }
 
 func TestConfigJsonEnv(t *testing.T) {
@@ -146,12 +144,12 @@ func TestConfigJsonEnv(t *testing.T) {
 }
 
 func createTempFile(ext, text string) (string, error) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
+	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
 	if err != nil {
 		return "", err
 	}
 
-	if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+	if err := os.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
 		return "", err
 	}
 

core/conf/readme.md

@@ -3,10 +3,10 @@
 1. Define a config structure, like below:
 
 ```go
-RestfulConf struct {
+type RestfulConf struct {
 	Host         string        `json:",default=0.0.0.0"`
 	Port         int
-	LogMode      string        `json:",options=[file,console]"
+	LogMode      string        `json:",options=[file,console]"`
 	Verbose      bool          `json:",optional"`
 	MaxConns     int           `json:",default=10000"`
 	MaxBytes     int64         `json:",default=1048576"`
@@ -15,7 +15,9 @@ RestfulConf struct {
 }
 ```
 
-2. Write the yaml or json config file:
+2. Write the yaml, toml or json config file:
+
+- yaml example
 
 ```yaml
 # most fields are optional or have default values
@@ -25,6 +27,16 @@ LogMode: console
 MaxBytes: ${MAX_BYTES}
 ```
 
+- toml example
+
+```toml
+# most fields are optional or have default values
+Port = 8_080
+LogMode = "console"
+# you can use env settings
+MaxBytes = "${MAX_BYTES}"
+```
+
 3. Load the config from a file:
 
 ```go

core/discov/internal/accountmanager.go

@@ -3,7 +3,7 @@ package internal
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"io/ioutil"
+	"os"
 	"sync"
 )
 
@@ -37,7 +37,7 @@ func AddTLS(endpoints []string, certFile, certKeyFile, caFile string, insecureSk
 		return err
 	}
 
-	caData, err := ioutil.ReadFile(caFile)
+	caData, err := os.ReadFile(caFile)
 	if err != nil {
 		return err
 	}

core/discov/internal/registry.go

@@ -191,9 +191,11 @@ func (c *cluster) handleWatchEvents(key string, events []*clientv3.Event) {
 				})
 			}
 		case clientv3.EventTypeDelete:
+			c.lock.Lock()
 			if vals, ok := c.values[key]; ok {
 				delete(vals, string(ev.Kv.Key))
 			}
+			c.lock.Unlock()
 			for _, l := range listeners {
 				l.OnDelete(KV{
 					Key: string(ev.Kv.Key),
@@ -206,7 +208,7 @@ func (c *cluster) handleWatchEvents(key string, events []*clientv3.Event) {
 	}
 }
 
-func (c *cluster) load(cli EtcdClient, key string) {
+func (c *cluster) load(cli EtcdClient, key string) int64 {
 	var resp *clientv3.GetResponse
 	for {
 		var err error
@@ -230,6 +232,8 @@ func (c *cluster) load(cli EtcdClient, key string) {
 	}
 
 	c.handleChanges(key, kvs)
+
+	return resp.Header.Revision
 }
 
 func (c *cluster) monitor(key string, l UpdateListener) error {
@@ -242,9 +246,9 @@ func (c *cluster) monitor(key string, l UpdateListener) error {
 		return err
 	}
 
-	c.load(cli, key)
+	rev := c.load(cli, key)
 	c.watchGroup.Run(func() {
-		c.watch(cli, key)
+		c.watch(cli, key, rev)
 	})
 
 	return nil
@@ -276,22 +280,29 @@ func (c *cluster) reload(cli EtcdClient) {
 	for _, key := range keys {
 		k := key
 		c.watchGroup.Run(func() {
-			c.load(cli, k)
-			c.watch(cli, k)
+			rev := c.load(cli, k)
+			c.watch(cli, k, rev)
 		})
 	}
 }
 
-func (c *cluster) watch(cli EtcdClient, key string) {
+func (c *cluster) watch(cli EtcdClient, key string, rev int64) {
 	for {
-		if c.watchStream(cli, key) {
+		if c.watchStream(cli, key, rev) {
 			return
 		}
 	}
 }
 
-func (c *cluster) watchStream(cli EtcdClient, key string) bool {
-	rch := cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
+func (c *cluster) watchStream(cli EtcdClient, key string, rev int64) bool {
+	var rch clientv3.WatchChan
+	if rev != 0 {
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix(),
+			clientv3.WithRev(rev+1))
+	} else {
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
+	}
+
 	for {
 		select {
 		case wresp, ok := <-rch:
@@ -332,6 +343,7 @@ func DialClient(endpoints []string) (EtcdClient, error) {
 		DialKeepAliveTime:    dialKeepAliveTime,
 		DialKeepAliveTimeout: DialTimeout,
 		RejectOldCluster:     true,
+		PermitWithoutStream:  true,
 	}
 	if account, ok := GetAccount(endpoints); ok {
 		cfg.Username = account.User

core/discov/internal/registry_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/zeromicro/go-zero/core/lang"
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/stringx"
+	"go.etcd.io/etcd/api/v3/etcdserverpb"
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
@@ -112,6 +113,7 @@ func TestCluster_Load(t *testing.T) {
 	restore := setMockClient(cli)
 	defer restore()
 	cli.EXPECT().Get(gomock.Any(), "any/", gomock.Any()).Return(&clientv3.GetResponse{
+		Header: &etcdserverpb.ResponseHeader{},
 		Kvs: []*mvccpb.KeyValue{
 			{
 				Key:   []byte("hello"),
@@ -168,7 +170,7 @@ func TestCluster_Watch(t *testing.T) {
 			listener.EXPECT().OnDelete(gomock.Any()).Do(func(_ interface{}) {
 				wg.Done()
 			}).MaxTimes(1)
-			go c.watch(cli, "any")
+			go c.watch(cli, "any", 0)
 			ch <- clientv3.WatchResponse{
 				Events: []*clientv3.Event{
 					{
@@ -212,7 +214,7 @@ func TestClusterWatch_RespFailures(t *testing.T) {
 				ch <- resp
 				close(c.done)
 			}()
-			c.watch(cli, "any")
+			c.watch(cli, "any", 0)
 		})
 	}
 }
@@ -232,7 +234,7 @@ func TestClusterWatch_CloseChan(t *testing.T) {
 		close(ch)
 		close(c.done)
 	}()
-	c.watch(cli, "any")
+	c.watch(cli, "any", 0)
 }
 
 func TestValueOnlyContext(t *testing.T) {

core/errorx/wrap.go

@@ -0,0 +1,21 @@
+package errorx
+
+import "fmt"
+
+// Wrap returns an error that wraps err with given message.
+func Wrap(err error, message string) error {
+	if err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("%s: %w", message, err)
+}
+
+// Wrapf returns an error that wraps err with given format and args.
+func Wrapf(err error, format string, args ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), err)
+}

core/errorx/wrap_test.go

@@ -0,0 +1,24 @@
+package errorx
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWrap(t *testing.T) {
+	assert.Nil(t, Wrap(nil, "test"))
+	assert.Equal(t, "foo: bar", Wrap(errors.New("bar"), "foo").Error())
+
+	err := errors.New("foo")
+	assert.True(t, errors.Is(Wrap(err, "bar"), err))
+}
+
+func TestWrapf(t *testing.T) {
+	assert.Nil(t, Wrapf(nil, "%s", "test"))
+	assert.Equal(t, "foo bar: quz", Wrapf(errors.New("quz"), "foo %s", "bar").Error())
+
+	err := errors.New("foo")
+	assert.True(t, errors.Is(Wrapf(err, "foo %s", "bar"), err))
+}

core/fs/temps.go

@@ -1,7 +1,6 @@
 package fs
 
 import (
-	"io/ioutil"
 	"os"
 
 	"github.com/zeromicro/go-zero/core/hash"
@@ -12,12 +11,12 @@ import (
 // The file is kept as open, the caller should close the file handle,
 // and remove the file by name.
 func TempFileWithText(text string) (*os.File, error) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text)))
+	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text)))
 	if err != nil {
 		return nil, err
 	}
 
-	if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+	if err := os.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
 		return nil, err
 	}
 

core/fs/temps_test.go

@@ -1,7 +1,7 @@
 package fs
 
 import (
-	"io/ioutil"
+	"io"
 	"os"
 	"testing"
 
@@ -21,7 +21,7 @@ func TestTempFileWithText(t *testing.T) {
 	}
 	defer os.Remove(f.Name())
 
-	bs, err := ioutil.ReadAll(f)
+	bs, err := io.ReadAll(f)
 	assert.Nil(t, err)
 	if len(bs) != 4 {
 		t.Error("TempFileWithText returned wrong file size")
@@ -41,7 +41,7 @@ func TestTempFilenameWithText(t *testing.T) {
 	}
 	defer os.Remove(f)
 
-	bs, err := ioutil.ReadFile(f)
+	bs, err := os.ReadFile(f)
 	assert.Nil(t, err)
 	if len(bs) != 4 {
 		t.Error("TempFilenameWithText returned wrong file size")

core/fx/stream_test.go

@@ -1,7 +1,7 @@
 package fx
 
 import (
-	"io/ioutil"
+	"io"
 	"log"
 	"math/rand"
 	"reflect"
@@ -238,7 +238,7 @@ func TestLast(t *testing.T) {
 
 func TestMap(t *testing.T) {
 	runCheckedTest(t, func(t *testing.T) {
-		log.SetOutput(ioutil.Discard)
+		log.SetOutput(io.Discard)
 
 		tests := []struct {
 			mapper MapFunc

core/iox/read.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 )
@@ -26,7 +25,7 @@ type (
 func DupReadCloser(reader io.ReadCloser) (io.ReadCloser, io.ReadCloser) {
 	var buf bytes.Buffer
 	tee := io.TeeReader(reader, &buf)
-	return ioutil.NopCloser(tee), ioutil.NopCloser(&buf)
+	return io.NopCloser(tee), io.NopCloser(&buf)
 }
 
 // KeepSpace customizes the reading functions to keep leading and tailing spaces.
@@ -54,7 +53,7 @@ func ReadBytes(reader io.Reader, buf []byte) error {
 
 // ReadText reads content from the given file with leading and tailing spaces trimmed.
 func ReadText(filename string) (string, error) {
-	content, err := ioutil.ReadFile(filename)
+	content, err := os.ReadFile(filename)
 	if err != nil {
 		return "", err
 	}

core/iox/read_test.go

@@ -3,7 +3,6 @@ package iox
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -97,10 +96,10 @@ func TestReadTextLines(t *testing.T) {
 
 func TestDupReadCloser(t *testing.T) {
 	input := "hello"
-	reader := ioutil.NopCloser(bytes.NewBufferString(input))
+	reader := io.NopCloser(bytes.NewBufferString(input))
 	r1, r2 := DupReadCloser(reader)
 	verify := func(r io.Reader) {
-		output, err := ioutil.ReadAll(r)
+		output, err := io.ReadAll(r)
 		assert.Nil(t, err)
 		assert.Equal(t, input, string(output))
 	}
@@ -110,7 +109,7 @@ func TestDupReadCloser(t *testing.T) {
 }
 
 func TestReadBytes(t *testing.T) {
-	reader := ioutil.NopCloser(bytes.NewBufferString("helloworld"))
+	reader := io.NopCloser(bytes.NewBufferString("helloworld"))
 	buf := make([]byte, 5)
 	err := ReadBytes(reader, buf)
 	assert.Nil(t, err)
@@ -118,7 +117,7 @@ func TestReadBytes(t *testing.T) {
 }
 
 func TestReadBytesNotEnough(t *testing.T) {
-	reader := ioutil.NopCloser(bytes.NewBufferString("hell"))
+	reader := io.NopCloser(bytes.NewBufferString("hell"))
 	buf := make([]byte, 5)
 	err := ReadBytes(reader, buf)
 	assert.Equal(t, io.EOF, err)

core/iox/textfile_test.go

@@ -1,7 +1,6 @@
 package iox
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -13,7 +12,7 @@ func TestCountLines(t *testing.T) {
 2
 3
 4`
-	file, err := ioutil.TempFile(os.TempDir(), "test-")
+	file, err := os.CreateTemp(os.TempDir(), "test-")
 	if err != nil {
 		t.Fatal(err)
 	}

core/jsonx/json.go

@@ -13,6 +13,16 @@ func Marshal(v interface{}) ([]byte, error) {
 	return json.Marshal(v)
 }
 
+// MarshalToString marshals v into a string.
+func MarshalToString(v interface{}) (string, error) {
+	data, err := Marshal(v)
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
+}
+
 // Unmarshal unmarshals data bytes into v.
 func Unmarshal(data []byte, v interface{}) error {
 	decoder := json.NewDecoder(bytes.NewReader(data))

core/jsonx/json_test.go

@@ -20,6 +20,22 @@ func TestMarshal(t *testing.T) {
 	assert.Equal(t, `{"name":"John","age":30}`, string(bs))
 }
 
+func TestMarshalToString(t *testing.T) {
+	var v = struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}{
+		Name: "John",
+		Age:  30,
+	}
+	toString, err := MarshalToString(v)
+	assert.Nil(t, err)
+	assert.Equal(t, `{"name":"John","age":30}`, toString)
+
+	_, err = MarshalToString(make(chan int))
+	assert.NotNil(t, err)
+}
+
 func TestUnmarshal(t *testing.T) {
 	const s = `{"name":"John","age":30}`
 	var v struct {

core/limit/periodlimit.go

@@ -1,6 +1,7 @@
 package limit
 
 import (
+	"context"
 	"errors"
 	"strconv"
 	"time"
@@ -74,7 +75,12 @@ func NewPeriodLimit(period, quota int, limitStore *redis.Redis, keyPrefix string
 
 // Take requests a permit, it returns the permit state.
 func (h *PeriodLimit) Take(key string) (int, error) {
-	resp, err := h.limitStore.Eval(periodScript, []string{h.keyPrefix + key}, []string{
+	return h.TakeCtx(context.Background(), key)
+}
+
+// TakeCtx requests a permit with context, it returns the permit state.
+func (h *PeriodLimit) TakeCtx(ctx context.Context, key string) (int, error) {
+	resp, err := h.limitStore.EvalCtx(ctx, periodScript, []string{h.keyPrefix + key}, []string{
 		strconv.Itoa(h.quota),
 		strconv.Itoa(h.calcExpireSeconds()),
 	})

core/limit/tokenlimit.go

@@ -1,6 +1,8 @@
 package limit
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"strconv"
 	"sync"
@@ -58,8 +60,8 @@ type TokenLimiter struct {
 	timestampKey   string
 	rescueLock     sync.Mutex
 	redisAlive     uint32
-	rescueLimiter  *xrate.Limiter
 	monitorStarted bool
+	rescueLimiter  *xrate.Limiter
 }
 
 // NewTokenLimiter returns a new TokenLimiter that allows events up to rate and permits
@@ -84,19 +86,31 @@ func (lim *TokenLimiter) Allow() bool {
 	return lim.AllowN(time.Now(), 1)
 }
 
+// AllowCtx is shorthand for AllowNCtx(ctx,time.Now(), 1) with incoming context.
+func (lim *TokenLimiter) AllowCtx(ctx context.Context) bool {
+	return lim.AllowNCtx(ctx, time.Now(), 1)
+}
+
 // AllowN reports whether n events may happen at time now.
 // Use this method if you intend to drop / skip events that exceed the rate.
 // Otherwise, use Reserve or Wait.
 func (lim *TokenLimiter) AllowN(now time.Time, n int) bool {
-	return lim.reserveN(now, n)
+	return lim.reserveN(context.Background(), now, n)
 }
 
-func (lim *TokenLimiter) reserveN(now time.Time, n int) bool {
+// AllowNCtx reports whether n events may happen at time now with incoming context.
+// Use this method if you intend to drop / skip events that exceed the rate.
+// Otherwise, use Reserve or Wait.
+func (lim *TokenLimiter) AllowNCtx(ctx context.Context, now time.Time, n int) bool {
+	return lim.reserveN(ctx, now, n)
+}
+
+func (lim *TokenLimiter) reserveN(ctx context.Context, now time.Time, n int) bool {
 	if atomic.LoadUint32(&lim.redisAlive) == 0 {
 		return lim.rescueLimiter.AllowN(now, n)
 	}
 
-	resp, err := lim.store.Eval(
+	resp, err := lim.store.EvalCtx(ctx,
 		script,
 		[]string{
 			lim.tokenKey,
@@ -113,6 +127,10 @@ func (lim *TokenLimiter) reserveN(now time.Time, n int) bool {
 	if err == redis.Nil {
 		return false
 	}
+	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+		logx.Errorf("fail to use rate limiter: %s", err)
+		return false
+	}
 	if err != nil {
 		logx.Errorf("fail to use rate limiter: %s, use in-process limiter for rescue", err)
 		lim.startMonitor()

core/limit/tokenlimit_test.go

@@ -1,6 +1,7 @@
 package limit
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -15,6 +16,30 @@ func init() {
 	logx.Disable()
 }
 
+func TestTokenLimit_WithCtx(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+
+	const (
+		total = 100
+		rate  = 5
+		burst = 10
+	)
+	l := NewTokenLimiter(rate, burst, redis.New(s.Addr()), "tokenlimit")
+	defer s.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	ok := l.AllowCtx(ctx)
+	assert.True(t, ok)
+
+	cancel()
+	for i := 0; i < total; i++ {
+		ok := l.AllowCtx(ctx)
+		assert.False(t, ok)
+		assert.False(t, l.monitorStarted)
+	}
+}
+
 func TestTokenLimit_Rescue(t *testing.T) {
 	s, err := miniredis.Run()
 	assert.Nil(t, err)

core/load/adaptiveshedder.go

@@ -17,7 +17,7 @@ import (
 const (
 	defaultBuckets = 50
 	defaultWindow  = time.Second * 5
-	// using 1000m notation, 900m is like 80%, keep it as var for unit test
+	// using 1000m notation, 900m is like 90%, keep it as var for unit test
 	defaultCpuThreshold = 900
 	defaultMinRt        = float64(time.Second / time.Millisecond)
 	// moving average hyperparameter beta for calculating requests on the fly
@@ -70,7 +70,7 @@ type (
 		flying          int64
 		avgFlying       float64
 		avgFlyingLock   syncx.SpinLock
-		dropTime        *syncx.AtomicDuration
+		overloadTime    *syncx.AtomicDuration
 		droppedRecently *syncx.AtomicBool
 		passCounter     *collection.RollingWindow
 		rtCounter       *collection.RollingWindow
@@ -106,7 +106,7 @@ func NewAdaptiveShedder(opts ...ShedderOption) Shedder {
 	return &adaptiveShedder{
 		cpuThreshold:    options.cpuThreshold,
 		windows:         int64(time.Second / bucketDuration),
-		dropTime:        syncx.NewAtomicDuration(),
+		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.NewAtomicBool(),
 		passCounter: collection.NewRollingWindow(options.buckets, bucketDuration,
 			collection.IgnoreCurrentBucket()),
@@ -118,7 +118,6 @@ func NewAdaptiveShedder(opts ...ShedderOption) Shedder {
 // Allow implements Shedder.Allow.
 func (as *adaptiveShedder) Allow() (Promise, error) {
 	if as.shouldDrop() {
-		as.dropTime.Set(timex.Now())
 		as.droppedRecently.Set(true)
 
 		return nil, ErrServiceOverloaded
@@ -215,21 +214,26 @@ func (as *adaptiveShedder) stillHot() bool {
 		return false
 	}
 
-	dropTime := as.dropTime.Load()
-	if dropTime == 0 {
+	overloadTime := as.overloadTime.Load()
+	if overloadTime == 0 {
 		return false
 	}
 
-	hot := timex.Since(dropTime) < coolOffDuration
-	if !hot {
-		as.droppedRecently.Set(false)
+	if timex.Since(overloadTime) < coolOffDuration {
+		return true
 	}
 
-	return hot
+	as.droppedRecently.Set(false)
+	return false
 }
 
 func (as *adaptiveShedder) systemOverloaded() bool {
-	return systemOverloadChecker(as.cpuThreshold)
+	if !systemOverloadChecker(as.cpuThreshold) {
+		return false
+	}
+
+	as.overloadTime.Set(timex.Now())
+	return true
 }
 
 // WithBuckets customizes the Shedder with given number of buckets.

core/load/adaptiveshedder_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/zeromicro/go-zero/core/mathx"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/syncx"
+	"github.com/zeromicro/go-zero/core/timex"
 )
 
 const (
@@ -136,7 +137,7 @@ func TestAdaptiveShedderShouldDrop(t *testing.T) {
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
 		windows:         buckets,
-		dropTime:        syncx.NewAtomicDuration(),
+		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.NewAtomicBool(),
 	}
 	// cpu >=  800, inflight < maxPass
@@ -190,12 +191,15 @@ func TestAdaptiveShedderStillHot(t *testing.T) {
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
 		windows:         buckets,
-		dropTime:        syncx.NewAtomicDuration(),
+		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.ForAtomicBool(true),
 	}
 	assert.False(t, shedder.stillHot())
-	shedder.dropTime.Set(-coolOffDuration * 2)
+	shedder.overloadTime.Set(-coolOffDuration * 2)
 	assert.False(t, shedder.stillHot())
+	shedder.droppedRecently.Set(true)
+	shedder.overloadTime.Set(timex.Now())
+	assert.True(t, shedder.stillHot())
 }
 
 func BenchmarkAdaptiveShedder_Allow(b *testing.B) {

core/logc/logs.go

@@ -0,0 +1,122 @@
+package logc
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+type (
+	LogConf  = logx.LogConf
+	LogField = logx.LogField
+)
+
+// AddGlobalFields adds global fields.
+func AddGlobalFields(fields ...LogField) {
+	logx.AddGlobalFields(fields...)
+}
+
+// Alert alerts v in alert level, and the message is written to error log.
+func Alert(_ context.Context, v string) {
+	logx.Alert(v)
+}
+
+// Close closes the logging.
+func Close() error {
+	return logx.Close()
+}
+
+// Error writes v into error log.
+func Error(ctx context.Context, v ...interface{}) {
+	getLogger(ctx).Error(v...)
+}
+
+// Errorf writes v with format into error log.
+func Errorf(ctx context.Context, format string, v ...interface{}) {
+	getLogger(ctx).Errorf(fmt.Errorf(format, v...).Error())
+}
+
+// Errorv writes v into error log with json content.
+// No call stack attached, because not elegant to pack the messages.
+func Errorv(ctx context.Context, v interface{}) {
+	getLogger(ctx).Errorv(v)
+}
+
+// Errorw writes msg along with fields into error log.
+func Errorw(ctx context.Context, msg string, fields ...LogField) {
+	getLogger(ctx).Errorw(msg, fields...)
+}
+
+// Field returns a LogField for the given key and value.
+func Field(key string, value interface{}) LogField {
+	return logx.Field(key, value)
+}
+
+// Info writes v into access log.
+func Info(ctx context.Context, v ...interface{}) {
+	getLogger(ctx).Info(v...)
+}
+
+// Infof writes v with format into access log.
+func Infof(ctx context.Context, format string, v ...interface{}) {
+	getLogger(ctx).Infof(format, v...)
+}
+
+// Infov writes v into access log with json content.
+func Infov(ctx context.Context, v interface{}) {
+	getLogger(ctx).Infov(v)
+}
+
+// Infow writes msg along with fields into access log.
+func Infow(ctx context.Context, msg string, fields ...LogField) {
+	getLogger(ctx).Infow(msg, fields...)
+}
+
+// Must checks if err is nil, otherwise logs the error and exits.
+func Must(err error) {
+	logx.Must(err)
+}
+
+// MustSetup sets up logging with given config c. It exits on error.
+func MustSetup(c logx.LogConf) {
+	logx.MustSetup(c)
+}
+
+// SetLevel sets the logging level. It can be used to suppress some logs.
+func SetLevel(level uint32) {
+	logx.SetLevel(level)
+}
+
+// SetUp sets up the logx. If already set up, just return nil.
+// we allow SetUp to be called multiple times, because for example
+// we need to allow different service frameworks to initialize logx respectively.
+// the same logic for SetUp
+func SetUp(c LogConf) error {
+	return logx.SetUp(c)
+}
+
+// Slow writes v into slow log.
+func Slow(ctx context.Context, v ...interface{}) {
+	getLogger(ctx).Slow(v...)
+}
+
+// Slowf writes v with format into slow log.
+func Slowf(ctx context.Context, format string, v ...interface{}) {
+	getLogger(ctx).Slowf(format, v...)
+}
+
+// Slowv writes v into slow log with json content.
+func Slowv(ctx context.Context, v interface{}) {
+	getLogger(ctx).Slowv(v)
+}
+
+// Sloww writes msg along with fields into slow log.
+func Sloww(ctx context.Context, msg string, fields ...LogField) {
+	getLogger(ctx).Sloww(msg, fields...)
+}
+
+// getLogger returns the logx.Logger with the given ctx and correct caller.
+func getLogger(ctx context.Context) logx.Logger {
+	return logx.WithContext(ctx).WithCallerSkip(1)
+}

core/logc/logs_test.go

@@ -0,0 +1,218 @@
+package logc
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+func TestAddGlobalFields(t *testing.T) {
+	var buf bytes.Buffer
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	Info(context.Background(), "hello")
+	buf.Reset()
+
+	AddGlobalFields(Field("a", "1"), Field("b", "2"))
+	AddGlobalFields(Field("c", "3"))
+	Info(context.Background(), "world")
+	var m map[string]interface{}
+	assert.NoError(t, json.Unmarshal(buf.Bytes(), &m))
+	assert.Equal(t, "1", m["a"])
+	assert.Equal(t, "2", m["b"])
+	assert.Equal(t, "3", m["c"])
+}
+
+func TestAlert(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	Alert(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), "foo"), buf.String())
+}
+
+func TestError(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Error(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestErrorf(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Errorf(context.Background(), "foo %s", "bar")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestErrorv(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Errorv(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestErrorw(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Errorw(context.Background(), "foo", Field("a", "b"))
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfo(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Info(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfof(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Infof(context.Background(), "foo %s", "bar")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfov(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Infov(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfow(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Infow(context.Background(), "foo", Field("a", "b"))
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestMust(t *testing.T) {
+	assert.NotPanics(t, func() {
+		Must(nil)
+	})
+	assert.NotPanics(t, func() {
+		MustSetup(LogConf{})
+	})
+}
+
+func TestMisc(t *testing.T) {
+	SetLevel(logx.DebugLevel)
+	assert.NoError(t, SetUp(LogConf{}))
+	assert.NoError(t, Close())
+}
+
+func TestSlow(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Slow(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func TestSlowf(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Slowf(context.Background(), "foo %s", "bar")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func TestSlowv(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Slowv(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func TestSloww(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Sloww(context.Background(), "foo", Field("a", "b"))
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func getFileLine() (string, int) {
+	_, file, line, _ := runtime.Caller(1)
+	short := file
+
+	for i := len(file) - 1; i > 0; i-- {
+		if file[i] == '/' {
+			short = file[i+1:]
+			break
+		}
+	}
+
+	return short, line
+}

core/logx/config.go

@@ -7,8 +7,20 @@ type LogConf struct {
 	Encoding            string `json:",default=json,options=[json,plain]"`
 	TimeFormat          string `json:",optional"`
 	Path                string `json:",default=logs"`
-	Level               string `json:",default=info,options=[info,error,severe]"`
+	Level               string `json:",default=info,options=[debug,info,error,severe]"`
 	Compress            bool   `json:",optional"`
 	KeepDays            int    `json:",optional"`
 	StackCooldownMillis int    `json:",default=100"`
+	// MaxBackups represents how many backup log files will be kept. 0 means all files will be kept forever.
+	// Only take effect when RotationRuleType is `size`.
+	// Even thougth `MaxBackups` sets 0, log files will still be removed
+	// if the `KeepDays` limitation is reached.
+	MaxBackups int `json:",default=0"`
+	// MaxSize represents how much space the writing log file takes up. 0 means no limit. The unit is `MB`.
+	// Only take effect when RotationRuleType is `size`
+	MaxSize int `json:",default=0"`
+	// RotationRuleType represents the type of log rotation rule. Default is `daily`.
+	// daily: daily rotation.
+	// size: size limited rotation.
+	Rotation string `json:",default=daily,options=[daily,size]"`
 }

core/logx/durationlogger.go

@@ -1,101 +0,0 @@
-package logx
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/zeromicro/go-zero/core/timex"
-)
-
-// WithDuration returns a Logger which logs the given duration.
-func WithDuration(d time.Duration) Logger {
-	return &durationLogger{
-		Duration: timex.ReprOfDuration(d),
-	}
-}
-
-type durationLogger logEntry
-
-func (l *durationLogger) Error(v ...interface{}) {
-	l.err(fmt.Sprint(v...))
-}
-
-func (l *durationLogger) Errorf(format string, v ...interface{}) {
-	l.err(fmt.Sprintf(format, v...))
-}
-
-func (l *durationLogger) Errorv(v interface{}) {
-	l.err(v)
-}
-
-func (l *durationLogger) Errorw(msg string, fields ...LogField) {
-	l.err(msg, fields...)
-}
-
-func (l *durationLogger) Info(v ...interface{}) {
-	l.info(fmt.Sprint(v...))
-}
-
-func (l *durationLogger) Infof(format string, v ...interface{}) {
-	l.info(fmt.Sprintf(format, v...))
-}
-
-func (l *durationLogger) Infov(v interface{}) {
-	l.info(v)
-}
-
-func (l *durationLogger) Infow(msg string, fields ...LogField) {
-	l.info(msg, fields...)
-}
-
-func (l *durationLogger) Slow(v ...interface{}) {
-	l.slow(fmt.Sprint(v...))
-}
-
-func (l *durationLogger) Slowf(format string, v ...interface{}) {
-	l.slow(fmt.Sprintf(format, v...))
-}
-
-func (l *durationLogger) Slowv(v interface{}) {
-	l.slow(v)
-}
-
-func (l *durationLogger) Sloww(msg string, fields ...LogField) {
-	l.slow(msg, fields...)
-}
-
-func (l *durationLogger) WithContext(ctx context.Context) Logger {
-	return &traceLogger{
-		ctx: ctx,
-		logEntry: logEntry{
-			Duration: l.Duration,
-		},
-	}
-}
-
-func (l *durationLogger) WithDuration(duration time.Duration) Logger {
-	l.Duration = timex.ReprOfDuration(duration)
-	return l
-}
-
-func (l *durationLogger) err(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		fields = append(fields, Field(durationKey, l.Duration))
-		getWriter().Error(v, fields...)
-	}
-}
-
-func (l *durationLogger) info(v interface{}, fields ...LogField) {
-	if shallLog(InfoLevel) {
-		fields = append(fields, Field(durationKey, l.Duration))
-		getWriter().Info(v, fields...)
-	}
-}
-
-func (l *durationLogger) slow(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		fields = append(fields, Field(durationKey, l.Duration))
-		getWriter().Slow(v, fields...)
-	}
-}

core/logx/durationlogger_test.go

@@ -1,161 +0,0 @@
-package logx
-
-import (
-	"context"
-	"strings"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"go.opentelemetry.io/otel"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-)
-
-func TestWithDurationError(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Error("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationErrorf(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Errorf("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationErrorv(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Errorv("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationErrorw(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Errorw("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-}
-
-func TestWithDurationInfo(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Info("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationInfoConsole(t *testing.T) {
-	old := atomic.LoadUint32(&encoding)
-	atomic.StoreUint32(&encoding, plainEncodingType)
-	defer func() {
-		atomic.StoreUint32(&encoding, old)
-	}()
-
-	w := new(mockWriter)
-	o := writer.Swap(w)
-	defer writer.Store(o)
-
-	WithDuration(time.Second).Info("foo")
-	assert.True(t, strings.Contains(w.String(), "ms"), w.String())
-}
-
-func TestWithDurationInfof(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Infof("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationInfov(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Infov("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationInfow(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Infow("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-}
-
-func TestWithDurationWithContextInfow(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	otp := otel.GetTracerProvider()
-	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
-	otel.SetTracerProvider(tp)
-	defer otel.SetTracerProvider(otp)
-
-	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
-	WithDuration(time.Second).WithContext(ctx).Infow("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-	assert.True(t, strings.Contains(w.String(), "trace"), w.String())
-	assert.True(t, strings.Contains(w.String(), "span"), w.String())
-}
-
-func TestWithDurationSlow(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Slow("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationSlowf(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).WithDuration(time.Hour).Slowf("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationSlowv(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).WithDuration(time.Hour).Slowv("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationSloww(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).WithDuration(time.Hour).Sloww("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-}

core/logx/fields.go

@@ -0,0 +1,45 @@
+package logx
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+)
+
+var (
+	fieldsContextKey contextKey
+	globalFields     atomic.Value
+	globalFieldsLock sync.Mutex
+)
+
+type contextKey struct{}
+
+// AddGlobalFields adds global fields.
+func AddGlobalFields(fields ...LogField) {
+	globalFieldsLock.Lock()
+	defer globalFieldsLock.Unlock()
+
+	old := globalFields.Load()
+	if old == nil {
+		globalFields.Store(append([]LogField(nil), fields...))
+	} else {
+		globalFields.Store(append(old.([]LogField), fields...))
+	}
+}
+
+// ContextWithFields returns a new context with the given fields.
+func ContextWithFields(ctx context.Context, fields ...LogField) context.Context {
+	if val := ctx.Value(fieldsContextKey); val != nil {
+		if arr, ok := val.([]LogField); ok {
+			return context.WithValue(ctx, fieldsContextKey, append(arr, fields...))
+		}
+	}
+
+	return context.WithValue(ctx, fieldsContextKey, fields)
+}
+
+// WithFields returns a new logger with the given fields.
+// deprecated: use ContextWithFields instead.
+func WithFields(ctx context.Context, fields ...LogField) context.Context {
+	return ContextWithFields(ctx, fields...)
+}

core/logx/fields_test.go

@@ -0,0 +1,104 @@
+package logx
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAddGlobalFields(t *testing.T) {
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	old := Reset()
+	SetWriter(writer)
+	defer SetWriter(old)
+
+	Info("hello")
+	buf.Reset()
+
+	AddGlobalFields(Field("a", "1"), Field("b", "2"))
+	AddGlobalFields(Field("c", "3"))
+	Info("world")
+	var m map[string]interface{}
+	assert.NoError(t, json.Unmarshal(buf.Bytes(), &m))
+	assert.Equal(t, "1", m["a"])
+	assert.Equal(t, "2", m["b"])
+	assert.Equal(t, "3", m["c"])
+}
+
+func TestContextWithFields(t *testing.T) {
+	ctx := ContextWithFields(context.Background(), Field("a", 1), Field("b", 2))
+	vals := ctx.Value(fieldsContextKey)
+	assert.NotNil(t, vals)
+	fields, ok := vals.([]LogField)
+	assert.True(t, ok)
+	assert.EqualValues(t, []LogField{Field("a", 1), Field("b", 2)}, fields)
+}
+
+func TestWithFields(t *testing.T) {
+	ctx := WithFields(context.Background(), Field("a", 1), Field("b", 2))
+	vals := ctx.Value(fieldsContextKey)
+	assert.NotNil(t, vals)
+	fields, ok := vals.([]LogField)
+	assert.True(t, ok)
+	assert.EqualValues(t, []LogField{Field("a", 1), Field("b", 2)}, fields)
+}
+
+func TestWithFieldsAppend(t *testing.T) {
+	var dummyKey struct{}
+	ctx := context.WithValue(context.Background(), dummyKey, "dummy")
+	ctx = ContextWithFields(ctx, Field("a", 1), Field("b", 2))
+	ctx = ContextWithFields(ctx, Field("c", 3), Field("d", 4))
+	vals := ctx.Value(fieldsContextKey)
+	assert.NotNil(t, vals)
+	fields, ok := vals.([]LogField)
+	assert.True(t, ok)
+	assert.Equal(t, "dummy", ctx.Value(dummyKey))
+	assert.EqualValues(t, []LogField{
+		Field("a", 1),
+		Field("b", 2),
+		Field("c", 3),
+		Field("d", 4),
+	}, fields)
+}
+
+func BenchmarkAtomicValue(b *testing.B) {
+	b.ReportAllocs()
+
+	var container atomic.Value
+	vals := []LogField{
+		Field("a", "b"),
+		Field("c", "d"),
+		Field("e", "f"),
+	}
+	container.Store(&vals)
+
+	for i := 0; i < b.N; i++ {
+		val := container.Load()
+		if val != nil {
+			_ = *val.(*[]LogField)
+		}
+	}
+}
+
+func BenchmarkRWMutex(b *testing.B) {
+	b.ReportAllocs()
+
+	var lock sync.RWMutex
+	vals := []LogField{
+		Field("a", "b"),
+		Field("c", "d"),
+		Field("e", "f"),
+	}
+
+	for i := 0; i < b.N; i++ {
+		lock.RLock()
+		_ = vals
+		lock.RUnlock()
+	}
+}

core/logx/logger.go

@@ -7,6 +7,14 @@ import (
 
 // A Logger represents a logger.
 type Logger interface {
+	// Debug logs a message at info level.
+	Debug(...interface{})
+	// Debugf logs a message at info level.
+	Debugf(string, ...interface{})
+	// Debugv logs a message at info level.
+	Debugv(interface{})
+	// Debugw logs a message at info level.
+	Debugw(string, ...LogField)
 	// Error logs a message at error level.
 	Error(...interface{})
 	// Errorf logs a message at error level.
@@ -31,8 +39,12 @@ type Logger interface {
 	Slowv(interface{})
 	// Sloww logs a message at slow level.
 	Sloww(string, ...LogField)
+	// WithCallerSkip returns a new logger with the given caller skip.
+	WithCallerSkip(skip int) Logger
 	// WithContext returns a new logger with the given context.
-	WithContext(context.Context) Logger
+	WithContext(ctx context.Context) Logger
 	// WithDuration returns a new logger with the given duration.
-	WithDuration(time.Duration) Logger
+	WithDuration(d time.Duration) Logger
+	// WithFields returns a new logger with the given fields.
+	WithFields(fields ...LogField) Logger
 }

core/logx/logs.go

@@ -7,42 +7,28 @@ import (
 	"os"
 	"path"
 	"runtime/debug"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/zeromicro/go-zero/core/sysx"
 )
 
-const callerDepth = 5
+const callerDepth = 4
 
 var (
 	timeFormat = "2006-01-02T15:04:05.000Z07:00"
 	logLevel   uint32
 	encoding   uint32 = jsonEncodingType
 	// use uint32 for atomic operations
+	disableLog  uint32
 	disableStat uint32
-
-	options logOptions
-	writer  = new(atomicWriter)
+	options     logOptions
+	writer      = new(atomicWriter)
+	setupOnce   sync.Once
 )
 
 type (
-	logEntry struct {
-		Timestamp string      `json:"@timestamp"`
-		Level     string      `json:"level"`
-		Duration  string      `json:"duration,omitempty"`
-		Caller    string      `json:"caller,omitempty"`
-		Content   interface{} `json:"content"`
-	}
-
-	logEntryWithFields map[string]interface{}
-
-	logOptions struct {
-		gzipEnabled           bool
-		logStackCooldownMills int
-		keepDays              int
-	}
-
 	// LogField is a key-value pair that will be added to the log entry.
 	LogField struct {
 		Key   string
@@ -51,6 +37,17 @@ type (
 
 	// LogOption defines the method to customize the logging.
 	LogOption func(options *logOptions)
+
+	logEntry map[string]interface{}
+
+	logOptions struct {
+		gzipEnabled           bool
+		logStackCooldownMills int
+		keepDays              int
+		maxBackups            int
+		maxSize               int
+		rotationRule          string
+	}
 )
 
 // Alert alerts v in alert level, and the message is written to error log.
@@ -67,8 +64,29 @@ func Close() error {
 	return nil
 }
 
+// Debug writes v into access log.
+func Debug(v ...interface{}) {
+	writeDebug(fmt.Sprint(v...))
+}
+
+// Debugf writes v with format into access log.
+func Debugf(format string, v ...interface{}) {
+	writeDebug(fmt.Sprintf(format, v...))
+}
+
+// Debugv writes v into access log with json content.
+func Debugv(v interface{}) {
+	writeDebug(v)
+}
+
+// Debugw writes msg along with fields into access log.
+func Debugw(msg string, fields ...LogField) {
+	writeDebug(msg, fields...)
+}
+
 // Disable disables the logging.
 func Disable() {
+	atomic.StoreUint32(&disableLog, 1)
 	writer.Store(nopWriter{})
 }
 
@@ -79,35 +97,35 @@ func DisableStat() {
 
 // Error writes v into error log.
 func Error(v ...interface{}) {
-	errorTextSync(fmt.Sprint(v...))
+	writeError(fmt.Sprint(v...))
 }
 
 // Errorf writes v with format into error log.
 func Errorf(format string, v ...interface{}) {
-	errorTextSync(fmt.Errorf(format, v...).Error())
+	writeError(fmt.Errorf(format, v...).Error())
 }
 
 // ErrorStack writes v along with call stack into error log.
 func ErrorStack(v ...interface{}) {
 	// there is newline in stack string
-	stackSync(fmt.Sprint(v...))
+	writeStack(fmt.Sprint(v...))
 }
 
 // ErrorStackf writes v along with call stack in format into error log.
 func ErrorStackf(format string, v ...interface{}) {
 	// there is newline in stack string
-	stackSync(fmt.Sprintf(format, v...))
+	writeStack(fmt.Sprintf(format, v...))
 }
 
 // Errorv writes v into error log with json content.
 // No call stack attached, because not elegant to pack the messages.
 func Errorv(v interface{}) {
-	errorAnySync(v)
+	writeError(v)
 }
 
 // Errorw writes msg along with fields into error log.
 func Errorw(msg string, fields ...LogField) {
-	errorFieldsSync(msg, fields...)
+	writeError(msg, fields...)
 }
 
 // Field returns a LogField for the given key and value.
@@ -150,22 +168,22 @@ func Field(key string, value interface{}) LogField {
 
 // Info writes v into access log.
 func Info(v ...interface{}) {
-	infoTextSync(fmt.Sprint(v...))
+	writeInfo(fmt.Sprint(v...))
 }
 
 // Infof writes v with format into access log.
 func Infof(format string, v ...interface{}) {
-	infoTextSync(fmt.Sprintf(format, v...))
+	writeInfo(fmt.Sprintf(format, v...))
 }
 
 // Infov writes v into access log with json content.
 func Infov(v interface{}) {
-	infoAnySync(v)
+	writeInfo(v)
 }
 
 // Infow writes msg along with fields into access log.
 func Infow(msg string, fields ...LogField) {
-	infoFieldsSync(msg, fields...)
+	writeInfo(msg, fields...)
 }
 
 // Must checks if err is nil, otherwise logs the error and exits.
@@ -187,7 +205,6 @@ func MustSetup(c LogConf) {
 
 // Reset clears the writer and resets the log level.
 func Reset() Writer {
-	SetLevel(InfoLevel)
 	return writer.Swap(nil)
 }
 
@@ -197,9 +214,8 @@ func SetLevel(level uint32) {
 }
 
 // SetWriter sets the logging writer. It can be used to customize the logging.
-// Call Reset before calling SetWriter again.
 func SetWriter(w Writer) {
-	if writer.Load() == nil {
+	if atomic.LoadUint32(&disableLog) == 0 {
 		writer.Store(w)
 	}
 }
@@ -207,70 +223,75 @@ func SetWriter(w Writer) {
 // SetUp sets up the logx. If already set up, just return nil.
 // we allow SetUp to be called multiple times, because for example
 // we need to allow different service frameworks to initialize logx respectively.
-// the same logic for SetUp
-func SetUp(c LogConf) error {
-	setupLogLevel(c)
+func SetUp(c LogConf) (err error) {
+	// Just ignore the subsequent SetUp calls.
+	// Because multiple services in one process might call SetUp respectively.
+	// Need to wait for the first caller to complete the execution.
+	setupOnce.Do(func() {
+		setupLogLevel(c)
 
-	if len(c.TimeFormat) > 0 {
-		timeFormat = c.TimeFormat
-	}
+		if len(c.TimeFormat) > 0 {
+			timeFormat = c.TimeFormat
+		}
 
-	switch c.Encoding {
-	case plainEncoding:
-		atomic.StoreUint32(&encoding, plainEncodingType)
-	default:
-		atomic.StoreUint32(&encoding, jsonEncodingType)
-	}
+		switch c.Encoding {
+		case plainEncoding:
+			atomic.StoreUint32(&encoding, plainEncodingType)
+		default:
+			atomic.StoreUint32(&encoding, jsonEncodingType)
+		}
 
-	switch c.Mode {
-	case fileMode:
-		return setupWithFiles(c)
-	case volumeMode:
-		return setupWithVolume(c)
-	default:
-		setupWithConsole()
-		return nil
-	}
+		switch c.Mode {
+		case fileMode:
+			err = setupWithFiles(c)
+		case volumeMode:
+			err = setupWithVolume(c)
+		default:
+			setupWithConsole()
+		}
+	})
+
+	return
 }
 
 // Severe writes v into severe log.
 func Severe(v ...interface{}) {
-	severeSync(fmt.Sprint(v...))
+	writeSevere(fmt.Sprint(v...))
 }
 
 // Severef writes v with format into severe log.
 func Severef(format string, v ...interface{}) {
-	severeSync(fmt.Sprintf(format, v...))
+	writeSevere(fmt.Sprintf(format, v...))
 }
 
 // Slow writes v into slow log.
 func Slow(v ...interface{}) {
-	slowTextSync(fmt.Sprint(v...))
+	writeSlow(fmt.Sprint(v...))
 }
 
 // Slowf writes v with format into slow log.
 func Slowf(format string, v ...interface{}) {
-	slowTextSync(fmt.Sprintf(format, v...))
+	writeSlow(fmt.Sprintf(format, v...))
 }
 
 // Slowv writes v into slow log with json content.
 func Slowv(v interface{}) {
-	slowAnySync(v)
+	writeSlow(v)
 }
 
 // Sloww writes msg along with fields into slow log.
 func Sloww(msg string, fields ...LogField) {
-	slowFieldsSync(msg, fields...)
+	writeSlow(msg, fields...)
 }
 
 // Stat writes v into stat log.
 func Stat(v ...interface{}) {
-	statSync(fmt.Sprint(v...))
+	writeStat(fmt.Sprint(v...))
 }
 
 // Statf writes v with format into stat log.
 func Statf(format string, v ...interface{}) {
-	statSync(fmt.Sprintf(format, v...))
+	writeStat(fmt.Sprintf(format, v...))
 }
 
 // WithCooldownMillis customizes logging on writing call stack interval.
@@ -294,38 +315,50 @@ func WithGzip() LogOption {
 	}
 }
 
+// WithMaxBackups customizes how many log files backups will be kept.
+func WithMaxBackups(count int) LogOption {
+	return func(opts *logOptions) {
+		opts.maxBackups = count
+	}
+}
+
+// WithMaxSize customizes how much space the writing log file can take up.
+func WithMaxSize(size int) LogOption {
+	return func(opts *logOptions) {
+		opts.maxSize = size
+	}
+}
+
+// WithRotation customizes which log rotation rule to use.
+func WithRotation(r string) LogOption {
+	return func(opts *logOptions) {
+		opts.rotationRule = r
+	}
+}
+
+func addCaller(fields ...LogField) []LogField {
+	return append(fields, Field(callerKey, getCaller(callerDepth)))
+}
+
 func createOutput(path string) (io.WriteCloser, error) {
 	if len(path) == 0 {
 		return nil, ErrLogPathNotSet
 	}
 
-	return NewLogger(path, DefaultRotateRule(path, backupFileDelimiter, options.keepDays,
-		options.gzipEnabled), options.gzipEnabled)
-}
-
-func errorAnySync(v interface{}) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(v)
-	}
-}
-
-func errorFieldsSync(content string, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(content, fields...)
-	}
-}
-
-func errorTextSync(msg string) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(msg)
+	switch options.rotationRule {
+	case sizeRotationRule:
+		return NewLogger(path, NewSizeLimitRotateRule(path, backupFileDelimiter, options.keepDays,
+			options.maxSize, options.maxBackups, options.gzipEnabled), options.gzipEnabled)
+	default:
+		return NewLogger(path, DefaultRotateRule(path, backupFileDelimiter, options.keepDays,
+			options.gzipEnabled), options.gzipEnabled)
 	}
 }
 
 func getWriter() Writer {
 	w := writer.Load()
 	if w == nil {
-		w = newConsoleWriter()
-		writer.Store(w)
+		w = writer.StoreIfNil(newConsoleWriter())
 	}
 
 	return w
@@ -337,26 +370,10 @@ func handleOptions(opts []LogOption) {
 	}
 }
 
-func infoAnySync(val interface{}) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(val)
-	}
-}
-
-func infoFieldsSync(content string, fields ...LogField) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(content, fields...)
-	}
-}
-
-func infoTextSync(msg string) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(msg)
-	}
-}
-
 func setupLogLevel(c LogConf) {
 	switch c.Level {
+	case levelDebug:
+		SetLevel(DebugLevel)
 	case levelInfo:
 		SetLevel(InfoLevel)
 	case levelError:
@@ -389,12 +406,6 @@ func setupWithVolume(c LogConf) error {
 	return setupWithFiles(c)
 }
 
-func severeSync(msg string) {
-	if shallLog(SevereLevel) {
-		getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
-	}
-}
-
 func shallLog(level uint32) bool {
 	return atomic.LoadUint32(&logLevel) <= level
 }
@@ -403,32 +414,44 @@ func shallLogStat() bool {
 	return atomic.LoadUint32(&disableStat) == 0
 }
 
-func slowAnySync(v interface{}) {
-	if shallLog(ErrorLevel) {
-		getWriter().Slow(v)
+func writeDebug(val interface{}, fields ...LogField) {
+	if shallLog(DebugLevel) {
+		getWriter().Debug(val, addCaller(fields...)...)
 	}
 }
 
-func slowFieldsSync(content string, fields ...LogField) {
+func writeError(val interface{}, fields ...LogField) {
 	if shallLog(ErrorLevel) {
-		getWriter().Slow(content, fields...)
+		getWriter().Error(val, addCaller(fields...)...)
 	}
 }
 
-func slowTextSync(msg string) {
-	if shallLog(ErrorLevel) {
-		getWriter().Slow(msg)
+func writeInfo(val interface{}, fields ...LogField) {
+	if shallLog(InfoLevel) {
+		getWriter().Info(val, addCaller(fields...)...)
 	}
 }
 
-func stackSync(msg string) {
+func writeSevere(msg string) {
+	if shallLog(SevereLevel) {
+		getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+	}
+}
+
+func writeSlow(val interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Slow(val, addCaller(fields...)...)
+	}
+}
+
+func writeStack(msg string) {
 	if shallLog(ErrorLevel) {
 		getWriter().Stack(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
-func statSync(msg string) {
+func writeStat(msg string) {
 	if shallLogStat() && shallLog(InfoLevel) {
-		getWriter().Stat(msg)
+		getWriter().Stat(msg, addCaller()...)
 	}
 }

core/logx/logs_test.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
 	"reflect"
@@ -35,6 +35,12 @@ func (mw *mockWriter) Alert(v interface{}) {
 	output(&mw.builder, levelAlert, v)
 }
 
+func (mw *mockWriter) Debug(v interface{}, fields ...LogField) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelDebug, v, fields...)
+}
+
 func (mw *mockWriter) Error(v interface{}, fields ...LogField) {
 	mw.lock.Lock()
 	defer mw.lock.Unlock()
@@ -212,6 +218,46 @@ func TestStructedLogAlert(t *testing.T) {
 	})
 }
 
+func TestStructedLogDebug(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debug(v...)
+	})
+}
+
+func TestStructedLogDebugf(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debugf(fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogDebugv(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debugv(fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogDebugw(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debugw(fmt.Sprint(v...), Field("foo", time.Second))
+	})
+}
+
 func TestStructedLogError(t *testing.T) {
 	w := new(mockWriter)
 	old := writer.Swap(w)
@@ -461,13 +507,13 @@ func TestStructedLogWithDuration(t *testing.T) {
 	defer writer.Store(old)
 
 	WithDuration(time.Second).Info(message)
-	var entry logEntry
+	var entry map[string]interface{}
 	if err := json.Unmarshal([]byte(w.String()), &entry); err != nil {
 		t.Error(err)
 	}
-	assert.Equal(t, levelInfo, entry.Level)
-	assert.Equal(t, message, entry.Content)
-	assert.Equal(t, "1000.0ms", entry.Duration)
+	assert.Equal(t, levelInfo, entry[levelKey])
+	assert.Equal(t, message, entry[contentKey])
+	assert.Equal(t, "1000.0ms", entry[durationKey])
 }
 
 func TestSetLevel(t *testing.T) {
@@ -531,6 +577,7 @@ func TestSetup(t *testing.T) {
 	MustSetup(LogConf{
 		ServiceName: "any",
 		Mode:        "console",
+		TimeFormat:  timeFormat,
 	})
 	MustSetup(LogConf{
 		ServiceName: "any",
@@ -553,13 +600,23 @@ func TestSetup(t *testing.T) {
 		Encoding:    plainEncoding,
 	})
 
+	defer os.RemoveAll("CD01CB7D-2705-4F3F-889E-86219BF56F10")
 	assert.NotNil(t, setupWithVolume(LogConf{}))
+	assert.Nil(t, setupWithVolume(LogConf{
+		ServiceName: "CD01CB7D-2705-4F3F-889E-86219BF56F10",
+	}))
+	assert.Nil(t, setupWithVolume(LogConf{
+		ServiceName: "CD01CB7D-2705-4F3F-889E-86219BF56F10",
+		Rotation:    sizeRotationRule,
+	}))
 	assert.NotNil(t, setupWithFiles(LogConf{}))
 	assert.Nil(t, setupWithFiles(LogConf{
 		ServiceName: "any",
 		Path:        os.TempDir(),
 		Compress:    true,
 		KeepDays:    1,
+		MaxBackups:  3,
+		MaxSize:     1024 * 1024,
 	}))
 	setupLogLevel(LogConf{
 		Level: levelInfo,
@@ -583,6 +640,8 @@ func TestDisable(t *testing.T) {
 	var opt logOptions
 	WithKeepDays(1)(&opt)
 	WithGzip()(&opt)
+	WithMaxBackups(1)(&opt)
+	WithMaxSize(1024)(&opt)
 	assert.Nil(t, Close())
 	assert.Nil(t, Close())
 }
@@ -599,12 +658,14 @@ func TestDisableStat(t *testing.T) {
 }
 
 func TestSetWriter(t *testing.T) {
+	atomic.StoreUint32(&disableLog, 0)
 	Reset()
 	SetWriter(nopWriter{})
 	assert.NotNil(t, writer.Load())
 	assert.True(t, writer.Load() == nopWriter{})
-	SetWriter(new(mockWriter))
-	assert.True(t, writer.Load() == nopWriter{})
+	mocked := new(mockWriter)
+	SetWriter(mocked)
+	assert.Equal(t, mocked, writer.Load())
 }
 
 func TestWithGzip(t *testing.T) {
@@ -647,7 +708,7 @@ func BenchmarkCopyByteSlice(b *testing.B) {
 		buf = make([]byte, len(s))
 		copy(buf, s)
 	}
-	fmt.Fprint(ioutil.Discard, buf)
+	fmt.Fprint(io.Discard, buf)
 }
 
 func BenchmarkCopyOnWriteByteSlice(b *testing.B) {
@@ -656,7 +717,7 @@ func BenchmarkCopyOnWriteByteSlice(b *testing.B) {
 		size := len(s)
 		buf = s[:size:size]
 	}
-	fmt.Fprint(ioutil.Discard, buf)
+	fmt.Fprint(io.Discard, buf)
 }
 
 func BenchmarkCacheByteSlice(b *testing.B) {
@@ -670,7 +731,7 @@ func BenchmarkCacheByteSlice(b *testing.B) {
 func BenchmarkLogs(b *testing.B) {
 	b.ReportAllocs()
 
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 	for i := 0; i < b.N; i++ {
 		Info(i)
 	}
@@ -709,14 +770,16 @@ func put(b []byte) {
 func doTestStructedLog(t *testing.T, level string, w *mockWriter, write func(...interface{})) {
 	const message = "hello there"
 	write(message)
-	var entry logEntry
+
+	var entry map[string]interface{}
 	if err := json.Unmarshal([]byte(w.String()), &entry); err != nil {
 		t.Error(err)
 	}
-	assert.Equal(t, level, entry.Level)
-	val, ok := entry.Content.(string)
+
+	assert.Equal(t, level, entry[levelKey])
+	val, ok := entry[contentKey]
 	assert.True(t, ok)
-	assert.True(t, strings.Contains(val, message))
+	assert.True(t, strings.Contains(val.(string), message))
 }
 
 func doTestStructedLogConsole(t *testing.T, w *mockWriter, write func(...interface{})) {

core/logx/readme-cn.md

@@ -8,15 +8,18 @@
 
 ```go
 type LogConf struct {
-	ServiceName         string `json:",optional"`
-	Mode                string `json:",default=console,options=[console,file,volume]"`
-	Encoding            string `json:",default=json,options=[json,plain]"`
-	TimeFormat          string `json:",optional"`
-	Path                string `json:",default=logs"`
-	Level               string `json:",default=info,options=[info,error,severe]"`
-	Compress            bool   `json:",optional"`
-	KeepDays            int    `json:",optional"`
-	StackCooldownMillis int    `json:",default=100"`
+	ServiceName         string              `json:",optional"`
+	Mode                string              `json:",default=console,options=[console,file,volume]"`
+	Encoding            string              `json:",default=json,options=[json,plain]"`
+	TimeFormat          string              `json:",optional"`
+	Path                string              `json:",default=logs"`
+	Level               string              `json:",default=info,options=[info,error,severe]"`
+	Compress            bool                `json:",optional"`
+	KeepDays            int                 `json:",optional"`
+	StackCooldownMillis int                 `json:",default=100"`
+	MaxBackups          int                 `json:",default=0"`
+	MaxSize             int                 `json:",default=0"`
+	Rotation            string              `json:",default=daily,options=[daily,size]"`
 }
 ```
 
@@ -37,6 +40,12 @@ type LogConf struct {
 - `Compress`: 是否压缩日志文件，只在 `file` 模式下工作
 - `KeepDays`：日志文件被保留多少天，在给定的天数之后，过期的文件将被自动删除。对 `console` 模式没有影响
 - `StackCooldownMillis`：多少毫秒后再次写入堆栈跟踪。用来避免堆栈跟踪日志过多
+- `MaxBackups`: 多少个日志文件备份将被保存。0代表所有备份都被保存。当`Rotation`被设置为`size`时才会起作用。注意：`KeepDays`选项的优先级会比`MaxBackups`高，即使`MaxBackups`被设置为0，当达到`KeepDays`上限时备份文件同样会被删除。
+- `MaxSize`: 当前被写入的日志文件最大可占用多少空间。0代表没有上限。单位为`MB`。当`Rotation`被设置为`size`时才会起作用。
+- `Rotation`: 日志轮转策略类型。默认为`daily`（按天轮转）。
+  - `daily` 按天轮转。
+  - `size` 按日志大小轮转。
+
 
 ## 打印日志方法
 

core/logx/readme.md

@@ -8,15 +8,18 @@ English | [简体中文](readme-cn.md)
 
 ```go
 type LogConf struct {
-	ServiceName         string `json:",optional"`
-	Mode                string `json:",default=console,options=[console,file,volume]"`
-	Encoding            string `json:",default=json,options=[json,plain]"`
-	TimeFormat          string `json:",optional"`
-	Path                string `json:",default=logs"`
-	Level               string `json:",default=info,options=[info,error,severe]"`
-	Compress            bool   `json:",optional"`
-	KeepDays            int    `json:",optional"`
-	StackCooldownMillis int    `json:",default=100"`
+	ServiceName         string              `json:",optional"`
+	Mode                string              `json:",default=console,options=[console,file,volume]"`
+	Encoding            string              `json:",default=json,options=[json,plain]"`
+	TimeFormat          string              `json:",optional"`
+	Path                string              `json:",default=logs"`
+	Level               string              `json:",default=info,options=[info,error,severe]"`
+	Compress            bool                `json:",optional"`
+	KeepDays            int                 `json:",optional"`
+	StackCooldownMillis int                 `json:",default=100"`
+	MaxBackups          int                 `json:",default=0"`
+	MaxSize             int                 `json:",default=0"`
+	Rotation            string              `json:",default=daily,options=[daily,size]"`
 }
 ```
 
@@ -37,6 +40,11 @@ type LogConf struct {
 - `Compress`: whether or not to compress log files, only works with `file` mode.
 - `KeepDays`: how many days that the log files are kept, after the given days, the outdated files will be deleted automatically. It has no effect on `console` mode.
 - `StackCooldownMillis`: how many milliseconds to rewrite stacktrace again. It’s used to avoid stacktrace flooding.
+- `MaxBackups`: represents how many backup log files will be kept. 0 means all files will be kept forever. Only take effect when `Rotation` is `size`. NOTE: the level of option `KeepDays` will be higher. Even thougth `MaxBackups` sets 0, log files will still be removed if the `KeepDays` limitation is reached.
+- `MaxSize`: represents how much space the writing log file takes up. 0 means no limit. The unit is `MB`. Only take effect when `Rotation` is `size`.
+- `Rotation`: represents the type of log rotation rule. Default is `daily`.
+  - `daily` rotate the logs by day.
+  - `size` rotate the logs by size of logs.
 
 ## Logging methods
 

core/logx/richlogger.go

@@ -0,0 +1,199 @@
+package logx
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/zeromicro/go-zero/core/timex"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// WithCallerSkip returns a Logger with given caller skip.
+func WithCallerSkip(skip int) Logger {
+	if skip <= 0 {
+		return new(richLogger)
+	}
+
+	return &richLogger{
+		callerSkip: skip,
+	}
+}
+
+// WithContext sets ctx to log, for keeping tracing information.
+func WithContext(ctx context.Context) Logger {
+	return &richLogger{
+		ctx: ctx,
+	}
+}
+
+// WithDuration returns a Logger with given duration.
+func WithDuration(d time.Duration) Logger {
+	return &richLogger{
+		fields: []LogField{Field(durationKey, timex.ReprOfDuration(d))},
+	}
+}
+
+type richLogger struct {
+	ctx        context.Context
+	callerSkip int
+	fields     []LogField
+}
+
+func (l *richLogger) Debug(v ...interface{}) {
+	l.debug(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Debugf(format string, v ...interface{}) {
+	l.debug(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Debugv(v interface{}) {
+	l.debug(v)
+}
+
+func (l *richLogger) Debugw(msg string, fields ...LogField) {
+	l.debug(msg, fields...)
+}
+
+func (l *richLogger) Error(v ...interface{}) {
+	l.err(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Errorf(format string, v ...interface{}) {
+	l.err(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Errorv(v interface{}) {
+	l.err(fmt.Sprint(v))
+}
+
+func (l *richLogger) Errorw(msg string, fields ...LogField) {
+	l.err(msg, fields...)
+}
+
+func (l *richLogger) Info(v ...interface{}) {
+	l.info(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Infof(format string, v ...interface{}) {
+	l.info(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Infov(v interface{}) {
+	l.info(v)
+}
+
+func (l *richLogger) Infow(msg string, fields ...LogField) {
+	l.info(msg, fields...)
+}
+
+func (l *richLogger) Slow(v ...interface{}) {
+	l.slow(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Slowf(format string, v ...interface{}) {
+	l.slow(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Slowv(v interface{}) {
+	l.slow(v)
+}
+
+func (l *richLogger) Sloww(msg string, fields ...LogField) {
+	l.slow(msg, fields...)
+}
+
+func (l *richLogger) WithCallerSkip(skip int) Logger {
+	if skip <= 0 {
+		return l
+	}
+
+	l.callerSkip = skip
+	return l
+}
+
+func (l *richLogger) WithContext(ctx context.Context) Logger {
+	l.ctx = ctx
+	return l
+}
+
+func (l *richLogger) WithDuration(duration time.Duration) Logger {
+	l.fields = append(l.fields, Field(durationKey, timex.ReprOfDuration(duration)))
+	return l
+}
+
+func (l *richLogger) WithFields(fields ...LogField) Logger {
+	l.fields = append(l.fields, fields...)
+	return l
+}
+
+func (l *richLogger) buildFields(fields ...LogField) []LogField {
+	fields = append(l.fields, fields...)
+	fields = append(fields, Field(callerKey, getCaller(callerDepth+l.callerSkip)))
+
+	if l.ctx == nil {
+		return fields
+	}
+
+	traceID := traceIdFromContext(l.ctx)
+	if len(traceID) > 0 {
+		fields = append(fields, Field(traceKey, traceID))
+	}
+
+	spanID := spanIdFromContext(l.ctx)
+	if len(spanID) > 0 {
+		fields = append(fields, Field(spanKey, spanID))
+	}
+
+	val := l.ctx.Value(fieldsContextKey)
+	if val != nil {
+		if arr, ok := val.([]LogField); ok {
+			fields = append(fields, arr...)
+		}
+	}
+
+	return fields
+}
+
+func (l *richLogger) debug(v interface{}, fields ...LogField) {
+	if shallLog(DebugLevel) {
+		getWriter().Debug(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *richLogger) err(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Error(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *richLogger) info(v interface{}, fields ...LogField) {
+	if shallLog(InfoLevel) {
+		getWriter().Info(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *richLogger) slow(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Slow(v, l.buildFields(fields...)...)
+	}
+}
+
+func spanIdFromContext(ctx context.Context) string {
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasSpanID() {
+		return spanCtx.SpanID().String()
+	}
+
+	return ""
+}
+
+func traceIdFromContext(ctx context.Context) string {
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasTraceID() {
+		return spanCtx.TraceID().String()
+	}
+
+	return ""
+}

core/logx/richlogger_test.go

@@ -3,6 +3,7 @@ package logx
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"strings"
 	"sync/atomic"
@@ -29,13 +30,48 @@ func TestTraceLog(t *testing.T) {
 	otel.SetTracerProvider(tp)
 	defer otel.SetTracerProvider(otp)
 
-	ctx, span := tp.Tracer("foo").Start(context.Background(), "bar")
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
 	defer span.End()
 
 	WithContext(ctx).Info(testlog)
 	validate(t, w.String(), true, true)
 }
 
+func TestTraceDebug(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, span := tp.Tracer("foo").Start(context.Background(), "bar")
+	defer span.End()
+
+	l := WithContext(ctx)
+	SetLevel(DebugLevel)
+	l.WithDuration(time.Second).Debug(testlog)
+	assert.True(t, strings.Contains(w.String(), traceKey))
+	assert.True(t, strings.Contains(w.String(), spanKey))
+	w.Reset()
+	l.WithDuration(time.Second).Debugf(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Debugv(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Debugw(testlog, Field("foo", "bar"))
+	validate(t, w.String(), true, true)
+	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
+	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
+}
+
 func TestTraceError(t *testing.T) {
 	w := new(mockWriter)
 	old := writer.Swap(w)
@@ -50,7 +86,7 @@ func TestTraceError(t *testing.T) {
 	otel.SetTracerProvider(tp)
 	defer otel.SetTracerProvider(otp)
 
-	ctx, span := tp.Tracer("foo").Start(context.Background(), "bar")
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
 	defer span.End()
 
 	var nilCtx context.Context
@@ -67,10 +103,10 @@ func TestTraceError(t *testing.T) {
 	l.WithDuration(time.Second).Errorv(testlog)
 	validate(t, w.String(), true, true)
 	w.Reset()
-	l.WithDuration(time.Second).Errorw(testlog, Field("foo", "bar"))
+	l.WithDuration(time.Second).Errorw(testlog, Field("basket", "ball"))
 	validate(t, w.String(), true, true)
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
+	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
+	assert.True(t, strings.Contains(w.String(), "ball"), w.String())
 }
 
 func TestTraceInfo(t *testing.T) {
@@ -87,7 +123,7 @@ func TestTraceInfo(t *testing.T) {
 	otel.SetTracerProvider(tp)
 	defer otel.SetTracerProvider(otp)
 
-	ctx, span := tp.Tracer("foo").Start(context.Background(), "bar")
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
 	defer span.End()
 
 	SetLevel(InfoLevel)
@@ -101,10 +137,10 @@ func TestTraceInfo(t *testing.T) {
 	l.WithDuration(time.Second).Infov(testlog)
 	validate(t, w.String(), true, true)
 	w.Reset()
-	l.WithDuration(time.Second).Infow(testlog, Field("foo", "bar"))
+	l.WithDuration(time.Second).Infow(testlog, Field("basket", "ball"))
 	validate(t, w.String(), true, true)
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
+	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
+	assert.True(t, strings.Contains(w.String(), "ball"), w.String())
 }
 
 func TestTraceInfoConsole(t *testing.T) {
@@ -124,7 +160,7 @@ func TestTraceInfoConsole(t *testing.T) {
 	otel.SetTracerProvider(tp)
 	defer otel.SetTracerProvider(otp)
 
-	ctx, span := tp.Tracer("foo").Start(context.Background(), "bar")
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
 	defer span.End()
 
 	l := WithContext(ctx)
@@ -153,7 +189,7 @@ func TestTraceSlow(t *testing.T) {
 	otel.SetTracerProvider(tp)
 	defer otel.SetTracerProvider(otp)
 
-	ctx, span := tp.Tracer("foo").Start(context.Background(), "bar")
+	ctx, span := tp.Tracer("trace-id").Start(context.Background(), "span-id")
 	defer span.End()
 
 	l := WithContext(ctx)
@@ -168,10 +204,10 @@ func TestTraceSlow(t *testing.T) {
 	l.WithDuration(time.Second).Slowv(testlog)
 	validate(t, w.String(), true, true)
 	w.Reset()
-	l.WithDuration(time.Second).Sloww(testlog, Field("foo", "bar"))
+	l.WithDuration(time.Second).Sloww(testlog, Field("basket", "ball"))
 	validate(t, w.String(), true, true)
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
+	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
+	assert.True(t, strings.Contains(w.String(), "ball"), w.String())
 }
 
 func TestTraceWithoutContext(t *testing.T) {
@@ -192,6 +228,67 @@ func TestTraceWithoutContext(t *testing.T) {
 	validate(t, w.String(), false, false)
 }
 
+func TestLogWithFields(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	ctx := ContextWithFields(context.Background(), Field("foo", "bar"))
+	l := WithContext(ctx)
+	SetLevel(InfoLevel)
+	l.Info(testlog)
+
+	var val mockValue
+	assert.Nil(t, json.Unmarshal([]byte(w.String()), &val))
+	assert.Equal(t, "bar", val.Foo)
+}
+
+func TestLogWithCallerSkip(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	l := WithCallerSkip(1).WithCallerSkip(0)
+	p := func(v string) {
+		l.Info(v)
+	}
+
+	file, line := getFileLine()
+	p(testlog)
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+
+	w.Reset()
+	l = WithCallerSkip(0).WithCallerSkip(1)
+	file, line = getFileLine()
+	p(testlog)
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestLoggerWithFields(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	l := WithContext(context.Background()).WithFields(Field("foo", "bar"))
+	l.Info(testlog)
+
+	var val mockValue
+	assert.Nil(t, json.Unmarshal([]byte(w.String()), &val))
+	assert.Equal(t, "bar", val.Foo)
+}
+
 func validate(t *testing.T, body string, expectedTrace, expectedSpan bool) {
 	var val mockValue
 	dec := json.NewDecoder(strings.NewReader(body))
@@ -217,4 +314,5 @@ func validate(t *testing.T, body string, expectedTrace, expectedSpan bool) {
 type mockValue struct {
 	Trace string `json:"trace"`
 	Span  string `json:"span"`
+	Foo   string `json:"foo"`
 }

core/logx/rotatelogger.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -19,10 +20,13 @@ import (
 
 const (
 	dateFormat      = "2006-01-02"
+	fileTimeFormat  = time.RFC3339
 	hoursPerDay     = 24
 	bufferSize      = 100
 	defaultDirMode  = 0o755
 	defaultFileMode = 0o600
+	gzipExt         = ".gz"
+	megaBytes       = 1 << 20
 )
 
 // ErrLogFileClosed is an error that indicates the log file is already closed.
@@ -34,7 +38,7 @@ type (
 		BackupFileName() string
 		MarkRotated()
 		OutdatedFiles() []string
-		ShallRotate() bool
+		ShallRotate(size int64) bool
 	}
 
 	// A RotateLogger is a Logger that can rotate log files with given rules.
@@ -47,8 +51,9 @@ type (
 		rule     RotateRule
 		compress bool
 		// can't use threading.RoutineGroup because of cycle import
-		waitGroup sync.WaitGroup
-		closeOnce sync.Once
+		waitGroup   sync.WaitGroup
+		closeOnce   sync.Once
+		currentSize int64
 	}
 
 	// A DailyRotateRule is a rule to daily rotate the log files.
@@ -59,6 +64,13 @@ type (
 		days        int
 		gzip        bool
 	}
+
+	// SizeLimitRotateRule a rotation rule that make the log file rotated base on size
+	SizeLimitRotateRule struct {
+		DailyRotateRule
+		maxSize    int64
+		maxBackups int
+	}
 )
 
 // DefaultRotateRule is a default log rotating rule, currently DailyRotateRule.
@@ -90,7 +102,7 @@ func (r *DailyRotateRule) OutdatedFiles() []string {
 
 	var pattern string
 	if r.gzip {
-		pattern = fmt.Sprintf("%s%s*.gz", r.filename, r.delimiter)
+		pattern = fmt.Sprintf("%s%s*%s", r.filename, r.delimiter, gzipExt)
 	} else {
 		pattern = fmt.Sprintf("%s%s*", r.filename, r.delimiter)
 	}
@@ -103,9 +115,11 @@ func (r *DailyRotateRule) OutdatedFiles() []string {
 
 	var buf strings.Builder
 	boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay*r.days)).Format(dateFormat)
-	fmt.Fprintf(&buf, "%s%s%s", r.filename, r.delimiter, boundary)
+	buf.WriteString(r.filename)
+	buf.WriteString(r.delimiter)
+	buf.WriteString(boundary)
 	if r.gzip {
-		buf.WriteString(".gz")
+		buf.WriteString(gzipExt)
 	}
 	boundaryFile := buf.String()
 
@@ -120,10 +134,100 @@ func (r *DailyRotateRule) OutdatedFiles() []string {
 }
 
 // ShallRotate checks if the file should be rotated.
-func (r *DailyRotateRule) ShallRotate() bool {
+func (r *DailyRotateRule) ShallRotate(_ int64) bool {
 	return len(r.rotatedTime) > 0 && getNowDate() != r.rotatedTime
 }
 
+// NewSizeLimitRotateRule returns the rotation rule with size limit
+func NewSizeLimitRotateRule(filename, delimiter string, days, maxSize, maxBackups int, gzip bool) RotateRule {
+	return &SizeLimitRotateRule{
+		DailyRotateRule: DailyRotateRule{
+			rotatedTime: getNowDateInRFC3339Format(),
+			filename:    filename,
+			delimiter:   delimiter,
+			days:        days,
+			gzip:        gzip,
+		},
+		maxSize:    int64(maxSize) * megaBytes,
+		maxBackups: maxBackups,
+	}
+}
+
+func (r *SizeLimitRotateRule) BackupFileName() string {
+	dir := filepath.Dir(r.filename)
+	prefix, ext := r.parseFilename()
+	timestamp := getNowDateInRFC3339Format()
+	return filepath.Join(dir, fmt.Sprintf("%s%s%s%s", prefix, r.delimiter, timestamp, ext))
+}
+
+func (r *SizeLimitRotateRule) MarkRotated() {
+	r.rotatedTime = getNowDateInRFC3339Format()
+}
+
+func (r *SizeLimitRotateRule) OutdatedFiles() []string {
+	dir := filepath.Dir(r.filename)
+	prefix, ext := r.parseFilename()
+
+	var pattern string
+	if r.gzip {
+		pattern = fmt.Sprintf("%s%s%s%s*%s%s", dir, string(filepath.Separator),
+			prefix, r.delimiter, ext, gzipExt)
+	} else {
+		pattern = fmt.Sprintf("%s%s%s%s*%s", dir, string(filepath.Separator),
+			prefix, r.delimiter, ext)
+	}
+
+	files, err := filepath.Glob(pattern)
+	if err != nil {
+		Errorf("failed to delete outdated log files, error: %s", err)
+		return nil
+	}
+
+	sort.Strings(files)
+
+	outdated := make(map[string]lang.PlaceholderType)
+
+	// test if too many backups
+	if r.maxBackups > 0 && len(files) > r.maxBackups {
+		for _, f := range files[:len(files)-r.maxBackups] {
+			outdated[f] = lang.Placeholder
+		}
+		files = files[len(files)-r.maxBackups:]
+	}
+
+	// test if any too old backups
+	if r.days > 0 {
+		boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay*r.days)).Format(fileTimeFormat)
+		boundaryFile := filepath.Join(dir, fmt.Sprintf("%s%s%s%s", prefix, r.delimiter, boundary, ext))
+		if r.gzip {
+			boundaryFile += gzipExt
+		}
+		for _, f := range files {
+			if f >= boundaryFile {
+				break
+			}
+			outdated[f] = lang.Placeholder
+		}
+	}
+
+	var result []string
+	for k := range outdated {
+		result = append(result, k)
+	}
+	return result
+}
+
+func (r *SizeLimitRotateRule) ShallRotate(size int64) bool {
+	return r.maxSize > 0 && r.maxSize < size
+}
+
+func (r *SizeLimitRotateRule) parseFilename() (prefix, ext string) {
+	logName := filepath.Base(r.filename)
+	ext = filepath.Ext(r.filename)
+	prefix = logName[:len(logName)-len(ext)]
+	return
+}
+
 // NewLogger returns a RotateLogger with given filename and rule, etc.
 func NewLogger(filename string, rule RotateRule, compress bool) (*RotateLogger, error) {
 	l := &RotateLogger{
@@ -282,15 +386,17 @@ func (l *RotateLogger) startWorker() {
 }
 
 func (l *RotateLogger) write(v []byte) {
-	if l.rule.ShallRotate() {
+	if l.rule.ShallRotate(l.currentSize + int64(len(v))) {
 		if err := l.rotate(); err != nil {
 			log.Println(err)
 		} else {
 			l.rule.MarkRotated()
+			l.currentSize = 0
 		}
 	}
 	if l.fp != nil {
 		l.fp.Write(v)
+		l.currentSize += int64(len(v))
 	}
 }
 
@@ -308,6 +414,10 @@ func getNowDate() string {
 	return time.Now().Format(dateFormat)
 }
 
+func getNowDateInRFC3339Format() string {
+	return time.Now().Format(fileTimeFormat)
+}
+
 func gzipFile(file string) error {
 	in, err := os.Open(file)
 	if err != nil {
@@ -315,7 +425,7 @@ func gzipFile(file string) error {
 	}
 	defer in.Close()
 
-	out, err := os.Create(fmt.Sprintf("%s.gz", file))
+	out, err := os.Create(fmt.Sprintf("%s%s", file, gzipExt))
 	if err != nil {
 		return err
 	}

core/logx/rotatelogger_test.go

@@ -29,7 +29,34 @@ func TestDailyRotateRuleOutdatedFiles(t *testing.T) {
 func TestDailyRotateRuleShallRotate(t *testing.T) {
 	var rule DailyRotateRule
 	rule.rotatedTime = time.Now().Add(time.Hour * 24).Format(dateFormat)
-	assert.True(t, rule.ShallRotate())
+	assert.True(t, rule.ShallRotate(0))
+}
+
+func TestSizeLimitRotateRuleMarkRotated(t *testing.T) {
+	var rule SizeLimitRotateRule
+	rule.MarkRotated()
+	assert.Equal(t, getNowDateInRFC3339Format(), rule.rotatedTime)
+}
+
+func TestSizeLimitRotateRuleOutdatedFiles(t *testing.T) {
+	var rule SizeLimitRotateRule
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.days = 1
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.gzip = true
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.maxBackups = 0
+	assert.Empty(t, rule.OutdatedFiles())
+}
+
+func TestSizeLimitRotateRuleShallRotate(t *testing.T) {
+	var rule SizeLimitRotateRule
+	rule.rotatedTime = time.Now().Add(time.Hour * 24).Format(fileTimeFormat)
+	rule.maxSize = 0
+	assert.False(t, rule.ShallRotate(0))
+	rule.maxSize = 100
+	assert.False(t, rule.ShallRotate(0))
+	assert.True(t, rule.ShallRotate(101*megaBytes))
 }
 
 func TestRotateLoggerClose(t *testing.T) {
@@ -142,3 +169,162 @@ func TestRotateLoggerWrite(t *testing.T) {
 func TestLogWriterClose(t *testing.T) {
 	assert.Nil(t, newLogWriter(nil).Close())
 }
+
+func TestRotateLoggerWithSizeLimitRotateRuleClose(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), false)
+	assert.Nil(t, err)
+	assert.Nil(t, logger.Close())
+}
+
+func TestRotateLoggerGetBackupWithSizeLimitRotateRuleFilename(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), false)
+	assert.Nil(t, err)
+	assert.True(t, len(logger.getBackupFilename()) > 0)
+	logger.backup = ""
+	assert.True(t, len(logger.getBackupFilename()) > 0)
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleMayCompressFile(t *testing.T) {
+	old := os.Stdout
+	os.Stdout = os.NewFile(0, os.DevNull)
+	defer func() {
+		os.Stdout = old
+	}()
+
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), false)
+	assert.Nil(t, err)
+	logger.maybeCompressFile(filename)
+	_, err = os.Stat(filename)
+	assert.Nil(t, err)
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleMayCompressFileTrue(t *testing.T) {
+	old := os.Stdout
+	os.Stdout = os.NewFile(0, os.DevNull)
+	defer func() {
+		os.Stdout = old
+	}()
+
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+	}
+	logger.maybeCompressFile(filename)
+	_, err = os.Stat(filename)
+	assert.NotNil(t, err)
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleRotate(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	logger, err := NewLogger(filename, new(SizeLimitRotateRule), true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer func() {
+			os.Remove(logger.getBackupFilename())
+			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+		}()
+	}
+	err = logger.rotate()
+	switch v := err.(type) {
+	case *os.LinkError:
+		// avoid rename error on docker container
+		assert.Equal(t, syscall.EXDEV, v.Err)
+	case *os.PathError:
+		// ignore remove error for tests,
+		// files are cleaned in GitHub actions.
+		assert.Equal(t, "remove", v.Op)
+	default:
+		assert.Nil(t, err)
+	}
+}
+
+func TestRotateLoggerWithSizeLimitRotateRuleWrite(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	rule := new(SizeLimitRotateRule)
+	logger, err := NewLogger(filename, rule, true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer func() {
+			os.Remove(logger.getBackupFilename())
+			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+		}()
+	}
+	// the following write calls cannot be changed to Write, because of DATA RACE.
+	logger.write([]byte(`foo`))
+	rule.rotatedTime = time.Now().Add(-time.Hour * 24).Format(dateFormat)
+	logger.write([]byte(`bar`))
+	logger.Close()
+	logger.write([]byte(`baz`))
+}
+
+func BenchmarkRotateLogger(b *testing.B) {
+	filename := "./test.log"
+	filename2 := "./test2.log"
+	dailyRotateRuleLogger, err1 := NewLogger(
+		filename,
+		DefaultRotateRule(
+			filename,
+			backupFileDelimiter,
+			1,
+			true,
+		),
+		true,
+	)
+	if err1 != nil {
+		b.Logf("Failed to new daily rotate rule logger: %v", err1)
+		b.FailNow()
+	}
+	sizeLimitRotateRuleLogger, err2 := NewLogger(
+		filename2,
+		NewSizeLimitRotateRule(
+			filename,
+			backupFileDelimiter,
+			1,
+			100,
+			10,
+			true,
+		),
+		true,
+	)
+	if err2 != nil {
+		b.Logf("Failed to new size limit rotate rule logger: %v", err1)
+		b.FailNow()
+	}
+	defer func() {
+		dailyRotateRuleLogger.Close()
+		sizeLimitRotateRuleLogger.Close()
+		os.Remove(filename)
+		os.Remove(filename2)
+	}()
+
+	b.Run("daily rotate rule", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			dailyRotateRuleLogger.write([]byte("testing\ntesting\n"))
+		}
+	})
+	b.Run("size limit rotate rule", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			sizeLimitRotateRuleLogger.write([]byte("testing\ntesting\n"))
+		}
+	})
+}

core/logx/syslog_test.go

@@ -42,11 +42,18 @@ func captureOutput(f func()) string {
 }
 
 func getContent(jsonStr string) string {
-	var entry logEntry
+	var entry map[string]interface{}
 	json.Unmarshal([]byte(jsonStr), &entry)
-	val, ok := entry.Content.(string)
-	if ok {
-		return val
+
+	val, ok := entry[contentKey]
+	if !ok {
+		return ""
 	}
-	return ""
+
+	str, ok := val.(string)
+	if !ok {
+		return ""
+	}
+
+	return str
 }

core/logx/tracelogger.go

@@ -1,136 +0,0 @@
-package logx
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/zeromicro/go-zero/core/timex"
-	"go.opentelemetry.io/otel/trace"
-)
-
-// WithContext sets ctx to log, for keeping tracing information.
-func WithContext(ctx context.Context) Logger {
-	return &traceLogger{
-		ctx: ctx,
-	}
-}
-
-type traceLogger struct {
-	logEntry
-	ctx context.Context
-}
-
-func (l *traceLogger) Error(v ...interface{}) {
-	l.err(fmt.Sprint(v...))
-}
-
-func (l *traceLogger) Errorf(format string, v ...interface{}) {
-	l.err(fmt.Sprintf(format, v...))
-}
-
-func (l *traceLogger) Errorv(v interface{}) {
-	l.err(fmt.Sprint(v))
-}
-
-func (l *traceLogger) Errorw(msg string, fields ...LogField) {
-	l.err(msg, fields...)
-}
-
-func (l *traceLogger) Info(v ...interface{}) {
-	l.info(fmt.Sprint(v...))
-}
-
-func (l *traceLogger) Infof(format string, v ...interface{}) {
-	l.info(fmt.Sprintf(format, v...))
-}
-
-func (l *traceLogger) Infov(v interface{}) {
-	l.info(v)
-}
-
-func (l *traceLogger) Infow(msg string, fields ...LogField) {
-	l.info(msg, fields...)
-}
-
-func (l *traceLogger) Slow(v ...interface{}) {
-	l.slow(fmt.Sprint(v...))
-}
-
-func (l *traceLogger) Slowf(format string, v ...interface{}) {
-	l.slow(fmt.Sprintf(format, v...))
-}
-
-func (l *traceLogger) Slowv(v interface{}) {
-	l.slow(v)
-}
-
-func (l *traceLogger) Sloww(msg string, fields ...LogField) {
-	l.slow(msg, fields...)
-}
-
-func (l *traceLogger) WithContext(ctx context.Context) Logger {
-	if ctx == nil {
-		return l
-	}
-
-	l.ctx = ctx
-	return l
-}
-
-func (l *traceLogger) WithDuration(duration time.Duration) Logger {
-	l.Duration = timex.ReprOfDuration(duration)
-	return l
-}
-
-func (l *traceLogger) buildFields(fields ...LogField) []LogField {
-	if len(l.Duration) > 0 {
-		fields = append(fields, Field(durationKey, l.Duration))
-	}
-	traceID := traceIdFromContext(l.ctx)
-	if len(traceID) > 0 {
-		fields = append(fields, Field(traceKey, traceID))
-	}
-	spanID := spanIdFromContext(l.ctx)
-	if len(spanID) > 0 {
-		fields = append(fields, Field(spanKey, spanID))
-	}
-
-	return fields
-}
-
-func (l *traceLogger) err(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(v, l.buildFields(fields...)...)
-	}
-}
-
-func (l *traceLogger) info(v interface{}, fields ...LogField) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(v, l.buildFields(fields...)...)
-	}
-}
-
-func (l *traceLogger) slow(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Slow(v, l.buildFields(fields...)...)
-	}
-}
-
-func spanIdFromContext(ctx context.Context) string {
-	spanCtx := trace.SpanContextFromContext(ctx)
-	if spanCtx.HasSpanID() {
-		return spanCtx.SpanID().String()
-	}
-
-	return ""
-}
-
-func traceIdFromContext(ctx context.Context) string {
-	spanCtx := trace.SpanContextFromContext(ctx)
-	if spanCtx.HasTraceID() {
-		return spanCtx.TraceID().String()
-	}
-
-	return ""
-}

core/logx/vars.go

@@ -3,8 +3,10 @@ package logx
 import "errors"
 
 const (
-	// InfoLevel logs everything
-	InfoLevel uint32 = iota
+	// DebugLevel logs everything
+	DebugLevel uint32 = iota
+	// InfoLevel does not include debugs
+	InfoLevel
 	// ErrorLevel includes errors, slows, stacks
 	ErrorLevel
 	// SevereLevel only log severe messages
@@ -15,9 +17,9 @@ const (
 	jsonEncodingType = iota
 	plainEncodingType
 
-	jsonEncoding     = "json"
 	plainEncoding    = "plain"
 	plainEncodingSep = '\t'
+	sizeRotationRule = "size"
 )
 
 const (
@@ -27,9 +29,8 @@ const (
 	slowFilename   = "slow.log"
 	statFilename   = "stat.log"
 
-	consoleMode = "console"
-	fileMode    = "file"
-	volumeMode  = "volume"
+	fileMode   = "file"
+	volumeMode = "volume"
 
 	levelAlert  = "alert"
 	levelInfo   = "info"
@@ -38,6 +39,7 @@ const (
 	levelFatal  = "fatal"
 	levelSlow   = "slow"
 	levelStat   = "stat"
+	levelDebug  = "debug"
 
 	backupFileDelimiter = "-"
 	flags               = 0x0

core/logx/writer.go

@@ -1,16 +1,16 @@
 package logx
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log"
-	"os"
 	"path"
-	"strings"
 	"sync"
 	"sync/atomic"
 
+	fatihcolor "github.com/fatih/color"
 	"github.com/zeromicro/go-zero/core/color"
 )
 
@@ -18,6 +18,7 @@ type (
 	Writer interface {
 		Alert(v interface{})
 		Close() error
+		Debug(v interface{}, fields ...LogField)
 		Error(v interface{}, fields ...LogField)
 		Info(v interface{}, fields ...LogField)
 		Severe(v interface{})
@@ -63,21 +64,32 @@ func (w *atomicWriter) Load() Writer {
 
 func (w *atomicWriter) Store(v Writer) {
 	w.lock.Lock()
+	defer w.lock.Unlock()
 	w.writer = v
-	w.lock.Unlock()
+}
+
+func (w *atomicWriter) StoreIfNil(v Writer) Writer {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if w.writer == nil {
+		w.writer = v
+	}
+
+	return w.writer
 }
 
 func (w *atomicWriter) Swap(v Writer) Writer {
 	w.lock.Lock()
+	defer w.lock.Unlock()
 	old := w.writer
 	w.writer = v
-	w.lock.Unlock()
 	return old
 }
 
 func newConsoleWriter() Writer {
-	outLog := newLogWriter(log.New(os.Stdout, "", flags))
-	errLog := newLogWriter(log.New(os.Stderr, "", flags))
+	outLog := newLogWriter(log.New(fatihcolor.Output, "", flags))
+	errLog := newLogWriter(log.New(fatihcolor.Error, "", flags))
 	return &concreteWriter{
 		infoLog:   outLog,
 		errorLog:  errLog,
@@ -109,6 +121,14 @@ func newFileWriter(c LogConf) (Writer, error) {
 	if c.KeepDays > 0 {
 		opts = append(opts, WithKeepDays(c.KeepDays))
 	}
+	if c.MaxBackups > 0 {
+		opts = append(opts, WithMaxBackups(c.MaxBackups))
+	}
+	if c.MaxSize > 0 {
+		opts = append(opts, WithMaxSize(c.MaxSize))
+	}
+
+	opts = append(opts, WithRotation(c.Rotation))
 
 	accessFile := path.Join(c.Path, accessFilename)
 	errorFile := path.Join(c.Path, errorFilename)
@@ -175,6 +195,10 @@ func (w *concreteWriter) Close() error {
 	return w.statLog.Close()
 }
 
+func (w *concreteWriter) Debug(v interface{}, fields ...LogField) {
+	output(w.infoLog, levelDebug, v, fields...)
+}
+
 func (w *concreteWriter) Error(v interface{}, fields ...LogField) {
 	output(w.errorLog, levelError, v, fields...)
 }
@@ -208,6 +232,9 @@ func (n nopWriter) Close() error {
 	return nil
 }
 
+func (n nopWriter) Debug(_ interface{}, _ ...LogField) {
+}
+
 func (n nopWriter) Error(_ interface{}, _ ...LogField) {
 }
 
@@ -236,14 +263,23 @@ func buildFields(fields ...LogField) []string {
 	return items
 }
 
+func combineGlobalFields(fields []LogField) []LogField {
+	globals := globalFields.Load()
+	if globals == nil {
+		return fields
+	}
+
+	return append(globals.([]LogField), fields...)
+}
+
 func output(writer io.Writer, level string, val interface{}, fields ...LogField) {
-	fields = append(fields, Field(callerKey, getCaller(callerDepth)))
+	fields = combineGlobalFields(fields)
 
 	switch atomic.LoadUint32(&encoding) {
 	case plainEncodingType:
 		writePlainAny(writer, level, val, buildFields(fields...)...)
 	default:
-		entry := make(logEntryWithFields)
+		entry := make(logEntry)
 		for _, field := range fields {
 			entry[field.Key] = field.Value
 		}
@@ -267,6 +303,8 @@ func wrapLevelWithColor(level string) string {
 		colour = color.FgBlue
 	case levelSlow:
 		colour = color.FgYellow
+	case levelDebug:
+		colour = color.FgYellow
 	case levelStat:
 		colour = color.FgGreen
 	}
@@ -299,34 +337,12 @@ func writePlainAny(writer io.Writer, level string, val interface{}, fields ...st
 	case fmt.Stringer:
 		writePlainText(writer, level, v.String(), fields...)
 	default:
-		var buf strings.Builder
-		buf.WriteString(getTimestamp())
-		buf.WriteByte(plainEncodingSep)
-		buf.WriteString(level)
-		buf.WriteByte(plainEncodingSep)
-		if err := json.NewEncoder(&buf).Encode(val); err != nil {
-			log.Println(err.Error())
-			return
-		}
-
-		for _, item := range fields {
-			buf.WriteByte(plainEncodingSep)
-			buf.WriteString(item)
-		}
-		buf.WriteByte('\n')
-		if writer == nil {
-			log.Println(buf.String())
-			return
-		}
-
-		if _, err := fmt.Fprint(writer, buf.String()); err != nil {
-			log.Println(err.Error())
-		}
+		writePlainValue(writer, level, v, fields...)
 	}
 }
 
 func writePlainText(writer io.Writer, level, msg string, fields ...string) {
-	var buf strings.Builder
+	var buf bytes.Buffer
 	buf.WriteString(getTimestamp())
 	buf.WriteByte(plainEncodingSep)
 	buf.WriteString(level)
@@ -342,7 +358,33 @@ func writePlainText(writer io.Writer, level, msg string, fields ...string) {
 		return
 	}
 
-	if _, err := fmt.Fprint(writer, buf.String()); err != nil {
+	if _, err := writer.Write(buf.Bytes()); err != nil {
+		log.Println(err.Error())
+	}
+}
+
+func writePlainValue(writer io.Writer, level string, val interface{}, fields ...string) {
+	var buf bytes.Buffer
+	buf.WriteString(getTimestamp())
+	buf.WriteByte(plainEncodingSep)
+	buf.WriteString(level)
+	buf.WriteByte(plainEncodingSep)
+	if err := json.NewEncoder(&buf).Encode(val); err != nil {
+		log.Println(err.Error())
+		return
+	}
+
+	for _, item := range fields {
+		buf.WriteByte(plainEncodingSep)
+		buf.WriteString(item)
+	}
+	buf.WriteByte('\n')
+	if writer == nil {
+		log.Println(buf.String())
+		return
+	}
+
+	if _, err := writer.Write(buf.Bytes()); err != nil {
 		log.Println(err.Error())
 	}
 }

core/logx/writer_test.go

@@ -16,6 +16,9 @@ func TestNewWriter(t *testing.T) {
 	w := NewWriter(&buf)
 	w.Info(literal)
 	assert.Contains(t, buf.String(), literal)
+	buf.Reset()
+	w.Debug(literal)
+	assert.Contains(t, buf.String(), literal)
 }
 
 func TestConsoleWriter(t *testing.T) {
@@ -97,6 +100,7 @@ func TestNopWriter(t *testing.T) {
 	assert.NotPanics(t, func() {
 		var w nopWriter
 		w.Alert("foo")
+		w.Debug("foo")
 		w.Error("foo")
 		w.Info("foo")
 		w.Severe("foo")
@@ -123,6 +127,12 @@ func TestWritePlainAny(t *testing.T) {
 	writePlainAny(nil, levelInfo, "foo")
 	assert.Contains(t, buf.String(), "foo")
 
+	buf.Reset()
+	writePlainAny(nil, levelDebug, make(chan int))
+	assert.Contains(t, buf.String(), "unsupported type")
+	writePlainAny(nil, levelDebug, 100)
+	assert.Contains(t, buf.String(), "100")
+
 	buf.Reset()
 	writePlainAny(nil, levelError, make(chan int))
 	assert.Contains(t, buf.String(), "unsupported type")

core/mapping/marshaler_test.go

@@ -261,6 +261,78 @@ func TestMarshal_RangeOut(t *testing.T) {
 	}
 }
 
+func TestMarshal_RangeIllegal(t *testing.T) {
+	tests := []interface{}{
+		struct {
+			Int int `json:"int,range=[3:1]"`
+		}{
+			Int: 2,
+		},
+		struct {
+			Int int `json:"int,range=(3:1]"`
+		}{
+			Int: 2,
+		},
+	}
+
+	for _, test := range tests {
+		_, err := Marshal(test)
+		assert.Equal(t, err, errNumberRange)
+	}
+}
+
+func TestMarshal_RangeLeftEqualsToRight(t *testing.T) {
+	tests := []struct {
+		name  string
+		value interface{}
+		err   error
+	}{
+		{
+			name: "left inclusive, right inclusive",
+			value: struct {
+				Int int `json:"int,range=[2:2]"`
+			}{
+				Int: 2,
+			},
+		},
+		{
+			name: "left inclusive, right exclusive",
+			value: struct {
+				Int int `json:"int,range=[2:2)"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+		{
+			name: "left exclusive, right inclusive",
+			value: struct {
+				Int int `json:"int,range=(2:2]"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+		{
+			name: "left exclusive, right exclusive",
+			value: struct {
+				Int int `json:"int,range=(2:2)"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			_, err := Marshal(test.value)
+			assert.Equal(t, test.err, err)
+		})
+	}
+}
+
 func TestMarshal_FromString(t *testing.T) {
 	v := struct {
 		Age int `json:"age,string"`

core/mapping/unmarshaler.go

@@ -1,6 +1,7 @@
 package mapping
 
 import (
+	"encoding"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -25,9 +26,9 @@ var (
 	errValueNotStruct   = errors.New("value type is not struct")
 	keyUnmarshaler      = NewUnmarshaler(defaultKeyName)
 	durationType        = reflect.TypeOf(time.Duration(0))
-	cacheKeys           map[string][]string
+	cacheKeys           = make(map[string][]string)
 	cacheKeysLock       sync.Mutex
-	defaultCache        map[string]interface{}
+	defaultCache        = make(map[string]interface{})
 	defaultCacheLock    sync.Mutex
 	emptyMap            = map[string]interface{}{}
 	emptyValue          = reflect.ValueOf(lang.Placeholder)
@@ -49,11 +50,6 @@ type (
 	}
 )
 
-func init() {
-	cacheKeys = make(map[string][]string)
-	defaultCache = make(map[string]interface{})
-}
-
 // NewUnmarshaler returns a Unmarshaler.
 func NewUnmarshaler(key string, opts ...UnmarshalOption) *Unmarshaler {
 	unmarshaler := Unmarshaler{
@@ -144,7 +140,6 @@ func (u *Unmarshaler) processAnonymousFieldOptional(field reflect.StructField, v
 				filled = true
 				maybeNewValue(field, value)
 				indirectValue = reflect.Indirect(value)
-
 			}
 			if err = u.processField(subField, indirectValue.Field(i), m, fullName); err != nil {
 				return err
@@ -205,6 +200,8 @@ func (u *Unmarshaler) processFieldNotFromString(field reflect.StructField, value
 		return u.processFieldStruct(field, value, mapValue, fullName)
 	case valueKind == reflect.Map && typeKind == reflect.Map:
 		return u.fillMap(field, value, mapValue)
+	case valueKind == reflect.String && typeKind == reflect.Map:
+		return u.fillMapFromString(value, mapValue)
 	case valueKind == reflect.String && typeKind == reflect.Slice:
 		return u.fillSliceFromString(fieldType, value, mapValue)
 	case valueKind == reflect.String && derefedFieldType == durationType:
@@ -322,6 +319,28 @@ func (u *Unmarshaler) processFieldStructWithMap(field reflect.StructField, value
 	return nil
 }
 
+func (u *Unmarshaler) processFieldTextUnmarshaler(field reflect.StructField, value reflect.Value,
+	mapValue interface{}) (bool, error) {
+	var tval encoding.TextUnmarshaler
+	var ok bool
+
+	if field.Type.Kind() == reflect.Ptr {
+		tval, ok = value.Interface().(encoding.TextUnmarshaler)
+	} else {
+		tval, ok = value.Addr().Interface().(encoding.TextUnmarshaler)
+	}
+	if ok {
+		switch mv := mapValue.(type) {
+		case string:
+			return true, tval.UnmarshalText([]byte(mv))
+		case []byte:
+			return true, tval.UnmarshalText(mv)
+		}
+	}
+
+	return false, nil
+}
+
 func (u *Unmarshaler) processNamedField(field reflect.StructField, value reflect.Value,
 	m Valuer, fullName string) error {
 	key, opts, err := u.parseOptionsWithContext(field, m, fullName)
@@ -352,8 +371,16 @@ func (u *Unmarshaler) processNamedFieldWithValue(field reflect.StructField, valu
 		return fmt.Errorf("field %s mustn't be nil", key)
 	}
 
+	if !value.CanSet() {
+		return fmt.Errorf("field %s is not settable", key)
+	}
+
 	maybeNewValue(field, value)
 
+	if yes, err := u.processFieldTextUnmarshaler(field, value, mapValue); yes {
+		return err
+	}
+
 	fieldKind := Deref(field.Type).Kind()
 	switch fieldKind {
 	case reflect.Array, reflect.Map, reflect.Slice, reflect.Struct:
@@ -442,6 +469,27 @@ func (u *Unmarshaler) fillMap(field reflect.StructField, value reflect.Value, ma
 	return nil
 }
 
+func (u *Unmarshaler) fillMapFromString(value reflect.Value, mapValue interface{}) error {
+	if !value.CanSet() {
+		return errValueNotSettable
+	}
+
+	switch v := mapValue.(type) {
+	case fmt.Stringer:
+		if err := jsonx.UnmarshalFromString(v.String(), value.Addr().Interface()); err != nil {
+			return err
+		}
+	case string:
+		if err := jsonx.UnmarshalFromString(v, value.Addr().Interface()); err != nil {
+			return err
+		}
+	default:
+		return errUnsupportedType
+	}
+
+	return nil
+}
+
 func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, mapValue interface{}) error {
 	if !value.CanSet() {
 		return errValueNotSettable
@@ -534,8 +582,10 @@ func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int,
 	baseKind reflect.Kind, value interface{}) error {
 	ithVal := slice.Index(index)
 	switch v := value.(type) {
-	case json.Number:
+	case fmt.Stringer:
 		return setValue(baseKind, ithVal, v.String())
+	case string:
+		return setValue(baseKind, ithVal, v)
 	default:
 		// don't need to consider the difference between int, int8, int16, int32, int64,
 		// uint, uint8, uint16, uint32, uint64, because they're handled as json.Number.

core/mapping/unmarshaler_test.go

@@ -2,11 +2,13 @@ package mapping
 
 import (
 	"encoding/json"
+	"fmt"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/zeromicro/go-zero/core/stringx"
 )
@@ -467,6 +469,146 @@ func TestUnmarshalIntSliceFromString(t *testing.T) {
 	ast.Equal(2, v.Values[1])
 }
 
+func TestUnmarshalIntMapFromString(t *testing.T) {
+	var v struct {
+		Sort map[string]int `key:"sort"`
+	}
+	m := map[string]interface{}{
+		"sort": `{"value":12345,"zeroVal":0,"nullVal":null}`,
+	}
+
+	ast := assert.New(t)
+	ast.Nil(UnmarshalKey(m, &v))
+	ast.Equal(3, len(v.Sort))
+	ast.Equal(12345, v.Sort["value"])
+	ast.Equal(0, v.Sort["zeroVal"])
+	ast.Equal(0, v.Sort["nullVal"])
+}
+
+func TestUnmarshalBoolMapFromString(t *testing.T) {
+	var v struct {
+		Sort map[string]bool `key:"sort"`
+	}
+	m := map[string]interface{}{
+		"sort": `{"value":true,"zeroVal":false,"nullVal":null}`,
+	}
+
+	ast := assert.New(t)
+	ast.Nil(UnmarshalKey(m, &v))
+	ast.Equal(3, len(v.Sort))
+	ast.Equal(true, v.Sort["value"])
+	ast.Equal(false, v.Sort["zeroVal"])
+	ast.Equal(false, v.Sort["nullVal"])
+}
+
+type CustomStringer string
+
+type UnsupportedStringer string
+
+func (c CustomStringer) String() string {
+	return fmt.Sprintf("{%s}", string(c))
+}
+
+func TestUnmarshalStringMapFromStringer(t *testing.T) {
+	var v struct {
+		Sort map[string]string `key:"sort"`
+	}
+	m := map[string]interface{}{
+		"sort": CustomStringer(`"value":"ascend","emptyStr":""`),
+	}
+
+	ast := assert.New(t)
+	ast.Nil(UnmarshalKey(m, &v))
+	ast.Equal(2, len(v.Sort))
+	ast.Equal("ascend", v.Sort["value"])
+	ast.Equal("", v.Sort["emptyStr"])
+}
+
+func TestUnmarshalStringMapFromUnsupportedType(t *testing.T) {
+	var v struct {
+		Sort map[string]string `key:"sort"`
+	}
+	m := map[string]interface{}{
+		"sort": UnsupportedStringer(`{"value":"ascend","emptyStr":""}`),
+	}
+
+	ast := assert.New(t)
+	ast.NotNil(UnmarshalKey(m, &v))
+}
+
+func TestUnmarshalStringMapFromNotSettableValue(t *testing.T) {
+	var v struct {
+		sort  map[string]string  `key:"sort"`
+		psort *map[string]string `key:"sort"`
+	}
+	m := map[string]interface{}{
+		"sort":  `{"value":"ascend","emptyStr":""}`,
+		"psort": `{"value":"ascend","emptyStr":""}`,
+	}
+
+	ast := assert.New(t)
+	ast.NotNil(UnmarshalKey(m, &v))
+}
+
+func TestUnmarshalStringMapFromString(t *testing.T) {
+	var v struct {
+		Sort map[string]string `key:"sort"`
+	}
+	m := map[string]interface{}{
+		"sort": `{"value":"ascend","emptyStr":""}`,
+	}
+
+	ast := assert.New(t)
+	ast.Nil(UnmarshalKey(m, &v))
+	ast.Equal(2, len(v.Sort))
+	ast.Equal("ascend", v.Sort["value"])
+	ast.Equal("", v.Sort["emptyStr"])
+}
+
+func TestUnmarshalStructMapFromString(t *testing.T) {
+	var v struct {
+		Filter map[string]struct {
+			Field1 bool     `json:"field1"`
+			Field2 int64    `json:"field2,string"`
+			Field3 string   `json:"field3"`
+			Field4 *string  `json:"field4"`
+			Field5 []string `json:"field5"`
+		} `key:"filter"`
+	}
+	m := map[string]interface{}{
+		"filter": `{"obj":{"field1":true,"field2":"1573570455447539712","field3":"this is a string",
+			"field4":"this is a string pointer","field5":["str1","str2"]}}`,
+	}
+
+	ast := assert.New(t)
+	ast.Nil(UnmarshalKey(m, &v))
+	ast.Equal(1, len(v.Filter))
+	ast.NotNil(v.Filter["obj"])
+	ast.Equal(true, v.Filter["obj"].Field1)
+	ast.Equal(int64(1573570455447539712), v.Filter["obj"].Field2)
+	ast.Equal("this is a string", v.Filter["obj"].Field3)
+	ast.Equal("this is a string pointer", *v.Filter["obj"].Field4)
+	ast.ElementsMatch([]string{"str1", "str2"}, v.Filter["obj"].Field5)
+}
+
+func TestUnmarshalStringSliceMapFromString(t *testing.T) {
+	var v struct {
+		Filter map[string][]string `key:"filter"`
+	}
+	m := map[string]interface{}{
+		"filter": `{"assignType":null,"status":["process","comment"],"rate":[]}`,
+	}
+
+	ast := assert.New(t)
+	ast.Nil(UnmarshalKey(m, &v))
+	ast.Equal(3, len(v.Filter))
+	ast.Equal([]string(nil), v.Filter["assignType"])
+	ast.Equal(2, len(v.Filter["status"]))
+	ast.Equal("process", v.Filter["status"][0])
+	ast.Equal("comment", v.Filter["status"][1])
+	ast.Equal(0, len(v.Filter["rate"]))
+}
+
 func TestUnmarshalStruct(t *testing.T) {
 	type address struct {
 		City          string `key:"city"`
@@ -2681,7 +2823,7 @@ func TestUnmarshalJsonReaderMultiArray(t *testing.T) {
 	assert.Equal(t, 2, len(res.B))
 }
 
-func TestUnmarshalJsonReaderPtrMultiArray(t *testing.T) {
+func TestUnmarshalJsonReaderPtrMultiArrayString(t *testing.T) {
 	payload := `{"a": "133", "b": [["add", "cccd"], ["eeee"]]}`
 	var res struct {
 		A string      `json:"a"`
@@ -2694,6 +2836,32 @@ func TestUnmarshalJsonReaderPtrMultiArray(t *testing.T) {
 	assert.Equal(t, 2, len(res.B[0]))
 }
 
+func TestUnmarshalJsonReaderPtrMultiArrayString_Int(t *testing.T) {
+	payload := `{"a": "133", "b": [[11, 22], [33]]}`
+	var res struct {
+		A string      `json:"a"`
+		B [][]*string `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(res.B))
+	assert.Equal(t, 2, len(res.B[0]))
+}
+
+func TestUnmarshalJsonReaderPtrMultiArrayInt(t *testing.T) {
+	payload := `{"a": "133", "b": [[11, 22], [33]]}`
+	var res struct {
+		A string   `json:"a"`
+		B [][]*int `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(res.B))
+	assert.Equal(t, 2, len(res.B[0]))
+}
+
 func TestUnmarshalJsonReaderPtrArray(t *testing.T) {
 	payload := `{"a": "133", "b": ["add", "cccd", "eeee"]}`
 	var res struct {
@@ -2706,6 +2874,30 @@ func TestUnmarshalJsonReaderPtrArray(t *testing.T) {
 	assert.Equal(t, 3, len(res.B))
 }
 
+func TestUnmarshalJsonReaderPtrArray_Int(t *testing.T) {
+	payload := `{"a": "133", "b": [11, 22, 33]}`
+	var res struct {
+		A string    `json:"a"`
+		B []*string `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(res.B))
+}
+
+func TestUnmarshalJsonReaderPtrInt(t *testing.T) {
+	payload := `{"a": "133", "b": [11, 22, 33]}`
+	var res struct {
+		A string    `json:"a"`
+		B []*string `json:"b"`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(res.B))
+}
+
 func TestUnmarshalJsonWithoutKey(t *testing.T) {
 	payload := `{"A": "1", "B": "2"}`
 	var res struct {
@@ -2829,6 +3021,24 @@ func TestUnmarshalJsonReaderArrayString(t *testing.T) {
 	assert.NotNil(t, err)
 }
 
+func TestGoogleUUID(t *testing.T) {
+	var val struct {
+		Uid  uuid.UUID  `json:"uid,optional"`
+		Uidp *uuid.UUID `json:"uidp,optional"`
+	}
+	assert.NoError(t, UnmarshalJsonBytes([]byte(`{
+	"uid": "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
+	"uidp": "6ba7b810-9dad-11d1-80b4-00c04fd430c9"}`), &val))
+	assert.Equal(t, "6ba7b810-9dad-11d1-80b4-00c04fd430c8", val.Uid.String())
+	assert.Equal(t, "6ba7b810-9dad-11d1-80b4-00c04fd430c9", val.Uidp.String())
+	assert.NoError(t, UnmarshalJsonMap(map[string]interface{}{
+		"uid":  []byte("6ba7b810-9dad-11d1-80b4-00c04fd430c1"),
+		"uidp": []byte("6ba7b810-9dad-11d1-80b4-00c04fd430c2"),
+	}, &val))
+	assert.Equal(t, "6ba7b810-9dad-11d1-80b4-00c04fd430c1", val.Uid.String())
+	assert.Equal(t, "6ba7b810-9dad-11d1-80b4-00c04fd430c2", val.Uidp.String())
+}
+
 func BenchmarkDefaultValue(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		var a struct {

core/mapping/utils.go

@@ -143,6 +143,23 @@ func doParseKeyAndOptions(field reflect.StructField, value string) (string, *fie
 	return key, &fieldOpts, nil
 }
 
+// ensureValue ensures nested members not to be nil.
+// If pointer value is nil, set to a new value.
+func ensureValue(v reflect.Value) reflect.Value {
+	for {
+		if v.Kind() != reflect.Ptr {
+			break
+		}
+
+		if v.IsNil() {
+			v.Set(reflect.New(v.Type().Elem()))
+		}
+		v = v.Elem()
+	}
+
+	return v
+}
+
 func implicitValueRequiredStruct(tag string, tp reflect.Type) (bool, error) {
 	numFields := tp.NumField()
 	for i := 0; i < numFields; i++ {
@@ -294,6 +311,20 @@ func parseNumberRange(str string) (*numberRange, error) {
 		right = math.MaxFloat64
 	}
 
+	if left > right {
+		return nil, errNumberRange
+	}
+
+	// [2:2] valid
+	// [2:2) invalid
+	// (2:2] invalid
+	// (2:2) invalid
+	if left == right {
+		if !leftInclude || !rightInclude {
+			return nil, errNumberRange
+		}
+	}
+
 	return &numberRange{
 		left:         left,
 		leftInclude:  leftInclude,
@@ -478,6 +509,7 @@ func setValue(kind reflect.Kind, value reflect.Value, str string) error {
 		return errValueNotSettable
 	}
 
+	value = ensureValue(value)
 	v, err := convertType(kind, str)
 	if err != nil {
 		return err

core/mapping/utils_test.go

@@ -15,7 +15,7 @@ type Foo struct {
 	StrWithTagAndOption string `key:"stringwithtag,string"`
 }
 
-func TestDeferInt(t *testing.T) {
+func TestDerefInt(t *testing.T) {
 	i := 1
 	s := "hello"
 	number := struct {
@@ -60,6 +60,51 @@ func TestDeferInt(t *testing.T) {
 	}
 }
 
+func TestDerefValInt(t *testing.T) {
+	i := 1
+	s := "hello"
+	number := struct {
+		f float64
+	}{
+		f: 6.4,
+	}
+	cases := []struct {
+		t      reflect.Value
+		expect reflect.Kind
+	}{
+		{
+			t:      reflect.ValueOf(i),
+			expect: reflect.Int,
+		},
+		{
+			t:      reflect.ValueOf(&i),
+			expect: reflect.Int,
+		},
+		{
+			t:      reflect.ValueOf(s),
+			expect: reflect.String,
+		},
+		{
+			t:      reflect.ValueOf(&s),
+			expect: reflect.String,
+		},
+		{
+			t:      reflect.ValueOf(number.f),
+			expect: reflect.Float64,
+		},
+		{
+			t:      reflect.ValueOf(&number.f),
+			expect: reflect.Float64,
+		},
+	}
+
+	for _, each := range cases {
+		t.Run(each.t.String(), func(t *testing.T) {
+			assert.Equal(t, each.expect, ensureValue(each.t).Kind())
+		})
+	}
+}
+
 func TestParseKeyAndOptionWithoutTag(t *testing.T) {
 	var foo Foo
 	rte := reflect.TypeOf(&foo).Elem()

core/mapping/yamlunmarshaler.go

@@ -63,7 +63,7 @@ func cleanupMapValue(v interface{}) interface{} {
 	}
 }
 
-func unmarshal(unmarshaler *Unmarshaler, o interface{}, v interface{}) error {
+func unmarshal(unmarshaler *Unmarshaler, o, v interface{}) error {
 	if m, ok := o.(map[string]interface{}); ok {
 		return unmarshaler.Unmarshal(m, v)
 	}

core/metric/counter.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -47,10 +48,18 @@ func NewCounterVec(cfg *CounterVecOpts) CounterVec {
 }
 
 func (cv *promCounterVec) Inc(labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	cv.counter.WithLabelValues(labels...).Inc()
 }
 
 func (cv *promCounterVec) Add(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	cv.counter.WithLabelValues(labels...).Add(v)
 }
 

core/metric/counter_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 func TestNewCounterVec(t *testing.T) {
@@ -21,6 +22,7 @@ func TestNewCounterVec(t *testing.T) {
 }
 
 func TestCounterIncr(t *testing.T) {
+	startAgent()
 	counterVec := NewCounterVec(&CounterVecOpts{
 		Namespace: "http_client",
 		Subsystem: "call",
@@ -37,6 +39,7 @@ func TestCounterIncr(t *testing.T) {
 }
 
 func TestCounterAdd(t *testing.T) {
+	startAgent()
 	counterVec := NewCounterVec(&CounterVecOpts{
 		Namespace: "rpc_server",
 		Subsystem: "requests",
@@ -51,3 +54,11 @@ func TestCounterAdd(t *testing.T) {
 	r := testutil.ToFloat64(cv.counter)
 	assert.Equal(t, float64(33), r)
 }
+
+func startAgent() {
+	prometheus.StartAgent(prometheus.Config{
+		Host: "127.0.0.1",
+		Port: 9101,
+		Path: "/metrics",
+	})
+}

core/metric/gauge.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -50,14 +51,26 @@ func NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {
 }
 
 func (gv *promGaugeVec) Inc(labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Inc()
 }
 
 func (gv *promGaugeVec) Add(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Add(v)
 }
 
 func (gv *promGaugeVec) Set(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Set(v)
 }
 

core/metric/gauge_test.go

@@ -21,6 +21,7 @@ func TestNewGaugeVec(t *testing.T) {
 }
 
 func TestGaugeInc(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "rpc_client2",
 		Subsystem: "requests",
@@ -37,6 +38,7 @@ func TestGaugeInc(t *testing.T) {
 }
 
 func TestGaugeAdd(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "rpc_client",
 		Subsystem: "request",
@@ -53,6 +55,7 @@ func TestGaugeAdd(t *testing.T) {
 }
 
 func TestGaugeSet(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "http_client",
 		Subsystem: "request",

core/metric/histogram.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -53,6 +54,10 @@ func NewHistogramVec(cfg *HistogramVecOpts) HistogramVec {
 }
 
 func (hv *promHistogramVec) Observe(v int64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	hv.histogram.WithLabelValues(labels...).Observe(float64(v))
 }
 

core/metric/histogram_test.go

@@ -21,6 +21,7 @@ func TestNewHistogramVec(t *testing.T) {
 }
 
 func TestHistogramObserve(t *testing.T) {
+	startAgent()
 	histogramVec := NewHistogramVec(&HistogramVecOpts{
 		Name:    "counts",
 		Help:    "rpc server requests duration(ms).",

core/mr/mapreduce.go

@@ -145,7 +145,7 @@ func MapReduceChan(source <-chan interface{}, mapper MapperFunc, reducer Reducer
 	return mapReduceWithPanicChan(source, panicChan, mapper, reducer, opts...)
 }
 
-// MapReduceChan maps all elements from source, and reduce the output elements with given reducer.
+// mapReduceWithPanicChan maps all elements from source, and reduce the output elements with given reducer.
 func mapReduceWithPanicChan(source <-chan interface{}, panicChan *onceChan, mapper MapperFunc,
 	reducer ReducerFunc, opts ...Option) (interface{}, error) {
 	options := buildOptions(opts...)
@@ -212,6 +212,8 @@ func mapReduceWithPanicChan(source <-chan interface{}, panicChan *onceChan, mapp
 		cancel(context.DeadlineExceeded)
 		return nil, context.DeadlineExceeded
 	case v := <-panicChan.channel:
+		// drain output here, otherwise for loop panic in defer
+		drain(output)
 		panic(v)
 	case v, ok := <-output:
 		if err := retErr.Load(); err != nil {
@@ -376,9 +378,7 @@ type onceChan struct {
 }
 
 func (oc *onceChan) write(val interface{}) {
-	if atomic.AddInt32(&oc.wrote, 1) > 1 {
-		return
+	if atomic.CompareAndSwapInt32(&oc.wrote, 0, 1) {
+		oc.channel <- val
 	}
-
-	oc.channel <- val
 }

core/mr/mapreduce_fuzz_test.go

@@ -19,7 +19,7 @@ func FuzzMapReduce(f *testing.F) {
 	rand.Seed(time.Now().UnixNano())
 
 	f.Add(uint(10), uint(runtime.NumCPU()))
-	f.Fuzz(func(t *testing.T, num uint, workers uint) {
+	f.Fuzz(func(t *testing.T, num, workers uint) {
 		n := int64(num)%5000 + 5000
 		genPanic := rand.Intn(100) == 0
 		mapperPanic := rand.Intn(100) == 0

core/mr/mapreduce_test.go

@@ -3,7 +3,7 @@ package mr
 import (
 	"context"
 	"errors"
-	"io/ioutil"
+	"io"
 	"log"
 	"runtime"
 	"sync/atomic"
@@ -17,7 +17,7 @@ import (
 var errDummy = errors.New("dummy")
 
 func init() {
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 }
 
 func TestFinish(t *testing.T) {

core/service/serviceconf.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/zeromicro/go-zero/core/load"
 	"github.com/zeromicro/go-zero/core/logx"
+	"github.com/zeromicro/go-zero/core/proc"
 	"github.com/zeromicro/go-zero/core/prometheus"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/trace"
@@ -56,6 +57,9 @@ func (sc ServiceConf) SetUp() error {
 		sc.Telemetry.Name = sc.Name
 	}
 	trace.StartAgent(sc.Telemetry)
+	proc.AddShutdownListener(func() {
+		trace.StopAgent()
+	})
 
 	if len(sc.MetricsUrl) > 0 {
 		stat.SetReportWriter(stat.NewRemoteWriter(sc.MetricsUrl))

core/stat/alert.go

@@ -46,14 +46,14 @@ func Report(msg string) {
 	if fn != nil {
 		reported := lessExecutor.DoOrDiscard(func() {
 			var builder strings.Builder
-			fmt.Fprintf(&builder, "%s\n", time.Now().Format(timeFormat))
+			builder.WriteString(fmt.Sprintln(time.Now().Format(timeFormat)))
 			if len(clusterName) > 0 {
-				fmt.Fprintf(&builder, "cluster: %s\n", clusterName)
+				builder.WriteString(fmt.Sprintf("cluster: %s\n", clusterName))
 			}
-			fmt.Fprintf(&builder, "host: %s\n", sysx.Hostname())
+			builder.WriteString(fmt.Sprintf("host: %s\n", sysx.Hostname()))
 			dp := atomic.SwapInt32(&dropped, 0)
 			if dp > 0 {
-				fmt.Fprintf(&builder, "dropped: %d\n", dp)
+				builder.WriteString(fmt.Sprintf("dropped: %d\n", dp))
 			}
 			builder.WriteString(strings.TrimSpace(msg))
 			fn(builder.String())

core/stat/internal/cpu_linux.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/zeromicro/go-zero/core/iox"
@@ -20,10 +21,11 @@ var (
 	preTotal  uint64
 	quota     float64
 	cores     uint64
+	initOnce  sync.Once
 )
 
 // if /proc not present, ignore the cpu calculation, like wsl linux
-func init() {
+func initialize() {
 	cpus, err := cpuSets()
 	if err != nil {
 		logx.Error(err)
@@ -69,10 +71,13 @@ func init() {
 
 // RefreshCpu refreshes cpu usage and returns.
 func RefreshCpu() uint64 {
+	initOnce.Do(initialize)
+
 	total, err := totalCpuUsage()
 	if err != nil {
 		return 0
 	}
+
 	system, err := systemCpuUsage()
 	if err != nil {
 		return 0

core/stores/cache/cachenode.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"math/rand"
 	"sync"
 	"time"
@@ -130,7 +131,7 @@ func (c cacheNode) SetWithExpireCtx(ctx context.Context, key string, val interfa
 		return err
 	}
 
-	return c.rds.SetexCtx(ctx, key, string(data), int(expire.Seconds()))
+	return c.rds.SetexCtx(ctx, key, string(data), int(math.Ceil(expire.Seconds())))
 }
 
 // String returns a string that represents the cacheNode.
@@ -275,5 +276,6 @@ func (c cacheNode) processCache(ctx context.Context, key, data string, v interfa
 }
 
 func (c cacheNode) setCacheWithNotFound(ctx context.Context, key string) error {
-	return c.rds.SetexCtx(ctx, key, notFoundPlaceholder, int(c.aroundDuration(c.notFoundExpiry).Seconds()))
+	seconds := int(math.Ceil(c.aroundDuration(c.notFoundExpiry).Seconds()))
+	return c.rds.SetexCtx(ctx, key, notFoundPlaceholder, seconds)
 }

core/stores/kv/store.go

@@ -110,7 +110,9 @@ type (
 		Ttl(key string) (int, error)
 		TtlCtx(ctx context.Context, key string) (int, error)
 		Zadd(key string, score int64, value string) (bool, error)
+		ZaddFloat(key string, score float64, value string) (bool, error)
 		ZaddCtx(ctx context.Context, key string, score int64, value string) (bool, error)
+		ZaddFloatCtx(ctx context.Context, key string, score float64, value string) (bool, error)
 		Zadds(key string, ps ...redis.Pair) (int64, error)
 		ZaddsCtx(ctx context.Context, key string, ps ...redis.Pair) (int64, error)
 		Zcard(key string) (int, error)
@@ -787,13 +789,21 @@ func (cs clusterStore) Zadd(key string, score int64, value string) (bool, error)
 	return cs.ZaddCtx(context.Background(), key, score, value)
 }
 
+func (cs clusterStore) ZaddFloat(key string, score float64, value string) (bool, error) {
+	return cs.ZaddFloatCtx(context.Background(), key, score, value)
+}
+
 func (cs clusterStore) ZaddCtx(ctx context.Context, key string, score int64, value string) (bool, error) {
+	return cs.ZaddFloatCtx(ctx, key, float64(score), value)
+}
+
+func (cs clusterStore) ZaddFloatCtx(ctx context.Context, key string, score float64, value string) (bool, error) {
 	node, err := cs.getRedis(key)
 	if err != nil {
 		return false, err
 	}
 
-	return node.ZaddCtx(ctx, key, score, value)
+	return node.ZaddFloatCtx(ctx, key, score, value)
 }
 
 func (cs clusterStore) Zadds(key string, ps ...redis.Pair) (int64, error) {

core/stores/kv/store_test.go

@@ -22,7 +22,7 @@ func TestRedis_Decr(t *testing.T) {
 	_, err := store.Decr("a")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		val, err := client.Decr("a")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(-1), val)
@@ -37,7 +37,7 @@ func TestRedis_DecrBy(t *testing.T) {
 	_, err := store.Incrby("a", 2)
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		val, err := client.Decrby("a", 2)
 		assert.Nil(t, err)
 		assert.Equal(t, int64(-2), val)
@@ -52,7 +52,7 @@ func TestRedis_Exists(t *testing.T) {
 	_, err := store.Exists("foo")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		ok, err := client.Exists("a")
 		assert.Nil(t, err)
 		assert.False(t, ok)
@@ -68,7 +68,7 @@ func TestRedis_Eval(t *testing.T) {
 	_, err := store.Eval(`redis.call("EXISTS", KEYS[1])`, "key1")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		_, err := client.Eval(`redis.call("EXISTS", KEYS[1])`, "notexist")
 		assert.Equal(t, redis.Nil, err)
 		err = client.Set("key1", "value1")
@@ -88,7 +88,7 @@ func TestRedis_Hgetall(t *testing.T) {
 	_, err = store.Hgetall("a")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
 		vals, err := client.Hgetall("a")
@@ -105,7 +105,7 @@ func TestRedis_Hvals(t *testing.T) {
 	_, err := store.Hvals("a")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
 		vals, err := client.Hvals("a")
@@ -119,7 +119,7 @@ func TestRedis_Hsetnx(t *testing.T) {
 	_, err := store.Hsetnx("a", "dd", "ddd")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
 		ok, err := client.Hsetnx("a", "bb", "ccc")
@@ -141,7 +141,7 @@ func TestRedis_HdelHlen(t *testing.T) {
 	_, err = store.Hlen("a")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
 		num, err := client.Hlen("a")
@@ -161,7 +161,7 @@ func TestRedis_HIncrBy(t *testing.T) {
 	_, err := store.Hincrby("key", "field", 3)
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		val, err := client.Hincrby("key", "field", 2)
 		assert.Nil(t, err)
 		assert.Equal(t, 2, val)
@@ -176,7 +176,7 @@ func TestRedis_Hkeys(t *testing.T) {
 	_, err := store.Hkeys("a")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
 		vals, err := client.Hkeys("a")
@@ -190,7 +190,7 @@ func TestRedis_Hmget(t *testing.T) {
 	_, err := store.Hmget("a", "aa", "bb")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
 		vals, err := client.Hmget("a", "aa", "bb")
@@ -209,7 +209,7 @@ func TestRedis_Hmset(t *testing.T) {
 	})
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		assert.Nil(t, client.Hmset("a", map[string]string{
 			"aa": "aaa",
 			"bb": "bbb",
@@ -225,7 +225,7 @@ func TestRedis_Incr(t *testing.T) {
 	_, err := store.Incr("a")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		val, err := client.Incr("a")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(1), val)
@@ -240,7 +240,7 @@ func TestRedis_IncrBy(t *testing.T) {
 	_, err := store.Incrby("a", 2)
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		val, err := client.Incrby("a", 2)
 		assert.Nil(t, err)
 		assert.Equal(t, int64(2), val)
@@ -267,7 +267,7 @@ func TestRedis_List(t *testing.T) {
 	_, err = store.Lindex("key", 0)
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		val, err := client.Lpush("key", "value1", "value2")
 		assert.Nil(t, err)
 		assert.Equal(t, 2, val)
@@ -316,7 +316,7 @@ func TestRedis_Persist(t *testing.T) {
 	err = store.Expireat("key", time.Now().Unix()+5)
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		ok, err := client.Persist("key")
 		assert.Nil(t, err)
 		assert.False(t, ok)
@@ -348,7 +348,7 @@ func TestRedis_Sscan(t *testing.T) {
 	_, err = store.Del(key)
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		var list []string
 		for i := 0; i < 1550; i++ {
 			list = append(list, stringx.Randn(i))
@@ -390,7 +390,7 @@ func TestRedis_Set(t *testing.T) {
 	_, err = store.Spop("key")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		num, err := client.Sadd("key", 1, 2, 3, 4)
 		assert.Nil(t, err)
 		assert.Equal(t, 4, num)
@@ -434,7 +434,7 @@ func TestRedis_SetGetDel(t *testing.T) {
 	_, err = store.Del("hello")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		err := client.Set("hello", "world")
 		assert.Nil(t, err)
 		val, err := client.Get("hello")
@@ -457,7 +457,7 @@ func TestRedis_SetExNx(t *testing.T) {
 	_, err = store.SetnxEx("newhello", "newworld", 5)
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		err := client.Setex("hello", "world", 5)
 		assert.Nil(t, err)
 		ok, err := client.Setnx("hello", "newworld")
@@ -495,7 +495,7 @@ func TestRedis_Getset(t *testing.T) {
 	_, err := store.GetSet("hello", "world")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		val, err := client.GetSet("hello", "world")
 		assert.Nil(t, err)
 		assert.Equal(t, "", val)
@@ -524,7 +524,7 @@ func TestRedis_SetGetDelHashField(t *testing.T) {
 	_, err = store.Hdel("key", "field")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
+	runOnCluster(func(client Store) {
 		err := client.Hset("key", "field", "value")
 		assert.Nil(t, err)
 		val, err := client.Hget("key", "field")
@@ -587,8 +587,8 @@ func TestRedis_SortedSet(t *testing.T) {
 	})
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(client Store) {
-		ok, err := client.Zadd("key", 1, "value1")
+	runOnCluster(func(client Store) {
+		ok, err := client.ZaddFloat("key", 1, "value1")
 		assert.Nil(t, err)
 		assert.True(t, ok)
 		ok, err = client.Zadd("key", 2, "value1")
@@ -724,7 +724,7 @@ func TestRedis_HyperLogLog(t *testing.T) {
 	_, err = store.Pfcount("key")
 	assert.NotNil(t, err)
 
-	runOnCluster(t, func(cluster Store) {
+	runOnCluster(func(cluster Store) {
 		ok, err := cluster.Pfadd("key", "value")
 		assert.Nil(t, err)
 		assert.True(t, ok)
@@ -734,7 +734,7 @@ func TestRedis_HyperLogLog(t *testing.T) {
 	})
 }
 
-func runOnCluster(t *testing.T, fn func(cluster Store)) {
+func runOnCluster(fn func(cluster Store)) {
 	s1.FlushAll()
 	s2.FlushAll()
 

core/stores/mon/collection.go

@@ -83,12 +83,12 @@ type (
 		// FindOneAndReplace returns at most one document that matches the filter. If the filter
 		// matches multiple documents, FindOneAndReplace returns the first document in the
 		// collection that matches the filter.
-		FindOneAndReplace(ctx context.Context, filter interface{}, replacement interface{},
+		FindOneAndReplace(ctx context.Context, filter, replacement interface{},
 			opts ...*mopt.FindOneAndReplaceOptions) (*mongo.SingleResult, error)
 		// FindOneAndUpdate returns at most one document that matches the filter. If the filter
 		// matches multiple documents, FindOneAndUpdate returns the first document in the
 		// collection that matches the filter.
-		FindOneAndUpdate(ctx context.Context, filter interface{}, update interface{},
+		FindOneAndUpdate(ctx context.Context, filter, update interface{},
 			opts ...*mopt.FindOneAndUpdateOptions) (*mongo.SingleResult, error)
 		// Indexes returns the index view for this collection.
 		Indexes() mongo.IndexView
@@ -99,16 +99,16 @@ type (
 		InsertOne(ctx context.Context, document interface{}, opts ...*mopt.InsertOneOptions) (
 			*mongo.InsertOneResult, error)
 		// ReplaceOne replaces at most one document that matches the filter.
-		ReplaceOne(ctx context.Context, filter interface{}, replacement interface{},
+		ReplaceOne(ctx context.Context, filter, replacement interface{},
 			opts ...*mopt.ReplaceOptions) (*mongo.UpdateResult, error)
 		// UpdateByID updates a single document matching the provided filter.
-		UpdateByID(ctx context.Context, id interface{}, update interface{},
+		UpdateByID(ctx context.Context, id, update interface{},
 			opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)
 		// UpdateMany updates the provided documents.
-		UpdateMany(ctx context.Context, filter interface{}, update interface{},
+		UpdateMany(ctx context.Context, filter, update interface{},
 			opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)
 		// UpdateOne updates a single document matching the provided filter.
-		UpdateOne(ctx context.Context, filter interface{}, update interface{},
+		UpdateOne(ctx context.Context, filter, update interface{},
 			opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)
 		// Watch returns a change stream cursor used to receive notifications of changes to the collection.
 		Watch(ctx context.Context, pipeline interface{}, opts ...*mopt.ChangeStreamOptions) (
@@ -359,7 +359,7 @@ func (c *decoratedCollection) FindOneAndReplace(ctx context.Context, filter inte
 	return
 }
 
-func (c *decoratedCollection) FindOneAndUpdate(ctx context.Context, filter interface{}, update interface{},
+func (c *decoratedCollection) FindOneAndUpdate(ctx context.Context, filter, update interface{},
 	opts ...*mopt.FindOneAndUpdateOptions) (res *mongo.SingleResult, err error) {
 	ctx, span := startSpan(ctx, findOneAndUpdate)
 	defer func() {
@@ -420,7 +420,7 @@ func (c *decoratedCollection) InsertOne(ctx context.Context, document interface{
 	return
 }
 
-func (c *decoratedCollection) ReplaceOne(ctx context.Context, filter interface{}, replacement interface{},
+func (c *decoratedCollection) ReplaceOne(ctx context.Context, filter, replacement interface{},
 	opts ...*mopt.ReplaceOptions) (res *mongo.UpdateResult, err error) {
 	ctx, span := startSpan(ctx, replaceOne)
 	defer func() {
@@ -440,7 +440,7 @@ func (c *decoratedCollection) ReplaceOne(ctx context.Context, filter interface{}
 	return
 }
 
-func (c *decoratedCollection) UpdateByID(ctx context.Context, id interface{}, update interface{},
+func (c *decoratedCollection) UpdateByID(ctx context.Context, id, update interface{},
 	opts ...*mopt.UpdateOptions) (res *mongo.UpdateResult, err error) {
 	ctx, span := startSpan(ctx, updateByID)
 	defer func() {
@@ -460,7 +460,7 @@ func (c *decoratedCollection) UpdateByID(ctx context.Context, id interface{}, up
 	return
 }
 
-func (c *decoratedCollection) UpdateMany(ctx context.Context, filter interface{}, update interface{},
+func (c *decoratedCollection) UpdateMany(ctx context.Context, filter, update interface{},
 	opts ...*mopt.UpdateOptions) (res *mongo.UpdateResult, err error) {
 	ctx, span := startSpan(ctx, updateMany)
 	defer func() {
@@ -480,7 +480,7 @@ func (c *decoratedCollection) UpdateMany(ctx context.Context, filter interface{}
 	return
 }
 
-func (c *decoratedCollection) UpdateOne(ctx context.Context, filter interface{}, update interface{},
+func (c *decoratedCollection) UpdateOne(ctx context.Context, filter, update interface{},
 	opts ...*mopt.UpdateOptions) (res *mongo.UpdateResult, err error) {
 	ctx, span := startSpan(ctx, updateOne)
 	defer func() {

core/stores/mon/collection_test.go

@@ -20,10 +20,6 @@ import (
 
 var errDummy = errors.New("dummy")
 
-func init() {
-	logx.Disable()
-}
-
 func TestKeepPromise_accept(t *testing.T) {
 	p := new(mockPromise)
 	kp := keepablePromise{
@@ -110,14 +106,14 @@ func TestCollection_BulkWrite(t *testing.T) {
 		}
 		mt.AddMockResponses(mtest.CreateSuccessResponse(bson.D{{Key: "ok", Value: 1}}...))
 		res, err := c.BulkWrite(context.Background(), []mongo.WriteModel{
-			mongo.NewInsertOneModel().SetDocument(bson.D{{Key: "foo", Value: 1}})},
-		)
+			mongo.NewInsertOneModel().SetDocument(bson.D{{Key: "foo", Value: 1}}),
+		})
 		assert.Nil(t, err)
 		assert.NotNil(t, res)
 		c.brk = new(dropBreaker)
 		_, err = c.BulkWrite(context.Background(), []mongo.WriteModel{
-			mongo.NewInsertOneModel().SetDocument(bson.D{{Key: "foo", Value: 1}})},
-		)
+			mongo.NewInsertOneModel().SetDocument(bson.D{{Key: "foo", Value: 1}}),
+		})
 		assert.Equal(t, errDummy, err)
 	})
 }
@@ -208,7 +204,7 @@ func TestCollection_EstimatedDocumentCount(t *testing.T) {
 	})
 }
 
-func TestCollectionFind(t *testing.T) {
+func TestCollection_Find(t *testing.T) {
 	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
 	defer mt.Close()
 
@@ -256,7 +252,7 @@ func TestCollectionFind(t *testing.T) {
 	})
 }
 
-func TestCollectionFindOne(t *testing.T) {
+func TestCollection_FindOne(t *testing.T) {
 	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
 	defer mt.Close()
 
@@ -440,7 +436,7 @@ func TestCollection_InsertMany(t *testing.T) {
 	})
 }
 
-func TestCollection_Remove(t *testing.T) {
+func TestCollection_DeleteOne(t *testing.T) {
 	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
 	defer mt.Close()
 
@@ -460,7 +456,7 @@ func TestCollection_Remove(t *testing.T) {
 	})
 }
 
-func TestCollectionRemoveAll(t *testing.T) {
+func TestCollection_DeleteMany(t *testing.T) {
 	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
 	defer mt.Close()
 
@@ -569,7 +565,7 @@ func TestCollection_UpdateMany(t *testing.T) {
 	})
 }
 
-func Test_DecoratedCollectionLogDuration(t *testing.T) {
+func TestDecoratedCollection_LogDuration(t *testing.T) {
 	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
 	defer mt.Close()
 	c := decoratedCollection{

core/stores/mon/model.go

@@ -159,7 +159,7 @@ func (m *Model) FindOneAndDelete(ctx context.Context, v, filter interface{},
 }
 
 // FindOneAndReplace finds a single document and replaces it.
-func (m *Model) FindOneAndReplace(ctx context.Context, v, filter interface{}, replacement interface{},
+func (m *Model) FindOneAndReplace(ctx context.Context, v, filter, replacement interface{},
 	opts ...*mopt.FindOneAndReplaceOptions) error {
 	res, err := m.Collection.FindOneAndReplace(ctx, filter, replacement, opts...)
 	if err != nil {
@@ -170,7 +170,7 @@ func (m *Model) FindOneAndReplace(ctx context.Context, v, filter interface{}, re
 }
 
 // FindOneAndUpdate finds a single document and updates it.
-func (m *Model) FindOneAndUpdate(ctx context.Context, v, filter interface{}, update interface{},
+func (m *Model) FindOneAndUpdate(ctx context.Context, v, filter, update interface{},
 	opts ...*mopt.FindOneAndUpdateOptions) error {
 	res, err := m.Collection.FindOneAndUpdate(ctx, filter, update, opts...)
 	if err != nil {

core/stores/monc/cachedmodel.go

@@ -192,7 +192,7 @@ func (mm *Model) InsertOneNoCache(ctx context.Context, document interface{},
 }
 
 // ReplaceOne replaces a single document in the collection, and remove the cache.
-func (mm *Model) ReplaceOne(ctx context.Context, key string, filter interface{}, replacement interface{},
+func (mm *Model) ReplaceOne(ctx context.Context, key string, filter, replacement interface{},
 	opts ...*mopt.ReplaceOptions) (*mongo.UpdateResult, error) {
 	res, err := mm.Model.ReplaceOne(ctx, filter, replacement, opts...)
 	if err != nil {
@@ -207,7 +207,7 @@ func (mm *Model) ReplaceOne(ctx context.Context, key string, filter interface{},
 }
 
 // ReplaceOneNoCache replaces a single document in the collection.
-func (mm *Model) ReplaceOneNoCache(ctx context.Context, filter interface{}, replacement interface{},
+func (mm *Model) ReplaceOneNoCache(ctx context.Context, filter, replacement interface{},
 	opts ...*mopt.ReplaceOptions) (*mongo.UpdateResult, error) {
 	return mm.Model.ReplaceOne(ctx, filter, replacement, opts...)
 }
@@ -218,7 +218,7 @@ func (mm *Model) SetCache(key string, v interface{}) error {
 }
 
 // UpdateByID updates the document with given id with update, and remove the cache.
-func (mm *Model) UpdateByID(ctx context.Context, key string, id interface{}, update interface{},
+func (mm *Model) UpdateByID(ctx context.Context, key string, id, update interface{},
 	opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error) {
 	res, err := mm.Model.UpdateByID(ctx, id, update, opts...)
 	if err != nil {
@@ -233,13 +233,13 @@ func (mm *Model) UpdateByID(ctx context.Context, key string, id interface{}, upd
 }
 
 // UpdateByIDNoCache updates the document with given id with update.
-func (mm *Model) UpdateByIDNoCache(ctx context.Context, id interface{}, update interface{},
+func (mm *Model) UpdateByIDNoCache(ctx context.Context, id, update interface{},
 	opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error) {
 	return mm.Model.UpdateByID(ctx, id, update, opts...)
 }
 
 // UpdateMany updates the documents that match filter with update, and remove the cache.
-func (mm *Model) UpdateMany(ctx context.Context, keys []string, filter interface{}, update interface{},
+func (mm *Model) UpdateMany(ctx context.Context, keys []string, filter, update interface{},
 	opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error) {
 	res, err := mm.Model.UpdateMany(ctx, filter, update, opts...)
 	if err != nil {
@@ -254,13 +254,13 @@ func (mm *Model) UpdateMany(ctx context.Context, keys []string, filter interface
 }
 
 // UpdateManyNoCache updates the documents that match filter with update.
-func (mm *Model) UpdateManyNoCache(ctx context.Context, filter interface{}, update interface{},
+func (mm *Model) UpdateManyNoCache(ctx context.Context, filter, update interface{},
 	opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error) {
 	return mm.Model.UpdateMany(ctx, filter, update, opts...)
 }
 
 // UpdateOne updates the first document that matches filter with update, and remove the cache.
-func (mm *Model) UpdateOne(ctx context.Context, key string, filter interface{}, update interface{},
+func (mm *Model) UpdateOne(ctx context.Context, key string, filter, update interface{},
 	opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error) {
 	res, err := mm.Model.UpdateOne(ctx, filter, update, opts...)
 	if err != nil {
@@ -275,7 +275,7 @@ func (mm *Model) UpdateOne(ctx context.Context, key string, filter interface{},
 }
 
 // UpdateOneNoCache updates the first document that matches filter with update.
-func (mm *Model) UpdateOneNoCache(ctx context.Context, filter interface{}, update interface{},
+func (mm *Model) UpdateOneNoCache(ctx context.Context, filter, update interface{},
 	opts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error) {
 	return mm.Model.UpdateOne(ctx, filter, update, opts...)
 }

core/stores/mongo/collection_test.go

@@ -17,10 +17,6 @@ import (
 
 var errDummy = errors.New("dummy")
 
-func init() {
-	logx.Disable()
-}
-
 func TestKeepPromise_accept(t *testing.T) {
 	p := new(mockPromise)
 	kp := keepablePromise{

core/stores/mongoc/cachedcollection_test.go

@@ -3,7 +3,7 @@ package mongoc
 import (
 	"encoding/json"
 	"errors"
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
 	"runtime"
@@ -117,7 +117,7 @@ func TestStat(t *testing.T) {
 
 func TestStatCacheFails(t *testing.T) {
 	resetStats()
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 	defer log.SetOutput(os.Stdout)
 
 	r := redis.New("localhost:59999")

core/stores/redis/hook.go

@@ -2,10 +2,13 @@ package redis
 
 import (
 	"context"
+	"io"
+	"net"
 	"strings"
 	"time"
 
 	red "github.com/go-redis/redis/v8"
+	"github.com/zeromicro/go-zero/core/breaker"
 	"github.com/zeromicro/go-zero/core/errorx"
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/mapping"
@@ -53,7 +56,12 @@ func (h hook) AfterProcess(ctx context.Context, cmd red.Cmder) error {
 
 	duration := timex.Since(start)
 	if duration > slowThreshold.Load() {
-		logDuration(ctx, cmd, duration)
+		logDuration(ctx, []red.Cmder{cmd}, duration)
+	}
+
+	metricReqDur.Observe(int64(duration/time.Millisecond), cmd.Name())
+	if msg := formatError(err); len(msg) > 0 {
+		metricReqErr.Inc(cmd.Name(), msg)
 	}
 
 	return nil
@@ -95,19 +103,53 @@ func (h hook) AfterProcessPipeline(ctx context.Context, cmds []red.Cmder) error
 
 	duration := timex.Since(start)
 	if duration > slowThreshold.Load()*time.Duration(len(cmds)) {
-		logDuration(ctx, cmds[0], duration)
+		logDuration(ctx, cmds, duration)
+	}
+
+	metricReqDur.Observe(int64(duration/time.Millisecond), "Pipeline")
+	if msg := formatError(batchError.Err()); len(msg) > 0 {
+		metricReqErr.Inc("Pipeline", msg)
 	}
 
 	return nil
 }
 
-func logDuration(ctx context.Context, cmd red.Cmder, duration time.Duration) {
+func formatError(err error) string {
+	if err == nil || err == red.Nil {
+		return ""
+	}
+
+	opErr, ok := err.(*net.OpError)
+	if ok && opErr.Timeout() {
+		return "timeout"
+	}
+
+	switch err {
+	case io.EOF:
+		return "eof"
+	case context.DeadlineExceeded:
+		return "context deadline"
+	case breaker.ErrServiceUnavailable:
+		return "breaker"
+	default:
+		return "unexpected error"
+	}
+}
+
+func logDuration(ctx context.Context, cmds []red.Cmder, duration time.Duration) {
 	var buf strings.Builder
-	for i, arg := range cmd.Args() {
-		if i > 0 {
-			buf.WriteByte(' ')
+	for k, cmd := range cmds {
+		if k > 0 {
+			buf.WriteByte('\n')
 		}
-		buf.WriteString(mapping.Repr(arg))
+		var build strings.Builder
+		for i, arg := range cmd.Args() {
+			if i > 0 {
+				build.WriteByte(' ')
+			}
+			build.WriteString(mapping.Repr(arg))
+		}
+		buf.WriteString(build.String())
 	}
 	logx.WithContext(ctx).WithDuration(duration).Slowf("[REDIS] slowcall on executing: %s", buf.String())
 }

core/stores/redis/hook_test.go

@@ -21,6 +21,7 @@ func TestHookProcessCase1(t *testing.T) {
 		Batcher:  "jaeger",
 		Sampler:  1.0,
 	})
+	defer ztrace.StopAgent()
 
 	writer := log.Writer()
 	var buf strings.Builder
@@ -44,6 +45,7 @@ func TestHookProcessCase2(t *testing.T) {
 		Batcher:  "jaeger",
 		Sampler:  1.0,
 	})
+	defer ztrace.StopAgent()
 
 	w, restore := injectLog()
 	defer restore()
@@ -89,10 +91,10 @@ func TestHookProcessPipelineCase1(t *testing.T) {
 	log.SetOutput(&buf)
 	defer log.SetOutput(writer)
 
-	ctx, err := durationHook.BeforeProcessPipeline(context.Background(), []red.Cmder{red.NewCmd(context.Background())})
-	if err != nil {
-		t.Fatal(err)
-	}
+	ctx, err := durationHook.BeforeProcessPipeline(context.Background(), []red.Cmder{
+		red.NewCmd(context.Background()),
+	})
+	assert.NoError(t, err)
 	assert.Equal(t, "redis", tracesdk.SpanFromContext(ctx).(interface{ Name() string }).Name())
 
 	assert.Nil(t, durationHook.AfterProcessPipeline(ctx, []red.Cmder{
@@ -108,14 +110,15 @@ func TestHookProcessPipelineCase2(t *testing.T) {
 		Batcher:  "jaeger",
 		Sampler:  1.0,
 	})
+	defer ztrace.StopAgent()
 
 	w, restore := injectLog()
 	defer restore()
 
-	ctx, err := durationHook.BeforeProcessPipeline(context.Background(), []red.Cmder{red.NewCmd(context.Background())})
-	if err != nil {
-		t.Fatal(err)
-	}
+	ctx, err := durationHook.BeforeProcessPipeline(context.Background(), []red.Cmder{
+		red.NewCmd(context.Background()),
+	})
+	assert.NoError(t, err)
 	assert.Equal(t, "redis", tracesdk.SpanFromContext(ctx).(interface{ Name() string }).Name())
 
 	time.Sleep(slowThreshold.Load() + time.Millisecond)
@@ -156,10 +159,28 @@ func TestHookProcessPipelineCase5(t *testing.T) {
 	defer log.SetOutput(writer)
 
 	ctx := context.WithValue(context.Background(), startTimeKey, "foo")
-	assert.Nil(t, durationHook.AfterProcessPipeline(ctx, []red.Cmder{red.NewCmd(context.Background())}))
+	assert.Nil(t, durationHook.AfterProcessPipeline(ctx, []red.Cmder{
+		red.NewCmd(context.Background()),
+	}))
 	assert.True(t, buf.Len() == 0)
 }
 
+func TestLogDuration(t *testing.T) {
+	w, restore := injectLog()
+	defer restore()
+
+	logDuration(context.Background(), []red.Cmder{
+		red.NewCmd(context.Background(), "get", "foo"),
+	}, 1*time.Second)
+	assert.True(t, strings.Contains(w.String(), "get foo"))
+
+	logDuration(context.Background(), []red.Cmder{
+		red.NewCmd(context.Background(), "get", "foo"),
+		red.NewCmd(context.Background(), "set", "bar", 0),
+	}, 1*time.Second)
+	assert.True(t, strings.Contains(w.String(), `get foo\nset bar 0`))
+}
+
 func injectLog() (r *strings.Builder, restore func()) {
 	var buf strings.Builder
 	w := logx.NewWriter(&buf)

core/stores/redis/metrics.go

@@ -0,0 +1,23 @@
+package redis
+
+import "github.com/zeromicro/go-zero/core/metric"
+
+const namespace = "redis_client"
+
+var (
+	metricReqDur = metric.NewHistogramVec(&metric.HistogramVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "duration_ms",
+		Help:      "redis client requests duration(ms).",
+		Labels:    []string{"command"},
+		Buckets:   []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500},
+	})
+	metricReqErr = metric.NewCounterVec(&metric.CounterVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "error_total",
+		Help:      "redis client requests error count.",
+		Labels:    []string{"command", "error"},
+	})
+)

core/stores/redis/redis.go

@@ -226,20 +226,7 @@ func (s *Redis) Blpop(node RedisNode, key string) (string, error) {
 // BlpopCtx uses passed in redis connection to execute blocking queries.
 // Doesn't benefit from pooling redis connections of blocking queries
 func (s *Redis) BlpopCtx(ctx context.Context, node RedisNode, key string) (string, error) {
-	if node == nil {
-		return "", ErrNilNode
-	}
-
-	vals, err := node.BLPop(ctx, blockingQueryTimeout, key).Result()
-	if err != nil {
-		return "", err
-	}
-
-	if len(vals) < 2 {
-		return "", fmt.Errorf("no value on key: %s", key)
-	}
-
-	return vals[1], nil
+	return s.BlpopWithTimeoutCtx(ctx, node, blockingQueryTimeout, key)
 }
 
 // BlpopEx uses passed in redis connection to execute blpop command.
@@ -267,6 +254,32 @@ func (s *Redis) BlpopExCtx(ctx context.Context, node RedisNode, key string) (str
 	return vals[1], true, nil
 }
 
+// BlpopWithTimeout uses passed in redis connection to execute blpop command.
+// Control blocking query timeout
+func (s *Redis) BlpopWithTimeout(node RedisNode, timeout time.Duration, key string) (string, error) {
+	return s.BlpopWithTimeoutCtx(context.Background(), node, timeout, key)
+}
+
+// BlpopWithTimeoutCtx uses passed in redis connection to execute blpop command.
+// Control blocking query timeout
+func (s *Redis) BlpopWithTimeoutCtx(ctx context.Context, node RedisNode, timeout time.Duration,
+	key string) (string, error) {
+	if node == nil {
+		return "", ErrNilNode
+	}
+
+	vals, err := node.BLPop(ctx, timeout, key).Result()
+	if err != nil {
+		return "", err
+	}
+
+	if len(vals) < 2 {
+		return "", fmt.Errorf("no value on key: %s", key)
+	}
+
+	return vals[1], nil
+}
+
 // Decr is the implementation of redis decr command.
 func (s *Redis) Decr(key string) (int64, error) {
 	return s.DecrCtx(context.Background(), key)
@@ -1836,8 +1849,19 @@ func (s *Redis) Zadd(key string, score int64, value string) (bool, error) {
 	return s.ZaddCtx(context.Background(), key, score, value)
 }
 
+// ZaddFloat is the implementation of redis zadd command.
+func (s *Redis) ZaddFloat(key string, score float64, value string) (bool, error) {
+	return s.ZaddFloatCtx(context.Background(), key, score, value)
+}
+
 // ZaddCtx is the implementation of redis zadd command.
 func (s *Redis) ZaddCtx(ctx context.Context, key string, score int64, value string) (
+	val bool, err error) {
+	return s.ZaddFloatCtx(ctx, key, float64(score), value)
+}
+
+// ZaddFloatCtx is the implementation of redis zadd command.
+func (s *Redis) ZaddFloatCtx(ctx context.Context, key string, score float64, value string) (
 	val bool, err error) {
 	err = s.brk.DoWithAcceptable(func() error {
 		conn, err := getRedis(s)
@@ -1846,7 +1870,7 @@ func (s *Redis) ZaddCtx(ctx context.Context, key string, score int64, value stri
 		}
 
 		v, err := conn.ZAdd(ctx, key, &red.Z{
-			Score:  float64(score),
+			Score:  score,
 			Member: value,
 		}).Result()
 		if err != nil {

core/stores/redis/redis_test.go

@@ -810,7 +810,7 @@ func TestRedis_SetGetDelHashField(t *testing.T) {
 
 func TestRedis_SortedSet(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
-		ok, err := client.Zadd("key", 1, "value1")
+		ok, err := client.ZaddFloat("key", 1, "value1")
 		assert.Nil(t, err)
 		assert.True(t, ok)
 		ok, err = client.Zadd("key", 2, "value1")
@@ -988,8 +988,8 @@ func TestRedis_SortedSet(t *testing.T) {
 		assert.Equal(t, 0, len(pairs))
 		_, err = New(client.Addr, badType()).Zrevrank("key", "value")
 		assert.NotNil(t, err)
-		client.Zadd("second", 2, "aa")
-		client.Zadd("third", 3, "bbb")
+		_, _ = client.Zadd("second", 2, "aa")
+		_, _ = client.Zadd("third", 3, "bbb")
 		val, err = client.Zunionstore("union", &ZStore{
 			Keys:      []string{"second", "third"},
 			Weights:   []float64{1, 2},
@@ -1117,6 +1117,17 @@ func TestRedisBlpopEx(t *testing.T) {
 	})
 }
 
+func TestRedisBlpopWithTimeout(t *testing.T) {
+	runOnRedis(t, func(client *Redis) {
+		client.Ping()
+		var node mockedNode
+		_, err := client.BlpopWithTimeout(nil, 10*time.Second, "foo")
+		assert.NotNil(t, err)
+		_, err = client.BlpopWithTimeout(node, 10*time.Second, "foo")
+		assert.NotNil(t, err)
+	})
+}
+
 func TestRedisGeo(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		client.Ping()
@@ -1176,7 +1187,7 @@ func runOnRedis(t *testing.T, fn func(client *Redis)) {
 		}
 
 		if client != nil {
-			client.Close()
+			_ = client.Close()
 		}
 	}()
 	fn(New(s.Addr()))
@@ -1198,7 +1209,7 @@ func runOnRedisTLS(t *testing.T, fn func(client *Redis)) {
 			t.Error(err)
 		}
 		if client != nil {
-			client.Close()
+			_ = client.Close()
 		}
 	}()
 	fn(New(s.Addr(), WithTLS()))
@@ -1214,6 +1225,6 @@ type mockedNode struct {
 	RedisNode
 }
 
-func (n mockedNode) BLPop(ctx context.Context, timeout time.Duration, keys ...string) *red.StringSliceCmd {
+func (n mockedNode) BLPop(_ context.Context, _ time.Duration, _ ...string) *red.StringSliceCmd {
 	return red.NewStringSliceCmd(context.Background(), "foo", "bar")
 }

core/stores/redis/redislock.go

@@ -1,12 +1,14 @@
 package redis
 
 import (
+	"context"
 	"math/rand"
 	"strconv"
 	"sync/atomic"
 	"time"
 
 	red "github.com/go-redis/redis/v8"
+
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/stringx"
 )
@@ -51,8 +53,13 @@ func NewRedisLock(store *Redis, key string) *RedisLock {
 
 // Acquire acquires the lock.
 func (rl *RedisLock) Acquire() (bool, error) {
+	return rl.AcquireCtx(context.Background())
+}
+
+// AcquireCtx acquires the lock with the given ctx.
+func (rl *RedisLock) AcquireCtx(ctx context.Context) (bool, error) {
 	seconds := atomic.LoadUint32(&rl.seconds)
-	resp, err := rl.store.Eval(lockCommand, []string{rl.key}, []string{
+	resp, err := rl.store.EvalCtx(ctx, lockCommand, []string{rl.key}, []string{
 		rl.id, strconv.Itoa(int(seconds)*millisPerSecond + tolerance),
 	})
 	if err == red.Nil {
@@ -75,7 +82,12 @@ func (rl *RedisLock) Acquire() (bool, error) {
 
 // Release releases the lock.
 func (rl *RedisLock) Release() (bool, error) {
-	resp, err := rl.store.Eval(delCommand, []string{rl.key}, []string{rl.id})
+	return rl.ReleaseCtx(context.Background())
+}
+
+// ReleaseCtx releases the lock with the given ctx.
+func (rl *RedisLock) ReleaseCtx(ctx context.Context) (bool, error) {
+	resp, err := rl.store.EvalCtx(ctx, delCommand, []string{rl.key}, []string{rl.id})
 	if err != nil {
 		return false, err
 	}

core/stores/redis/redislock_test.go

@@ -1,33 +1,65 @@
 package redis
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
 	"github.com/zeromicro/go-zero/core/stringx"
 )
 
 func TestRedisLock(t *testing.T) {
-	runOnRedis(t, func(client *Redis) {
-		key := stringx.Rand()
-		firstLock := NewRedisLock(client, key)
-		firstLock.SetExpire(5)
-		firstAcquire, err := firstLock.Acquire()
-		assert.Nil(t, err)
-		assert.True(t, firstAcquire)
+	testFn := func(ctx context.Context) func(client *Redis) {
+		return func(client *Redis) {
+			key := stringx.Rand()
+			firstLock := NewRedisLock(client, key)
+			firstLock.SetExpire(5)
+			firstAcquire, err := firstLock.Acquire()
+			assert.Nil(t, err)
+			assert.True(t, firstAcquire)
 
-		secondLock := NewRedisLock(client, key)
-		secondLock.SetExpire(5)
-		againAcquire, err := secondLock.Acquire()
-		assert.Nil(t, err)
-		assert.False(t, againAcquire)
+			secondLock := NewRedisLock(client, key)
+			secondLock.SetExpire(5)
+			againAcquire, err := secondLock.Acquire()
+			assert.Nil(t, err)
+			assert.False(t, againAcquire)
 
-		release, err := firstLock.Release()
-		assert.Nil(t, err)
-		assert.True(t, release)
+			release, err := firstLock.Release()
+			assert.Nil(t, err)
+			assert.True(t, release)
 
-		endAcquire, err := secondLock.Acquire()
-		assert.Nil(t, err)
-		assert.True(t, endAcquire)
+			endAcquire, err := secondLock.Acquire()
+			assert.Nil(t, err)
+			assert.True(t, endAcquire)
+		}
+	}
+
+	t.Run("normal", func(t *testing.T) {
+		runOnRedis(t, testFn(nil))
+	})
+
+	t.Run("withContext", func(t *testing.T) {
+		runOnRedis(t, testFn(context.Background()))
+	})
+}
+
+func TestRedisLock_Expired(t *testing.T) {
+	runOnRedis(t, func(client *Redis) {
+		key := stringx.Rand()
+		redisLock := NewRedisLock(client, key)
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel()
+		_, err := redisLock.AcquireCtx(ctx)
+		assert.NotNil(t, err)
+	})
+
+	runOnRedis(t, func(client *Redis) {
+		key := stringx.Rand()
+		redisLock := NewRedisLock(client, key)
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel()
+		_, err := redisLock.ReleaseCtx(ctx)
+		assert.NotNil(t, err)
 	})
 }

core/stores/sqlc/cachedsql_test.go

@@ -6,7 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
 	"runtime"
@@ -284,7 +284,7 @@ func TestCachedConn_QueryRowIndex_HasWrongCache(t *testing.T) {
 
 func TestStatCacheFails(t *testing.T) {
 	resetStats()
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 	defer log.SetOutput(os.Stdout)
 
 	r := redis.New("localhost:59999")

core/stores/sqlx/metrics.go

@@ -0,0 +1,23 @@
+package sqlx
+
+import "github.com/zeromicro/go-zero/core/metric"
+
+const namespace = "sql_client"
+
+var (
+	metricReqDur = metric.NewHistogramVec(&metric.HistogramVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "duration_ms",
+		Help:      "mysql client requests duration(ms).",
+		Labels:    []string{"command"},
+		Buckets:   []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500},
+	})
+	metricReqErr = metric.NewCounterVec(&metric.CounterVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "error_total",
+		Help:      "mysql client requests error count.",
+		Labels:    []string{"command", "error"},
+	})
+)

core/stores/sqlx/sqlconn.go

@@ -153,6 +153,9 @@ func (db *commonSqlConn) ExecCtx(ctx context.Context, q string, args ...interfac
 		result, err = exec(ctx, conn, q, args...)
 		return err
 	}, db.acceptable)
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("Exec", "breaker")
+	}
 
 	return
 }
@@ -186,6 +189,9 @@ func (db *commonSqlConn) PrepareCtx(ctx context.Context, query string) (stmt Stm
 		}
 		return nil
 	}, db.acceptable)
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("Prepare", "breaker")
+	}
 
 	return
 }
@@ -270,9 +276,14 @@ func (db *commonSqlConn) TransactCtx(ctx context.Context, fn func(context.Contex
 		endSpan(span, err)
 	}()
 
-	return db.brk.DoWithAcceptable(func() error {
+	err = db.brk.DoWithAcceptable(func() error {
 		return transact(ctx, db, db.beginTx, fn)
 	}, db.acceptable)
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("Transact", "breaker")
+	}
+
+	return
 }
 
 func (db *commonSqlConn) acceptable(err error) bool {
@@ -287,7 +298,7 @@ func (db *commonSqlConn) acceptable(err error) bool {
 func (db *commonSqlConn) queryRows(ctx context.Context, scanner func(*sql.Rows) error,
 	q string, args ...interface{}) (err error) {
 	var qerr error
-	return db.brk.DoWithAcceptable(func() error {
+	err = db.brk.DoWithAcceptable(func() error {
 		conn, err := db.connProv()
 		if err != nil {
 			db.onError(err)
@@ -301,6 +312,11 @@ func (db *commonSqlConn) queryRows(ctx context.Context, scanner func(*sql.Rows)
 	}, func(err error) bool {
 		return qerr == err || db.acceptable(err)
 	})
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("queryRows", "breaker")
+	}
+
+	return
 }
 
 func (s statement) Close() error {

core/stores/sqlx/sqlconn_test.go

@@ -17,7 +17,8 @@ func init() {
 }
 
 func TestSqlConn(t *testing.T) {
-	mock := buildConn()
+	mock, err := buildConn()
+	assert.Nil(t, err)
 	mock.ExpectExec("any")
 	mock.ExpectQuery("any").WillReturnRows(sqlmock.NewRows([]string{"foo"}))
 	conn := NewMysql(mockedDatasource)
@@ -50,8 +51,8 @@ func TestSqlConn(t *testing.T) {
 	}))
 }
 
-func buildConn() (mock sqlmock.Sqlmock) {
-	connManager.GetResource(mockedDatasource, func() (io.Closer, error) {
+func buildConn() (mock sqlmock.Sqlmock, err error) {
+	_, err = connManager.GetResource(mockedDatasource, func() (io.Closer, error) {
 		var db *sql.DB
 		var err error
 		db, mock, err = sqlmock.New()

core/stores/sqlx/stmt.go

@@ -12,7 +12,22 @@ import (
 
 const defaultSlowThreshold = time.Millisecond * 500
 
-var slowThreshold = syncx.ForAtomicDuration(defaultSlowThreshold)
+var (
+	slowThreshold = syncx.ForAtomicDuration(defaultSlowThreshold)
+	logSql        = syncx.ForAtomicBool(true)
+	logSlowSql    = syncx.ForAtomicBool(true)
+)
+
+// DisableLog disables logging of sql statements, includes info and slow logs.
+func DisableLog() {
+	logSql.Set(false)
+	logSlowSql.Set(false)
+}
+
+// DisableStmtLog disables info logging of sql statements, but keeps slow logs.
+func DisableStmtLog() {
+	logSql.Set(false)
+}
 
 // SetSlowThreshold sets the slow threshold.
 func SetSlowThreshold(threshold time.Duration) {
@@ -20,64 +35,39 @@ func SetSlowThreshold(threshold time.Duration) {
 }
 
 func exec(ctx context.Context, conn sessionConn, q string, args ...interface{}) (sql.Result, error) {
-	stmt, err := format(q, args...)
-	if err != nil {
+	guard := newGuard("exec")
+	if err := guard.start(q, args...); err != nil {
 		return nil, err
 	}
 
-	startTime := timex.Now()
 	result, err := conn.ExecContext(ctx, q, args...)
-	duration := timex.Since(startTime)
-	if duration > slowThreshold.Load() {
-		logx.WithContext(ctx).WithDuration(duration).Slowf("[SQL] exec: slowcall - %s", stmt)
-	} else {
-		logx.WithContext(ctx).WithDuration(duration).Infof("sql exec: %s", stmt)
-	}
-	if err != nil {
-		logSqlError(ctx, stmt, err)
-	}
+	guard.finish(ctx, err)
 
 	return result, err
 }
 
 func execStmt(ctx context.Context, conn stmtConn, q string, args ...interface{}) (sql.Result, error) {
-	stmt, err := format(q, args...)
-	if err != nil {
+	guard := newGuard("execStmt")
+	if err := guard.start(q, args...); err != nil {
 		return nil, err
 	}
 
-	startTime := timex.Now()
 	result, err := conn.ExecContext(ctx, args...)
-	duration := timex.Since(startTime)
-	if duration > slowThreshold.Load() {
-		logx.WithContext(ctx).WithDuration(duration).Slowf("[SQL] execStmt: slowcall - %s", stmt)
-	} else {
-		logx.WithContext(ctx).WithDuration(duration).Infof("sql execStmt: %s", stmt)
-	}
-	if err != nil {
-		logSqlError(ctx, stmt, err)
-	}
+	guard.finish(ctx, err)
 
 	return result, err
 }
 
 func query(ctx context.Context, conn sessionConn, scanner func(*sql.Rows) error,
 	q string, args ...interface{}) error {
-	stmt, err := format(q, args...)
-	if err != nil {
+	guard := newGuard("query")
+	if err := guard.start(q, args...); err != nil {
 		return err
 	}
 
-	startTime := timex.Now()
 	rows, err := conn.QueryContext(ctx, q, args...)
-	duration := timex.Since(startTime)
-	if duration > slowThreshold.Load() {
-		logx.WithContext(ctx).WithDuration(duration).Slowf("[SQL] query: slowcall - %s", stmt)
-	} else {
-		logx.WithContext(ctx).WithDuration(duration).Infof("sql query: %s", stmt)
-	}
+	guard.finish(ctx, err)
 	if err != nil {
-		logSqlError(ctx, stmt, err)
 		return err
 	}
 	defer rows.Close()
@@ -87,24 +77,76 @@ func query(ctx context.Context, conn sessionConn, scanner func(*sql.Rows) error,
 
 func queryStmt(ctx context.Context, conn stmtConn, scanner func(*sql.Rows) error,
 	q string, args ...interface{}) error {
-	stmt, err := format(q, args...)
-	if err != nil {
+	guard := newGuard("queryStmt")
+	if err := guard.start(q, args...); err != nil {
 		return err
 	}
 
-	startTime := timex.Now()
 	rows, err := conn.QueryContext(ctx, args...)
-	duration := timex.Since(startTime)
-	if duration > slowThreshold.Load() {
-		logx.WithContext(ctx).WithDuration(duration).Slowf("[SQL] queryStmt: slowcall - %s", stmt)
-	} else {
-		logx.WithContext(ctx).WithDuration(duration).Infof("sql queryStmt: %s", stmt)
-	}
+	guard.finish(ctx, err)
 	if err != nil {
-		logSqlError(ctx, stmt, err)
 		return err
 	}
 	defer rows.Close()
 
 	return scanner(rows)
 }
+
+type (
+	sqlGuard interface {
+		start(q string, args ...interface{}) error
+		finish(ctx context.Context, err error)
+	}
+
+	nilGuard struct{}
+
+	realSqlGuard struct {
+		command   string
+		stmt      string
+		startTime time.Duration
+	}
+)
+
+func newGuard(command string) sqlGuard {
+	if logSql.True() || logSlowSql.True() {
+		return &realSqlGuard{
+			command: command,
+		}
+	}
+
+	return nilGuard{}
+}
+
+func (n nilGuard) start(_ string, _ ...interface{}) error {
+	return nil
+}
+
+func (n nilGuard) finish(_ context.Context, _ error) {
+}
+
+func (e *realSqlGuard) finish(ctx context.Context, err error) {
+	duration := timex.Since(e.startTime)
+	if duration > slowThreshold.Load() {
+		logx.WithContext(ctx).WithDuration(duration).Slowf("[SQL] %s: slowcall - %s", e.command, e.stmt)
+	} else if logSql.True() {
+		logx.WithContext(ctx).WithDuration(duration).Infof("sql %s: %s", e.command, e.stmt)
+	}
+
+	if err != nil {
+		logSqlError(ctx, e.stmt, err)
+	}
+
+	metricReqDur.Observe(int64(duration/time.Millisecond), e.command)
+}
+
+func (e *realSqlGuard) start(q string, args ...interface{}) error {
+	stmt, err := format(q, args...)
+	if err != nil {
+		return err
+	}
+
+	e.stmt = stmt
+	e.startTime = timex.Now()
+
+	return nil
+}

core/stores/sqlx/stmt_test.go

@@ -178,6 +178,47 @@ func TestSetSlowThreshold(t *testing.T) {
 	assert.Equal(t, time.Second, slowThreshold.Load())
 }
 
+func TestDisableLog(t *testing.T) {
+	assert.True(t, logSql.True())
+	assert.True(t, logSlowSql.True())
+	defer func() {
+		logSql.Set(true)
+		logSlowSql.Set(true)
+	}()
+
+	DisableLog()
+	assert.False(t, logSql.True())
+	assert.False(t, logSlowSql.True())
+}
+
+func TestDisableStmtLog(t *testing.T) {
+	assert.True(t, logSql.True())
+	assert.True(t, logSlowSql.True())
+	defer func() {
+		logSql.Set(true)
+		logSlowSql.Set(true)
+	}()
+
+	DisableStmtLog()
+	assert.False(t, logSql.True())
+	assert.True(t, logSlowSql.True())
+}
+
+func TestNilGuard(t *testing.T) {
+	assert.True(t, logSql.True())
+	assert.True(t, logSlowSql.True())
+	defer func() {
+		logSql.Set(true)
+		logSlowSql.Set(true)
+	}()
+
+	DisableLog()
+	guard := newGuard("any")
+	assert.Nil(t, guard.start("foo", "bar"))
+	guard.finish(context.Background(), nil)
+	assert.Equal(t, nilGuard{}, guard)
+}
+
 type mockedSessionConn struct {
 	lastInsertId int64
 	rowsAffected int64

core/stores/sqlx/utils.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/mapping"
@@ -152,6 +153,14 @@ func writeValue(buf *strings.Builder, arg interface{}) {
 		buf.WriteByte('\'')
 		buf.WriteString(escape(v))
 		buf.WriteByte('\'')
+	case time.Time:
+		buf.WriteByte('\'')
+		buf.WriteString(v.String())
+		buf.WriteByte('\'')
+	case *time.Time:
+		buf.WriteByte('\'')
+		buf.WriteString(v.String())
+		buf.WriteByte('\'')
 	default:
 		buf.WriteString(mapping.Repr(v))
 	}

core/stores/sqlx/utils_test.go

@@ -3,6 +3,7 @@ package sqlx
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -138,3 +139,14 @@ func TestFormat(t *testing.T) {
 		})
 	}
 }
+
+func TestWriteValue(t *testing.T) {
+	var buf strings.Builder
+	tm := time.Now()
+	writeValue(&buf, &tm)
+	assert.Equal(t, "'"+tm.String()+"'", buf.String())
+
+	buf.Reset()
+	writeValue(&buf, tm)
+	assert.Equal(t, "'"+tm.String()+"'", buf.String())
+}

core/syncx/resourcemanager.go

@@ -57,8 +57,8 @@ func (manager *ResourceManager) GetResource(key string, create func() (io.Closer
 		}
 
 		manager.lock.Lock()
+		defer manager.lock.Unlock()
 		manager.resources[key] = resource
-		manager.lock.Unlock()
 
 		return resource, nil
 	})

core/syncx/resourcemanager_test.go

@@ -74,6 +74,12 @@ func TestResourceManager_UseAfterClose(t *testing.T) {
 			return nil, errors.New("fail")
 		})
 		assert.NotNil(t, err)
+
+		assert.Panics(t, func() {
+			_, err = manager.GetResource("key", func() (io.Closer, error) {
+				return &dummyResource{age: 123}, nil
+			})
+		})
 	}
 }
 

core/threading/routinegroup_test.go

@@ -1,7 +1,7 @@
 package threading
 
 import (
-	"io/ioutil"
+	"io"
 	"log"
 	"sync"
 	"sync/atomic"
@@ -25,7 +25,7 @@ func TestRoutineGroupRun(t *testing.T) {
 }
 
 func TestRoutingGroupRunSafe(t *testing.T) {
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 
 	var count int32
 	group := NewRoutineGroup()

core/threading/routines_test.go

@@ -1,7 +1,7 @@
 package threading
 
 import (
-	"io/ioutil"
+	"io"
 	"log"
 	"testing"
 
@@ -14,7 +14,7 @@ func TestRoutineId(t *testing.T) {
 }
 
 func TestRunSafe(t *testing.T) {
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 
 	i := 0