chore: update goctl version to 1.4.4 (#2811)

* chore: refactor func name

* chore: make plain log clearer

.codecov.yml

@@ -1,3 +1,6 @@
-comment: false
+comment:
+  layout: "flags, files"
+  behavior: once
+  require_changes: true
 ignore:
   - "tools"

.github/workflows/go.yml

@@ -11,15 +11,17 @@ jobs:
     name: Linux
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.x
-        uses: actions/setup-go@v2
-        with:
-          go-version: ^1.15
-        id: go
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
 
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: ^1.16
+          check-latest: true
+          cache: true
+        id: go
+
       - name: Get dependencies
         run: |
           go get -v -t -d ./...
@@ -34,20 +36,23 @@ jobs:
         run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
 
       - name: Codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
 
   test-win:
     name: Windows
     runs-on: windows-latest
     steps:
-      - name: Set up Go 1.x
-        uses: actions/setup-go@v2
-        with:
-          go-version: ^1.15
-
       - name: Checkout codebase
         uses: actions/checkout@v3
 
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          # use 1.16 to guarantee Go 1.16 compatibility
+          go-version: 1.16
+          check-latest: true
+          cache: true
+
       - name: Test
         run: |
           go mod verify

.github/workflows/issue-translator.yml

@@ -9,7 +9,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: tomsun28/issues-translate-action@v2.6
+      - uses: usthe/issues-translate-action@v2.7
         with:
           IS_MODIFY_TITLE: true
           # not require, default false, . Decide whether to modify the issue title

.github/workflows/issues.yml

@@ -7,10 +7,10 @@ jobs:
   close-issues:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v6
         with:
-          days-before-issue-stale: 30
+          days-before-issue-stale: 365
-          days-before-issue-close: 14
+          days-before-issue-close: 90
           stale-issue-label: "stale"
           stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
           close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."

.github/workflows/release.yaml

@@ -25,4 +25,4 @@ jobs:
           goversion: "https://dl.google.com/go/go1.17.5.linux-amd64.tar.gz"
           project_path: "tools/goctl"
           binary_name: "goctl"
-          extra_files: tools/goctl/goctl.md
+          extra_files: tools/goctl/readme.md tools/goctl/readme-cn.md

.gitignore

@@ -22,6 +22,7 @@ go.work.sum
 
 # gitlab ci
 .cache
+.golangci.yml
 
 # vim auto backup file
 *~

ROADMAP.md

@@ -1,28 +0,0 @@
-# go-zero Roadmap
-
-This document defines a high level roadmap for go-zero development and upcoming releases.
-Community and contributor involvement is vital for successfully implementing all desired items for each release.
-We hope that the items listed below will inspire further engagement from the community to keep go-zero progressing and shipping exciting and valuable features.
-
-## 2021 Q2
-- [x] Support service discovery through K8S client api
-- [x] Log full sql statements for easier sql problem solving
-
-## 2021 Q3
-- [x] Support `goctl model pg` to support PostgreSQL code generation
-- [x] Adapt builtin tracing mechanism to opentracing solutions
-
-## 2021 Q4
-- [x] Support `username/password` authentication in ETCD
-- [x] Support `SSL/TLS` in ETCD
-- [x] Support `SSL/TLS` in `zRPC`
-- [x] Support `TLS` in redis connections
-- [x] Support `goctl bug` to report bugs conveniently
-
-## 2022
-- [x] Support `context` in redis related methods for timeout and tracing
-- [x] Support `context` in sql related methods for timeout and tracing
-- [x] Support `context` in mongodb related methods for timeout and tracing
-- [x] Add `httpc.Do` with HTTP call governance, like circuit breaker etc.
-- [ ] Support `goctl doctor` command to report potential issues for given service
-- [ ] Support `goctl mock` command to start a mocking server with given `.api` file

core/breaker/nopbreaker.go

@@ -20,16 +20,16 @@ func (b noOpBreaker) Do(req func() error) error {
 	return req()
 }
 
-func (b noOpBreaker) DoWithAcceptable(req func() error, acceptable Acceptable) error {
+func (b noOpBreaker) DoWithAcceptable(req func() error, _ Acceptable) error {
 	return req()
 }
 
-func (b noOpBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
+func (b noOpBreaker) DoWithFallback(req func() error, _ func(err error) error) error {
 	return req()
 }
 
-func (b noOpBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
+func (b noOpBreaker) DoWithFallbackAcceptable(req func() error, _ func(err error) error,
-	acceptable Acceptable) error {
+	_ Acceptable) error {
 	return req()
 }
 
@@ -38,5 +38,5 @@ type nopPromise struct{}
 func (p nopPromise) Accept() {
 }
 
-func (p nopPromise) Reject(reason string) {
+func (p nopPromise) Reject(_ string) {
 }

core/codec/aesecb.go

@@ -32,9 +32,11 @@ func NewECBEncrypter(b cipher.Block) cipher.BlockMode {
 	return (*ecbEncrypter)(newECB(b))
 }
 
+// BlockSize returns the mode's block size.
 func (x *ecbEncrypter) BlockSize() int { return x.blockSize }
 
-// why we don't return error is because cipher.BlockMode doesn't allow this
+// CryptBlocks encrypts a number of blocks. The length of src must be a multiple of
+// the block size. Dst and src must overlap entirely or not at all.
 func (x *ecbEncrypter) CryptBlocks(dst, src []byte) {
 	if len(src)%x.blockSize != 0 {
 		logx.Error("crypto/cipher: input not full blocks")
@@ -59,11 +61,13 @@ func NewECBDecrypter(b cipher.Block) cipher.BlockMode {
 	return (*ecbDecrypter)(newECB(b))
 }
 
+// BlockSize returns the mode's block size.
 func (x *ecbDecrypter) BlockSize() int {
 	return x.blockSize
 }
 
-// why we don't return error is because cipher.BlockMode doesn't allow this
+// CryptBlocks decrypts a number of blocks. The length of src must be a multiple of
+// the block size. Dst and src must overlap entirely or not at all.
 func (x *ecbDecrypter) CryptBlocks(dst, src []byte) {
 	if len(src)%x.blockSize != 0 {
 		logx.Error("crypto/cipher: input not full blocks")

core/codec/aesecb_test.go

@@ -1,6 +1,7 @@
 package codec
 
 import (
+	"crypto/aes"
 	"encoding/base64"
 	"testing"
 
@@ -10,7 +11,8 @@ import (
 func TestAesEcb(t *testing.T) {
 	var (
 		key     = []byte("q4t7w!z%C*F-JaNdRgUjXn2r5u8x/A?D")
-		val     = []byte("hello")
+		val     = []byte("helloworld")
+		valLong = []byte("helloworldlong..")
 		badKey1 = []byte("aaaaaaaaa")
 		// more than 32 chars
 		badKey2 = []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
@@ -31,6 +33,39 @@ func TestAesEcb(t *testing.T) {
 	src, err := EcbDecrypt(key, dst)
 	assert.Nil(t, err)
 	assert.Equal(t, val, src)
+	block, err := aes.NewCipher(key)
+	assert.NoError(t, err)
+	encrypter := NewECBEncrypter(block)
+	assert.Equal(t, 16, encrypter.BlockSize())
+	decrypter := NewECBDecrypter(block)
+	assert.Equal(t, 16, decrypter.BlockSize())
+
+	dst = make([]byte, 8)
+	encrypter.CryptBlocks(dst, val)
+	for _, b := range dst {
+		assert.Equal(t, byte(0), b)
+	}
+
+	dst = make([]byte, 8)
+	encrypter.CryptBlocks(dst, valLong)
+	for _, b := range dst {
+		assert.Equal(t, byte(0), b)
+	}
+
+	dst = make([]byte, 8)
+	decrypter.CryptBlocks(dst, val)
+	for _, b := range dst {
+		assert.Equal(t, byte(0), b)
+	}
+
+	dst = make([]byte, 8)
+	decrypter.CryptBlocks(dst, valLong)
+	for _, b := range dst {
+		assert.Equal(t, byte(0), b)
+	}
+
+	_, err = EcbEncryptBase64("cTR0N3dDKkYtSmFOZFJnVWpYbjJyNXU4eC9BP0QK", "aGVsbG93b3JsZGxvbmcuLgo=")
+	assert.Error(t, err)
 }
 
 func TestAesEcbBase64(t *testing.T) {

core/codec/dh_test.go

@@ -80,3 +80,17 @@ func TestKeyBytes(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, len(key.Bytes()) > 0)
 }
+
+func TestDHOnErrors(t *testing.T) {
+	key, err := GenerateKey()
+	assert.Nil(t, err)
+	assert.NotEmpty(t, key.Bytes())
+	_, err = ComputeKey(key.PubKey, key.PriKey)
+	assert.NoError(t, err)
+	_, err = ComputeKey(nil, key.PriKey)
+	assert.Error(t, err)
+	_, err = ComputeKey(key.PubKey, nil)
+	assert.Error(t, err)
+
+	assert.NotNil(t, NewPublicKey([]byte("")))
+}

core/codec/rsa.go

@@ -7,7 +7,7 @@ import (
 	"encoding/base64"
 	"encoding/pem"
 	"errors"
-	"io/ioutil"
+	"os"
 )
 
 var (
@@ -48,7 +48,7 @@ type (
 
 // NewRsaDecrypter returns a RsaDecrypter with the given file.
 func NewRsaDecrypter(file string) (RsaDecrypter, error) {
-	content, err := ioutil.ReadFile(file)
+	content, err := os.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}

core/collection/ring.go

@@ -6,7 +6,7 @@ import "sync"
 type Ring struct {
 	elements []interface{}
 	index    int
-	lock     sync.Mutex
+	lock     sync.RWMutex
 }
 
 // NewRing returns a Ring object with the given size n.
@@ -31,8 +31,8 @@ func (r *Ring) Add(v interface{}) {
 
 // Take takes all items from r.
 func (r *Ring) Take() []interface{} {
-	r.lock.Lock()
+	r.lock.RLock()
-	defer r.lock.Unlock()
+	defer r.lock.RUnlock()
 
 	var size int
 	var start int

core/collection/safemap.go

@@ -68,6 +68,24 @@ func (m *SafeMap) Get(key interface{}) (interface{}, bool) {
 	return val, ok
 }
 
+// Range calls f sequentially for each key and value present in the map.
+// If f returns false, range stops the iteration.
+func (m *SafeMap) Range(f func(key, val interface{}) bool) {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	for k, v := range m.dirtyOld {
+		if !f(k, v) {
+			return
+		}
+	}
+	for k, v := range m.dirtyNew {
+		if !f(k, v) {
+			return
+		}
+	}
+}
+
 // Set sets the value into m with the given key.
 func (m *SafeMap) Set(key, value interface{}) {
 	m.lock.Lock()

core/collection/safemap_test.go

@@ -1,6 +1,7 @@
 package collection
 
 import (
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -107,3 +108,42 @@ func testSafeMapWithParameters(t *testing.T, size, exception int) {
 		}
 	}
 }
+
+func TestSafeMap_Range(t *testing.T) {
+	const (
+		size       = 100000
+		exception1 = 5
+		exception2 = 500
+	)
+
+	m := NewSafeMap()
+	newMap := NewSafeMap()
+
+	for i := 0; i < size; i++ {
+		m.Set(i, i)
+	}
+	for i := 0; i < size; i++ {
+		if i%exception1 == 0 {
+			m.Del(i)
+		}
+	}
+
+	for i := size; i < size<<1; i++ {
+		m.Set(i, i)
+	}
+	for i := size; i < size<<1; i++ {
+		if i%exception2 != 0 {
+			m.Del(i)
+		}
+	}
+
+	var count int32
+	m.Range(func(k, v interface{}) bool {
+		atomic.AddInt32(&count, 1)
+		newMap.Set(k, v)
+		return true
+	})
+	assert.Equal(t, int(atomic.LoadInt32(&count)), m.Size())
+	assert.Equal(t, m.dirtyNew, newMap.dirtyNew)
+	assert.Equal(t, m.dirtyOld, newMap.dirtyOld)
+}

core/collection/set.go

@@ -29,7 +29,7 @@ func NewSet() *Set {
 	}
 }
 
-// NewUnmanagedSet returns a unmanaged Set, which can put values with different types.
+// NewUnmanagedSet returns an unmanaged Set, which can put values with different types.
 func NewUnmanagedSet() *Set {
 	return &Set{
 		data: make(map[interface{}]lang.PlaceholderType),

core/collection/timingwheel.go

@@ -69,10 +69,11 @@ func NewTimingWheel(interval time.Duration, numSlots int, execute Execute) (*Tim
 			interval, numSlots, execute)
 	}
 
-	return newTimingWheelWithClock(interval, numSlots, execute, timex.NewTicker(interval))
+	return NewTimingWheelWithTicker(interval, numSlots, execute, timex.NewTicker(interval))
 }
 
-func newTimingWheelWithClock(interval time.Duration, numSlots int, execute Execute,
+// NewTimingWheelWithTicker returns a TimingWheel with the given ticker.
+func NewTimingWheelWithTicker(interval time.Duration, numSlots int, execute Execute,
 	ticker timex.Ticker) (*TimingWheel, error) {
 	tw := &TimingWheel{
 		interval:      interval,

core/collection/timingwheel_test.go

@@ -26,7 +26,7 @@ func TestNewTimingWheel(t *testing.T) {
 
 func TestTimingWheel_Drain(t *testing.T) {
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {
 	}, ticker)
 	tw.SetTimer("first", 3, testStep*4)
 	tw.SetTimer("second", 5, testStep*7)
@@ -62,7 +62,7 @@ func TestTimingWheel_Drain(t *testing.T) {
 func TestTimingWheel_SetTimerSoon(t *testing.T) {
 	run := syncx.NewAtomicBool()
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {
 		assert.True(t, run.CompareAndSwap(false, true))
 		assert.Equal(t, "any", k)
 		assert.Equal(t, 3, v.(int))
@@ -78,7 +78,7 @@ func TestTimingWheel_SetTimerSoon(t *testing.T) {
 func TestTimingWheel_SetTimerTwice(t *testing.T) {
 	run := syncx.NewAtomicBool()
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {
 		assert.True(t, run.CompareAndSwap(false, true))
 		assert.Equal(t, "any", k)
 		assert.Equal(t, 5, v.(int))
@@ -96,7 +96,7 @@ func TestTimingWheel_SetTimerTwice(t *testing.T) {
 
 func TestTimingWheel_SetTimerWrongDelay(t *testing.T) {
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {}, ticker)
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {}, ticker)
 	defer tw.Stop()
 	assert.NotPanics(t, func() {
 		tw.SetTimer("any", 3, -testStep)
@@ -105,7 +105,7 @@ func TestTimingWheel_SetTimerWrongDelay(t *testing.T) {
 
 func TestTimingWheel_SetTimerAfterClose(t *testing.T) {
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {}, ticker)
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {}, ticker)
 	tw.Stop()
 	assert.Equal(t, ErrClosed, tw.SetTimer("any", 3, testStep))
 }
@@ -113,7 +113,7 @@ func TestTimingWheel_SetTimerAfterClose(t *testing.T) {
 func TestTimingWheel_MoveTimer(t *testing.T) {
 	run := syncx.NewAtomicBool()
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 3, func(k, v interface{}) {
+	tw, _ := NewTimingWheelWithTicker(testStep, 3, func(k, v interface{}) {
 		assert.True(t, run.CompareAndSwap(false, true))
 		assert.Equal(t, "any", k)
 		assert.Equal(t, 3, v.(int))
@@ -139,7 +139,7 @@ func TestTimingWheel_MoveTimer(t *testing.T) {
 func TestTimingWheel_MoveTimerSoon(t *testing.T) {
 	run := syncx.NewAtomicBool()
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 3, func(k, v interface{}) {
+	tw, _ := NewTimingWheelWithTicker(testStep, 3, func(k, v interface{}) {
 		assert.True(t, run.CompareAndSwap(false, true))
 		assert.Equal(t, "any", k)
 		assert.Equal(t, 3, v.(int))
@@ -155,7 +155,7 @@ func TestTimingWheel_MoveTimerSoon(t *testing.T) {
 func TestTimingWheel_MoveTimerEarlier(t *testing.T) {
 	run := syncx.NewAtomicBool()
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {
 		assert.True(t, run.CompareAndSwap(false, true))
 		assert.Equal(t, "any", k)
 		assert.Equal(t, 3, v.(int))
@@ -173,7 +173,7 @@ func TestTimingWheel_MoveTimerEarlier(t *testing.T) {
 
 func TestTimingWheel_RemoveTimer(t *testing.T) {
 	ticker := timex.NewFakeTicker()
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {}, ticker)
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {}, ticker)
 	tw.SetTimer("any", 3, testStep)
 	assert.NotPanics(t, func() {
 		tw.RemoveTimer("any")
@@ -236,7 +236,7 @@ func TestTimingWheel_SetTimer(t *testing.T) {
 			}
 			var actual int32
 			done := make(chan lang.PlaceholderType)
-			tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
+			tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value interface{}) {
 				assert.Equal(t, 1, key.(int))
 				assert.Equal(t, 2, value.(int))
 				actual = atomic.LoadInt32(&count)
@@ -317,7 +317,7 @@ func TestTimingWheel_SetAndMoveThenStart(t *testing.T) {
 			}
 			var actual int32
 			done := make(chan lang.PlaceholderType)
-			tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
+			tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value interface{}) {
 				actual = atomic.LoadInt32(&count)
 				close(done)
 			}, ticker)
@@ -405,7 +405,7 @@ func TestTimingWheel_SetAndMoveTwice(t *testing.T) {
 			}
 			var actual int32
 			done := make(chan lang.PlaceholderType)
-			tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
+			tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value interface{}) {
 				actual = atomic.LoadInt32(&count)
 				close(done)
 			}, ticker)
@@ -486,7 +486,7 @@ func TestTimingWheel_ElapsedAndSet(t *testing.T) {
 			}
 			var actual int32
 			done := make(chan lang.PlaceholderType)
-			tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
+			tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value interface{}) {
 				actual = atomic.LoadInt32(&count)
 				close(done)
 			}, ticker)
@@ -577,7 +577,7 @@ func TestTimingWheel_ElapsedAndSetThenMove(t *testing.T) {
 			}
 			var actual int32
 			done := make(chan lang.PlaceholderType)
-			tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
+			tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value interface{}) {
 				actual = atomic.LoadInt32(&count)
 				close(done)
 			}, ticker)
@@ -612,7 +612,7 @@ func TestMoveAndRemoveTask(t *testing.T) {
 		}
 	}
 	var keys []int
-	tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
+	tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v interface{}) {
 		assert.Equal(t, "any", k)
 		assert.Equal(t, 3, v.(int))
 		keys = append(keys, v.(int))

core/conf/config.go

@@ -2,13 +2,15 @@ package conf
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
+	"reflect"
 	"strings"
 
+	"github.com/zeromicro/go-zero/core/jsonx"
 	"github.com/zeromicro/go-zero/core/mapping"
+	"github.com/zeromicro/go-zero/internal/encoding"
 )
 
 var loaders = map[string]func([]byte, interface{}) error{
@@ -18,9 +20,15 @@ var loaders = map[string]func([]byte, interface{}) error{
 	".yml":  LoadFromYamlBytes,
 }
 
+type fieldInfo struct {
+	name     string
+	kind     reflect.Kind
+	children map[string]fieldInfo
+}
+
 // Load loads config into v from file, .json, .yaml and .yml are acceptable.
 func Load(file string, v interface{}, opts ...Option) error {
-	content, err := ioutil.ReadFile(file)
+	content, err := os.ReadFile(file)
 	if err != nil {
 		return err
 	}
@@ -50,7 +58,15 @@ func LoadConfig(file string, v interface{}, opts ...Option) error {
 
 // LoadFromJsonBytes loads config into v from content json bytes.
 func LoadFromJsonBytes(content []byte, v interface{}) error {
-	return mapping.UnmarshalJsonBytes(content, v)
+	var m map[string]interface{}
+	if err := jsonx.Unmarshal(content, &m); err != nil {
+		return err
+	}
+
+	finfo := buildFieldsInfo(reflect.TypeOf(v))
+	lowerCaseKeyMap := toLowerCaseKeyMap(m, finfo)
+
+	return mapping.UnmarshalJsonMap(lowerCaseKeyMap, v, mapping.WithCanonicalKeyFunc(toLowerCase))
 }
 
 // LoadConfigFromJsonBytes loads config into v from content json bytes.
@@ -61,12 +77,22 @@ func LoadConfigFromJsonBytes(content []byte, v interface{}) error {
 
 // LoadFromTomlBytes loads config into v from content toml bytes.
 func LoadFromTomlBytes(content []byte, v interface{}) error {
-	return mapping.UnmarshalTomlBytes(content, v)
+	b, err := encoding.TomlToJson(content)
+	if err != nil {
+		return err
+	}
+
+	return LoadFromJsonBytes(b, v)
 }
 
 // LoadFromYamlBytes loads config into v from content yaml bytes.
 func LoadFromYamlBytes(content []byte, v interface{}) error {
-	return mapping.UnmarshalYamlBytes(content, v)
+	b, err := encoding.YamlToJson(content)
+	if err != nil {
+		return err
+	}
+
+	return LoadFromJsonBytes(b, v)
 }
 
 // LoadConfigFromYamlBytes loads config into v from content yaml bytes.
@@ -81,3 +107,101 @@ func MustLoad(path string, v interface{}, opts ...Option) {
 		log.Fatalf("error: config file %s, %s", path, err.Error())
 	}
 }
+
+func buildFieldsInfo(tp reflect.Type) map[string]fieldInfo {
+	tp = mapping.Deref(tp)
+
+	switch tp.Kind() {
+	case reflect.Struct:
+		return buildStructFieldsInfo(tp)
+	case reflect.Array, reflect.Slice:
+		return buildFieldsInfo(mapping.Deref(tp.Elem()))
+	default:
+		return nil
+	}
+}
+
+func buildStructFieldsInfo(tp reflect.Type) map[string]fieldInfo {
+	info := make(map[string]fieldInfo)
+
+	for i := 0; i < tp.NumField(); i++ {
+		field := tp.Field(i)
+		name := field.Name
+		lowerCaseName := toLowerCase(name)
+		ft := mapping.Deref(field.Type)
+
+		// flatten anonymous fields
+		if field.Anonymous {
+			if ft.Kind() == reflect.Struct {
+				fields := buildFieldsInfo(ft)
+				for k, v := range fields {
+					info[k] = v
+				}
+			} else {
+				info[lowerCaseName] = fieldInfo{
+					name: name,
+					kind: ft.Kind(),
+				}
+			}
+			continue
+		}
+
+		var fields map[string]fieldInfo
+		switch ft.Kind() {
+		case reflect.Struct:
+			fields = buildFieldsInfo(ft)
+		case reflect.Array, reflect.Slice:
+			fields = buildFieldsInfo(ft.Elem())
+		case reflect.Map:
+			fields = buildFieldsInfo(ft.Elem())
+		}
+
+		info[lowerCaseName] = fieldInfo{
+			name:     name,
+			kind:     ft.Kind(),
+			children: fields,
+		}
+	}
+
+	return info
+}
+
+func toLowerCase(s string) string {
+	return strings.ToLower(s)
+}
+
+func toLowerCaseInterface(v interface{}, info map[string]fieldInfo) interface{} {
+	switch vv := v.(type) {
+	case map[string]interface{}:
+		return toLowerCaseKeyMap(vv, info)
+	case []interface{}:
+		var arr []interface{}
+		for _, vvv := range vv {
+			arr = append(arr, toLowerCaseInterface(vvv, info))
+		}
+		return arr
+	default:
+		return v
+	}
+}
+
+func toLowerCaseKeyMap(m map[string]interface{}, info map[string]fieldInfo) map[string]interface{} {
+	res := make(map[string]interface{})
+
+	for k, v := range m {
+		ti, ok := info[k]
+		if ok {
+			res[k] = toLowerCaseInterface(v, ti.children)
+			continue
+		}
+
+		lk := toLowerCase(k)
+		if ti, ok = info[lk]; ok {
+			res[lk] = toLowerCaseInterface(v, ti.children)
+		} else {
+			res[k] = v
+		}
+	}
+
+	return res
+}

core/conf/config_test.go

@@ -1,7 +1,6 @@
 package conf
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -57,6 +56,22 @@ func TestConfigJson(t *testing.T) {
 	}
 }
 
+func TestLoadFromJsonBytesArray(t *testing.T) {
+	input := []byte(`{"users": [{"name": "foo"}, {"Name": "bar"}]}`)
+	var val struct {
+		Users []struct {
+			Name string
+		}
+	}
+
+	assert.NoError(t, LoadFromJsonBytes(input, &val))
+	var expect []string
+	for _, user := range val.Users {
+		expect = append(expect, user.Name)
+	}
+	assert.EqualValues(t, []string{"foo", "bar"}, expect)
+}
+
 func TestConfigToml(t *testing.T) {
 	text := `a = "foo"
 b = 1
@@ -82,6 +97,89 @@ d = "abcd!@#$112"
 	assert.Equal(t, "abcd!@#$112", val.D)
 }
 
+func TestConfigOptional(t *testing.T) {
+	text := `a = "foo"
+b = 1
+c = "FOO"
+d = "abcd"
+`
+	tmpfile, err := createTempFile(".toml", text)
+	assert.Nil(t, err)
+	defer os.Remove(tmpfile)
+
+	var val struct {
+		A string `json:"a"`
+		B int    `json:"b,optional"`
+		C string `json:"c,optional=B"`
+		D string `json:"d,optional=b"`
+	}
+	if assert.NoError(t, Load(tmpfile, &val)) {
+		assert.Equal(t, "foo", val.A)
+		assert.Equal(t, 1, val.B)
+		assert.Equal(t, "FOO", val.C)
+		assert.Equal(t, "abcd", val.D)
+	}
+}
+
+func TestConfigJsonCanonical(t *testing.T) {
+	text := []byte(`{"a": "foo", "B": "bar"}`)
+
+	var val1 struct {
+		A string `json:"a"`
+		B string `json:"b"`
+	}
+	var val2 struct {
+		A string
+		B string
+	}
+	assert.NoError(t, LoadFromJsonBytes(text, &val1))
+	assert.Equal(t, "foo", val1.A)
+	assert.Equal(t, "bar", val1.B)
+	assert.NoError(t, LoadFromJsonBytes(text, &val2))
+	assert.Equal(t, "foo", val2.A)
+	assert.Equal(t, "bar", val2.B)
+}
+
+func TestConfigTomlCanonical(t *testing.T) {
+	text := []byte(`a = "foo"
+B = "bar"`)
+
+	var val1 struct {
+		A string `json:"a"`
+		B string `json:"b"`
+	}
+	var val2 struct {
+		A string
+		B string
+	}
+	assert.NoError(t, LoadFromTomlBytes(text, &val1))
+	assert.Equal(t, "foo", val1.A)
+	assert.Equal(t, "bar", val1.B)
+	assert.NoError(t, LoadFromTomlBytes(text, &val2))
+	assert.Equal(t, "foo", val2.A)
+	assert.Equal(t, "bar", val2.B)
+}
+
+func TestConfigYamlCanonical(t *testing.T) {
+	text := []byte(`a: foo
+B: bar`)
+
+	var val1 struct {
+		A string `json:"a"`
+		B string `json:"b"`
+	}
+	var val2 struct {
+		A string
+		B string
+	}
+	assert.NoError(t, LoadFromYamlBytes(text, &val1))
+	assert.Equal(t, "foo", val1.A)
+	assert.Equal(t, "bar", val1.B)
+	assert.NoError(t, LoadFromYamlBytes(text, &val2))
+	assert.Equal(t, "foo", val2.A)
+	assert.Equal(t, "bar", val2.B)
+}
+
 func TestConfigTomlEnv(t *testing.T) {
 	text := `a = "foo"
 b = 1
@@ -106,7 +204,6 @@ d = "abcd!@#112"
 	assert.Equal(t, 1, val.B)
 	assert.Equal(t, "2", val.C)
 	assert.Equal(t, "abcd!@#112", val.D)
-
 }
 
 func TestConfigJsonEnv(t *testing.T) {
@@ -145,13 +242,221 @@ func TestConfigJsonEnv(t *testing.T) {
 	}
 }
 
+func TestToCamelCase(t *testing.T) {
+	tests := []struct {
+		input  string
+		expect string
+	}{
+		{
+			input:  "",
+			expect: "",
+		},
+		{
+			input:  "A",
+			expect: "a",
+		},
+		{
+			input:  "a",
+			expect: "a",
+		},
+		{
+			input:  "hello_world",
+			expect: "hello_world",
+		},
+		{
+			input:  "Hello_world",
+			expect: "hello_world",
+		},
+		{
+			input:  "hello_World",
+			expect: "hello_world",
+		},
+		{
+			input:  "helloWorld",
+			expect: "helloworld",
+		},
+		{
+			input:  "HelloWorld",
+			expect: "helloworld",
+		},
+		{
+			input:  "hello World",
+			expect: "hello world",
+		},
+		{
+			input:  "Hello World",
+			expect: "hello world",
+		},
+		{
+			input:  "Hello World",
+			expect: "hello world",
+		},
+		{
+			input:  "Hello World foo_bar",
+			expect: "hello world foo_bar",
+		},
+		{
+			input:  "Hello World foo_Bar",
+			expect: "hello world foo_bar",
+		},
+		{
+			input:  "Hello World Foo_bar",
+			expect: "hello world foo_bar",
+		},
+		{
+			input:  "Hello World Foo_Bar",
+			expect: "hello world foo_bar",
+		},
+		{
+			input:  "Hello.World Foo_Bar",
+			expect: "hello.world foo_bar",
+		},
+		{
+			input:  "你好 World Foo_Bar",
+			expect: "你好 world foo_bar",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.input, func(t *testing.T) {
+			assert.Equal(t, test.expect, toLowerCase(test.input))
+		})
+	}
+}
+
+func TestLoadFromJsonBytesError(t *testing.T) {
+	var val struct{}
+	assert.Error(t, LoadFromJsonBytes([]byte(`hello`), &val))
+}
+
+func TestLoadFromTomlBytesError(t *testing.T) {
+	var val struct{}
+	assert.Error(t, LoadFromTomlBytes([]byte(`hello`), &val))
+}
+
+func TestLoadFromYamlBytesError(t *testing.T) {
+	var val struct{}
+	assert.Error(t, LoadFromYamlBytes([]byte(`':hello`), &val))
+}
+
+func TestLoadFromYamlBytes(t *testing.T) {
+	input := []byte(`layer1:
+  layer2:
+    layer3: foo`)
+	var val struct {
+		Layer1 struct {
+			Layer2 struct {
+				Layer3 string
+			}
+		}
+	}
+
+	assert.NoError(t, LoadFromYamlBytes(input, &val))
+	assert.Equal(t, "foo", val.Layer1.Layer2.Layer3)
+}
+
+func TestLoadFromYamlBytesTerm(t *testing.T) {
+	input := []byte(`layer1:
+  layer2:
+    tls_conf: foo`)
+	var val struct {
+		Layer1 struct {
+			Layer2 struct {
+				Layer3 string `json:"tls_conf"`
+			}
+		}
+	}
+
+	assert.NoError(t, LoadFromYamlBytes(input, &val))
+	assert.Equal(t, "foo", val.Layer1.Layer2.Layer3)
+}
+
+func TestLoadFromYamlBytesLayers(t *testing.T) {
+	input := []byte(`layer1:
+  layer2:
+    layer3: foo`)
+	var val struct {
+		Value string `json:"Layer1.Layer2.Layer3"`
+	}
+
+	assert.NoError(t, LoadFromYamlBytes(input, &val))
+	assert.Equal(t, "foo", val.Value)
+}
+
+func TestUnmarshalJsonBytesMap(t *testing.T) {
+	input := []byte(`{"foo":{"/mtproto.RPCTos": "bff.bff","bar":"baz"}}`)
+
+	var val struct {
+		Foo map[string]string
+	}
+
+	assert.NoError(t, LoadFromJsonBytes(input, &val))
+	assert.Equal(t, "bff.bff", val.Foo["/mtproto.RPCTos"])
+	assert.Equal(t, "baz", val.Foo["bar"])
+}
+
+func TestUnmarshalJsonBytesMapWithSliceElements(t *testing.T) {
+	input := []byte(`{"foo":{"/mtproto.RPCTos": ["bff.bff", "any"],"bar":["baz", "qux"]}}`)
+
+	var val struct {
+		Foo map[string][]string
+	}
+
+	assert.NoError(t, LoadFromJsonBytes(input, &val))
+	assert.EqualValues(t, []string{"bff.bff", "any"}, val.Foo["/mtproto.RPCTos"])
+	assert.EqualValues(t, []string{"baz", "qux"}, val.Foo["bar"])
+}
+
+func TestUnmarshalJsonBytesMapWithSliceOfStructs(t *testing.T) {
+	input := []byte(`{"foo":{
+	"/mtproto.RPCTos": [{"bar": "any"}],
+	"bar":[{"bar": "qux"}, {"bar": "ever"}]}}`)
+
+	var val struct {
+		Foo map[string][]struct {
+			Bar string
+		}
+	}
+
+	assert.NoError(t, LoadFromJsonBytes(input, &val))
+	assert.Equal(t, 1, len(val.Foo["/mtproto.RPCTos"]))
+	assert.Equal(t, "any", val.Foo["/mtproto.RPCTos"][0].Bar)
+	assert.Equal(t, 2, len(val.Foo["bar"]))
+	assert.Equal(t, "qux", val.Foo["bar"][0].Bar)
+	assert.Equal(t, "ever", val.Foo["bar"][1].Bar)
+}
+
+func TestUnmarshalJsonBytesWithAnonymousField(t *testing.T) {
+	type (
+		Int int
+
+		InnerConf struct {
+			Name string
+		}
+
+		Conf struct {
+			Int
+			InnerConf
+		}
+	)
+
+	var (
+		input = []byte(`{"Name": "hello", "int": 3}`)
+		c     Conf
+	)
+	assert.NoError(t, LoadFromJsonBytes(input, &c))
+	assert.Equal(t, "hello", c.Name)
+	assert.Equal(t, Int(3), c.Int)
+}
+
 func createTempFile(ext, text string) (string, error) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
+	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
 	if err != nil {
 		return "", err
 	}
 
-	if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+	if err := os.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
 		return "", err
 	}
 

core/discov/internal/accountmanager.go

@@ -3,7 +3,7 @@ package internal
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"io/ioutil"
+	"os"
 	"sync"
 )
 
@@ -37,7 +37,7 @@ func AddTLS(endpoints []string, certFile, certKeyFile, caFile string, insecureSk
 		return err
 	}
 
-	caData, err := ioutil.ReadFile(caFile)
+	caData, err := os.ReadFile(caFile)
 	if err != nil {
 		return err
 	}

core/discov/internal/registry.go

@@ -208,7 +208,7 @@ func (c *cluster) handleWatchEvents(key string, events []*clientv3.Event) {
 	}
 }
 
-func (c *cluster) load(cli EtcdClient, key string) {
+func (c *cluster) load(cli EtcdClient, key string) int64 {
 	var resp *clientv3.GetResponse
 	for {
 		var err error
@@ -232,6 +232,8 @@ func (c *cluster) load(cli EtcdClient, key string) {
 	}
 
 	c.handleChanges(key, kvs)
+
+	return resp.Header.Revision
 }
 
 func (c *cluster) monitor(key string, l UpdateListener) error {
@@ -244,9 +246,9 @@ func (c *cluster) monitor(key string, l UpdateListener) error {
 		return err
 	}
 
-	c.load(cli, key)
+	rev := c.load(cli, key)
 	c.watchGroup.Run(func() {
-		c.watch(cli, key)
+		c.watch(cli, key, rev)
 	})
 
 	return nil
@@ -278,22 +280,29 @@ func (c *cluster) reload(cli EtcdClient) {
 	for _, key := range keys {
 		k := key
 		c.watchGroup.Run(func() {
-			c.load(cli, k)
+			rev := c.load(cli, k)
-			c.watch(cli, k)
+			c.watch(cli, k, rev)
 		})
 	}
 }
 
-func (c *cluster) watch(cli EtcdClient, key string) {
+func (c *cluster) watch(cli EtcdClient, key string, rev int64) {
 	for {
-		if c.watchStream(cli, key) {
+		if c.watchStream(cli, key, rev) {
 			return
 		}
 	}
 }
 
-func (c *cluster) watchStream(cli EtcdClient, key string) bool {
+func (c *cluster) watchStream(cli EtcdClient, key string, rev int64) bool {
-	rch := cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
+	var rch clientv3.WatchChan
+	if rev != 0 {
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix(),
+			clientv3.WithRev(rev+1))
+	} else {
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
+	}
+
 	for {
 		select {
 		case wresp, ok := <-rch:
@@ -334,6 +343,7 @@ func DialClient(endpoints []string) (EtcdClient, error) {
 		DialKeepAliveTime:    dialKeepAliveTime,
 		DialKeepAliveTimeout: DialTimeout,
 		RejectOldCluster:     true,
+		PermitWithoutStream:  true,
 	}
 	if account, ok := GetAccount(endpoints); ok {
 		cfg.Username = account.User

core/discov/internal/registry_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/zeromicro/go-zero/core/lang"
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/stringx"
+	"go.etcd.io/etcd/api/v3/etcdserverpb"
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
@@ -112,6 +113,7 @@ func TestCluster_Load(t *testing.T) {
 	restore := setMockClient(cli)
 	defer restore()
 	cli.EXPECT().Get(gomock.Any(), "any/", gomock.Any()).Return(&clientv3.GetResponse{
+		Header: &etcdserverpb.ResponseHeader{},
 		Kvs: []*mvccpb.KeyValue{
 			{
 				Key:   []byte("hello"),
@@ -168,7 +170,7 @@ func TestCluster_Watch(t *testing.T) {
 			listener.EXPECT().OnDelete(gomock.Any()).Do(func(_ interface{}) {
 				wg.Done()
 			}).MaxTimes(1)
-			go c.watch(cli, "any")
+			go c.watch(cli, "any", 0)
 			ch <- clientv3.WatchResponse{
 				Events: []*clientv3.Event{
 					{
@@ -212,7 +214,7 @@ func TestClusterWatch_RespFailures(t *testing.T) {
 				ch <- resp
 				close(c.done)
 			}()
-			c.watch(cli, "any")
+			c.watch(cli, "any", 0)
 		})
 	}
 }
@@ -232,7 +234,7 @@ func TestClusterWatch_CloseChan(t *testing.T) {
 		close(ch)
 		close(c.done)
 	}()
-	c.watch(cli, "any")
+	c.watch(cli, "any", 0)
 }
 
 func TestValueOnlyContext(t *testing.T) {

core/discov/publisher.go

@@ -1,6 +1,8 @@
 package discov
 
 import (
+	"time"
+
 	"github.com/zeromicro/go-zero/core/discov/internal"
 	"github.com/zeromicro/go-zero/core/lang"
 	"github.com/zeromicro/go-zero/core/logx"
@@ -51,12 +53,7 @@ func NewPublisher(endpoints []string, key, value string, opts ...PubOption) *Pub
 
 // KeepAlive keeps key:value alive.
 func (p *Publisher) KeepAlive() error {
-	cli, err := internal.GetRegistry().GetConn(p.endpoints)
+	cli, err := p.doRegister()
-	if err != nil {
-		return err
-	}
-
-	p.lease, err = p.register(cli)
 	if err != nil {
 		return err
 	}
@@ -83,6 +80,43 @@ func (p *Publisher) Stop() {
 	p.quit.Close()
 }
 
+func (p *Publisher) doKeepAlive() error {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		select {
+		case <-p.quit.Done():
+			return nil
+		default:
+			cli, err := p.doRegister()
+			if err != nil {
+				logx.Errorf("etcd publisher doRegister: %s", err.Error())
+				break
+			}
+
+			if err := p.keepAliveAsync(cli); err != nil {
+				logx.Errorf("etcd publisher keepAliveAsync: %s", err.Error())
+				break
+			}
+
+			return nil
+		}
+	}
+
+	return nil
+}
+
+func (p *Publisher) doRegister() (internal.EtcdClient, error) {
+	cli, err := internal.GetRegistry().GetConn(p.endpoints)
+	if err != nil {
+		return nil, err
+	}
+
+	p.lease, err = p.register(cli)
+	return cli, err
+}
+
 func (p *Publisher) keepAliveAsync(cli internal.EtcdClient) error {
 	ch, err := cli.KeepAlive(cli.Ctx(), p.lease)
 	if err != nil {
@@ -95,8 +129,8 @@ func (p *Publisher) keepAliveAsync(cli internal.EtcdClient) error {
 			case _, ok := <-ch:
 				if !ok {
 					p.revoke(cli)
-					if err := p.KeepAlive(); err != nil {
+					if err := p.doKeepAlive(); err != nil {
-						logx.Errorf("KeepAlive: %s", err.Error())
+						logx.Errorf("etcd publisher KeepAlive: %s", err.Error())
 					}
 					return
 				}
@@ -105,8 +139,8 @@ func (p *Publisher) keepAliveAsync(cli internal.EtcdClient) error {
 				p.revoke(cli)
 				select {
 				case <-p.resumeChan:
-					if err := p.KeepAlive(); err != nil {
+					if err := p.doKeepAlive(); err != nil {
-						logx.Errorf("KeepAlive: %s", err.Error())
+						logx.Errorf("etcd publisher KeepAlive: %s", err.Error())
 					}
 					return
 				case <-p.quit.Done():
@@ -141,7 +175,7 @@ func (p *Publisher) register(client internal.EtcdClient) (clientv3.LeaseID, erro
 
 func (p *Publisher) revoke(cli internal.EtcdClient) {
 	if _, err := cli.Revoke(cli.Ctx(), p.lease); err != nil {
-		logx.Error(err)
+		logx.Errorf("etcd publisher revoke: %s", err.Error())
 	}
 }
 

core/discov/subscriber.go

@@ -13,7 +13,7 @@ type (
 	// SubOption defines the method to customize a Subscriber.
 	SubOption func(sub *Subscriber)
 
-	// A Subscriber is used to subscribe the given key on a etcd cluster.
+	// A Subscriber is used to subscribe the given key on an etcd cluster.
 	Subscriber struct {
 		endpoints []string
 		exclusive bool

core/fs/files_test.go

@@ -0,0 +1,15 @@
+package fs
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCloseOnExec(t *testing.T) {
+	file := os.NewFile(0, os.DevNull)
+	assert.NotPanics(t, func() {
+		CloseOnExec(file)
+	})
+}

core/fs/temps.go

@@ -1,7 +1,6 @@
 package fs
 
 import (
-	"io/ioutil"
 	"os"
 
 	"github.com/zeromicro/go-zero/core/hash"
@@ -12,12 +11,12 @@ import (
 // The file is kept as open, the caller should close the file handle,
 // and remove the file by name.
 func TempFileWithText(text string) (*os.File, error) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text)))
+	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text)))
 	if err != nil {
 		return nil, err
 	}
 
-	if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+	if err := os.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
 		return nil, err
 	}
 

core/fs/temps_test.go

@@ -1,7 +1,7 @@
 package fs
 
 import (
-	"io/ioutil"
+	"io"
 	"os"
 	"testing"
 
@@ -21,7 +21,7 @@ func TestTempFileWithText(t *testing.T) {
 	}
 	defer os.Remove(f.Name())
 
-	bs, err := ioutil.ReadAll(f)
+	bs, err := io.ReadAll(f)
 	assert.Nil(t, err)
 	if len(bs) != 4 {
 		t.Error("TempFileWithText returned wrong file size")
@@ -41,7 +41,7 @@ func TestTempFilenameWithText(t *testing.T) {
 	}
 	defer os.Remove(f)
 
-	bs, err := ioutil.ReadFile(f)
+	bs, err := os.ReadFile(f)
 	assert.Nil(t, err)
 	if len(bs) != 4 {
 		t.Error("TempFilenameWithText returned wrong file size")

core/fx/stream.go

@@ -328,7 +328,7 @@ func (s Stream) Parallel(fn ParallelFunc, opts ...Option) {
 	}, opts...).Done()
 }
 
-// Reduce is a utility method to let the caller deal with the underlying channel.
+// Reduce is an utility method to let the caller deal with the underlying channel.
 func (s Stream) Reduce(fn ReduceFunc) (interface{}, error) {
 	return fn(s.source)
 }

core/fx/stream_test.go

@@ -1,7 +1,7 @@
 package fx
 
 import (
-	"io/ioutil"
+	"io"
 	"log"
 	"math/rand"
 	"reflect"
@@ -238,7 +238,7 @@ func TestLast(t *testing.T) {
 
 func TestMap(t *testing.T) {
 	runCheckedTest(t, func(t *testing.T) {
-		log.SetOutput(ioutil.Discard)
+		log.SetOutput(io.Discard)
 
 		tests := []struct {
 			mapper MapFunc

core/hash/consistenthash.go

@@ -7,7 +7,6 @@ import (
 	"sync"
 
 	"github.com/zeromicro/go-zero/core/lang"
-	"github.com/zeromicro/go-zero/core/mapping"
 )
 
 const (
@@ -183,5 +182,5 @@ func innerRepr(node interface{}) string {
 }
 
 func repr(node interface{}) string {
-	return mapping.Repr(node)
+	return lang.Repr(node)
 }

core/iox/nopcloser.go

@@ -10,7 +10,7 @@ func (nopCloser) Close() error {
 	return nil
 }
 
-// NopCloser returns a io.WriteCloser that does nothing on calling Close.
+// NopCloser returns an io.WriteCloser that does nothing on calling Close.
 func NopCloser(w io.Writer) io.WriteCloser {
 	return nopCloser{w}
 }

core/iox/read.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 )
@@ -26,7 +25,7 @@ type (
 func DupReadCloser(reader io.ReadCloser) (io.ReadCloser, io.ReadCloser) {
 	var buf bytes.Buffer
 	tee := io.TeeReader(reader, &buf)
-	return ioutil.NopCloser(tee), ioutil.NopCloser(&buf)
+	return io.NopCloser(tee), io.NopCloser(&buf)
 }
 
 // KeepSpace customizes the reading functions to keep leading and tailing spaces.
@@ -54,7 +53,7 @@ func ReadBytes(reader io.Reader, buf []byte) error {
 
 // ReadText reads content from the given file with leading and tailing spaces trimmed.
 func ReadText(filename string) (string, error) {
-	content, err := ioutil.ReadFile(filename)
+	content, err := os.ReadFile(filename)
 	if err != nil {
 		return "", err
 	}

core/iox/read_test.go

@@ -3,7 +3,6 @@ package iox
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -97,10 +96,10 @@ func TestReadTextLines(t *testing.T) {
 
 func TestDupReadCloser(t *testing.T) {
 	input := "hello"
-	reader := ioutil.NopCloser(bytes.NewBufferString(input))
+	reader := io.NopCloser(bytes.NewBufferString(input))
 	r1, r2 := DupReadCloser(reader)
 	verify := func(r io.Reader) {
-		output, err := ioutil.ReadAll(r)
+		output, err := io.ReadAll(r)
 		assert.Nil(t, err)
 		assert.Equal(t, input, string(output))
 	}
@@ -110,7 +109,7 @@ func TestDupReadCloser(t *testing.T) {
 }
 
 func TestReadBytes(t *testing.T) {
-	reader := ioutil.NopCloser(bytes.NewBufferString("helloworld"))
+	reader := io.NopCloser(bytes.NewBufferString("helloworld"))
 	buf := make([]byte, 5)
 	err := ReadBytes(reader, buf)
 	assert.Nil(t, err)
@@ -118,7 +117,7 @@ func TestReadBytes(t *testing.T) {
 }
 
 func TestReadBytesNotEnough(t *testing.T) {
-	reader := ioutil.NopCloser(bytes.NewBufferString("hell"))
+	reader := io.NopCloser(bytes.NewBufferString("hell"))
 	buf := make([]byte, 5)
 	err := ReadBytes(reader, buf)
 	assert.Equal(t, io.EOF, err)

core/iox/textfile_test.go

@@ -1,7 +1,6 @@
 package iox
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -13,7 +12,7 @@ func TestCountLines(t *testing.T) {
 2
 3
 4`
-	file, err := ioutil.TempFile(os.TempDir(), "test-")
+	file, err := os.CreateTemp(os.TempDir(), "test-")
 	if err != nil {
 		t.Fatal(err)
 	}

core/jsontype/time.go

@@ -1,44 +0,0 @@
-package jsontype
-
-import (
-	"encoding/json"
-	"time"
-
-	"github.com/globalsign/mgo/bson"
-)
-
-// MilliTime represents time.Time that works better with mongodb.
-type MilliTime struct {
-	time.Time
-}
-
-// MarshalJSON marshals mt to json bytes.
-func (mt MilliTime) MarshalJSON() ([]byte, error) {
-	return json.Marshal(mt.Milli())
-}
-
-// UnmarshalJSON unmarshals data into mt.
-func (mt *MilliTime) UnmarshalJSON(data []byte) error {
-	var milli int64
-	if err := json.Unmarshal(data, &milli); err != nil {
-		return err
-	}
-
-	mt.Time = time.Unix(0, milli*int64(time.Millisecond))
-	return nil
-}
-
-// GetBSON returns BSON base on mt.
-func (mt MilliTime) GetBSON() (interface{}, error) {
-	return mt.Time, nil
-}
-
-// SetBSON sets raw into mt.
-func (mt *MilliTime) SetBSON(raw bson.Raw) error {
-	return raw.Unmarshal(&mt.Time)
-}
-
-// Milli returns milliseconds for mt.
-func (mt MilliTime) Milli() int64 {
-	return mt.UnixNano() / int64(time.Millisecond)
-}

core/jsontype/time_test.go

@@ -1,126 +0,0 @@
-package jsontype
-
-import (
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/globalsign/mgo/bson"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestMilliTime_GetBSON(t *testing.T) {
-	tests := []struct {
-		name string
-		tm   time.Time
-	}{
-		{
-			name: "now",
-			tm:   time.Now(),
-		},
-		{
-			name: "future",
-			tm:   time.Now().Add(time.Hour),
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			got, err := MilliTime{test.tm}.GetBSON()
-			assert.Nil(t, err)
-			assert.Equal(t, test.tm, got)
-		})
-	}
-}
-
-func TestMilliTime_MarshalJSON(t *testing.T) {
-	tests := []struct {
-		name string
-		tm   time.Time
-	}{
-		{
-			name: "now",
-			tm:   time.Now(),
-		},
-		{
-			name: "future",
-			tm:   time.Now().Add(time.Hour),
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			b, err := MilliTime{test.tm}.MarshalJSON()
-			assert.Nil(t, err)
-			assert.Equal(t, strconv.FormatInt(test.tm.UnixNano()/1e6, 10), string(b))
-		})
-	}
-}
-
-func TestMilliTime_Milli(t *testing.T) {
-	tests := []struct {
-		name string
-		tm   time.Time
-	}{
-		{
-			name: "now",
-			tm:   time.Now(),
-		},
-		{
-			name: "future",
-			tm:   time.Now().Add(time.Hour),
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			n := MilliTime{test.tm}.Milli()
-			assert.Equal(t, test.tm.UnixNano()/1e6, n)
-		})
-	}
-}
-
-func TestMilliTime_UnmarshalJSON(t *testing.T) {
-	tests := []struct {
-		name string
-		tm   time.Time
-	}{
-		{
-			name: "now",
-			tm:   time.Now(),
-		},
-		{
-			name: "future",
-			tm:   time.Now().Add(time.Hour),
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			var mt MilliTime
-			s := strconv.FormatInt(test.tm.UnixNano()/1e6, 10)
-			err := mt.UnmarshalJSON([]byte(s))
-			assert.Nil(t, err)
-			s1, err := mt.MarshalJSON()
-			assert.Nil(t, err)
-			assert.Equal(t, s, string(s1))
-		})
-	}
-}
-
-func TestUnmarshalWithError(t *testing.T) {
-	var mt MilliTime
-	assert.NotNil(t, mt.UnmarshalJSON([]byte("hello")))
-}
-
-func TestSetBSON(t *testing.T) {
-	data, err := bson.Marshal(time.Now())
-	assert.Nil(t, err)
-
-	var raw bson.Raw
-	assert.Nil(t, bson.Unmarshal(data, &raw))
-
-	var mt MilliTime
-	assert.Nil(t, mt.SetBSON(raw))
-	assert.NotNil(t, mt.SetBSON(bson.Raw{}))
-}

core/lang/lang.go

@@ -1,5 +1,11 @@
 package lang
 
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+)
+
 // Placeholder is a placeholder object that can be used globally.
 var Placeholder PlaceholderType
 
@@ -9,3 +15,64 @@ type (
 	// PlaceholderType represents a placeholder type.
 	PlaceholderType = struct{}
 )
+
+// Repr returns the string representation of v.
+func Repr(v interface{}) string {
+	if v == nil {
+		return ""
+	}
+
+	// if func (v *Type) String() string, we can't use Elem()
+	switch vt := v.(type) {
+	case fmt.Stringer:
+		return vt.String()
+	}
+
+	val := reflect.ValueOf(v)
+	for val.Kind() == reflect.Ptr && !val.IsNil() {
+		val = val.Elem()
+	}
+
+	return reprOfValue(val)
+}
+
+func reprOfValue(val reflect.Value) string {
+	switch vt := val.Interface().(type) {
+	case bool:
+		return strconv.FormatBool(vt)
+	case error:
+		return vt.Error()
+	case float32:
+		return strconv.FormatFloat(float64(vt), 'f', -1, 32)
+	case float64:
+		return strconv.FormatFloat(vt, 'f', -1, 64)
+	case fmt.Stringer:
+		return vt.String()
+	case int:
+		return strconv.Itoa(vt)
+	case int8:
+		return strconv.Itoa(int(vt))
+	case int16:
+		return strconv.Itoa(int(vt))
+	case int32:
+		return strconv.Itoa(int(vt))
+	case int64:
+		return strconv.FormatInt(vt, 10)
+	case string:
+		return vt
+	case uint:
+		return strconv.FormatUint(uint64(vt), 10)
+	case uint8:
+		return strconv.FormatUint(uint64(vt), 10)
+	case uint16:
+		return strconv.FormatUint(uint64(vt), 10)
+	case uint32:
+		return strconv.FormatUint(uint64(vt), 10)
+	case uint64:
+		return strconv.FormatUint(vt, 10)
+	case []byte:
+		return string(vt)
+	default:
+		return fmt.Sprint(val.Interface())
+	}
+}

core/lang/lang_test.go

@@ -0,0 +1,156 @@
+package lang
+
+import (
+	"encoding/json"
+	"errors"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRepr(t *testing.T) {
+	var (
+		f32 float32 = 1.1
+		f64         = 2.2
+		i8  int8    = 1
+		i16 int16   = 2
+		i32 int32   = 3
+		i64 int64   = 4
+		u8  uint8   = 5
+		u16 uint16  = 6
+		u32 uint32  = 7
+		u64 uint64  = 8
+	)
+	tests := []struct {
+		v      interface{}
+		expect string
+	}{
+		{
+			nil,
+			"",
+		},
+		{
+			mockStringable{},
+			"mocked",
+		},
+		{
+			new(mockStringable),
+			"mocked",
+		},
+		{
+			newMockPtr(),
+			"mockptr",
+		},
+		{
+			&mockOpacity{
+				val: 1,
+			},
+			"{1}",
+		},
+		{
+			true,
+			"true",
+		},
+		{
+			false,
+			"false",
+		},
+		{
+			f32,
+			"1.1",
+		},
+		{
+			f64,
+			"2.2",
+		},
+		{
+			i8,
+			"1",
+		},
+		{
+			i16,
+			"2",
+		},
+		{
+			i32,
+			"3",
+		},
+		{
+			i64,
+			"4",
+		},
+		{
+			u8,
+			"5",
+		},
+		{
+			u16,
+			"6",
+		},
+		{
+			u32,
+			"7",
+		},
+		{
+			u64,
+			"8",
+		},
+		{
+			[]byte(`abcd`),
+			"abcd",
+		},
+		{
+			mockOpacity{val: 1},
+			"{1}",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.expect, func(t *testing.T) {
+			assert.Equal(t, test.expect, Repr(test.v))
+		})
+	}
+}
+
+func TestReprOfValue(t *testing.T) {
+	t.Run("error", func(t *testing.T) {
+		assert.Equal(t, "error", reprOfValue(reflect.ValueOf(errors.New("error"))))
+	})
+
+	t.Run("stringer", func(t *testing.T) {
+		assert.Equal(t, "1.23", reprOfValue(reflect.ValueOf(json.Number("1.23"))))
+	})
+
+	t.Run("int", func(t *testing.T) {
+		assert.Equal(t, "1", reprOfValue(reflect.ValueOf(1)))
+	})
+
+	t.Run("int", func(t *testing.T) {
+		assert.Equal(t, "1", reprOfValue(reflect.ValueOf("1")))
+	})
+
+	t.Run("int", func(t *testing.T) {
+		assert.Equal(t, "1", reprOfValue(reflect.ValueOf(uint(1))))
+	})
+}
+
+type mockStringable struct{}
+
+func (m mockStringable) String() string {
+	return "mocked"
+}
+
+type mockPtr struct{}
+
+func newMockPtr() *mockPtr {
+	return new(mockPtr)
+}
+
+func (m *mockPtr) String() string {
+	return "mockptr"
+}
+
+type mockOpacity struct {
+	val int
+}

core/limit/tokenlimit.go

@@ -1,6 +1,8 @@
 package limit
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"strconv"
 	"sync"
@@ -58,8 +60,8 @@ type TokenLimiter struct {
 	timestampKey   string
 	rescueLock     sync.Mutex
 	redisAlive     uint32
-	rescueLimiter  *xrate.Limiter
 	monitorStarted bool
+	rescueLimiter  *xrate.Limiter
 }
 
 // NewTokenLimiter returns a new TokenLimiter that allows events up to rate and permits
@@ -84,19 +86,31 @@ func (lim *TokenLimiter) Allow() bool {
 	return lim.AllowN(time.Now(), 1)
 }
 
+// AllowCtx is shorthand for AllowNCtx(ctx,time.Now(), 1) with incoming context.
+func (lim *TokenLimiter) AllowCtx(ctx context.Context) bool {
+	return lim.AllowNCtx(ctx, time.Now(), 1)
+}
+
 // AllowN reports whether n events may happen at time now.
 // Use this method if you intend to drop / skip events that exceed the rate.
 // Otherwise, use Reserve or Wait.
 func (lim *TokenLimiter) AllowN(now time.Time, n int) bool {
-	return lim.reserveN(now, n)
+	return lim.reserveN(context.Background(), now, n)
 }
 
-func (lim *TokenLimiter) reserveN(now time.Time, n int) bool {
+// AllowNCtx reports whether n events may happen at time now with incoming context.
+// Use this method if you intend to drop / skip events that exceed the rate.
+// Otherwise, use Reserve or Wait.
+func (lim *TokenLimiter) AllowNCtx(ctx context.Context, now time.Time, n int) bool {
+	return lim.reserveN(ctx, now, n)
+}
+
+func (lim *TokenLimiter) reserveN(ctx context.Context, now time.Time, n int) bool {
 	if atomic.LoadUint32(&lim.redisAlive) == 0 {
 		return lim.rescueLimiter.AllowN(now, n)
 	}
 
-	resp, err := lim.store.Eval(
+	resp, err := lim.store.EvalCtx(ctx,
 		script,
 		[]string{
 			lim.tokenKey,
@@ -113,6 +127,10 @@ func (lim *TokenLimiter) reserveN(now time.Time, n int) bool {
 	if err == redis.Nil {
 		return false
 	}
+	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+		logx.Errorf("fail to use rate limiter: %s", err)
+		return false
+	}
 	if err != nil {
 		logx.Errorf("fail to use rate limiter: %s, use in-process limiter for rescue", err)
 		lim.startMonitor()

core/limit/tokenlimit_test.go

@@ -1,6 +1,7 @@
 package limit
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -15,6 +16,30 @@ func init() {
 	logx.Disable()
 }
 
+func TestTokenLimit_WithCtx(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+
+	const (
+		total = 100
+		rate  = 5
+		burst = 10
+	)
+	l := NewTokenLimiter(rate, burst, redis.New(s.Addr()), "tokenlimit")
+	defer s.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	ok := l.AllowCtx(ctx)
+	assert.True(t, ok)
+
+	cancel()
+	for i := 0; i < total; i++ {
+		ok := l.AllowCtx(ctx)
+		assert.False(t, ok)
+		assert.False(t, l.monitorStarted)
+	}
+}
+
 func TestTokenLimit_Rescue(t *testing.T) {
 	s, err := miniredis.Run()
 	assert.Nil(t, err)

core/load/adaptiveshedder.go

@@ -17,7 +17,7 @@ import (
 const (
 	defaultBuckets = 50
 	defaultWindow  = time.Second * 5
-	// using 1000m notation, 900m is like 80%, keep it as var for unit test
+	// using 1000m notation, 900m is like 90%, keep it as var for unit test
 	defaultCpuThreshold = 900
 	defaultMinRt        = float64(time.Second / time.Millisecond)
 	// moving average hyperparameter beta for calculating requests on the fly
@@ -70,7 +70,7 @@ type (
 		flying          int64
 		avgFlying       float64
 		avgFlyingLock   syncx.SpinLock
-		dropTime        *syncx.AtomicDuration
+		overloadTime    *syncx.AtomicDuration
 		droppedRecently *syncx.AtomicBool
 		passCounter     *collection.RollingWindow
 		rtCounter       *collection.RollingWindow
@@ -106,7 +106,7 @@ func NewAdaptiveShedder(opts ...ShedderOption) Shedder {
 	return &adaptiveShedder{
 		cpuThreshold:    options.cpuThreshold,
 		windows:         int64(time.Second / bucketDuration),
-		dropTime:        syncx.NewAtomicDuration(),
+		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.NewAtomicBool(),
 		passCounter: collection.NewRollingWindow(options.buckets, bucketDuration,
 			collection.IgnoreCurrentBucket()),
@@ -118,7 +118,6 @@ func NewAdaptiveShedder(opts ...ShedderOption) Shedder {
 // Allow implements Shedder.Allow.
 func (as *adaptiveShedder) Allow() (Promise, error) {
 	if as.shouldDrop() {
-		as.dropTime.Set(timex.Now())
 		as.droppedRecently.Set(true)
 
 		return nil, ErrServiceOverloaded
@@ -215,21 +214,26 @@ func (as *adaptiveShedder) stillHot() bool {
 		return false
 	}
 
-	dropTime := as.dropTime.Load()
+	overloadTime := as.overloadTime.Load()
-	if dropTime == 0 {
+	if overloadTime == 0 {
 		return false
 	}
 
-	hot := timex.Since(dropTime) < coolOffDuration
+	if timex.Since(overloadTime) < coolOffDuration {
-	if !hot {
+		return true
-		as.droppedRecently.Set(false)
 	}
 
-	return hot
+	as.droppedRecently.Set(false)
+	return false
 }
 
 func (as *adaptiveShedder) systemOverloaded() bool {
-	return systemOverloadChecker(as.cpuThreshold)
+	if !systemOverloadChecker(as.cpuThreshold) {
+		return false
+	}
+
+	as.overloadTime.Set(timex.Now())
+	return true
 }
 
 // WithBuckets customizes the Shedder with given number of buckets.

core/load/adaptiveshedder_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/zeromicro/go-zero/core/mathx"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/syncx"
+	"github.com/zeromicro/go-zero/core/timex"
 )
 
 const (
@@ -136,7 +137,7 @@ func TestAdaptiveShedderShouldDrop(t *testing.T) {
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
 		windows:         buckets,
-		dropTime:        syncx.NewAtomicDuration(),
+		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.NewAtomicBool(),
 	}
 	// cpu >=  800, inflight < maxPass
@@ -190,12 +191,15 @@ func TestAdaptiveShedderStillHot(t *testing.T) {
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
 		windows:         buckets,
-		dropTime:        syncx.NewAtomicDuration(),
+		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.ForAtomicBool(true),
 	}
 	assert.False(t, shedder.stillHot())
-	shedder.dropTime.Set(-coolOffDuration * 2)
+	shedder.overloadTime.Set(-coolOffDuration * 2)
 	assert.False(t, shedder.stillHot())
+	shedder.droppedRecently.Set(true)
+	shedder.overloadTime.Set(timex.Now())
+	assert.True(t, shedder.stillHot())
 }
 
 func BenchmarkAdaptiveShedder_Allow(b *testing.B) {

core/logc/logs.go

@@ -0,0 +1,122 @@
+package logc
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+type (
+	LogConf  = logx.LogConf
+	LogField = logx.LogField
+)
+
+// AddGlobalFields adds global fields.
+func AddGlobalFields(fields ...LogField) {
+	logx.AddGlobalFields(fields...)
+}
+
+// Alert alerts v in alert level, and the message is written to error log.
+func Alert(_ context.Context, v string) {
+	logx.Alert(v)
+}
+
+// Close closes the logging.
+func Close() error {
+	return logx.Close()
+}
+
+// Error writes v into error log.
+func Error(ctx context.Context, v ...interface{}) {
+	getLogger(ctx).Error(v...)
+}
+
+// Errorf writes v with format into error log.
+func Errorf(ctx context.Context, format string, v ...interface{}) {
+	getLogger(ctx).Errorf(fmt.Errorf(format, v...).Error())
+}
+
+// Errorv writes v into error log with json content.
+// No call stack attached, because not elegant to pack the messages.
+func Errorv(ctx context.Context, v interface{}) {
+	getLogger(ctx).Errorv(v)
+}
+
+// Errorw writes msg along with fields into error log.
+func Errorw(ctx context.Context, msg string, fields ...LogField) {
+	getLogger(ctx).Errorw(msg, fields...)
+}
+
+// Field returns a LogField for the given key and value.
+func Field(key string, value interface{}) LogField {
+	return logx.Field(key, value)
+}
+
+// Info writes v into access log.
+func Info(ctx context.Context, v ...interface{}) {
+	getLogger(ctx).Info(v...)
+}
+
+// Infof writes v with format into access log.
+func Infof(ctx context.Context, format string, v ...interface{}) {
+	getLogger(ctx).Infof(format, v...)
+}
+
+// Infov writes v into access log with json content.
+func Infov(ctx context.Context, v interface{}) {
+	getLogger(ctx).Infov(v)
+}
+
+// Infow writes msg along with fields into access log.
+func Infow(ctx context.Context, msg string, fields ...LogField) {
+	getLogger(ctx).Infow(msg, fields...)
+}
+
+// Must checks if err is nil, otherwise logs the error and exits.
+func Must(err error) {
+	logx.Must(err)
+}
+
+// MustSetup sets up logging with given config c. It exits on error.
+func MustSetup(c logx.LogConf) {
+	logx.MustSetup(c)
+}
+
+// SetLevel sets the logging level. It can be used to suppress some logs.
+func SetLevel(level uint32) {
+	logx.SetLevel(level)
+}
+
+// SetUp sets up the logx. If already set up, just return nil.
+// we allow SetUp to be called multiple times, because for example
+// we need to allow different service frameworks to initialize logx respectively.
+// the same logic for SetUp
+func SetUp(c LogConf) error {
+	return logx.SetUp(c)
+}
+
+// Slow writes v into slow log.
+func Slow(ctx context.Context, v ...interface{}) {
+	getLogger(ctx).Slow(v...)
+}
+
+// Slowf writes v with format into slow log.
+func Slowf(ctx context.Context, format string, v ...interface{}) {
+	getLogger(ctx).Slowf(format, v...)
+}
+
+// Slowv writes v into slow log with json content.
+func Slowv(ctx context.Context, v interface{}) {
+	getLogger(ctx).Slowv(v)
+}
+
+// Sloww writes msg along with fields into slow log.
+func Sloww(ctx context.Context, msg string, fields ...LogField) {
+	getLogger(ctx).Sloww(msg, fields...)
+}
+
+// getLogger returns the logx.Logger with the given ctx and correct caller.
+func getLogger(ctx context.Context) logx.Logger {
+	return logx.WithContext(ctx).WithCallerSkip(1)
+}

core/logc/logs_test.go

@@ -0,0 +1,218 @@
+package logc
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+func TestAddGlobalFields(t *testing.T) {
+	var buf bytes.Buffer
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	Info(context.Background(), "hello")
+	buf.Reset()
+
+	AddGlobalFields(Field("a", "1"), Field("b", "2"))
+	AddGlobalFields(Field("c", "3"))
+	Info(context.Background(), "world")
+	var m map[string]interface{}
+	assert.NoError(t, json.Unmarshal(buf.Bytes(), &m))
+	assert.Equal(t, "1", m["a"])
+	assert.Equal(t, "2", m["b"])
+	assert.Equal(t, "3", m["c"])
+}
+
+func TestAlert(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	Alert(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), "foo"), buf.String())
+}
+
+func TestError(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Error(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestErrorf(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Errorf(context.Background(), "foo %s", "bar")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestErrorv(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Errorv(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestErrorw(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Errorw(context.Background(), "foo", Field("a", "b"))
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfo(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Info(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfof(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Infof(context.Background(), "foo %s", "bar")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfov(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Infov(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestInfow(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Infow(context.Background(), "foo", Field("a", "b"))
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestMust(t *testing.T) {
+	assert.NotPanics(t, func() {
+		Must(nil)
+	})
+	assert.NotPanics(t, func() {
+		MustSetup(LogConf{})
+	})
+}
+
+func TestMisc(t *testing.T) {
+	SetLevel(logx.DebugLevel)
+	assert.NoError(t, SetUp(LogConf{}))
+	assert.NoError(t, Close())
+}
+
+func TestSlow(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Slow(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func TestSlowf(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Slowf(context.Background(), "foo %s", "bar")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func TestSlowv(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Slowv(context.Background(), "foo")
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func TestSloww(t *testing.T) {
+	var buf strings.Builder
+	writer := logx.NewWriter(&buf)
+	old := logx.Reset()
+	logx.SetWriter(writer)
+	defer logx.SetWriter(old)
+
+	file, line := getFileLine()
+	Sloww(context.Background(), "foo", Field("a", "b"))
+	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
+}
+
+func getFileLine() (string, int) {
+	_, file, line, _ := runtime.Caller(1)
+	short := file
+
+	for i := len(file) - 1; i > 0; i-- {
+		if file[i] == '/' {
+			short = file[i+1:]
+			break
+		}
+	}
+
+	return short, line
+}

core/logx/config.go

@@ -7,8 +7,10 @@ type LogConf struct {
 	Encoding            string `json:",default=json,options=[json,plain]"`
 	TimeFormat          string `json:",optional"`
 	Path                string `json:",default=logs"`
-	Level               string `json:",default=info,options=[info,error,severe]"`
+	Level               string `json:",default=info,options=[debug,info,error,severe]"`
+	MaxContentLength    uint32 `json:",optional"`
 	Compress            bool   `json:",optional"`
+	Stat                bool   `json:",default=true"`
 	KeepDays            int    `json:",optional"`
 	StackCooldownMillis int    `json:",default=100"`
 	// MaxBackups represents how many backup log files will be kept. 0 means all files will be kept forever.

core/logx/contextlogger.go

@@ -1,145 +0,0 @@
-package logx
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/zeromicro/go-zero/core/timex"
-	"go.opentelemetry.io/otel/trace"
-)
-
-// WithContext sets ctx to log, for keeping tracing information.
-func WithContext(ctx context.Context) Logger {
-	return &contextLogger{
-		ctx: ctx,
-	}
-}
-
-type contextLogger struct {
-	logEntry
-	ctx context.Context
-}
-
-func (l *contextLogger) Error(v ...interface{}) {
-	l.err(fmt.Sprint(v...))
-}
-
-func (l *contextLogger) Errorf(format string, v ...interface{}) {
-	l.err(fmt.Sprintf(format, v...))
-}
-
-func (l *contextLogger) Errorv(v interface{}) {
-	l.err(fmt.Sprint(v))
-}
-
-func (l *contextLogger) Errorw(msg string, fields ...LogField) {
-	l.err(msg, fields...)
-}
-
-func (l *contextLogger) Info(v ...interface{}) {
-	l.info(fmt.Sprint(v...))
-}
-
-func (l *contextLogger) Infof(format string, v ...interface{}) {
-	l.info(fmt.Sprintf(format, v...))
-}
-
-func (l *contextLogger) Infov(v interface{}) {
-	l.info(v)
-}
-
-func (l *contextLogger) Infow(msg string, fields ...LogField) {
-	l.info(msg, fields...)
-}
-
-func (l *contextLogger) Slow(v ...interface{}) {
-	l.slow(fmt.Sprint(v...))
-}
-
-func (l *contextLogger) Slowf(format string, v ...interface{}) {
-	l.slow(fmt.Sprintf(format, v...))
-}
-
-func (l *contextLogger) Slowv(v interface{}) {
-	l.slow(v)
-}
-
-func (l *contextLogger) Sloww(msg string, fields ...LogField) {
-	l.slow(msg, fields...)
-}
-
-func (l *contextLogger) WithContext(ctx context.Context) Logger {
-	if ctx == nil {
-		return l
-	}
-
-	l.ctx = ctx
-	return l
-}
-
-func (l *contextLogger) WithDuration(duration time.Duration) Logger {
-	l.Duration = timex.ReprOfDuration(duration)
-	return l
-}
-
-func (l *contextLogger) buildFields(fields ...LogField) []LogField {
-	if len(l.Duration) > 0 {
-		fields = append(fields, Field(durationKey, l.Duration))
-	}
-
-	traceID := traceIdFromContext(l.ctx)
-	if len(traceID) > 0 {
-		fields = append(fields, Field(traceKey, traceID))
-	}
-
-	spanID := spanIdFromContext(l.ctx)
-	if len(spanID) > 0 {
-		fields = append(fields, Field(spanKey, spanID))
-	}
-
-	val := l.ctx.Value(fieldsContextKey)
-	if val != nil {
-		if arr, ok := val.([]LogField); ok {
-			fields = append(fields, arr...)
-		}
-	}
-
-	return fields
-}
-
-func (l *contextLogger) err(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(v, l.buildFields(fields...)...)
-	}
-}
-
-func (l *contextLogger) info(v interface{}, fields ...LogField) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(v, l.buildFields(fields...)...)
-	}
-}
-
-func (l *contextLogger) slow(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Slow(v, l.buildFields(fields...)...)
-	}
-}
-
-func spanIdFromContext(ctx context.Context) string {
-	spanCtx := trace.SpanContextFromContext(ctx)
-	if spanCtx.HasSpanID() {
-		return spanCtx.SpanID().String()
-	}
-
-	return ""
-}
-
-func traceIdFromContext(ctx context.Context) string {
-	spanCtx := trace.SpanContextFromContext(ctx)
-	if spanCtx.HasTraceID() {
-		return spanCtx.TraceID().String()
-	}
-
-	return ""
-}

core/logx/durationlogger.go

@@ -1,101 +0,0 @@
-package logx
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/zeromicro/go-zero/core/timex"
-)
-
-// WithDuration returns a Logger which logs the given duration.
-func WithDuration(d time.Duration) Logger {
-	return &durationLogger{
-		Duration: timex.ReprOfDuration(d),
-	}
-}
-
-type durationLogger logEntry
-
-func (l *durationLogger) Error(v ...interface{}) {
-	l.err(fmt.Sprint(v...))
-}
-
-func (l *durationLogger) Errorf(format string, v ...interface{}) {
-	l.err(fmt.Sprintf(format, v...))
-}
-
-func (l *durationLogger) Errorv(v interface{}) {
-	l.err(v)
-}
-
-func (l *durationLogger) Errorw(msg string, fields ...LogField) {
-	l.err(msg, fields...)
-}
-
-func (l *durationLogger) Info(v ...interface{}) {
-	l.info(fmt.Sprint(v...))
-}
-
-func (l *durationLogger) Infof(format string, v ...interface{}) {
-	l.info(fmt.Sprintf(format, v...))
-}
-
-func (l *durationLogger) Infov(v interface{}) {
-	l.info(v)
-}
-
-func (l *durationLogger) Infow(msg string, fields ...LogField) {
-	l.info(msg, fields...)
-}
-
-func (l *durationLogger) Slow(v ...interface{}) {
-	l.slow(fmt.Sprint(v...))
-}
-
-func (l *durationLogger) Slowf(format string, v ...interface{}) {
-	l.slow(fmt.Sprintf(format, v...))
-}
-
-func (l *durationLogger) Slowv(v interface{}) {
-	l.slow(v)
-}
-
-func (l *durationLogger) Sloww(msg string, fields ...LogField) {
-	l.slow(msg, fields...)
-}
-
-func (l *durationLogger) WithContext(ctx context.Context) Logger {
-	return &contextLogger{
-		ctx: ctx,
-		logEntry: logEntry{
-			Duration: l.Duration,
-		},
-	}
-}
-
-func (l *durationLogger) WithDuration(duration time.Duration) Logger {
-	l.Duration = timex.ReprOfDuration(duration)
-	return l
-}
-
-func (l *durationLogger) err(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		fields = append(fields, Field(durationKey, l.Duration))
-		getWriter().Error(v, fields...)
-	}
-}
-
-func (l *durationLogger) info(v interface{}, fields ...LogField) {
-	if shallLog(InfoLevel) {
-		fields = append(fields, Field(durationKey, l.Duration))
-		getWriter().Info(v, fields...)
-	}
-}
-
-func (l *durationLogger) slow(v interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		fields = append(fields, Field(durationKey, l.Duration))
-		getWriter().Slow(v, fields...)
-	}
-}

core/logx/durationlogger_test.go

@@ -1,161 +0,0 @@
-package logx
-
-import (
-	"context"
-	"strings"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"go.opentelemetry.io/otel"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-)
-
-func TestWithDurationError(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Error("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationErrorf(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Errorf("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationErrorv(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Errorv("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationErrorw(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Errorw("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-}
-
-func TestWithDurationInfo(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Info("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationInfoConsole(t *testing.T) {
-	old := atomic.LoadUint32(&encoding)
-	atomic.StoreUint32(&encoding, plainEncodingType)
-	defer func() {
-		atomic.StoreUint32(&encoding, old)
-	}()
-
-	w := new(mockWriter)
-	o := writer.Swap(w)
-	defer writer.Store(o)
-
-	WithDuration(time.Second).Info("foo")
-	assert.True(t, strings.Contains(w.String(), "ms"), w.String())
-}
-
-func TestWithDurationInfof(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Infof("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationInfov(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Infov("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationInfow(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Infow("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-}
-
-func TestWithDurationWithContextInfow(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	otp := otel.GetTracerProvider()
-	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
-	otel.SetTracerProvider(tp)
-	defer otel.SetTracerProvider(otp)
-
-	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
-	WithDuration(time.Second).WithContext(ctx).Infow("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-	assert.True(t, strings.Contains(w.String(), "trace"), w.String())
-	assert.True(t, strings.Contains(w.String(), "span"), w.String())
-}
-
-func TestWithDurationSlow(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).Slow("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationSlowf(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).WithDuration(time.Hour).Slowf("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationSlowv(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).WithDuration(time.Hour).Slowv("foo")
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-}
-
-func TestWithDurationSloww(t *testing.T) {
-	w := new(mockWriter)
-	old := writer.Swap(w)
-	defer writer.Store(old)
-
-	WithDuration(time.Second).WithDuration(time.Hour).Sloww("foo", Field("foo", "bar"))
-	assert.True(t, strings.Contains(w.String(), "duration"), w.String())
-	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
-	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
-}

core/logx/fields.go

@@ -1,18 +1,48 @@
 package logx
 
-import "context"
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+)
 
-var fieldsContextKey contextKey
+var (
+	fieldsContextKey contextKey
+	globalFields     atomic.Value
+	globalFieldsLock sync.Mutex
+)
 
 type contextKey struct{}
 
-// WithFields returns a new context with the given fields.
+// AddGlobalFields adds global fields.
-func WithFields(ctx context.Context, fields ...LogField) context.Context {
+func AddGlobalFields(fields ...LogField) {
+	globalFieldsLock.Lock()
+	defer globalFieldsLock.Unlock()
+
+	old := globalFields.Load()
+	if old == nil {
+		globalFields.Store(append([]LogField(nil), fields...))
+	} else {
+		globalFields.Store(append(old.([]LogField), fields...))
+	}
+}
+
+// ContextWithFields returns a new context with the given fields.
+func ContextWithFields(ctx context.Context, fields ...LogField) context.Context {
 	if val := ctx.Value(fieldsContextKey); val != nil {
 		if arr, ok := val.([]LogField); ok {
-			return context.WithValue(ctx, fieldsContextKey, append(arr, fields...))
+			allFields := make([]LogField, 0, len(arr)+len(fields))
+			allFields = append(allFields, arr...)
+			allFields = append(allFields, fields...)
+			return context.WithValue(ctx, fieldsContextKey, allFields)
 		}
 	}
 
 	return context.WithValue(ctx, fieldsContextKey, fields)
 }
+
+// WithFields returns a new logger with the given fields.
+// deprecated: use ContextWithFields instead.
+func WithFields(ctx context.Context, fields ...LogField) context.Context {
+	return ContextWithFields(ctx, fields...)
+}

core/logx/fields_test.go

@@ -1,12 +1,46 @@
 package logx
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"strconv"
+	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
+func TestAddGlobalFields(t *testing.T) {
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	old := Reset()
+	SetWriter(writer)
+	defer SetWriter(old)
+
+	Info("hello")
+	buf.Reset()
+
+	AddGlobalFields(Field("a", "1"), Field("b", "2"))
+	AddGlobalFields(Field("c", "3"))
+	Info("world")
+	var m map[string]interface{}
+	assert.NoError(t, json.Unmarshal(buf.Bytes(), &m))
+	assert.Equal(t, "1", m["a"])
+	assert.Equal(t, "2", m["b"])
+	assert.Equal(t, "3", m["c"])
+}
+
+func TestContextWithFields(t *testing.T) {
+	ctx := ContextWithFields(context.Background(), Field("a", 1), Field("b", 2))
+	vals := ctx.Value(fieldsContextKey)
+	assert.NotNil(t, vals)
+	fields, ok := vals.([]LogField)
+	assert.True(t, ok)
+	assert.EqualValues(t, []LogField{Field("a", 1), Field("b", 2)}, fields)
+}
+
 func TestWithFields(t *testing.T) {
 	ctx := WithFields(context.Background(), Field("a", 1), Field("b", 2))
 	vals := ctx.Value(fieldsContextKey)
@@ -19,8 +53,8 @@ func TestWithFields(t *testing.T) {
 func TestWithFieldsAppend(t *testing.T) {
 	var dummyKey struct{}
 	ctx := context.WithValue(context.Background(), dummyKey, "dummy")
-	ctx = WithFields(ctx, Field("a", 1), Field("b", 2))
+	ctx = ContextWithFields(ctx, Field("a", 1), Field("b", 2))
-	ctx = WithFields(ctx, Field("c", 3), Field("d", 4))
+	ctx = ContextWithFields(ctx, Field("c", 3), Field("d", 4))
 	vals := ctx.Value(fieldsContextKey)
 	assert.NotNil(t, vals)
 	fields, ok := vals.([]LogField)
@@ -33,3 +67,55 @@ func TestWithFieldsAppend(t *testing.T) {
 		Field("d", 4),
 	}, fields)
 }
+
+func TestWithFieldsAppendCopy(t *testing.T) {
+	const count = 10
+	ctx := context.Background()
+	for i := 0; i < count; i++ {
+		ctx = ContextWithFields(ctx, Field(strconv.Itoa(i), 1))
+	}
+
+	af := Field("foo", 1)
+	bf := Field("bar", 2)
+	ctxa := ContextWithFields(ctx, af)
+	ctxb := ContextWithFields(ctx, bf)
+
+	assert.EqualValues(t, af, ctxa.Value(fieldsContextKey).([]LogField)[count])
+	assert.EqualValues(t, bf, ctxb.Value(fieldsContextKey).([]LogField)[count])
+}
+
+func BenchmarkAtomicValue(b *testing.B) {
+	b.ReportAllocs()
+
+	var container atomic.Value
+	vals := []LogField{
+		Field("a", "b"),
+		Field("c", "d"),
+		Field("e", "f"),
+	}
+	container.Store(&vals)
+
+	for i := 0; i < b.N; i++ {
+		val := container.Load()
+		if val != nil {
+			_ = *val.(*[]LogField)
+		}
+	}
+}
+
+func BenchmarkRWMutex(b *testing.B) {
+	b.ReportAllocs()
+
+	var lock sync.RWMutex
+	vals := []LogField{
+		Field("a", "b"),
+		Field("c", "d"),
+		Field("e", "f"),
+	}
+
+	for i := 0; i < b.N; i++ {
+		lock.RLock()
+		_ = vals
+		lock.RUnlock()
+	}
+}

core/logx/logger.go

@@ -7,6 +7,14 @@ import (
 
 // A Logger represents a logger.
 type Logger interface {
+	// Debug logs a message at info level.
+	Debug(...interface{})
+	// Debugf logs a message at info level.
+	Debugf(string, ...interface{})
+	// Debugv logs a message at info level.
+	Debugv(interface{})
+	// Debugw logs a message at info level.
+	Debugw(string, ...LogField)
 	// Error logs a message at error level.
 	Error(...interface{})
 	// Errorf logs a message at error level.
@@ -31,8 +39,12 @@ type Logger interface {
 	Slowv(interface{})
 	// Sloww logs a message at slow level.
 	Sloww(string, ...LogField)
+	// WithCallerSkip returns a new logger with the given caller skip.
+	WithCallerSkip(skip int) Logger
 	// WithContext returns a new logger with the given context.
-	WithContext(context.Context) Logger
+	WithContext(ctx context.Context) Logger
 	// WithDuration returns a new logger with the given duration.
-	WithDuration(time.Duration) Logger
+	WithDuration(d time.Duration) Logger
+	// WithFields returns a new logger with the given fields.
+	WithFields(fields ...LogField) Logger
 }

core/logx/logs.go

@@ -14,13 +14,16 @@ import (
 	"github.com/zeromicro/go-zero/core/sysx"
 )
 
-const callerDepth = 5
+const callerDepth = 4
 
 var (
 	timeFormat = "2006-01-02T15:04:05.000Z07:00"
 	logLevel   uint32
 	encoding   uint32 = jsonEncodingType
+	// maxContentLength is used to truncate the log content, 0 for not truncating.
+	maxContentLength uint32
 	// use uint32 for atomic operations
+	disableLog  uint32
 	disableStat uint32
 	options     logOptions
 	writer      = new(atomicWriter)
@@ -28,15 +31,16 @@ var (
 )
 
 type (
-	logEntry struct {
+	// LogField is a key-value pair that will be added to the log entry.
-		Timestamp string      `json:"@timestamp"`
+	LogField struct {
-		Level     string      `json:"level"`
+		Key   string
-		Duration  string      `json:"duration,omitempty"`
+		Value interface{}
-		Caller    string      `json:"caller,omitempty"`
-		Content   interface{} `json:"content"`
 	}
 
-	logEntryWithFields map[string]interface{}
+	// LogOption defines the method to customize the logging.
+	LogOption func(options *logOptions)
+
+	logEntry map[string]interface{}
 
 	logOptions struct {
 		gzipEnabled           bool
@@ -46,15 +50,6 @@ type (
 		maxSize               int
 		rotationRule          string
 	}
-
-	// LogField is a key-value pair that will be added to the log entry.
-	LogField struct {
-		Key   string
-		Value interface{}
-	}
-
-	// LogOption defines the method to customize the logging.
-	LogOption func(options *logOptions)
 )
 
 // Alert alerts v in alert level, and the message is written to error log.
@@ -71,8 +66,29 @@ func Close() error {
 	return nil
 }
 
+// Debug writes v into access log.
+func Debug(v ...interface{}) {
+	writeDebug(fmt.Sprint(v...))
+}
+
+// Debugf writes v with format into access log.
+func Debugf(format string, v ...interface{}) {
+	writeDebug(fmt.Sprintf(format, v...))
+}
+
+// Debugv writes v into access log with json content.
+func Debugv(v interface{}) {
+	writeDebug(v)
+}
+
+// Debugw writes msg along with fields into access log.
+func Debugw(msg string, fields ...LogField) {
+	writeDebug(msg, fields...)
+}
+
 // Disable disables the logging.
 func Disable() {
+	atomic.StoreUint32(&disableLog, 1)
 	writer.Store(nopWriter{})
 }
 
@@ -83,35 +99,35 @@ func DisableStat() {
 
 // Error writes v into error log.
 func Error(v ...interface{}) {
-	errorTextSync(fmt.Sprint(v...))
+	writeError(fmt.Sprint(v...))
 }
 
 // Errorf writes v with format into error log.
 func Errorf(format string, v ...interface{}) {
-	errorTextSync(fmt.Errorf(format, v...).Error())
+	writeError(fmt.Errorf(format, v...).Error())
 }
 
 // ErrorStack writes v along with call stack into error log.
 func ErrorStack(v ...interface{}) {
 	// there is newline in stack string
-	stackSync(fmt.Sprint(v...))
+	writeStack(fmt.Sprint(v...))
 }
 
 // ErrorStackf writes v along with call stack in format into error log.
 func ErrorStackf(format string, v ...interface{}) {
 	// there is newline in stack string
-	stackSync(fmt.Sprintf(format, v...))
+	writeStack(fmt.Sprintf(format, v...))
 }
 
 // Errorv writes v into error log with json content.
 // No call stack attached, because not elegant to pack the messages.
 func Errorv(v interface{}) {
-	errorAnySync(v)
+	writeError(v)
 }
 
 // Errorw writes msg along with fields into error log.
 func Errorw(msg string, fields ...LogField) {
-	errorFieldsSync(msg, fields...)
+	writeError(msg, fields...)
 }
 
 // Field returns a LogField for the given key and value.
@@ -154,22 +170,22 @@ func Field(key string, value interface{}) LogField {
 
 // Info writes v into access log.
 func Info(v ...interface{}) {
-	infoTextSync(fmt.Sprint(v...))
+	writeInfo(fmt.Sprint(v...))
 }
 
 // Infof writes v with format into access log.
 func Infof(format string, v ...interface{}) {
-	infoTextSync(fmt.Sprintf(format, v...))
+	writeInfo(fmt.Sprintf(format, v...))
 }
 
 // Infov writes v into access log with json content.
 func Infov(v interface{}) {
-	infoAnySync(v)
+	writeInfo(v)
 }
 
 // Infow writes msg along with fields into access log.
 func Infow(msg string, fields ...LogField) {
-	infoFieldsSync(msg, fields...)
+	writeInfo(msg, fields...)
 }
 
 // Must checks if err is nil, otherwise logs the error and exits.
@@ -201,7 +217,9 @@ func SetLevel(level uint32) {
 
 // SetWriter sets the logging writer. It can be used to customize the logging.
 func SetWriter(w Writer) {
-	writer.Store(w)
+	if atomic.LoadUint32(&disableLog) == 0 {
+		writer.Store(w)
+	}
 }
 
 // SetUp sets up the logx. If already set up, just return nil.
@@ -214,10 +232,16 @@ func SetUp(c LogConf) (err error) {
 	setupOnce.Do(func() {
 		setupLogLevel(c)
 
+		if !c.Stat {
+			DisableStat()
+		}
+
 		if len(c.TimeFormat) > 0 {
 			timeFormat = c.TimeFormat
 		}
 
+		atomic.StoreUint32(&maxContentLength, c.MaxContentLength)
+
 		switch c.Encoding {
 		case plainEncoding:
 			atomic.StoreUint32(&encoding, plainEncodingType)
@@ -240,42 +264,42 @@ func SetUp(c LogConf) (err error) {
 
 // Severe writes v into severe log.
 func Severe(v ...interface{}) {
-	severeSync(fmt.Sprint(v...))
+	writeSevere(fmt.Sprint(v...))
 }
 
 // Severef writes v with format into severe log.
 func Severef(format string, v ...interface{}) {
-	severeSync(fmt.Sprintf(format, v...))
+	writeSevere(fmt.Sprintf(format, v...))
 }
 
 // Slow writes v into slow log.
 func Slow(v ...interface{}) {
-	slowTextSync(fmt.Sprint(v...))
+	writeSlow(fmt.Sprint(v...))
 }
 
 // Slowf writes v with format into slow log.
 func Slowf(format string, v ...interface{}) {
-	slowTextSync(fmt.Sprintf(format, v...))
+	writeSlow(fmt.Sprintf(format, v...))
 }
 
 // Slowv writes v into slow log with json content.
 func Slowv(v interface{}) {
-	slowAnySync(v)
+	writeSlow(v)
 }
 
 // Sloww writes msg along with fields into slow log.
 func Sloww(msg string, fields ...LogField) {
-	slowFieldsSync(msg, fields...)
+	writeSlow(msg, fields...)
 }
 
 // Stat writes v into stat log.
 func Stat(v ...interface{}) {
-	statSync(fmt.Sprint(v...))
+	writeStat(fmt.Sprint(v...))
 }
 
 // Statf writes v with format into stat log.
 func Statf(format string, v ...interface{}) {
-	statSync(fmt.Sprintf(format, v...))
+	writeStat(fmt.Sprintf(format, v...))
 }
 
 // WithCooldownMillis customizes logging on writing call stack interval.
@@ -320,6 +344,10 @@ func WithRotation(r string) LogOption {
 	}
 }
 
+func addCaller(fields ...LogField) []LogField {
+	return append(fields, Field(callerKey, getCaller(callerDepth)))
+}
+
 func createOutput(path string) (io.WriteCloser, error) {
 	if len(path) == 0 {
 		return nil, ErrLogPathNotSet
@@ -335,29 +363,10 @@ func createOutput(path string) (io.WriteCloser, error) {
 	}
 }
 
-func errorAnySync(v interface{}) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(v)
-	}
-}
-
-func errorFieldsSync(content string, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(content, fields...)
-	}
-}
-
-func errorTextSync(msg string) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(msg)
-	}
-}
-
 func getWriter() Writer {
 	w := writer.Load()
 	if w == nil {
-		w = newConsoleWriter()
+		w = writer.StoreIfNil(newConsoleWriter())
-		writer.Store(w)
 	}
 
 	return w
@@ -369,26 +378,10 @@ func handleOptions(opts []LogOption) {
 	}
 }
 
-func infoAnySync(val interface{}) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(val)
-	}
-}
-
-func infoFieldsSync(content string, fields ...LogField) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(content, fields...)
-	}
-}
-
-func infoTextSync(msg string) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(msg)
-	}
-}
-
 func setupLogLevel(c LogConf) {
 	switch c.Level {
+	case levelDebug:
+		SetLevel(DebugLevel)
 	case levelInfo:
 		SetLevel(InfoLevel)
 	case levelError:
@@ -421,12 +414,6 @@ func setupWithVolume(c LogConf) error {
 	return setupWithFiles(c)
 }
 
-func severeSync(msg string) {
-	if shallLog(SevereLevel) {
-		getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
-	}
-}
-
 func shallLog(level uint32) bool {
 	return atomic.LoadUint32(&logLevel) <= level
 }
@@ -435,32 +422,44 @@ func shallLogStat() bool {
 	return atomic.LoadUint32(&disableStat) == 0
 }
 
-func slowAnySync(v interface{}) {
+func writeDebug(val interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
+	if shallLog(DebugLevel) {
-		getWriter().Slow(v)
+		getWriter().Debug(val, addCaller(fields...)...)
 	}
 }
 
-func slowFieldsSync(content string, fields ...LogField) {
+func writeError(val interface{}, fields ...LogField) {
 	if shallLog(ErrorLevel) {
-		getWriter().Slow(content, fields...)
+		getWriter().Error(val, addCaller(fields...)...)
 	}
 }
 
-func slowTextSync(msg string) {
+func writeInfo(val interface{}, fields ...LogField) {
-	if shallLog(ErrorLevel) {
+	if shallLog(InfoLevel) {
-		getWriter().Slow(msg)
+		getWriter().Info(val, addCaller(fields...)...)
 	}
 }
 
-func stackSync(msg string) {
+func writeSevere(msg string) {
+	if shallLog(SevereLevel) {
+		getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+	}
+}
+
+func writeSlow(val interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Slow(val, addCaller(fields...)...)
+	}
+}
+
+func writeStack(msg string) {
 	if shallLog(ErrorLevel) {
 		getWriter().Stack(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
-func statSync(msg string) {
+func writeStat(msg string) {
 	if shallLogStat() && shallLog(InfoLevel) {
-		getWriter().Stat(msg)
+		getWriter().Stat(msg, addCaller()...)
 	}
 }

core/logx/logs_test.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
 	"reflect"
@@ -35,6 +35,12 @@ func (mw *mockWriter) Alert(v interface{}) {
 	output(&mw.builder, levelAlert, v)
 }
 
+func (mw *mockWriter) Debug(v interface{}, fields ...LogField) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	output(&mw.builder, levelDebug, v, fields...)
+}
+
 func (mw *mockWriter) Error(v interface{}, fields ...LogField) {
 	mw.lock.Lock()
 	defer mw.lock.Unlock()
@@ -212,6 +218,46 @@ func TestStructedLogAlert(t *testing.T) {
 	})
 }
 
+func TestStructedLogDebug(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debug(v...)
+	})
+}
+
+func TestStructedLogDebugf(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debugf(fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogDebugv(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debugv(fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogDebugw(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	defer writer.Store(old)
+
+	doTestStructedLog(t, levelDebug, w, func(v ...interface{}) {
+		Debugw(fmt.Sprint(v...), Field("foo", time.Second))
+	})
+}
+
 func TestStructedLogError(t *testing.T) {
 	w := new(mockWriter)
 	old := writer.Swap(w)
@@ -461,13 +507,13 @@ func TestStructedLogWithDuration(t *testing.T) {
 	defer writer.Store(old)
 
 	WithDuration(time.Second).Info(message)
-	var entry logEntry
+	var entry map[string]interface{}
 	if err := json.Unmarshal([]byte(w.String()), &entry); err != nil {
 		t.Error(err)
 	}
-	assert.Equal(t, levelInfo, entry.Level)
+	assert.Equal(t, levelInfo, entry[levelKey])
-	assert.Equal(t, message, entry.Content)
+	assert.Equal(t, message, entry[contentKey])
-	assert.Equal(t, "1000.0ms", entry.Duration)
+	assert.Equal(t, "1000.0ms", entry[durationKey])
 }
 
 func TestSetLevel(t *testing.T) {
@@ -483,9 +529,9 @@ func TestSetLevel(t *testing.T) {
 
 func TestSetLevelTwiceWithMode(t *testing.T) {
 	testModes := []string{
-		"mode",
 		"console",
 		"volumn",
+		"mode",
 	}
 	w := new(mockWriter)
 	old := writer.Swap(w)
@@ -531,6 +577,7 @@ func TestSetup(t *testing.T) {
 	MustSetup(LogConf{
 		ServiceName: "any",
 		Mode:        "console",
+		TimeFormat:  timeFormat,
 	})
 	MustSetup(LogConf{
 		ServiceName: "any",
@@ -553,13 +600,23 @@ func TestSetup(t *testing.T) {
 		Encoding:    plainEncoding,
 	})
 
+	defer os.RemoveAll("CD01CB7D-2705-4F3F-889E-86219BF56F10")
 	assert.NotNil(t, setupWithVolume(LogConf{}))
+	assert.Nil(t, setupWithVolume(LogConf{
+		ServiceName: "CD01CB7D-2705-4F3F-889E-86219BF56F10",
+	}))
+	assert.Nil(t, setupWithVolume(LogConf{
+		ServiceName: "CD01CB7D-2705-4F3F-889E-86219BF56F10",
+		Rotation:    sizeRotationRule,
+	}))
 	assert.NotNil(t, setupWithFiles(LogConf{}))
 	assert.Nil(t, setupWithFiles(LogConf{
 		ServiceName: "any",
 		Path:        os.TempDir(),
 		Compress:    true,
 		KeepDays:    1,
+		MaxBackups:  3,
+		MaxSize:     1024 * 1024,
 	}))
 	setupLogLevel(LogConf{
 		Level: levelInfo,
@@ -583,6 +640,8 @@ func TestDisable(t *testing.T) {
 	var opt logOptions
 	WithKeepDays(1)(&opt)
 	WithGzip()(&opt)
+	WithMaxBackups(1)(&opt)
+	WithMaxSize(1024)(&opt)
 	assert.Nil(t, Close())
 	assert.Nil(t, Close())
 }
@@ -599,6 +658,7 @@ func TestDisableStat(t *testing.T) {
 }
 
 func TestSetWriter(t *testing.T) {
+	atomic.StoreUint32(&disableLog, 0)
 	Reset()
 	SetWriter(nopWriter{})
 	assert.NotNil(t, writer.Load())
@@ -648,7 +708,7 @@ func BenchmarkCopyByteSlice(b *testing.B) {
 		buf = make([]byte, len(s))
 		copy(buf, s)
 	}
-	fmt.Fprint(ioutil.Discard, buf)
+	fmt.Fprint(io.Discard, buf)
 }
 
 func BenchmarkCopyOnWriteByteSlice(b *testing.B) {
@@ -657,7 +717,7 @@ func BenchmarkCopyOnWriteByteSlice(b *testing.B) {
 		size := len(s)
 		buf = s[:size:size]
 	}
-	fmt.Fprint(ioutil.Discard, buf)
+	fmt.Fprint(io.Discard, buf)
 }
 
 func BenchmarkCacheByteSlice(b *testing.B) {
@@ -671,7 +731,7 @@ func BenchmarkCacheByteSlice(b *testing.B) {
 func BenchmarkLogs(b *testing.B) {
 	b.ReportAllocs()
 
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 	for i := 0; i < b.N; i++ {
 		Info(i)
 	}
@@ -710,14 +770,16 @@ func put(b []byte) {
 func doTestStructedLog(t *testing.T, level string, w *mockWriter, write func(...interface{})) {
 	const message = "hello there"
 	write(message)
-	var entry logEntry
+
+	var entry map[string]interface{}
 	if err := json.Unmarshal([]byte(w.String()), &entry); err != nil {
 		t.Error(err)
 	}
-	assert.Equal(t, level, entry.Level)
+
-	val, ok := entry.Content.(string)
+	assert.Equal(t, level, entry[levelKey])
+	val, ok := entry[contentKey]
 	assert.True(t, ok)
-	assert.True(t, strings.Contains(val, message))
+	assert.True(t, strings.Contains(val.(string), message))
 }
 
 func doTestStructedLogConsole(t *testing.T, w *mockWriter, write func(...interface{})) {
@@ -729,9 +791,12 @@ func doTestStructedLogConsole(t *testing.T, w *mockWriter, write func(...interfa
 func testSetLevelTwiceWithMode(t *testing.T, mode string, w *mockWriter) {
 	writer.Store(nil)
 	SetUp(LogConf{
-		Mode:  mode,
+		Mode:       mode,
-		Level: "error",
+		Level:      "debug",
-		Path:  "/dev/null",
+		Path:       "/dev/null",
+		Encoding:   plainEncoding,
+		Stat:       false,
+		TimeFormat: time.RFC3339,
 	})
 	SetUp(LogConf{
 		Mode:  mode,

core/logx/richlogger.go

@@ -0,0 +1,181 @@
+package logx
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/zeromicro/go-zero/core/timex"
+	"github.com/zeromicro/go-zero/internal/trace"
+)
+
+// WithCallerSkip returns a Logger with given caller skip.
+func WithCallerSkip(skip int) Logger {
+	if skip <= 0 {
+		return new(richLogger)
+	}
+
+	return &richLogger{
+		callerSkip: skip,
+	}
+}
+
+// WithContext sets ctx to log, for keeping tracing information.
+func WithContext(ctx context.Context) Logger {
+	return &richLogger{
+		ctx: ctx,
+	}
+}
+
+// WithDuration returns a Logger with given duration.
+func WithDuration(d time.Duration) Logger {
+	return &richLogger{
+		fields: []LogField{Field(durationKey, timex.ReprOfDuration(d))},
+	}
+}
+
+type richLogger struct {
+	ctx        context.Context
+	callerSkip int
+	fields     []LogField
+}
+
+func (l *richLogger) Debug(v ...interface{}) {
+	l.debug(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Debugf(format string, v ...interface{}) {
+	l.debug(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Debugv(v interface{}) {
+	l.debug(v)
+}
+
+func (l *richLogger) Debugw(msg string, fields ...LogField) {
+	l.debug(msg, fields...)
+}
+
+func (l *richLogger) Error(v ...interface{}) {
+	l.err(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Errorf(format string, v ...interface{}) {
+	l.err(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Errorv(v interface{}) {
+	l.err(fmt.Sprint(v))
+}
+
+func (l *richLogger) Errorw(msg string, fields ...LogField) {
+	l.err(msg, fields...)
+}
+
+func (l *richLogger) Info(v ...interface{}) {
+	l.info(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Infof(format string, v ...interface{}) {
+	l.info(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Infov(v interface{}) {
+	l.info(v)
+}
+
+func (l *richLogger) Infow(msg string, fields ...LogField) {
+	l.info(msg, fields...)
+}
+
+func (l *richLogger) Slow(v ...interface{}) {
+	l.slow(fmt.Sprint(v...))
+}
+
+func (l *richLogger) Slowf(format string, v ...interface{}) {
+	l.slow(fmt.Sprintf(format, v...))
+}
+
+func (l *richLogger) Slowv(v interface{}) {
+	l.slow(v)
+}
+
+func (l *richLogger) Sloww(msg string, fields ...LogField) {
+	l.slow(msg, fields...)
+}
+
+func (l *richLogger) WithCallerSkip(skip int) Logger {
+	if skip <= 0 {
+		return l
+	}
+
+	l.callerSkip = skip
+	return l
+}
+
+func (l *richLogger) WithContext(ctx context.Context) Logger {
+	l.ctx = ctx
+	return l
+}
+
+func (l *richLogger) WithDuration(duration time.Duration) Logger {
+	l.fields = append(l.fields, Field(durationKey, timex.ReprOfDuration(duration)))
+	return l
+}
+
+func (l *richLogger) WithFields(fields ...LogField) Logger {
+	l.fields = append(l.fields, fields...)
+	return l
+}
+
+func (l *richLogger) buildFields(fields ...LogField) []LogField {
+	fields = append(l.fields, fields...)
+	fields = append(fields, Field(callerKey, getCaller(callerDepth+l.callerSkip)))
+
+	if l.ctx == nil {
+		return fields
+	}
+
+	traceID := trace.TraceIDFromContext(l.ctx)
+	if len(traceID) > 0 {
+		fields = append(fields, Field(traceKey, traceID))
+	}
+
+	spanID := trace.SpanIDFromContext(l.ctx)
+	if len(spanID) > 0 {
+		fields = append(fields, Field(spanKey, spanID))
+	}
+
+	val := l.ctx.Value(fieldsContextKey)
+	if val != nil {
+		if arr, ok := val.([]LogField); ok {
+			fields = append(fields, arr...)
+		}
+	}
+
+	return fields
+}
+
+func (l *richLogger) debug(v interface{}, fields ...LogField) {
+	if shallLog(DebugLevel) {
+		getWriter().Debug(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *richLogger) err(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Error(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *richLogger) info(v interface{}, fields ...LogField) {
+	if shallLog(InfoLevel) {
+		getWriter().Info(v, l.buildFields(fields...)...)
+	}
+}
+
+func (l *richLogger) slow(v interface{}, fields ...LogField) {
+	if shallLog(ErrorLevel) {
+		getWriter().Slow(v, l.buildFields(fields...)...)
+	}
+}

core/logx/richlogger_test.go

@@ -3,6 +3,7 @@ package logx
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"strings"
 	"sync/atomic"
@@ -36,6 +37,41 @@ func TestTraceLog(t *testing.T) {
 	validate(t, w.String(), true, true)
 }
 
+func TestTraceDebug(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, span := tp.Tracer("foo").Start(context.Background(), "bar")
+	defer span.End()
+
+	l := WithContext(ctx)
+	SetLevel(DebugLevel)
+	l.WithDuration(time.Second).Debug(testlog)
+	assert.True(t, strings.Contains(w.String(), traceKey))
+	assert.True(t, strings.Contains(w.String(), spanKey))
+	w.Reset()
+	l.WithDuration(time.Second).Debugf(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Debugv(testlog)
+	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Debugw(testlog, Field("foo", "bar"))
+	validate(t, w.String(), true, true)
+	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
+	assert.True(t, strings.Contains(w.String(), "bar"), w.String())
+}
+
 func TestTraceError(t *testing.T) {
 	w := new(mockWriter)
 	old := writer.Swap(w)
@@ -201,7 +237,7 @@ func TestLogWithFields(t *testing.T) {
 		writer.Store(old)
 	}()
 
-	ctx := WithFields(context.Background(), Field("foo", "bar"))
+	ctx := ContextWithFields(context.Background(), Field("foo", "bar"))
 	l := WithContext(ctx)
 	SetLevel(InfoLevel)
 	l.Info(testlog)
@@ -211,6 +247,48 @@ func TestLogWithFields(t *testing.T) {
 	assert.Equal(t, "bar", val.Foo)
 }
 
+func TestLogWithCallerSkip(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	l := WithCallerSkip(1).WithCallerSkip(0)
+	p := func(v string) {
+		l.Info(v)
+	}
+
+	file, line := getFileLine()
+	p(testlog)
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+
+	w.Reset()
+	l = WithCallerSkip(0).WithCallerSkip(1)
+	file, line = getFileLine()
+	p(testlog)
+	assert.True(t, w.Contains(fmt.Sprintf("%s:%d", file, line+1)))
+}
+
+func TestLoggerWithFields(t *testing.T) {
+	w := new(mockWriter)
+	old := writer.Swap(w)
+	writer.lock.RLock()
+	defer func() {
+		writer.lock.RUnlock()
+		writer.Store(old)
+	}()
+
+	l := WithContext(context.Background()).WithFields(Field("foo", "bar"))
+	l.Info(testlog)
+
+	var val mockValue
+	assert.Nil(t, json.Unmarshal([]byte(w.String()), &val))
+	assert.Equal(t, "bar", val.Foo)
+}
+
 func validate(t *testing.T, body string, expectedTrace, expectedSpan bool) {
 	var val mockValue
 	dec := json.NewDecoder(strings.NewReader(body))

core/logx/rotatelogger.go

@@ -115,7 +115,9 @@ func (r *DailyRotateRule) OutdatedFiles() []string {
 
 	var buf strings.Builder
 	boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay*r.days)).Format(dateFormat)
-	fmt.Fprintf(&buf, "%s%s%s", r.filename, r.delimiter, boundary)
+	buf.WriteString(r.filename)
+	buf.WriteString(r.delimiter)
+	buf.WriteString(boundary)
 	if r.gzip {
 		buf.WriteString(gzipExt)
 	}
@@ -282,7 +284,7 @@ func (l *RotateLogger) getBackupFilename() string {
 func (l *RotateLogger) init() error {
 	l.backup = l.rule.BackupFileName()
 
-	if _, err := os.Stat(l.filename); err != nil {
+	if fileInfo, err := os.Stat(l.filename); err != nil {
 		basePath := path.Dir(l.filename)
 		if _, err = os.Stat(basePath); err != nil {
 			if err = os.MkdirAll(basePath, defaultDirMode); err != nil {
@@ -293,8 +295,11 @@ func (l *RotateLogger) init() error {
 		if l.fp, err = os.Create(l.filename); err != nil {
 			return err
 		}
-	} else if l.fp, err = os.OpenFile(l.filename, os.O_APPEND|os.O_WRONLY, defaultFileMode); err != nil {
+	} else {
-		return err
+		if l.fp, err = os.OpenFile(l.filename, os.O_APPEND|os.O_WRONLY, defaultFileMode); err != nil {
+			return err
+		}
+		l.currentSize = fileInfo.Size()
 	}
 
 	fs.CloseOnExec(l.fp)

core/logx/syslog_test.go

@@ -42,11 +42,18 @@ func captureOutput(f func()) string {
 }
 
 func getContent(jsonStr string) string {
-	var entry logEntry
+	var entry map[string]interface{}
 	json.Unmarshal([]byte(jsonStr), &entry)
-	val, ok := entry.Content.(string)
+
-	if ok {
+	val, ok := entry[contentKey]
-		return val
+	if !ok {
+		return ""
 	}
-	return ""
+
+	str, ok := val.(string)
+	if !ok {
+		return ""
+	}
+
+	return str
 }

core/logx/vars.go

@@ -3,8 +3,10 @@ package logx
 import "errors"
 
 const (
-	// InfoLevel logs everything
+	// DebugLevel logs everything
-	InfoLevel uint32 = iota
+	DebugLevel uint32 = iota
+	// InfoLevel does not include debugs
+	InfoLevel
 	// ErrorLevel includes errors, slows, stacks
 	ErrorLevel
 	// SevereLevel only log severe messages
@@ -14,13 +16,13 @@ const (
 const (
 	jsonEncodingType = iota
 	plainEncodingType
-
-	plainEncoding    = "plain"
-	plainEncodingSep = '\t'
-	sizeRotationRule = "size"
 )
 
 const (
+	plainEncoding    = "plain"
+	plainEncodingSep = '\t'
+	sizeRotationRule = "size"
+
 	accessFilename = "access.log"
 	errorFilename  = "error.log"
 	severeFilename = "severe.log"
@@ -37,6 +39,7 @@ const (
 	levelFatal  = "fatal"
 	levelSlow   = "slow"
 	levelStat   = "stat"
+	levelDebug  = "debug"
 
 	backupFileDelimiter = "-"
 	flags               = 0x0
@@ -50,6 +53,7 @@ const (
 	spanKey      = "span"
 	timestampKey = "@timestamp"
 	traceKey     = "trace"
+	truncatedKey = "truncated"
 )
 
 var (
@@ -57,4 +61,6 @@ var (
 	ErrLogPathNotSet = errors.New("log path must be set")
 	// ErrLogServiceNameNotSet is an error that indicates that the service name is not set.
 	ErrLogServiceNameNotSet = errors.New("log service name must be set")
+
+	truncatedField = Field(truncatedKey, true)
 )

core/logx/writer.go

@@ -1,16 +1,16 @@
 package logx
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log"
-	"os"
 	"path"
-	"strings"
 	"sync"
 	"sync/atomic"
 
+	fatihcolor "github.com/fatih/color"
 	"github.com/zeromicro/go-zero/core/color"
 )
 
@@ -18,6 +18,7 @@ type (
 	Writer interface {
 		Alert(v interface{})
 		Close() error
+		Debug(v interface{}, fields ...LogField)
 		Error(v interface{}, fields ...LogField)
 		Info(v interface{}, fields ...LogField)
 		Severe(v interface{})
@@ -67,6 +68,17 @@ func (w *atomicWriter) Store(v Writer) {
 	w.writer = v
 }
 
+func (w *atomicWriter) StoreIfNil(v Writer) Writer {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if w.writer == nil {
+		w.writer = v
+	}
+
+	return w.writer
+}
+
 func (w *atomicWriter) Swap(v Writer) Writer {
 	w.lock.Lock()
 	defer w.lock.Unlock()
@@ -76,8 +88,8 @@ func (w *atomicWriter) Swap(v Writer) Writer {
 }
 
 func newConsoleWriter() Writer {
-	outLog := newLogWriter(log.New(os.Stdout, "", flags))
+	outLog := newLogWriter(log.New(fatihcolor.Output, "", flags))
-	errLog := newLogWriter(log.New(os.Stderr, "", flags))
+	errLog := newLogWriter(log.New(fatihcolor.Error, "", flags))
 	return &concreteWriter{
 		infoLog:   outLog,
 		errorLog:  errLog,
@@ -183,6 +195,10 @@ func (w *concreteWriter) Close() error {
 	return w.statLog.Close()
 }
 
+func (w *concreteWriter) Debug(v interface{}, fields ...LogField) {
+	output(w.infoLog, levelDebug, v, fields...)
+}
+
 func (w *concreteWriter) Error(v interface{}, fields ...LogField) {
 	output(w.errorLog, levelError, v, fields...)
 }
@@ -216,6 +232,9 @@ func (n nopWriter) Close() error {
 	return nil
 }
 
+func (n nopWriter) Debug(_ interface{}, _ ...LogField) {
+}
+
 func (n nopWriter) Error(_ interface{}, _ ...LogField) {
 }
 
@@ -234,24 +253,47 @@ func (n nopWriter) Stack(_ interface{}) {
 func (n nopWriter) Stat(_ interface{}, _ ...LogField) {
 }
 
-func buildFields(fields ...LogField) []string {
+func buildPlainFields(fields ...LogField) []string {
 	var items []string
 
 	for _, field := range fields {
-		items = append(items, fmt.Sprintf("%s=%v", field.Key, field.Value))
+		items = append(items, fmt.Sprintf("%s=%+v", field.Key, field.Value))
 	}
 
 	return items
 }
 
+func combineGlobalFields(fields []LogField) []LogField {
+	globals := globalFields.Load()
+	if globals == nil {
+		return fields
+	}
+
+	gf := globals.([]LogField)
+	ret := make([]LogField, 0, len(gf)+len(fields))
+	ret = append(ret, gf...)
+	ret = append(ret, fields...)
+
+	return ret
+}
+
 func output(writer io.Writer, level string, val interface{}, fields ...LogField) {
-	fields = append(fields, Field(callerKey, getCaller(callerDepth)))
+	// only truncate string content, don't know how to truncate the values of other types.
+	if v, ok := val.(string); ok {
+		maxLen := atomic.LoadUint32(&maxContentLength)
+		if maxLen > 0 && len(v) > int(maxLen) {
+			val = v[:maxLen]
+			fields = append(fields, truncatedField)
+		}
+	}
+
+	fields = combineGlobalFields(fields)
 
 	switch atomic.LoadUint32(&encoding) {
 	case plainEncodingType:
-		writePlainAny(writer, level, val, buildFields(fields...)...)
+		writePlainAny(writer, level, val, buildPlainFields(fields...)...)
 	default:
-		entry := make(logEntryWithFields)
+		entry := make(logEntry)
 		for _, field := range fields {
 			entry[field.Key] = field.Value
 		}
@@ -275,6 +317,8 @@ func wrapLevelWithColor(level string) string {
 		colour = color.FgBlue
 	case levelSlow:
 		colour = color.FgYellow
+	case levelDebug:
+		colour = color.FgYellow
 	case levelStat:
 		colour = color.FgGreen
 	}
@@ -307,34 +351,12 @@ func writePlainAny(writer io.Writer, level string, val interface{}, fields ...st
 	case fmt.Stringer:
 		writePlainText(writer, level, v.String(), fields...)
 	default:
-		var buf strings.Builder
+		writePlainValue(writer, level, v, fields...)
-		buf.WriteString(getTimestamp())
-		buf.WriteByte(plainEncodingSep)
-		buf.WriteString(level)
-		buf.WriteByte(plainEncodingSep)
-		if err := json.NewEncoder(&buf).Encode(val); err != nil {
-			log.Println(err.Error())
-			return
-		}
-
-		for _, item := range fields {
-			buf.WriteByte(plainEncodingSep)
-			buf.WriteString(item)
-		}
-		buf.WriteByte('\n')
-		if writer == nil {
-			log.Println(buf.String())
-			return
-		}
-
-		if _, err := fmt.Fprint(writer, buf.String()); err != nil {
-			log.Println(err.Error())
-		}
 	}
 }
 
 func writePlainText(writer io.Writer, level, msg string, fields ...string) {
-	var buf strings.Builder
+	var buf bytes.Buffer
 	buf.WriteString(getTimestamp())
 	buf.WriteByte(plainEncodingSep)
 	buf.WriteString(level)
@@ -350,7 +372,33 @@ func writePlainText(writer io.Writer, level, msg string, fields ...string) {
 		return
 	}
 
-	if _, err := fmt.Fprint(writer, buf.String()); err != nil {
+	if _, err := writer.Write(buf.Bytes()); err != nil {
+		log.Println(err.Error())
+	}
+}
+
+func writePlainValue(writer io.Writer, level string, val interface{}, fields ...string) {
+	var buf bytes.Buffer
+	buf.WriteString(getTimestamp())
+	buf.WriteByte(plainEncodingSep)
+	buf.WriteString(level)
+	buf.WriteByte(plainEncodingSep)
+	if err := json.NewEncoder(&buf).Encode(val); err != nil {
+		log.Println(err.Error())
+		return
+	}
+
+	for _, item := range fields {
+		buf.WriteByte(plainEncodingSep)
+		buf.WriteString(item)
+	}
+	buf.WriteByte('\n')
+	if writer == nil {
+		log.Println(buf.String())
+		return
+	}
+
+	if _, err := writer.Write(buf.Bytes()); err != nil {
 		log.Println(err.Error())
 	}
 }

core/logx/writer_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"log"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -16,6 +17,9 @@ func TestNewWriter(t *testing.T) {
 	w := NewWriter(&buf)
 	w.Info(literal)
 	assert.Contains(t, buf.String(), literal)
+	buf.Reset()
+	w.Debug(literal)
+	assert.Contains(t, buf.String(), literal)
 }
 
 func TestConsoleWriter(t *testing.T) {
@@ -97,6 +101,7 @@ func TestNopWriter(t *testing.T) {
 	assert.NotPanics(t, func() {
 		var w nopWriter
 		w.Alert("foo")
+		w.Debug("foo")
 		w.Error("foo")
 		w.Info("foo")
 		w.Severe("foo")
@@ -123,6 +128,12 @@ func TestWritePlainAny(t *testing.T) {
 	writePlainAny(nil, levelInfo, "foo")
 	assert.Contains(t, buf.String(), "foo")
 
+	buf.Reset()
+	writePlainAny(nil, levelDebug, make(chan int))
+	assert.Contains(t, buf.String(), "unsupported type")
+	writePlainAny(nil, levelDebug, 100)
+	assert.Contains(t, buf.String(), "100")
+
 	buf.Reset()
 	writePlainAny(nil, levelError, make(chan int))
 	assert.Contains(t, buf.String(), "unsupported type")
@@ -147,9 +158,40 @@ func TestWritePlainAny(t *testing.T) {
 
 }
 
+func TestLogWithLimitContentLength(t *testing.T) {
+	maxLen := atomic.LoadUint32(&maxContentLength)
+	atomic.StoreUint32(&maxContentLength, 10)
+
+	t.Cleanup(func() {
+		atomic.StoreUint32(&maxContentLength, maxLen)
+	})
+
+	t.Run("alert", func(t *testing.T) {
+		var buf bytes.Buffer
+		w := NewWriter(&buf)
+		w.Info("1234567890")
+		var v1 mockedEntry
+		if err := json.Unmarshal(buf.Bytes(), &v1); err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, "1234567890", v1.Content)
+		assert.False(t, v1.Truncated)
+
+		buf.Reset()
+		var v2 mockedEntry
+		w.Info("12345678901")
+		if err := json.Unmarshal(buf.Bytes(), &v2); err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, "1234567890", v2.Content)
+		assert.True(t, v2.Truncated)
+	})
+}
+
 type mockedEntry struct {
-	Level   string `json:"level"`
+	Level     string `json:"level"`
-	Content string `json:"content"`
+	Content   string `json:"content"`
+	Truncated bool   `json:"truncated"`
 }
 
 type easyToCloseWriter struct{}

core/mapping/fieldoptions.go

@@ -8,10 +8,12 @@ type (
 	// use context and OptionalDep option to determine the value of Optional
 	// nothing to do with context.Context
 	fieldOptionsWithContext struct {
+		Inherit    bool
 		FromString bool
 		Optional   bool
 		Options    []string
 		Default    string
+		EnvVar     string
 		Range      *numberRange
 	}
 
@@ -40,6 +42,10 @@ func (o *fieldOptionsWithContext) getDefault() (string, bool) {
 	return o.Default, len(o.Default) > 0
 }
 
+func (o *fieldOptionsWithContext) inherit() bool {
+	return o != nil && o.Inherit
+}
+
 func (o *fieldOptionsWithContext) optional() bool {
 	return o != nil && o.Optional
 }
@@ -101,5 +107,6 @@ func (o *fieldOptions) toOptionsWithContext(key string, m Valuer, fullName strin
 		Optional:   optional,
 		Options:    o.Options,
 		Default:    o.Default,
+		EnvVar:     o.EnvVar,
 	}, nil
 }

core/mapping/jsonunmarshaler.go

@@ -11,22 +11,30 @@ const jsonTagKey = "json"
 var jsonUnmarshaler = NewUnmarshaler(jsonTagKey)
 
 // UnmarshalJsonBytes unmarshals content into v.
-func UnmarshalJsonBytes(content []byte, v interface{}) error {
+func UnmarshalJsonBytes(content []byte, v interface{}, opts ...UnmarshalOption) error {
-	return unmarshalJsonBytes(content, v, jsonUnmarshaler)
+	return unmarshalJsonBytes(content, v, getJsonUnmarshaler(opts...))
 }
 
 // UnmarshalJsonMap unmarshals content from m into v.
-func UnmarshalJsonMap(m map[string]interface{}, v interface{}) error {
+func UnmarshalJsonMap(m map[string]interface{}, v interface{}, opts ...UnmarshalOption) error {
-	return jsonUnmarshaler.Unmarshal(m, v)
+	return getJsonUnmarshaler(opts...).Unmarshal(m, v)
 }
 
 // UnmarshalJsonReader unmarshals content from reader into v.
-func UnmarshalJsonReader(reader io.Reader, v interface{}) error {
+func UnmarshalJsonReader(reader io.Reader, v interface{}, opts ...UnmarshalOption) error {
-	return unmarshalJsonReader(reader, v, jsonUnmarshaler)
+	return unmarshalJsonReader(reader, v, getJsonUnmarshaler(opts...))
+}
+
+func getJsonUnmarshaler(opts ...UnmarshalOption) *Unmarshaler {
+	if len(opts) > 0 {
+		return NewUnmarshaler(jsonTagKey, opts...)
+	}
+
+	return jsonUnmarshaler
 }
 
 func unmarshalJsonBytes(content []byte, v interface{}, unmarshaler *Unmarshaler) error {
-	var m map[string]interface{}
+	var m interface{}
 	if err := jsonx.Unmarshal(content, &m); err != nil {
 		return err
 	}
@@ -35,7 +43,7 @@ func unmarshalJsonBytes(content []byte, v interface{}, unmarshaler *Unmarshaler)
 }
 
 func unmarshalJsonReader(reader io.Reader, v interface{}, unmarshaler *Unmarshaler) error {
-	var m map[string]interface{}
+	var m interface{}
 	if err := jsonx.UnmarshalFromReader(reader, &m); err != nil {
 		return err
 	}

core/mapping/jsonunmarshaler_test.go

@@ -856,8 +856,7 @@ func TestUnmarshalBytesError(t *testing.T) {
 	}
 
 	err := UnmarshalJsonBytes([]byte(payload), &v)
-	assert.NotNil(t, err)
+	assert.Equal(t, errTypeMismatch, err)
-	assert.True(t, strings.Contains(err.Error(), payload))
 }
 
 func TestUnmarshalReaderError(t *testing.T) {
@@ -867,9 +866,7 @@ func TestUnmarshalReaderError(t *testing.T) {
 		Any string
 	}
 
-	err := UnmarshalJsonReader(reader, &v)
+	assert.Equal(t, errTypeMismatch, UnmarshalJsonReader(reader, &v))
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), payload))
 }
 
 func TestUnmarshalMap(t *testing.T) {
@@ -900,7 +897,9 @@ func TestUnmarshalMap(t *testing.T) {
 			Any string `json:",optional"`
 		}
 
-		err := UnmarshalJsonMap(m, &v)
+		err := UnmarshalJsonMap(m, &v, WithCanonicalKeyFunc(func(s string) string {
+			return s
+		}))
 		assert.Nil(t, err)
 		assert.True(t, len(v.Any) == 0)
 	})
@@ -918,3 +917,26 @@ func TestUnmarshalMap(t *testing.T) {
 		assert.Equal(t, "foo", v.Any)
 	})
 }
+
+func TestUnmarshalJsonArray(t *testing.T) {
+	var v []struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+
+	body := `[{"name":"kevin", "age": 18}]`
+	assert.NoError(t, UnmarshalJsonBytes([]byte(body), &v))
+	assert.Equal(t, 1, len(v))
+	assert.Equal(t, "kevin", v[0].Name)
+	assert.Equal(t, 18, v[0].Age)
+}
+
+func TestUnmarshalJsonBytesError(t *testing.T) {
+	var v []struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+	}
+
+	assert.Error(t, UnmarshalJsonBytes([]byte((``)), &v))
+	assert.Error(t, UnmarshalJsonReader(strings.NewReader(``), &v))
+}

core/mapping/marshaler_test.go

@@ -261,6 +261,78 @@ func TestMarshal_RangeOut(t *testing.T) {
 	}
 }
 
+func TestMarshal_RangeIllegal(t *testing.T) {
+	tests := []interface{}{
+		struct {
+			Int int `json:"int,range=[3:1]"`
+		}{
+			Int: 2,
+		},
+		struct {
+			Int int `json:"int,range=(3:1]"`
+		}{
+			Int: 2,
+		},
+	}
+
+	for _, test := range tests {
+		_, err := Marshal(test)
+		assert.Equal(t, err, errNumberRange)
+	}
+}
+
+func TestMarshal_RangeLeftEqualsToRight(t *testing.T) {
+	tests := []struct {
+		name  string
+		value interface{}
+		err   error
+	}{
+		{
+			name: "left inclusive, right inclusive",
+			value: struct {
+				Int int `json:"int,range=[2:2]"`
+			}{
+				Int: 2,
+			},
+		},
+		{
+			name: "left inclusive, right exclusive",
+			value: struct {
+				Int int `json:"int,range=[2:2)"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+		{
+			name: "left exclusive, right inclusive",
+			value: struct {
+				Int int `json:"int,range=(2:2]"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+		{
+			name: "left exclusive, right exclusive",
+			value: struct {
+				Int int `json:"int,range=(2:2)"`
+			}{
+				Int: 2,
+			},
+			err: errNumberRange,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			_, err := Marshal(test.value)
+			assert.Equal(t, test.err, err)
+		})
+	}
+}
+
 func TestMarshal_FromString(t *testing.T) {
 	v := struct {
 		Age int `json:"age,string"`

core/mapping/tomlunmarshaler.go

@@ -1,29 +1,27 @@
 package mapping
 
 import (
-	"bytes"
-	"encoding/json"
 	"io"
 
-	"github.com/pelletier/go-toml/v2"
+	"github.com/zeromicro/go-zero/internal/encoding"
 )
 
 // UnmarshalTomlBytes unmarshals TOML bytes into the given v.
-func UnmarshalTomlBytes(content []byte, v interface{}) error {
+func UnmarshalTomlBytes(content []byte, v interface{}, opts ...UnmarshalOption) error {
-	return UnmarshalTomlReader(bytes.NewReader(content), v)
+	b, err := encoding.TomlToJson(content)
+	if err != nil {
+		return err
+	}
+
+	return UnmarshalJsonBytes(b, v, opts...)
 }
 
 // UnmarshalTomlReader unmarshals TOML from the given io.Reader into the given v.
-func UnmarshalTomlReader(r io.Reader, v interface{}) error {
+func UnmarshalTomlReader(r io.Reader, v interface{}, opts ...UnmarshalOption) error {
-	var val interface{}
+	b, err := io.ReadAll(r)
-	if err := toml.NewDecoder(r).Decode(&val); err != nil {
+	if err != nil {
 		return err
 	}
 
-	var buf bytes.Buffer
+	return UnmarshalTomlBytes(b, v, opts...)
-	if err := json.NewEncoder(&buf).Encode(val); err != nil {
-		return err
-	}
-
-	return UnmarshalJsonReader(&buf, v)
 }

core/mapping/tomlunmarshaler_test.go

@@ -1,6 +1,7 @@
 package mapping
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -18,7 +19,7 @@ d = "abcd!@#$112"
 		C string `json:"c"`
 		D string `json:"d"`
 	}
-	assert.Nil(t, UnmarshalTomlBytes([]byte(input), &val))
+	assert.NoError(t, UnmarshalTomlReader(strings.NewReader(input), &val))
 	assert.Equal(t, "foo", val.A)
 	assert.Equal(t, 1, val.B)
 	assert.Equal(t, "${FOO}", val.C)
@@ -37,5 +38,12 @@ d = "abcd!@#$112"
 		C string `json:"c"`
 		D string `json:"d"`
 	}
-	assert.NotNil(t, UnmarshalTomlBytes([]byte(input), &val))
+	assert.Error(t, UnmarshalTomlReader(strings.NewReader(input), &val))
+}
+
+func TestUnmarshalTomlBadReader(t *testing.T) {
+	var val struct {
+		A string `json:"a"`
+	}
+	assert.Error(t, UnmarshalTomlReader(new(badReader), &val))
 }

core/mapping/utils.go

@@ -10,11 +10,14 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/zeromicro/go-zero/core/lang"
 	"github.com/zeromicro/go-zero/core/stringx"
 )
 
 const (
 	defaultOption      = "default"
+	envOption          = "env"
+	inheritOption      = "inherit"
 	stringOption       = "string"
 	optionalOption     = "optional"
 	optionsOption      = "options"
@@ -53,7 +56,7 @@ type (
 
 // Deref dereferences a type, if pointer type, returns its element type.
 func Deref(t reflect.Type) reflect.Type {
-	if t.Kind() == reflect.Ptr {
+	for t.Kind() == reflect.Ptr {
 		t = t.Elem()
 	}
 
@@ -62,22 +65,17 @@ func Deref(t reflect.Type) reflect.Type {
 
 // Repr returns the string representation of v.
 func Repr(v interface{}) string {
-	if v == nil {
+	return lang.Repr(v)
-		return ""
+}
-	}
 
-	// if func (v *Type) String() string, we can't use Elem()
+// SetValue sets target to value, pointers are processed automatically.
-	switch vt := v.(type) {
+func SetValue(tp reflect.Type, value, target reflect.Value) {
-	case fmt.Stringer:
+	value.Set(convertTypeOfPtr(tp, target))
-		return vt.String()
+}
-	}
 
-	val := reflect.ValueOf(v)
+// SetMapIndexValue sets target to value at key position, pointers are processed automatically.
-	if val.Kind() == reflect.Ptr && !val.IsNil() {
+func SetMapIndexValue(tp reflect.Type, value, key, target reflect.Value) {
-		val = val.Elem()
+	value.SetMapIndex(key, convertTypeOfPtr(tp, target))
-	}
-
-	return reprOfValue(val)
 }
 
 // ValidatePtr validates v if it's a valid pointer.
@@ -91,10 +89,17 @@ func ValidatePtr(v *reflect.Value) error {
 	return nil
 }
 
-func convertType(kind reflect.Kind, str string) (interface{}, error) {
+func convertTypeFromString(kind reflect.Kind, str string) (interface{}, error) {
 	switch kind {
 	case reflect.Bool:
-		return str == "1" || strings.ToLower(str) == "true", nil
+		switch strings.ToLower(str) {
+		case "1", "true":
+			return true, nil
+		case "0", "false":
+			return false, nil
+		default:
+			return false, errTypeMismatch
+		}
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 		intValue, err := strconv.ParseInt(str, 10, 64)
 		if err != nil {
@@ -123,6 +128,23 @@ func convertType(kind reflect.Kind, str string) (interface{}, error) {
 	}
 }
 
+func convertTypeOfPtr(tp reflect.Type, target reflect.Value) reflect.Value {
+	// keep the original value is a pointer
+	if tp.Kind() == reflect.Ptr && target.CanAddr() {
+		tp = tp.Elem()
+		target = target.Addr()
+	}
+
+	for tp.Kind() == reflect.Ptr {
+		p := reflect.New(target.Type())
+		p.Elem().Set(target)
+		target = p
+		tp = tp.Elem()
+	}
+
+	return target
+}
+
 func doParseKeyAndOptions(field reflect.StructField, value string) (string, *fieldOptions, error) {
 	segments := parseSegments(value)
 	key := strings.TrimSpace(segments[0])
@@ -215,8 +237,8 @@ func isRightInclude(b byte) (bool, error) {
 	}
 }
 
-func maybeNewValue(field reflect.StructField, value reflect.Value) {
+func maybeNewValue(fieldType reflect.Type, value reflect.Value) {
-	if field.Type.Kind() == reflect.Ptr && value.IsNil() {
+	if fieldType.Kind() == reflect.Ptr && value.IsNil() {
 		value.Set(reflect.New(value.Type().Elem()))
 	}
 }
@@ -311,6 +333,20 @@ func parseNumberRange(str string) (*numberRange, error) {
 		right = math.MaxFloat64
 	}
 
+	if left > right {
+		return nil, errNumberRange
+	}
+
+	// [2:2] valid
+	// [2:2) invalid
+	// (2:2] invalid
+	// (2:2) invalid
+	if left == right {
+		if !leftInclude || !rightInclude {
+			return nil, errNumberRange
+		}
+	}
+
 	return &numberRange{
 		left:         left,
 		leftInclude:  leftInclude,
@@ -321,6 +357,8 @@ func parseNumberRange(str string) (*numberRange, error) {
 
 func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 	switch {
+	case option == inheritOption:
+		fieldOpts.Inherit = true
 	case option == stringOption:
 		fieldOpts.FromString = true
 	case strings.HasPrefix(option, optionalOption):
@@ -337,26 +375,33 @@ func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 	case option == optionalOption:
 		fieldOpts.Optional = true
 	case strings.HasPrefix(option, optionsOption):
-		segs := strings.Split(option, equalToken)
+		val, err := parseProperty(fieldName, optionsOption, option)
-		if len(segs) != 2 {
+		if err != nil {
-			return fmt.Errorf("field %s has wrong options", fieldName)
+			return err
 		}
 
-		fieldOpts.Options = parseOptions(segs[1])
+		fieldOpts.Options = parseOptions(val)
 	case strings.HasPrefix(option, defaultOption):
-		segs := strings.Split(option, equalToken)
+		val, err := parseProperty(fieldName, defaultOption, option)
-		if len(segs) != 2 {
+		if err != nil {
-			return fmt.Errorf("field %s has wrong default option", fieldName)
+			return err
 		}
 
-		fieldOpts.Default = strings.TrimSpace(segs[1])
+		fieldOpts.Default = val
+	case strings.HasPrefix(option, envOption):
+		val, err := parseProperty(fieldName, envOption, option)
+		if err != nil {
+			return err
+		}
+
+		fieldOpts.EnvVar = val
 	case strings.HasPrefix(option, rangeOption):
-		segs := strings.Split(option, equalToken)
+		val, err := parseProperty(fieldName, rangeOption, option)
-		if len(segs) != 2 {
+		if err != nil {
-			return fmt.Errorf("field %s has wrong range", fieldName)
+			return err
 		}
 
-		nr, err := parseNumberRange(segs[1])
+		nr, err := parseNumberRange(val)
 		if err != nil {
 			return err
 		}
@@ -381,6 +426,15 @@ func parseOptions(val string) []string {
 	return strings.Split(val, optionSeparator)
 }
 
+func parseProperty(field, tag, val string) (string, error) {
+	segs := strings.Split(val, equalToken)
+	if len(segs) != 2 {
+		return "", fmt.Errorf("field %s has wrong %s", field, tag)
+	}
+
+	return strings.TrimSpace(segs[1]), nil
+}
+
 func parseSegments(val string) []string {
 	var segments []string
 	var escaped, grouped bool
@@ -430,47 +484,6 @@ func parseSegments(val string) []string {
 	return segments
 }
 
-func reprOfValue(val reflect.Value) string {
-	switch vt := val.Interface().(type) {
-	case bool:
-		return strconv.FormatBool(vt)
-	case error:
-		return vt.Error()
-	case float32:
-		return strconv.FormatFloat(float64(vt), 'f', -1, 32)
-	case float64:
-		return strconv.FormatFloat(vt, 'f', -1, 64)
-	case fmt.Stringer:
-		return vt.String()
-	case int:
-		return strconv.Itoa(vt)
-	case int8:
-		return strconv.Itoa(int(vt))
-	case int16:
-		return strconv.Itoa(int(vt))
-	case int32:
-		return strconv.Itoa(int(vt))
-	case int64:
-		return strconv.FormatInt(vt, 10)
-	case string:
-		return vt
-	case uint:
-		return strconv.FormatUint(uint64(vt), 10)
-	case uint8:
-		return strconv.FormatUint(uint64(vt), 10)
-	case uint16:
-		return strconv.FormatUint(uint64(vt), 10)
-	case uint32:
-		return strconv.FormatUint(uint64(vt), 10)
-	case uint64:
-		return strconv.FormatUint(vt, 10)
-	case []byte:
-		return string(vt)
-	default:
-		return fmt.Sprint(val.Interface())
-	}
-}
-
 func setMatchedPrimitiveValue(kind reflect.Kind, value reflect.Value, v interface{}) error {
 	switch kind {
 	case reflect.Bool:
@@ -490,13 +503,13 @@ func setMatchedPrimitiveValue(kind reflect.Kind, value reflect.Value, v interfac
 	return nil
 }
 
-func setValue(kind reflect.Kind, value reflect.Value, str string) error {
+func setValueFromString(kind reflect.Kind, value reflect.Value, str string) error {
 	if !value.CanSet() {
 		return errValueNotSettable
 	}
 
 	value = ensureValue(value)
-	v, err := convertType(kind, str)
+	v, err := convertTypeFromString(kind, str)
 	if err != nil {
 		return err
 	}
@@ -569,7 +582,7 @@ func validateAndSetValue(kind reflect.Kind, value reflect.Value, str string, opt
 		return errValueNotSettable
 	}
 
-	v, err := convertType(kind, str)
+	v, err := convertTypeFromString(kind, str)
 	if err != nil {
 		return err
 	}

core/mapping/utils_test.go

@@ -237,7 +237,7 @@ func TestValidatePtrWithZeroValue(t *testing.T) {
 
 func TestSetValueNotSettable(t *testing.T) {
 	var i int
-	assert.NotNil(t, setValue(reflect.Int, reflect.ValueOf(i), "1"))
+	assert.NotNil(t, setValueFromString(reflect.Int, reflect.ValueOf(i), "1"))
 }
 
 func TestParseKeyAndOptionsErrors(t *testing.T) {
@@ -290,133 +290,9 @@ func TestSetValueFormatErrors(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.kind.String(), func(t *testing.T) {
-			err := setValue(test.kind, test.target, test.value)
+			err := setValueFromString(test.kind, test.target, test.value)
 			assert.NotEqual(t, errValueNotSettable, err)
 			assert.NotNil(t, err)
 		})
 	}
 }
-
-func TestRepr(t *testing.T) {
-	var (
-		f32 float32 = 1.1
-		f64         = 2.2
-		i8  int8    = 1
-		i16 int16   = 2
-		i32 int32   = 3
-		i64 int64   = 4
-		u8  uint8   = 5
-		u16 uint16  = 6
-		u32 uint32  = 7
-		u64 uint64  = 8
-	)
-	tests := []struct {
-		v      interface{}
-		expect string
-	}{
-		{
-			nil,
-			"",
-		},
-		{
-			mockStringable{},
-			"mocked",
-		},
-		{
-			new(mockStringable),
-			"mocked",
-		},
-		{
-			newMockPtr(),
-			"mockptr",
-		},
-		{
-			&mockOpacity{
-				val: 1,
-			},
-			"{1}",
-		},
-		{
-			true,
-			"true",
-		},
-		{
-			false,
-			"false",
-		},
-		{
-			f32,
-			"1.1",
-		},
-		{
-			f64,
-			"2.2",
-		},
-		{
-			i8,
-			"1",
-		},
-		{
-			i16,
-			"2",
-		},
-		{
-			i32,
-			"3",
-		},
-		{
-			i64,
-			"4",
-		},
-		{
-			u8,
-			"5",
-		},
-		{
-			u16,
-			"6",
-		},
-		{
-			u32,
-			"7",
-		},
-		{
-			u64,
-			"8",
-		},
-		{
-			[]byte(`abcd`),
-			"abcd",
-		},
-		{
-			mockOpacity{val: 1},
-			"{1}",
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.expect, func(t *testing.T) {
-			assert.Equal(t, test.expect, Repr(test.v))
-		})
-	}
-}
-
-type mockStringable struct{}
-
-func (m mockStringable) String() string {
-	return "mocked"
-}
-
-type mockPtr struct{}
-
-func newMockPtr() *mockPtr {
-	return new(mockPtr)
-}
-
-func (m *mockPtr) String() string {
-	return "mockptr"
-}
-
-type mockOpacity struct {
-	val int
-}

core/mapping/valuer.go

@@ -7,12 +7,106 @@ type (
 		Value(key string) (interface{}, bool)
 	}
 
-	// A MapValuer is a map that can use Value method to get values with given keys.
+	// A valuerWithParent defines a node that has a parent node.
-	MapValuer map[string]interface{}
+	valuerWithParent interface {
+		Valuer
+		// Parent get the parent valuer for current node.
+		Parent() valuerWithParent
+	}
+
+	// A node is a map that can use Value method to get values with given keys.
+	node struct {
+		current Valuer
+		parent  valuerWithParent
+	}
+
+	// A valueWithParent is used to wrap the value with its parent.
+	valueWithParent struct {
+		value  interface{}
+		parent valuerWithParent
+	}
+
+	// mapValuer is a type for map to meet the Valuer interface.
+	mapValuer map[string]interface{}
+	// simpleValuer is a type to get value from current node.
+	simpleValuer node
+	// recursiveValuer is a type to get the value recursively from current and parent nodes.
+	recursiveValuer node
 )
 
-// Value gets the value associated with the given key from mv.
+// Value gets the value assciated with the given key from mv.
-func (mv MapValuer) Value(key string) (interface{}, bool) {
+func (mv mapValuer) Value(key string) (interface{}, bool) {
 	v, ok := mv[key]
 	return v, ok
 }
+
+// Value gets the value associated with the given key from sv.
+func (sv simpleValuer) Value(key string) (interface{}, bool) {
+	v, ok := sv.current.Value(key)
+	return v, ok
+}
+
+// Parent get the parent valuer from sv.
+func (sv simpleValuer) Parent() valuerWithParent {
+	if sv.parent == nil {
+		return nil
+	}
+
+	return recursiveValuer{
+		current: sv.parent,
+		parent:  sv.parent.Parent(),
+	}
+}
+
+// Value gets the value associated with the given key from rv,
+// and it will inherit the value from parent nodes.
+func (rv recursiveValuer) Value(key string) (interface{}, bool) {
+	val, ok := rv.current.Value(key)
+	if !ok {
+		if parent := rv.Parent(); parent != nil {
+			return parent.Value(key)
+		}
+
+		return nil, false
+	}
+
+	vm, ok := val.(map[string]interface{})
+	if !ok {
+		return val, true
+	}
+
+	parent := rv.Parent()
+	if parent == nil {
+		return val, true
+	}
+
+	pv, ok := parent.Value(key)
+	if !ok {
+		return val, true
+	}
+
+	pm, ok := pv.(map[string]interface{})
+	if !ok {
+		return val, true
+	}
+
+	for k, v := range pm {
+		if _, ok := vm[k]; !ok {
+			vm[k] = v
+		}
+	}
+
+	return vm, true
+}
+
+// Parent get the parent valuer from rv.
+func (rv recursiveValuer) Parent() valuerWithParent {
+	if rv.parent == nil {
+		return nil
+	}
+
+	return recursiveValuer{
+		current: rv.parent,
+		parent:  rv.parent.Parent(),
+	}
+}

core/mapping/valuer_test.go

@@ -0,0 +1,57 @@
+package mapping
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMapValuerWithInherit_Value(t *testing.T) {
+	input := map[string]interface{}{
+		"discovery": map[string]interface{}{
+			"host": "localhost",
+			"port": 8080,
+		},
+		"component": map[string]interface{}{
+			"name": "test",
+		},
+	}
+	valuer := recursiveValuer{
+		current: mapValuer(input["component"].(map[string]interface{})),
+		parent: simpleValuer{
+			current: mapValuer(input),
+		},
+	}
+
+	val, ok := valuer.Value("discovery")
+	assert.True(t, ok)
+
+	m, ok := val.(map[string]interface{})
+	assert.True(t, ok)
+	assert.Equal(t, "localhost", m["host"])
+	assert.Equal(t, 8080, m["port"])
+}
+
+func TestRecursiveValuer_Value(t *testing.T) {
+	input := map[string]interface{}{
+		"component": map[string]interface{}{
+			"name": "test",
+			"foo": map[string]interface{}{
+				"bar": "baz",
+			},
+		},
+		"foo": "value",
+	}
+	valuer := recursiveValuer{
+		current: mapValuer(input["component"].(map[string]interface{})),
+		parent: simpleValuer{
+			current: mapValuer(input),
+		},
+	}
+
+	val, ok := valuer.Value("foo")
+	assert.True(t, ok)
+	assert.EqualValues(t, map[string]interface{}{
+		"bar": "baz",
+	}, val)
+}

core/mapping/yamlunmarshaler.go

@@ -1,101 +1,27 @@
 package mapping
 
 import (
-	"encoding/json"
-	"errors"
 	"io"
 
-	"gopkg.in/yaml.v2"
+	"github.com/zeromicro/go-zero/internal/encoding"
-)
-
-// To make .json & .yaml consistent, we just use json as the tag key.
-const yamlTagKey = "json"
-
-var (
-	// ErrUnsupportedType is an error that indicates the config format is not supported.
-	ErrUnsupportedType = errors.New("only map-like configs are supported")
-
-	yamlUnmarshaler = NewUnmarshaler(yamlTagKey)
 )
 
 // UnmarshalYamlBytes unmarshals content into v.
-func UnmarshalYamlBytes(content []byte, v interface{}) error {
+func UnmarshalYamlBytes(content []byte, v interface{}, opts ...UnmarshalOption) error {
-	return unmarshalYamlBytes(content, v, yamlUnmarshaler)
+	b, err := encoding.YamlToJson(content)
+	if err != nil {
+		return err
+	}
+
+	return UnmarshalJsonBytes(b, v, opts...)
 }
 
 // UnmarshalYamlReader unmarshals content from reader into v.
-func UnmarshalYamlReader(reader io.Reader, v interface{}) error {
+func UnmarshalYamlReader(reader io.Reader, v interface{}, opts ...UnmarshalOption) error {
-	return unmarshalYamlReader(reader, v, yamlUnmarshaler)
+	b, err := io.ReadAll(reader)
-}
+	if err != nil {
-
-func cleanupInterfaceMap(in map[interface{}]interface{}) map[string]interface{} {
-	res := make(map[string]interface{})
-	for k, v := range in {
-		res[Repr(k)] = cleanupMapValue(v)
-	}
-	return res
-}
-
-func cleanupInterfaceNumber(in interface{}) json.Number {
-	return json.Number(Repr(in))
-}
-
-func cleanupInterfaceSlice(in []interface{}) []interface{} {
-	res := make([]interface{}, len(in))
-	for i, v := range in {
-		res[i] = cleanupMapValue(v)
-	}
-	return res
-}
-
-func cleanupMapValue(v interface{}) interface{} {
-	switch v := v.(type) {
-	case []interface{}:
-		return cleanupInterfaceSlice(v)
-	case map[interface{}]interface{}:
-		return cleanupInterfaceMap(v)
-	case bool, string:
-		return v
-	case int, uint, int8, uint8, int16, uint16, int32, uint32, int64, uint64, float32, float64:
-		return cleanupInterfaceNumber(v)
-	default:
-		return Repr(v)
-	}
-}
-
-func unmarshal(unmarshaler *Unmarshaler, o interface{}, v interface{}) error {
-	if m, ok := o.(map[string]interface{}); ok {
-		return unmarshaler.Unmarshal(m, v)
-	}
-
-	return ErrUnsupportedType
-}
-
-func unmarshalYamlBytes(content []byte, v interface{}, unmarshaler *Unmarshaler) error {
-	var o interface{}
-	if err := yamlUnmarshal(content, &o); err != nil {
 		return err
 	}
 
-	return unmarshal(unmarshaler, o, v)
+	return UnmarshalYamlBytes(b, v, opts...)
-}
-
-func unmarshalYamlReader(reader io.Reader, v interface{}, unmarshaler *Unmarshaler) error {
-	var res interface{}
-	if err := yaml.NewDecoder(reader).Decode(&res); err != nil {
-		return err
-	}
-
-	return unmarshal(unmarshaler, cleanupMapValue(res), v)
-}
-
-// yamlUnmarshal YAML to map[string]interface{} instead of map[interface{}]interface{}.
-func yamlUnmarshal(in []byte, out interface{}) error {
-	var res interface{}
-	if err := yaml.Unmarshal(in, &res); err != nil {
-		return err
-	}
-
-	*out.(*interface{}) = cleanupMapValue(res)
-	return nil
 }

core/mapping/yamlunmarshaler_test.go

@@ -934,9 +934,8 @@ func TestUnmarshalYamlReaderError(t *testing.T) {
 	err := UnmarshalYamlReader(reader, &v)
 	assert.NotNil(t, err)
 
-	reader = strings.NewReader("chenquan")
+	reader = strings.NewReader("foo")
-	err = UnmarshalYamlReader(reader, &v)
+	assert.Error(t, UnmarshalYamlReader(reader, &v))
-	assert.ErrorIs(t, err, ErrUnsupportedType)
 }
 
 func TestUnmarshalYamlBadReader(t *testing.T) {
@@ -1012,6 +1011,13 @@ func TestUnmarshalYamlMapRune(t *testing.T) {
 	assert.Equal(t, rune(3), v.Machine["node3"])
 }
 
+func TestUnmarshalYamlBadInput(t *testing.T) {
+	var v struct {
+		Any string
+	}
+	assert.Error(t, UnmarshalYamlBytes([]byte("':foo"), &v))
+}
+
 type badReader struct{}
 
 func (b *badReader) Read(_ []byte) (n int, err error) {

core/mathx/unstable.go

@@ -6,14 +6,14 @@ import (
 	"time"
 )
 
-// A Unstable is used to generate random value around the mean value base on given deviation.
+// An Unstable is used to generate random value around the mean value base on given deviation.
 type Unstable struct {
 	deviation float64
 	r         *rand.Rand
 	lock      *sync.Mutex
 }
 
-// NewUnstable returns a Unstable.
+// NewUnstable returns an Unstable.
 func NewUnstable(deviation float64) Unstable {
 	if deviation < 0 {
 		deviation = 0

core/metric/counter.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -47,10 +48,18 @@ func NewCounterVec(cfg *CounterVecOpts) CounterVec {
 }
 
 func (cv *promCounterVec) Inc(labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	cv.counter.WithLabelValues(labels...).Inc()
 }
 
 func (cv *promCounterVec) Add(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	cv.counter.WithLabelValues(labels...).Add(v)
 }
 

core/metric/counter_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 func TestNewCounterVec(t *testing.T) {
@@ -21,6 +22,7 @@ func TestNewCounterVec(t *testing.T) {
 }
 
 func TestCounterIncr(t *testing.T) {
+	startAgent()
 	counterVec := NewCounterVec(&CounterVecOpts{
 		Namespace: "http_client",
 		Subsystem: "call",
@@ -37,6 +39,7 @@ func TestCounterIncr(t *testing.T) {
 }
 
 func TestCounterAdd(t *testing.T) {
+	startAgent()
 	counterVec := NewCounterVec(&CounterVecOpts{
 		Namespace: "rpc_server",
 		Subsystem: "requests",
@@ -51,3 +54,11 @@ func TestCounterAdd(t *testing.T) {
 	r := testutil.ToFloat64(cv.counter)
 	assert.Equal(t, float64(33), r)
 }
+
+func startAgent() {
+	prometheus.StartAgent(prometheus.Config{
+		Host: "127.0.0.1",
+		Port: 9101,
+		Path: "/metrics",
+	})
+}

core/metric/gauge.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -50,14 +51,26 @@ func NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {
 }
 
 func (gv *promGaugeVec) Inc(labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Inc()
 }
 
 func (gv *promGaugeVec) Add(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Add(v)
 }
 
 func (gv *promGaugeVec) Set(v float64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	gv.gauge.WithLabelValues(labels...).Set(v)
 }
 

core/metric/gauge_test.go

@@ -21,6 +21,7 @@ func TestNewGaugeVec(t *testing.T) {
 }
 
 func TestGaugeInc(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "rpc_client2",
 		Subsystem: "requests",
@@ -37,6 +38,7 @@ func TestGaugeInc(t *testing.T) {
 }
 
 func TestGaugeAdd(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "rpc_client",
 		Subsystem: "request",
@@ -53,6 +55,7 @@ func TestGaugeAdd(t *testing.T) {
 }
 
 func TestGaugeSet(t *testing.T) {
+	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
 		Namespace: "http_client",
 		Subsystem: "request",

core/metric/histogram.go

@@ -3,6 +3,7 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
+	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -53,6 +54,10 @@ func NewHistogramVec(cfg *HistogramVecOpts) HistogramVec {
 }
 
 func (hv *promHistogramVec) Observe(v int64, labels ...string) {
+	if !prometheus.Enabled() {
+		return
+	}
+
 	hv.histogram.WithLabelValues(labels...).Observe(float64(v))
 }
 

core/metric/histogram_test.go

@@ -21,6 +21,7 @@ func TestNewHistogramVec(t *testing.T) {
 }
 
 func TestHistogramObserve(t *testing.T) {
+	startAgent()
 	histogramVec := NewHistogramVec(&HistogramVecOpts{
 		Name:    "counts",
 		Help:    "rpc server requests duration(ms).",

core/mr/mapreduce.go

@@ -145,7 +145,7 @@ func MapReduceChan(source <-chan interface{}, mapper MapperFunc, reducer Reducer
 	return mapReduceWithPanicChan(source, panicChan, mapper, reducer, opts...)
 }
 
-// MapReduceChan maps all elements from source, and reduce the output elements with given reducer.
+// mapReduceWithPanicChan maps all elements from source, and reduce the output elements with given reducer.
 func mapReduceWithPanicChan(source <-chan interface{}, panicChan *onceChan, mapper MapperFunc,
 	reducer ReducerFunc, opts ...Option) (interface{}, error) {
 	options := buildOptions(opts...)
@@ -212,6 +212,8 @@ func mapReduceWithPanicChan(source <-chan interface{}, panicChan *onceChan, mapp
 		cancel(context.DeadlineExceeded)
 		return nil, context.DeadlineExceeded
 	case v := <-panicChan.channel:
+		// drain output here, otherwise for loop panic in defer
+		drain(output)
 		panic(v)
 	case v, ok := <-output:
 		if err := retErr.Load(); err != nil {

core/mr/mapreduce_fuzz_test.go

@@ -19,7 +19,7 @@ func FuzzMapReduce(f *testing.F) {
 	rand.Seed(time.Now().UnixNano())
 
 	f.Add(uint(10), uint(runtime.NumCPU()))
-	f.Fuzz(func(t *testing.T, num uint, workers uint) {
+	f.Fuzz(func(t *testing.T, num, workers uint) {
 		n := int64(num)%5000 + 5000
 		genPanic := rand.Intn(100) == 0
 		mapperPanic := rand.Intn(100) == 0

core/mr/mapreduce_test.go

@@ -3,7 +3,7 @@ package mr
 import (
 	"context"
 	"errors"
-	"io/ioutil"
+	"io"
 	"log"
 	"runtime"
 	"sync/atomic"
@@ -17,7 +17,7 @@ import (
 var errDummy = errors.New("dummy")
 
 func init() {
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 }
 
 func TestFinish(t *testing.T) {

core/rescue/recover.go

@@ -4,7 +4,8 @@ import "github.com/zeromicro/go-zero/core/logx"
 
 // Recover is used with defer to do cleanup on panics.
 // Use it like:
-//  defer Recover(func() {})
+//
+//	defer Recover(func() {})
 func Recover(cleanups ...func()) {
 	for _, cleanup := range cleanups {
 		cleanup()

core/service/serviceconf.go

@@ -5,9 +5,11 @@ import (
 
 	"github.com/zeromicro/go-zero/core/load"
 	"github.com/zeromicro/go-zero/core/logx"
+	"github.com/zeromicro/go-zero/core/proc"
 	"github.com/zeromicro/go-zero/core/prometheus"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/trace"
+	"github.com/zeromicro/go-zero/internal/devserver"
 )
 
 const (
@@ -27,10 +29,12 @@ const (
 type ServiceConf struct {
 	Name       string
 	Log        logx.LogConf
-	Mode       string            `json:",default=pro,options=dev|test|rt|pre|pro"`
+	Mode       string `json:",default=pro,options=dev|test|rt|pre|pro"`
-	MetricsUrl string            `json:",optional"`
+	MetricsUrl string `json:",optional"`
+	// Deprecated: please use DevServer
 	Prometheus prometheus.Config `json:",optional"`
 	Telemetry  trace.Config      `json:",optional"`
+	DevServer  devserver.Config  `json:",optional"`
 }
 
 // MustSetUp sets up the service, exits on error.
@@ -56,10 +60,14 @@ func (sc ServiceConf) SetUp() error {
 		sc.Telemetry.Name = sc.Name
 	}
 	trace.StartAgent(sc.Telemetry)
+	proc.AddShutdownListener(func() {
+		trace.StopAgent()
+	})
 
 	if len(sc.MetricsUrl) > 0 {
 		stat.SetReportWriter(stat.NewRemoteWriter(sc.MetricsUrl))
 	}
+	devserver.StartAgent(sc.DevServer)
 
 	return nil
 }

core/stat/alert.go

@@ -46,14 +46,14 @@ func Report(msg string) {
 	if fn != nil {
 		reported := lessExecutor.DoOrDiscard(func() {
 			var builder strings.Builder
-			fmt.Fprintf(&builder, "%s\n", time.Now().Format(timeFormat))
+			builder.WriteString(fmt.Sprintln(time.Now().Format(timeFormat)))
 			if len(clusterName) > 0 {
-				fmt.Fprintf(&builder, "cluster: %s\n", clusterName)
+				builder.WriteString(fmt.Sprintf("cluster: %s\n", clusterName))
 			}
-			fmt.Fprintf(&builder, "host: %s\n", sysx.Hostname())
+			builder.WriteString(fmt.Sprintf("host: %s\n", sysx.Hostname()))
 			dp := atomic.SwapInt32(&dropped, 0)
 			if dp > 0 {
-				fmt.Fprintf(&builder, "dropped: %d\n", dp)
+				builder.WriteString(fmt.Sprintf("dropped: %d\n", dp))
 			}
 			builder.WriteString(strings.TrimSpace(msg))
 			fn(builder.String())

core/stat/alert_test.go

@@ -4,6 +4,7 @@
 package stat
 
 import (
+	"os"
 	"strconv"
 	"sync/atomic"
 	"testing"
@@ -12,6 +13,9 @@ import (
 )
 
 func TestReport(t *testing.T) {
+	os.Setenv(clusterNameKey, "test-cluster")
+	defer os.Unsetenv(clusterNameKey)
+
 	var count int32
 	SetReporter(func(s string) {
 		atomic.AddInt32(&count, 1)

core/stat/internal/cgroup_linux.go

@@ -258,7 +258,7 @@ func parseUints(val string) ([]uint64, error) {
 	return sets, nil
 }
 
-// runningInUserNS detects whether we are currently running in a user namespace.
+// runningInUserNS detects whether we are currently running in an user namespace.
 func runningInUserNS() bool {
 	nsOnce.Do(func() {
 		file, err := os.Open("/proc/self/uid_map")

core/stat/internal/cpu_linux.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/zeromicro/go-zero/core/iox"
@@ -20,10 +21,11 @@ var (
 	preTotal  uint64
 	quota     float64
 	cores     uint64
+	initOnce  sync.Once
 )
 
 // if /proc not present, ignore the cpu calculation, like wsl linux
-func init() {
+func initialize() {
 	cpus, err := cpuSets()
 	if err != nil {
 		logx.Error(err)
@@ -31,13 +33,7 @@ func init() {
 	}
 
 	cores = uint64(len(cpus))
-	sets, err := cpuSets()
+	quota = float64(len(cpus))
-	if err != nil {
-		logx.Error(err)
-		return
-	}
-
-	quota = float64(len(sets))
 	cq, err := cpuQuota()
 	if err == nil {
 		if cq != -1 {
@@ -69,10 +65,13 @@ func init() {
 
 // RefreshCpu refreshes cpu usage and returns.
 func RefreshCpu() uint64 {
+	initOnce.Do(initialize)
+
 	total, err := totalCpuUsage()
 	if err != nil {
 		return 0
 	}
+
 	system, err := systemCpuUsage()
 	if err != nil {
 		return 0

core/stores/builder/builder.go

@@ -45,9 +45,13 @@ func RawFieldNames(in interface{}, postgresSql ...bool) []string {
 			// `db:"id"`
 			// `db:"id,type=char,length=16"`
 			// `db:",type=char,length=16"`
+			// `db:"-,type=char,length=16"`
 			if strings.Contains(tagv, ",") {
 				tagv = strings.TrimSpace(strings.Split(tagv, ",")[0])
 			}
+			if tagv == "-" {
+				continue
+			}
 			if len(tagv) == 0 {
 				tagv = fi.Name
 			}

core/stores/builder/builder_test.go

@@ -39,3 +39,33 @@ func TestFieldNamesWithTagOptions(t *testing.T) {
 		assert.Equal(t, expected, out)
 	})
 }
+
+type mockedUserWithDashTag struct {
+	ID       string `db:"id" json:"id,omitempty"`
+	UserName string `db:"user_name" json:"userName,omitempty"`
+	Mobile   string `db:"-" json:"mobile,omitempty"`
+}
+
+func TestFieldNamesWithDashTag(t *testing.T) {
+	t.Run("new", func(t *testing.T) {
+		var u mockedUserWithDashTag
+		out := RawFieldNames(&u)
+		expected := []string{"`id`", "`user_name`"}
+		assert.Equal(t, expected, out)
+	})
+}
+
+type mockedUserWithDashTagAndOptions struct {
+	ID       string `db:"id" json:"id,omitempty"`
+	UserName string `db:"user_name,type=varchar,length=255" json:"userName,omitempty"`
+	Mobile   string `db:"-,type=varchar,length=255" json:"mobile,omitempty"`
+}
+
+func TestFieldNamesWithDashTagAndOptions(t *testing.T) {
+	t.Run("new", func(t *testing.T) {
+		var u mockedUserWithDashTagAndOptions
+		out := RawFieldNames(&u)
+		expected := []string{"`id`", "`user_name`"}
+		assert.Equal(t, expected, out)
+	})
+}

core/stores/cache/cache_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alicebob/miniredis/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/zeromicro/go-zero/core/errorx"
 	"github.com/zeromicro/go-zero/core/hash"
@@ -109,51 +110,85 @@ func (mc *mockedNode) TakeWithExpireCtx(ctx context.Context, val interface{}, ke
 }
 
 func TestCache_SetDel(t *testing.T) {
-	const total = 1000
+	t.Run("test set del", func(t *testing.T) {
-	r1, clean1, err := redistest.CreateRedis()
+		const total = 1000
-	assert.Nil(t, err)
+		r1, clean1, err := redistest.CreateRedis()
-	defer clean1()
+		assert.Nil(t, err)
-	r2, clean2, err := redistest.CreateRedis()
+		defer clean1()
-	assert.Nil(t, err)
+		r2, clean2, err := redistest.CreateRedis()
-	defer clean2()
+		assert.Nil(t, err)
-	conf := ClusterConf{
+		defer clean2()
-		{
+		conf := ClusterConf{
-			RedisConf: redis.RedisConf{
+			{
-				Host: r1.Addr,
+				RedisConf: redis.RedisConf{
-				Type: redis.NodeType,
+					Host: r1.Addr,
+					Type: redis.NodeType,
+				},
+				Weight: 100,
 			},
-			Weight: 100,
+			{
-		},
+				RedisConf: redis.RedisConf{
-		{
+					Host: r2.Addr,
-			RedisConf: redis.RedisConf{
+					Type: redis.NodeType,
-				Host: r2.Addr,
+				},
-				Type: redis.NodeType,
+				Weight: 100,
 			},
-			Weight: 100,
-		},
-	}
-	c := New(conf, syncx.NewSingleFlight(), NewStat("mock"), errPlaceholder)
-	for i := 0; i < total; i++ {
-		if i%2 == 0 {
-			assert.Nil(t, c.Set(fmt.Sprintf("key/%d", i), i))
-		} else {
-			assert.Nil(t, c.SetWithExpire(fmt.Sprintf("key/%d", i), i, 0))
 		}
-	}
+		c := New(conf, syncx.NewSingleFlight(), NewStat("mock"), errPlaceholder)
-	for i := 0; i < total; i++ {
+		for i := 0; i < total; i++ {
-		var val int
+			if i%2 == 0 {
-		assert.Nil(t, c.Get(fmt.Sprintf("key/%d", i), &val))
+				assert.Nil(t, c.Set(fmt.Sprintf("key/%d", i), i))
-		assert.Equal(t, i, val)
+			} else {
-	}
+				assert.Nil(t, c.SetWithExpire(fmt.Sprintf("key/%d", i), i, 0))
-	assert.Nil(t, c.Del())
+			}
-	for i := 0; i < total; i++ {
+		}
-		assert.Nil(t, c.Del(fmt.Sprintf("key/%d", i)))
+		for i := 0; i < total; i++ {
-	}
+			var val int
-	for i := 0; i < total; i++ {
+			assert.Nil(t, c.Get(fmt.Sprintf("key/%d", i), &val))
-		var val int
+			assert.Equal(t, i, val)
-		assert.True(t, c.IsNotFound(c.Get(fmt.Sprintf("key/%d", i), &val)))
+		}
-		assert.Equal(t, 0, val)
+		assert.Nil(t, c.Del())
-	}
+		for i := 0; i < total; i++ {
+			assert.Nil(t, c.Del(fmt.Sprintf("key/%d", i)))
+		}
+		assert.Nil(t, c.Del("a", "b", "c"))
+		for i := 0; i < total; i++ {
+			var val int
+			assert.True(t, c.IsNotFound(c.Get(fmt.Sprintf("key/%d", i), &val)))
+			assert.Equal(t, 0, val)
+		}
+	})
+
+	t.Run("test set del error", func(t *testing.T) {
+		r1, err := miniredis.Run()
+		assert.NoError(t, err)
+		defer r1.Close()
+		r1.SetError("mock error")
+
+		r2, err := miniredis.Run()
+		assert.NoError(t, err)
+		defer r2.Close()
+		r2.SetError("mock error")
+
+		conf := ClusterConf{
+			{
+				RedisConf: redis.RedisConf{
+					Host: r1.Addr(),
+					Type: redis.NodeType,
+				},
+				Weight: 100,
+			},
+			{
+				RedisConf: redis.RedisConf{
+					Host: r2.Addr(),
+					Type: redis.NodeType,
+				},
+				Weight: 100,
+			},
+		}
+		c := New(conf, syncx.NewSingleFlight(), NewStat("mock"), errPlaceholder)
+		assert.NoError(t, c.Del("a", "b", "c"))
+	})
 }
 
 func TestCache_OneNode(t *testing.T) {

core/stores/cache/cachenode.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"math/rand"
 	"sync"
 	"time"
@@ -130,7 +131,7 @@ func (c cacheNode) SetWithExpireCtx(ctx context.Context, key string, val interfa
 		return err
 	}
 
-	return c.rds.SetexCtx(ctx, key, string(data), int(expire.Seconds()))
+	return c.rds.SetexCtx(ctx, key, string(data), int(math.Ceil(expire.Seconds())))
 }
 
 // String returns a string that represents the cacheNode.
@@ -275,5 +276,6 @@ func (c cacheNode) processCache(ctx context.Context, key, data string, v interfa
 }
 
 func (c cacheNode) setCacheWithNotFound(ctx context.Context, key string) error {
-	return c.rds.SetexCtx(ctx, key, notFoundPlaceholder, int(c.aroundDuration(c.notFoundExpiry).Seconds()))
+	seconds := int(math.Ceil(c.aroundDuration(c.notFoundExpiry).Seconds()))
+	return c.rds.SetexCtx(ctx, key, notFoundPlaceholder, seconds)
 }

core/stores/cache/cachenode_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"runtime"
 	"strconv"
 	"sync"
 	"testing"
@@ -11,12 +12,14 @@ import (
 
 	"github.com/alicebob/miniredis/v2"
 	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/collection"
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/mathx"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/stores/redis"
 	"github.com/zeromicro/go-zero/core/stores/redis/redistest"
 	"github.com/zeromicro/go-zero/core/syncx"
+	"github.com/zeromicro/go-zero/core/timex"
 )
 
 var errTestNotFound = errors.New("not found")
@@ -27,27 +30,54 @@ func init() {
 }
 
 func TestCacheNode_DelCache(t *testing.T) {
-	store, clean, err := redistest.CreateRedis()
+	t.Run("del cache", func(t *testing.T) {
-	assert.Nil(t, err)
+		store, clean, err := redistest.CreateRedis()
-	store.Type = redis.ClusterType
+		assert.Nil(t, err)
-	defer clean()
+		store.Type = redis.ClusterType
+		defer clean()
 
-	cn := cacheNode{
+		cn := cacheNode{
-		rds:            store,
+			rds:            store,
-		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+			r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		lock:           new(sync.Mutex),
+			lock:           new(sync.Mutex),
-		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+			unstableExpiry: mathx.NewUnstable(expiryDeviation),
-		stat:           NewStat("any"),
+			stat:           NewStat("any"),
-		errNotFound:    errTestNotFound,
+			errNotFound:    errTestNotFound,
-	}
+		}
-	assert.Nil(t, cn.Del())
+		assert.Nil(t, cn.Del())
-	assert.Nil(t, cn.Del([]string{}...))
+		assert.Nil(t, cn.Del([]string{}...))
-	assert.Nil(t, cn.Del(make([]string, 0)...))
+		assert.Nil(t, cn.Del(make([]string, 0)...))
-	cn.Set("first", "one")
+		cn.Set("first", "one")
-	assert.Nil(t, cn.Del("first"))
+		assert.Nil(t, cn.Del("first"))
-	cn.Set("first", "one")
+		cn.Set("first", "one")
-	cn.Set("second", "two")
+		cn.Set("second", "two")
-	assert.Nil(t, cn.Del("first", "second"))
+		assert.Nil(t, cn.Del("first", "second"))
+	})
+
+	t.Run("del cache with errors", func(t *testing.T) {
+		old := timingWheel
+		ticker := timex.NewFakeTicker()
+		var err error
+		timingWheel, err = collection.NewTimingWheelWithTicker(
+			time.Millisecond, timingWheelSlots, func(key, value interface{}) {
+				clean(key, value)
+			}, ticker)
+		assert.NoError(t, err)
+		t.Cleanup(func() {
+			timingWheel = old
+		})
+
+		r, err := miniredis.Run()
+		assert.NoError(t, err)
+		defer r.Close()
+		r.SetError("mock error")
+
+		node := NewNode(redis.New(r.Addr(), redis.Cluster()), syncx.NewSingleFlight(),
+			NewStat("any"), errTestNotFound)
+		assert.NoError(t, node.Del("foo", "bar"))
+		ticker.Tick()
+		runtime.Gosched()
+	})
 }
 
 func TestCacheNode_DelCacheWithErrors(t *testing.T) {
@@ -125,6 +155,21 @@ func TestCacheNode_Take(t *testing.T) {
 	assert.Equal(t, `"value"`, val)
 }
 
+func TestCacheNode_TakeBadRedis(t *testing.T) {
+	r, err := miniredis.Run()
+	assert.NoError(t, err)
+	defer r.Close()
+	r.SetError("mock error")
+
+	cn := NewNode(redis.New(r.Addr()), syncx.NewSingleFlight(), NewStat("any"),
+		errTestNotFound, WithExpiry(time.Second), WithNotFoundExpiry(time.Second))
+	var str string
+	assert.Error(t, cn.Take(&str, "any", func(v interface{}) error {
+		*v.(*string) = "value"
+		return nil
+	}))
+}
+
 func TestCacheNode_TakeNotFound(t *testing.T) {
 	store, clean, err := redistest.CreateRedis()
 	assert.Nil(t, err)

core/stores/cache/cacheopt.go

@@ -34,14 +34,14 @@ func newOptions(opts ...Option) Options {
 	return o
 }
 
-// WithExpiry returns a func to customize a Options with given expiry.
+// WithExpiry returns a func to customize an Options with given expiry.
 func WithExpiry(expiry time.Duration) Option {
 	return func(o *Options) {
 		o.Expiry = expiry
 	}
 }
 
-// WithNotFoundExpiry returns a func to customize a Options with given not found expiry.
+// WithNotFoundExpiry returns a func to customize an Options with given not found expiry.
 func WithNotFoundExpiry(expiry time.Duration) Option {
 	return func(o *Options) {
 		o.NotFoundExpiry = expiry

core/stores/cache/cachestat.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/zeromicro/go-zero/core/logx"
+	"github.com/zeromicro/go-zero/core/timex"
 )
 
 const statInterval = time.Minute
@@ -25,7 +26,13 @@ func NewStat(name string) *Stat {
 	ret := &Stat{
 		name: name,
 	}
-	go ret.statLoop()
+
+	go func() {
+		ticker := timex.NewTicker(statInterval)
+		defer ticker.Stop()
+
+		ret.statLoop(ticker)
+	}()
 
 	return ret
 }
@@ -50,11 +57,8 @@ func (s *Stat) IncrementDbFails() {
 	atomic.AddUint64(&s.DbFails, 1)
 }
 
-func (s *Stat) statLoop() {
+func (s *Stat) statLoop(ticker timex.Ticker) {
-	ticker := time.NewTicker(statInterval)
+	for range ticker.Chan() {
-	defer ticker.Stop()
-
-	for range ticker.C {
 		total := atomic.SwapUint64(&s.Total, 0)
 		if total == 0 {
 			continue

core/stores/cache/cachestat_test.go

@@ -0,0 +1,28 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/zeromicro/go-zero/core/timex"
+)
+
+func TestCacheStat_statLoop(t *testing.T) {
+	t.Run("stat loop total 0", func(t *testing.T) {
+		var stat Stat
+		ticker := timex.NewFakeTicker()
+		go stat.statLoop(ticker)
+		ticker.Tick()
+		ticker.Tick()
+		ticker.Stop()
+	})
+
+	t.Run("stat loop total not 0", func(t *testing.T) {
+		var stat Stat
+		stat.IncrementTotal()
+		ticker := timex.NewFakeTicker()
+		go stat.statLoop(ticker)
+		ticker.Tick()
+		ticker.Tick()
+		ticker.Stop()
+	})
+}

core/stores/kv/store.go

@@ -110,7 +110,9 @@ type (
 		Ttl(key string) (int, error)
 		TtlCtx(ctx context.Context, key string) (int, error)
 		Zadd(key string, score int64, value string) (bool, error)
+		ZaddFloat(key string, score float64, value string) (bool, error)
 		ZaddCtx(ctx context.Context, key string, score int64, value string) (bool, error)
+		ZaddFloatCtx(ctx context.Context, key string, score float64, value string) (bool, error)
 		Zadds(key string, ps ...redis.Pair) (int64, error)
 		ZaddsCtx(ctx context.Context, key string, ps ...redis.Pair) (int64, error)
 		Zcard(key string) (int, error)
@@ -787,13 +789,21 @@ func (cs clusterStore) Zadd(key string, score int64, value string) (bool, error)
 	return cs.ZaddCtx(context.Background(), key, score, value)
 }
 
+func (cs clusterStore) ZaddFloat(key string, score float64, value string) (bool, error) {
+	return cs.ZaddFloatCtx(context.Background(), key, score, value)
+}
+
 func (cs clusterStore) ZaddCtx(ctx context.Context, key string, score int64, value string) (bool, error) {
+	return cs.ZaddFloatCtx(ctx, key, float64(score), value)
+}
+
+func (cs clusterStore) ZaddFloatCtx(ctx context.Context, key string, score float64, value string) (bool, error) {
 	node, err := cs.getRedis(key)
 	if err != nil {
 		return false, err
 	}
 
-	return node.ZaddCtx(ctx, key, score, value)
+	return node.ZaddFloatCtx(ctx, key, score, value)
 }
 
 func (cs clusterStore) Zadds(key string, ps ...redis.Pair) (int64, error) {