ci: remove windows 386 binary (#1392)

* ci: remove windows 386 binary

* chore: update go-zero

* chore: update go-zero

.codecov.yml

@@ -1,4 +1,3 @@
+comment: false
 ignore:
-  - "doc"
-  - "example"
-  - "tools"
+  - "tools"

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: https://gitee.com/kevwan/static/raw/master/images/sponsor.jpg

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior, if applicable:
+
+1. The code is
+
+   ```go
+   
+   ```
+
+2. The error is
+
+   ```
+   
+   ```
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environments (please complete the following information):**
+ - OS: [e.g. Linux]
+ - go-zero version [e.g. 1.2.1]
+ - goctl version [e.g. 1.2.1, optional]
+
+**More description**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: Question
+about: Ask a question on using go-zero or goctl
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/workflows/go.yml

@@ -15,7 +15,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.13
+        go-version: ^1.14
       id: go
 
     - name: Check out code into the Go module directory
@@ -25,10 +25,14 @@ jobs:
       run: |
         go get -v -t -d ./...
 
+    - name: Lint
+      run: |
+        go vet -stdmethods=false $(go list ./...)
+        go install mvdan.cc/gofumpt@latest
+        test -z "$(gofumpt -s -l -extra .)" || echo "Please run 'gofumpt -l -w -extra .'"
+
     - name: Test
       run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
 
     - name: Codecov
-      uses: codecov/codecov-action@v1.0.6
-      with:
-        token: ${{secrets.CODECOV_TOKEN}}
+      uses: codecov/codecov-action@v2

.github/workflows/issues.yml

@@ -0,0 +1,19 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v3
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -0,0 +1,30 @@
+on:
+  release:
+    types: [created]
+
+jobs:
+  releases-matrix:
+    name: Release goctl binary
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # build and publish in parallel: linux/386, linux/amd64, linux/arm64,
+        # windows/386, windows/amd64, windows/arm64, darwin/amd64, darwin/arm64
+        goos: [linux, windows, darwin]
+        goarch: ["386", amd64, arm64]
+        exclude:
+          - goarch: "386"
+            goos: darwin
+          - goarch: "386"
+            goos: windows
+    steps:
+      - uses: actions/checkout@v2
+      - uses: wangyoucao577/go-release-action@v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          goversion: "https://dl.google.com/go/go1.17.5.linux-amd64.tar.gz"
+          project_path: "tools/goctl"
+          binary_name: "goctl"
+          extra_files: tools/goctl/goctl.md

.github/workflows/reviewdog.yml

@@ -0,0 +1,19 @@
+name: reviewdog
+on: [pull_request]
+jobs:
+  staticcheck:
+    name: runner / staticcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: reviewdog/action-staticcheck@v1
+        with:
+          github_token: ${{ secrets.github_token }}
+          # Change reviewdog reporter if you need [github-pr-check,github-check,github-pr-review].
+          reporter: github-pr-review
+          # Report all results.
+          filter_mode: nofilter
+          # Exit with 1 when it find at least one finding.
+          fail_on_error: true
+          # Set staticcheck flags
+          staticcheck_flags: -checks=inherit,-SA1019,-SA1029,-SA5008

.gitignore

@@ -10,10 +10,14 @@
 !*/
 !api
 
+# ignore
 .idea
 **/.DS_Store
 **/logs
 
+# for test purpose
+adhoc
+
 # gitlab ci
 .cache
 

CONTRIBUTING.md

@@ -90,7 +90,7 @@ After that, run these local verifications before submitting pull request to pred
 fail of continuous integration.
 
 * Format the code with `gofmt`
-* Run the test with data race enabled `go test -race ./…`
+* Run the test with data race enabled `go test -race ./...`
 
 ## Code Review
 

ROADMAP.md

@@ -5,17 +5,22 @@ Community and contributor involvement is vital for successfully implementing all
 We hope that the items listed below will inspire further engagement from the community to keep go-zero progressing and shipping exciting and valuable features.
 
 ## 2021 Q2
-- Support TLS in redis connections
-- Support service discovery through K8S watch api
-- Log full sql statements for easier sql problem solving
+- [x] Support service discovery through K8S client api
+- [x] Log full sql statements for easier sql problem solving
 
 ## 2021 Q3
-- Support `goctl mock` command to start a mocking server with given `.api` file
-- Adapt builtin tracing mechanism to opentracing solutions
-- Support `goctl model pg` to support PostgreSQL code generation
+- [x] Support `goctl model pg` to support PostgreSQL code generation
+- [x] Adapt builtin tracing mechanism to opentracing solutions
 
 ## 2021 Q4
-- Support `goctl doctor` command to report potential issues for given service
-- Support `context` in redis related methods for timeout and tracing
-- Support `context` in sql related methods for timeout and tracing
-- Support `context` in mongodb related methods for timeout and tracing
+- [x] Support `username/password` authentication in ETCD
+- [x] Support `SSL/TLS` in `zRPC`
+- [x] Support `TLS` in redis connections
+
+## 2022
+- [ ] Support `goctl mock` command to start a mocking server with given `.api` file
+- [ ] Add `httpx.Client` with governance, like circuit breaker etc.
+- [ ] Support `goctl doctor` command to report potential issues for given service
+- [ ] Support `context` in redis related methods for timeout and tracing
+- [ ] Support `context` in sql related methods for timeout and tracing
+- [ ] Support `context` in mongodb related methods for timeout and tracing

core/collection/cache.go

@@ -34,7 +34,7 @@ type (
 		expire         time.Duration
 		timingWheel    *TimingWheel
 		lruCache       lru
-		barrier        syncx.SharedCalls
+		barrier        syncx.SingleFlight
 		unstableExpiry mathx.Unstable
 		stats          *cacheStat
 	}
@@ -46,7 +46,7 @@ func NewCache(expire time.Duration, opts ...CacheOption) (*Cache, error) {
 		data:           make(map[string]interface{}),
 		expire:         expire,
 		lruCache:       emptyLruCache,
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 	}
 

core/contextx/unmarshaler_test.go

@@ -47,9 +47,11 @@ func TestUnmarshalContextWithMissing(t *testing.T) {
 		Name string `ctx:"name"`
 		Age  int    `ctx:"age"`
 	}
+	type name string
+	const PersonNameKey name = "name"
 
 	ctx := context.Background()
-	ctx = context.WithValue(ctx, "name", "kevin")
+	ctx = context.WithValue(ctx, PersonNameKey, "kevin")
 
 	var person Person
 	err := For(ctx, &person)

core/contextx/valueonlycontext_test.go

@@ -9,7 +9,9 @@ import (
 )
 
 func TestContextCancel(t *testing.T) {
-	c := context.WithValue(context.Background(), "key", "value")
+	type key string
+	var nameKey key = "name"
+	c := context.WithValue(context.Background(), nameKey, "value")
 	c1, cancel := context.WithCancel(c)
 	o := ValueOnlyFrom(c1)
 	c2, cancel2 := context.WithCancel(o)

core/discov/accountregistry.go

@@ -0,0 +1,7 @@
+package discov
+
+import "github.com/tal-tech/go-zero/core/discov/internal"
+
+func RegisterAccount(endpoints []string, user, pass string) {
+	internal.AddAccount(endpoints, user, pass)
+}

core/discov/accountregistry_test.go

@@ -0,0 +1,21 @@
+package discov
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/discov/internal"
+	"github.com/tal-tech/go-zero/core/stringx"
+)
+
+func TestRegisterAccount(t *testing.T) {
+	endpoints := []string{
+		"localhost:2379",
+	}
+	user := "foo" + stringx.Rand()
+	RegisterAccount(endpoints, user, "bar")
+	account, ok := internal.GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, user, account.User)
+	assert.Equal(t, "bar", account.Pass)
+}

core/discov/config.go

@@ -6,6 +6,13 @@ import "errors"
 type EtcdConf struct {
 	Hosts []string
 	Key   string
+	User  string `json:",optional"`
+	Pass  string `json:",optional"`
+}
+
+// HasAccount returns if account provided.
+func (c EtcdConf) HasAccount() bool {
+	return len(c.User) > 0 && len(c.Pass) > 0
 }
 
 // Validate validates c.

core/discov/config_test.go

@@ -44,3 +44,39 @@ func TestConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestEtcdConf_HasAccount(t *testing.T) {
+	tests := []struct {
+		EtcdConf
+		hasAccount bool
+	}{
+		{
+			EtcdConf: EtcdConf{
+				Hosts: []string{"any"},
+				Key:   "key",
+			},
+			hasAccount: false,
+		},
+		{
+			EtcdConf: EtcdConf{
+				Hosts: []string{"any"},
+				Key:   "key",
+				User:  "foo",
+			},
+			hasAccount: false,
+		},
+		{
+			EtcdConf: EtcdConf{
+				Hosts: []string{"any"},
+				Key:   "key",
+				User:  "foo",
+				Pass:  "bar",
+			},
+			hasAccount: true,
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.hasAccount, test.EtcdConf.HasAccount())
+	}
+}

core/discov/internal/accountmanager.go

@@ -0,0 +1,31 @@
+package internal
+
+import "sync"
+
+type Account struct {
+	User string
+	Pass string
+}
+
+var (
+	accounts = make(map[string]Account)
+	lock     sync.RWMutex
+)
+
+func AddAccount(endpoints []string, user, pass string) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	accounts[getClusterKey(endpoints)] = Account{
+		User: user,
+		Pass: pass,
+	}
+}
+
+func GetAccount(endpoints []string) (Account, bool) {
+	lock.RLock()
+	defer lock.RUnlock()
+
+	account, ok := accounts[getClusterKey(endpoints)]
+	return account, ok
+}

core/discov/internal/accountmanager_test.go

@@ -0,0 +1,34 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/stringx"
+)
+
+func TestAccount(t *testing.T) {
+	endpoints := []string{
+		"192.168.0.2:2379",
+		"192.168.0.3:2379",
+		"192.168.0.4:2379",
+	}
+	username := "foo" + stringx.Rand()
+	password := "bar"
+	anotherPassword := "any"
+
+	_, ok := GetAccount(endpoints)
+	assert.False(t, ok)
+
+	AddAccount(endpoints, username, password)
+	account, ok := GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, username, account.User)
+	assert.Equal(t, password, account.Pass)
+
+	AddAccount(endpoints, username, anotherPassword)
+	account, ok = GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, username, account.User)
+	assert.Equal(t, anotherPassword, account.Pass)
+}

core/discov/internal/registry.go

@@ -37,25 +37,35 @@ func GetRegistry() *Registry {
 
 // GetConn returns an etcd client connection associated with given endpoints.
 func (r *Registry) GetConn(endpoints []string) (EtcdClient, error) {
-	return r.getCluster(endpoints).getClient()
+	c, _ := r.getCluster(endpoints)
+	return c.getClient()
 }
 
 // Monitor monitors the key on given etcd endpoints, notify with the given UpdateListener.
 func (r *Registry) Monitor(endpoints []string, key string, l UpdateListener) error {
-	return r.getCluster(endpoints).monitor(key, l)
+	c, exists := r.getCluster(endpoints)
+	// if exists, the existing values should be updated to the listener.
+	if exists {
+		kvs := c.getCurrent(key)
+		for _, kv := range kvs {
+			l.OnAdd(kv)
+		}
+	}
+
+	return c.monitor(key, l)
 }
 
-func (r *Registry) getCluster(endpoints []string) *cluster {
+func (r *Registry) getCluster(endpoints []string) (c *cluster, exists bool) {
 	clusterKey := getClusterKey(endpoints)
 	r.lock.Lock()
 	defer r.lock.Unlock()
-	c, ok := r.clusters[clusterKey]
-	if !ok {
+	c, exists = r.clusters[clusterKey]
+	if !exists {
 		c = newCluster(endpoints)
 		r.clusters[clusterKey] = c
 	}
 
-	return c
+	return
 }
 
 type cluster struct {
@@ -94,6 +104,21 @@ func (c *cluster) getClient() (EtcdClient, error) {
 	return val.(EtcdClient), nil
 }
 
+func (c *cluster) getCurrent(key string) []KV {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	var kvs []KV
+	for k, v := range c.values[key] {
+		kvs = append(kvs, KV{
+			Key: k,
+			Val: v,
+		})
+	}
+
+	return kvs
+}
+
 func (c *cluster) handleChanges(key string, kvs []KV) {
 	var add []KV
 	var remove []KV
@@ -197,14 +222,12 @@ func (c *cluster) load(cli EtcdClient, key string) {
 	}
 
 	var kvs []KV
-	c.lock.Lock()
 	for _, ev := range resp.Kvs {
 		kvs = append(kvs, KV{
 			Key: string(ev.Key),
 			Val: string(ev.Value),
 		})
 	}
-	c.lock.Unlock()
 
 	c.handleChanges(key, kvs)
 }
@@ -302,14 +325,20 @@ func (c *cluster) watchConnState(cli EtcdClient) {
 
 // DialClient dials an etcd cluster with given endpoints.
 func DialClient(endpoints []string) (EtcdClient, error) {
-	return clientv3.New(clientv3.Config{
+	cfg := clientv3.Config{
 		Endpoints:            endpoints,
 		AutoSyncInterval:     autoSyncInterval,
 		DialTimeout:          DialTimeout,
 		DialKeepAliveTime:    dialKeepAliveTime,
 		DialKeepAliveTimeout: DialTimeout,
 		RejectOldCluster:     true,
-	})
+	}
+	if account, ok := GetAccount(endpoints); ok {
+		cfg.Username = account.User
+		cfg.Password = account.Pass
+	}
+
+	return clientv3.New(cfg)
 }
 
 func getClusterKey(endpoints []string) string {

core/discov/internal/registry_test.go

@@ -33,9 +33,10 @@ func setMockClient(cli EtcdClient) func() {
 }
 
 func TestGetCluster(t *testing.T) {
-	c1 := GetRegistry().getCluster([]string{"first"})
-	c2 := GetRegistry().getCluster([]string{"second"})
-	c3 := GetRegistry().getCluster([]string{"first"})
+	AddAccount([]string{"first"}, "foo", "bar")
+	c1, _ := GetRegistry().getCluster([]string{"first"})
+	c2, _ := GetRegistry().getCluster([]string{"second"})
+	c3, _ := GetRegistry().getCluster([]string{"first"})
 	assert.Equal(t, c1, c3)
 	assert.NotEqual(t, c1, c2)
 }

core/discov/internal/statewatcher_mock.go

@@ -6,9 +6,10 @@ package internal
 
 import (
 	context "context"
+	reflect "reflect"
+
 	gomock "github.com/golang/mock/gomock"
 	connectivity "google.golang.org/grpc/connectivity"
-	reflect "reflect"
 )
 
 // MocketcdConn is a mock of etcdConn interface

core/discov/internal/updatelistener_mock.go

@@ -5,8 +5,9 @@
 package internal
 
 import (
-	gomock "github.com/golang/mock/gomock"
 	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
 )
 
 // MockUpdateListener is a mock of UpdateListener interface

core/discov/publisher.go

@@ -11,8 +11,8 @@ import (
 )
 
 type (
-	// PublisherOption defines the method to customize a Publisher.
-	PublisherOption func(client *Publisher)
+	// PubOption defines the method to customize a Publisher.
+	PubOption func(client *Publisher)
 
 	// A Publisher can be used to publish the value to an etcd cluster on the given key.
 	Publisher struct {
@@ -32,7 +32,7 @@ type (
 // endpoints is the hosts of the etcd cluster.
 // key:value are a pair to be published.
 // opts are used to customize the Publisher.
-func NewPublisher(endpoints []string, key, value string, opts ...PublisherOption) *Publisher {
+func NewPublisher(endpoints []string, key, value string, opts ...PubOption) *Publisher {
 	publisher := &Publisher{
 		endpoints:  endpoints,
 		key:        key,
@@ -145,8 +145,15 @@ func (p *Publisher) revoke(cli internal.EtcdClient) {
 	}
 }
 
+// WithPubEtcdAccount provides the etcd username/password.
+func WithPubEtcdAccount(user, pass string) PubOption {
+	return func(pub *Publisher) {
+		internal.AddAccount(pub.endpoints, user, pass)
+	}
+}
+
 // WithId customizes a Publisher with the id.
-func WithId(id int64) PublisherOption {
+func WithId(id int64) PubOption {
 	return func(publisher *Publisher) {
 		publisher.id = id
 	}

core/discov/publisher_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/tal-tech/go-zero/core/discov/internal"
 	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/core/stringx"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
@@ -30,7 +31,8 @@ func TestPublisher_register(t *testing.T) {
 		ID: id,
 	}, nil)
 	cli.EXPECT().Put(gomock.Any(), makeEtcdKey("thekey", id), "thevalue", gomock.Any())
-	pub := NewPublisher(nil, "thekey", "thevalue")
+	pub := NewPublisher(nil, "thekey", "thevalue",
+		WithPubEtcdAccount(stringx.Rand(), "bar"))
 	_, err := pub.register(cli)
 	assert.Nil(t, err)
 }

core/discov/subscriber.go

@@ -9,16 +9,14 @@ import (
 )
 
 type (
-	subOptions struct {
-		exclusive bool
-	}
-
 	// SubOption defines the method to customize a Subscriber.
-	SubOption func(opts *subOptions)
+	SubOption func(sub *Subscriber)
 
 	// A Subscriber is used to subscribe the given key on a etcd cluster.
 	Subscriber struct {
-		items *container
+		endpoints []string
+		exclusive bool
+		items     *container
 	}
 )
 
@@ -27,14 +25,14 @@ type (
 // key is the key to subscribe.
 // opts are used to customize the Subscriber.
 func NewSubscriber(endpoints []string, key string, opts ...SubOption) (*Subscriber, error) {
-	var subOpts subOptions
-	for _, opt := range opts {
-		opt(&subOpts)
-	}
-
 	sub := &Subscriber{
-		items: newContainer(subOpts.exclusive),
+		endpoints: endpoints,
 	}
+	for _, opt := range opts {
+		opt(sub)
+	}
+	sub.items = newContainer(sub.exclusive)
+
 	if err := internal.GetRegistry().Monitor(endpoints, key, sub.items); err != nil {
 		return nil, err
 	}
@@ -55,8 +53,14 @@ func (s *Subscriber) Values() []string {
 // Exclusive means that key value can only be 1:1,
 // which means later added value will remove the keys associated with the same value previously.
 func Exclusive() SubOption {
-	return func(opts *subOptions) {
-		opts.exclusive = true
+	return func(sub *Subscriber) {
+		sub.exclusive = true
+	}
+}
+
+func WithSubEtcdAccount(user, pass string) SubOption {
+	return func(sub *Subscriber) {
+		internal.AddAccount(sub.endpoints, user, pass)
 	}
 }
 

core/discov/subscriber_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/tal-tech/go-zero/core/discov/internal"
+	"github.com/tal-tech/go-zero/core/stringx"
 )
 
 const (
@@ -201,11 +202,9 @@ func TestContainer(t *testing.T) {
 }
 
 func TestSubscriber(t *testing.T) {
-	var opt subOptions
-	Exclusive()(&opt)
-
 	sub := new(Subscriber)
-	sub.items = newContainer(opt.exclusive)
+	Exclusive()(sub)
+	sub.items = newContainer(sub.exclusive)
 	var count int32
 	sub.AddListener(func() {
 		atomic.AddInt32(&count, 1)
@@ -214,3 +213,15 @@ func TestSubscriber(t *testing.T) {
 	assert.Empty(t, sub.Values())
 	assert.Equal(t, int32(1), atomic.LoadInt32(&count))
 }
+
+func TestWithSubEtcdAccount(t *testing.T) {
+	endpoints := []string{"localhost:2379"}
+	user := stringx.Rand()
+	WithSubEtcdAccount(user, "bar")(&Subscriber{
+		endpoints: endpoints,
+	})
+	account, ok := internal.GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, user, account.User)
+	assert.Equal(t, "bar", account.Pass)
+}

core/errorx/atomicerror.go

@@ -9,7 +9,9 @@ type AtomicError struct {
 
 // Set sets the error.
 func (ae *AtomicError) Set(err error) {
-	ae.err.Store(err)
+	if err != nil {
+		ae.err.Store(err)
+	}
 }
 
 // Load returns the error.

core/errorx/atomicerror_test.go

@@ -17,6 +17,15 @@ func TestAtomicError(t *testing.T) {
 	assert.Equal(t, errDummy, err.Load())
 }
 
+func TestAtomicErrorSetNil(t *testing.T) {
+	var (
+		errNil error
+		err    AtomicError
+	)
+	err.Set(errNil)
+	assert.Equal(t, errNil, err.Load())
+}
+
 func TestAtomicErrorNil(t *testing.T) {
 	var err AtomicError
 	assert.Nil(t, err.Load())

core/fs/files+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package fs

core/fs/files.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package fs

core/fx/stream.go

@@ -246,7 +246,7 @@ func (s Stream) Head(n int64) Stream {
 			}
 			if n == 0 {
 				// let successive method go ASAP even we have more items to skip
-				// why we don't just break the loop, because if break,
+				// why we don't just break the loop, because if breaks,
 				// this former goroutine will block forever, which will cause goroutine leak.
 				close(source)
 			}

core/fx/stream_test.go

@@ -395,16 +395,16 @@ func assetEqual(t *testing.T, except, data interface{}) {
 
 func TestStream_AnyMach(t *testing.T) {
 	assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 4 == item.(int)
+		return item.(int) == 4
 	}))
 	assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 0 == item.(int)
+		return item.(int) == 0
 	}))
 	assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 2 == item.(int)
+		return item.(int) == 2
 	}))
 	assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 2 == item.(int)
+		return item.(int) == 2
 	}))
 }
 

core/fx/timeout.go

@@ -2,6 +2,9 @@ package fx
 
 import (
 	"context"
+	"fmt"
+	"runtime/debug"
+	"strings"
 	"time"
 )
 
@@ -30,7 +33,8 @@ func DoWithTimeout(fn func() error, timeout time.Duration, opts ...DoOption) err
 	go func() {
 		defer func() {
 			if p := recover(); p != nil {
-				panicChan <- p
+				// attach call stack to avoid missing in different goroutine
+				panicChan <- fmt.Sprintf("%+v\n\n%s", p, strings.TrimSpace(string(debug.Stack())))
 			}
 		}()
 		done <- fn()

core/hash/consistenthash_test.go

@@ -74,12 +74,12 @@ func TestConsistentHashIncrementalTransfer(t *testing.T) {
 		laterCh := create()
 		laterCh.AddWithWeight(node, 10*(i+1))
 
-		for i := 0; i < requestSize; i++ {
-			key, ok := laterCh.Get(requestSize + i)
+		for j := 0; j < requestSize; j++ {
+			key, ok := laterCh.Get(requestSize + j)
 			assert.True(t, ok)
 			assert.NotNil(t, key)
 			value := key.(string)
-			assert.True(t, value == keys[i] || value == node)
+			assert.True(t, value == keys[j] || value == node)
 		}
 	}
 }

core/hash/hash_test.go

@@ -20,6 +20,11 @@ func TestMd5(t *testing.T) {
 	assert.Equal(t, md5Digest, actual)
 }
 
+func TestMd5Hex(t *testing.T) {
+	actual := Md5Hex([]byte(text))
+	assert.Equal(t, md5Digest, actual)
+}
+
 func BenchmarkHashFnv(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		h := fnv.New32()

core/jsontype/time_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/globalsign/mgo/bson"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -106,3 +107,20 @@ func TestMilliTime_UnmarshalJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestUnmarshalWithError(t *testing.T) {
+	var mt MilliTime
+	assert.NotNil(t, mt.UnmarshalJSON([]byte("hello")))
+}
+
+func TestSetBSON(t *testing.T) {
+	data, err := bson.Marshal(time.Now())
+	assert.Nil(t, err)
+
+	var raw bson.Raw
+	assert.Nil(t, bson.Unmarshal(data, &raw))
+
+	var mt MilliTime
+	assert.Nil(t, mt.SetBSON(raw))
+	assert.NotNil(t, mt.SetBSON(bson.Raw{}))
+}

core/limit/tokenlimit_test.go

@@ -24,7 +24,7 @@ func TestTokenLimit_Rescue(t *testing.T) {
 		rate  = 5
 		burst = 10
 	)
-	l := NewTokenLimiter(rate, burst, redis.NewRedis(s.Addr(), redis.NodeType), "tokenlimit")
+	l := NewTokenLimiter(rate, burst, redis.New(s.Addr()), "tokenlimit")
 	s.Close()
 
 	var allowed int

core/load/adaptiveshedder.go

@@ -31,6 +31,8 @@ var (
 
 	// default to be enabled
 	enabled = syncx.ForAtomicBool(true)
+	// default to be enabled
+	logEnabled = syncx.ForAtomicBool(true)
 	// make it a variable for unit test
 	systemOverloadChecker = func(cpuThreshold int64) bool {
 		return stat.CpuUsage() >= cpuThreshold
@@ -80,6 +82,11 @@ func Disable() {
 	enabled.Set(false)
 }
 
+// DisableLog disables the stat logs for load shedding.
+func DisableLog() {
+	logEnabled.Set(false)
+}
+
 // NewAdaptiveShedder returns an adaptive shedder.
 // opts can be used to customize the Shedder.
 func NewAdaptiveShedder(opts ...ShedderOption) Shedder {

core/load/adaptiveshedder_test.go

@@ -25,6 +25,7 @@ func init() {
 }
 
 func TestAdaptiveShedder(t *testing.T) {
+	DisableLog()
 	shedder := NewAdaptiveShedder(WithWindow(bucketDuration), WithBuckets(buckets), WithCpuThreshold(100))
 	var wg sync.WaitGroup
 	var drop int64

core/load/sheddingstat.go

@@ -48,6 +48,25 @@ func (s *SheddingStat) IncrementDrop() {
 	atomic.AddInt64(&s.drop, 1)
 }
 
+func (s *SheddingStat) loop(c <-chan time.Time) {
+	for range c {
+		st := s.reset()
+
+		if !logEnabled.True() {
+			continue
+		}
+
+		c := stat.CpuUsage()
+		if st.Drop == 0 {
+			logx.Statf("(%s) shedding_stat [1m], cpu: %d, total: %d, pass: %d, drop: %d",
+				s.name, c, st.Total, st.Pass, st.Drop)
+		} else {
+			logx.Statf("(%s) shedding_stat_drop [1m], cpu: %d, total: %d, pass: %d, drop: %d",
+				s.name, c, st.Total, st.Pass, st.Drop)
+		}
+	}
+}
+
 func (s *SheddingStat) reset() snapshot {
 	return snapshot{
 		Total: atomic.SwapInt64(&s.total, 0),
@@ -59,15 +78,6 @@ func (s *SheddingStat) reset() snapshot {
 func (s *SheddingStat) run() {
 	ticker := time.NewTicker(time.Minute)
 	defer ticker.Stop()
-	for range ticker.C {
-		c := stat.CpuUsage()
-		st := s.reset()
-		if st.Drop == 0 {
-			logx.Statf("(%s) shedding_stat [1m], cpu: %d, total: %d, pass: %d, drop: %d",
-				s.name, c, st.Total, st.Pass, st.Drop)
-		} else {
-			logx.Statf("(%s) shedding_stat_drop [1m], cpu: %d, total: %d, pass: %d, drop: %d",
-				s.name, c, st.Total, st.Pass, st.Drop)
-		}
-	}
+
+	s.loop(ticker.C)
 }

core/load/sheddingstat_test.go

@@ -2,6 +2,7 @@ package load
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -22,3 +23,32 @@ func TestSheddingStat(t *testing.T) {
 	assert.Equal(t, int64(5), result.Pass)
 	assert.Equal(t, int64(7), result.Drop)
 }
+
+func TestLoopTrue(t *testing.T) {
+	ch := make(chan time.Time, 1)
+	ch <- time.Now()
+	close(ch)
+	st := new(SheddingStat)
+	logEnabled.Set(true)
+	st.loop(ch)
+}
+
+func TestLoopTrueAndDrop(t *testing.T) {
+	ch := make(chan time.Time, 1)
+	ch <- time.Now()
+	close(ch)
+	st := new(SheddingStat)
+	st.IncrementDrop()
+	logEnabled.Set(true)
+	st.loop(ch)
+}
+
+func TestLoopFalseAndDrop(t *testing.T) {
+	ch := make(chan time.Time, 1)
+	ch <- time.Now()
+	close(ch)
+	st := new(SheddingStat)
+	st.IncrementDrop()
+	logEnabled.Set(false)
+	st.loop(ch)
+}

core/logx/durationlogger.go

@@ -20,49 +20,69 @@ func WithDuration(d time.Duration) Logger {
 }
 
 func (l *durationLogger) Error(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
 	}
 }
 
 func (l *durationLogger) Errorf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
 	}
 }
 
+func (l *durationLogger) Errorv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(errorLog, levelError, v)
+	}
+}
+
 func (l *durationLogger) Info(v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprint(v...))
 	}
 }
 
 func (l *durationLogger) Infof(format string, v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *durationLogger) Infov(v interface{}) {
+	if shallLog(InfoLevel) {
+		l.write(infoLog, levelInfo, v)
+	}
+}
+
 func (l *durationLogger) Slow(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprint(v...))
 	}
 }
 
 func (l *durationLogger) Slowf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *durationLogger) Slowv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(slowLog, levelSlow, v)
+	}
+}
+
 func (l *durationLogger) WithDuration(duration time.Duration) Logger {
 	l.Duration = timex.ReprOfDuration(duration)
 	return l
 }
 
-func (l *durationLogger) write(writer io.Writer, level, content string) {
-	l.Timestamp = getTimestamp()
-	l.Level = level
-	l.Content = content
-	outputJson(writer, logEntry(*l))
+func (l *durationLogger) write(writer io.Writer, level string, val interface{}) {
+	outputJson(writer, &durationLogger{
+		Timestamp: getTimestamp(),
+		Level:     level,
+		Content:   val,
+		Duration:  l.Duration,
+	})
 }

core/logx/durationlogger_test.go

@@ -23,6 +23,13 @@ func TestWithDurationErrorf(t *testing.T) {
 	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
 }
 
+func TestWithDurationErrorv(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Errorv("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
 func TestWithDurationInfo(t *testing.T) {
 	var builder strings.Builder
 	log.SetOutput(&builder)
@@ -37,6 +44,13 @@ func TestWithDurationInfof(t *testing.T) {
 	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
 }
 
+func TestWithDurationInfov(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Infov("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
 func TestWithDurationSlow(t *testing.T) {
 	var builder strings.Builder
 	log.SetOutput(&builder)
@@ -50,3 +64,10 @@ func TestWithDurationSlowf(t *testing.T) {
 	WithDuration(time.Second).WithDuration(time.Hour).Slowf("foo")
 	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
 }
+
+func TestWithDurationSlowv(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).WithDuration(time.Hour).Slowv("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}

core/logx/limitedexecutor_test.go

@@ -0,0 +1,62 @@
+package logx
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/timex"
+)
+
+func TestLimitedExecutor_logOrDiscard(t *testing.T) {
+	tests := []struct {
+		name      string
+		threshold time.Duration
+		lastTime  time.Duration
+		discarded uint32
+		executed  bool
+	}{
+		{
+			name:     "nil executor",
+			executed: true,
+		},
+		{
+			name:      "regular",
+			threshold: time.Hour,
+			lastTime:  timex.Now(),
+			discarded: 10,
+			executed:  false,
+		},
+		{
+			name:      "slow",
+			threshold: time.Duration(1),
+			lastTime:  -1000,
+			discarded: 10,
+			executed:  true,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			executor := newLimitedExecutor(0)
+			executor.threshold = test.threshold
+			executor.discarded = test.discarded
+			executor.lastTime.Set(test.lastTime)
+
+			var run int32
+			executor.logOrDiscard(func() {
+				atomic.AddInt32(&run, 1)
+			})
+			if test.executed {
+				assert.Equal(t, int32(1), atomic.LoadInt32(&run))
+			} else {
+				assert.Equal(t, int32(0), atomic.LoadInt32(&run))
+				assert.Equal(t, test.discarded+1, atomic.LoadUint32(&executor.discarded))
+			}
+		})
+	}
+}

core/logx/logs.go

@@ -65,12 +65,14 @@ var (
 	timeFormat   = "2006-01-02T15:04:05.000Z07"
 	writeConsole bool
 	logLevel     uint32
-	infoLog      io.WriteCloser
-	errorLog     io.WriteCloser
-	severeLog    io.WriteCloser
-	slowLog      io.WriteCloser
-	statLog      io.WriteCloser
-	stackLog     io.Writer
+	// use uint32 for atomic operations
+	disableStat uint32
+	infoLog     io.WriteCloser
+	errorLog    io.WriteCloser
+	severeLog   io.WriteCloser
+	slowLog     io.WriteCloser
+	statLog     io.WriteCloser
+	stackLog    io.Writer
 
 	once        sync.Once
 	initialized uint32
@@ -79,10 +81,10 @@ var (
 
 type (
 	logEntry struct {
-		Timestamp string `json:"@timestamp"`
-		Level     string `json:"level"`
-		Duration  string `json:"duration,omitempty"`
-		Content   string `json:"content"`
+		Timestamp string      `json:"@timestamp"`
+		Level     string      `json:"level"`
+		Duration  string      `json:"duration,omitempty"`
+		Content   interface{} `json:"content"`
 	}
 
 	logOptions struct {
@@ -98,10 +100,13 @@ type (
 	Logger interface {
 		Error(...interface{})
 		Errorf(string, ...interface{})
+		Errorv(interface{})
 		Info(...interface{})
 		Infof(string, ...interface{})
+		Infov(interface{})
 		Slow(...interface{})
 		Slowf(string, ...interface{})
+		Slowv(interface{})
 		WithDuration(time.Duration) Logger
 	}
 )
@@ -133,7 +138,7 @@ func SetUp(c LogConf) error {
 
 // Alert alerts v in alert level, and the message is written to error log.
 func Alert(v string) {
-	output(errorLog, levelAlert, v)
+	outputText(errorLog, levelAlert, v)
 }
 
 // Close closes the logging.
@@ -195,24 +200,29 @@ func Disable() {
 	})
 }
 
+// DisableStat disables the stat logs.
+func DisableStat() {
+	atomic.StoreUint32(&disableStat, 1)
+}
+
 // Error writes v into error log.
 func Error(v ...interface{}) {
 	ErrorCaller(1, v...)
 }
 
-// Errorf writes v with format into error log.
-func Errorf(format string, v ...interface{}) {
-	ErrorCallerf(1, format, v...)
-}
-
 // ErrorCaller writes v with context into error log.
 func ErrorCaller(callDepth int, v ...interface{}) {
-	errorSync(fmt.Sprint(v...), callDepth+callerInnerDepth)
+	errorTextSync(fmt.Sprint(v...), callDepth+callerInnerDepth)
 }
 
 // ErrorCallerf writes v with context in format into error log.
 func ErrorCallerf(callDepth int, format string, v ...interface{}) {
-	errorSync(fmt.Sprintf(format, v...), callDepth+callerInnerDepth)
+	errorTextSync(fmt.Errorf(format, v...).Error(), callDepth+callerInnerDepth)
+}
+
+// Errorf writes v with format into error log.
+func Errorf(format string, v ...interface{}) {
+	ErrorCallerf(1, format, v...)
 }
 
 // ErrorStack writes v along with call stack into error log.
@@ -227,14 +237,25 @@ func ErrorStackf(format string, v ...interface{}) {
 	stackSync(fmt.Sprintf(format, v...))
 }
 
+// Errorv writes v into error log with json content.
+// No call stack attached, because not elegant to pack the messages.
+func Errorv(v interface{}) {
+	errorAnySync(v)
+}
+
 // Info writes v into access log.
 func Info(v ...interface{}) {
-	infoSync(fmt.Sprint(v...))
+	infoTextSync(fmt.Sprint(v...))
 }
 
 // Infof writes v with format into access log.
 func Infof(format string, v ...interface{}) {
-	infoSync(fmt.Sprintf(format, v...))
+	infoTextSync(fmt.Sprintf(format, v...))
+}
+
+// Infov writes v into access log with json content.
+func Infov(v interface{}) {
+	infoAnySync(v)
 }
 
 // Must checks if err is nil, otherwise logs the err and exits.
@@ -242,7 +263,7 @@ func Must(err error) {
 	if err != nil {
 		msg := formatWithCaller(err.Error(), 3)
 		log.Print(msg)
-		output(severeLog, levelFatal, msg)
+		outputText(severeLog, levelFatal, msg)
 		os.Exit(1)
 	}
 }
@@ -264,12 +285,17 @@ func Severef(format string, v ...interface{}) {
 
 // Slow writes v into slow log.
 func Slow(v ...interface{}) {
-	slowSync(fmt.Sprint(v...))
+	slowTextSync(fmt.Sprint(v...))
 }
 
 // Slowf writes v with format into slow log.
 func Slowf(format string, v ...interface{}) {
-	slowSync(fmt.Sprintf(format, v...))
+	slowTextSync(fmt.Sprintf(format, v...))
+}
+
+// Slowv writes v into slow log with json content.
+func Slowv(v interface{}) {
+	slowAnySync(v)
 }
 
 // Stat writes v into stat log.
@@ -312,8 +338,14 @@ func createOutput(path string) (io.WriteCloser, error) {
 		options.gzipEnabled), options.gzipEnabled)
 }
 
-func errorSync(msg string, callDepth int) {
-	if shouldLog(ErrorLevel) {
+func errorAnySync(v interface{}) {
+	if shallLog(ErrorLevel) {
+		outputAny(errorLog, levelError, v)
+	}
+}
+
+func errorTextSync(msg string, callDepth int) {
+	if shallLog(ErrorLevel) {
 		outputError(errorLog, msg, callDepth)
 	}
 }
@@ -362,13 +394,28 @@ func handleOptions(opts []LogOption) {
 	}
 }
 
-func infoSync(msg string) {
-	if shouldLog(InfoLevel) {
-		output(infoLog, levelInfo, msg)
+func infoAnySync(val interface{}) {
+	if shallLog(InfoLevel) {
+		outputAny(infoLog, levelInfo, val)
 	}
 }
 
-func output(writer io.Writer, level, msg string) {
+func infoTextSync(msg string) {
+	if shallLog(InfoLevel) {
+		outputText(infoLog, levelInfo, msg)
+	}
+}
+
+func outputAny(writer io.Writer, level string, val interface{}) {
+	info := logEntry{
+		Timestamp: getTimestamp(),
+		Level:     level,
+		Content:   val,
+	}
+	outputJson(writer, info)
+}
+
+func outputText(writer io.Writer, level, msg string) {
 	info := logEntry{
 		Timestamp: getTimestamp(),
 		Level:     level,
@@ -379,7 +426,7 @@ func output(writer io.Writer, level, msg string) {
 
 func outputError(writer io.Writer, msg string, callDepth int) {
 	content := formatWithCaller(msg, callDepth)
-	output(writer, levelError, content)
+	outputText(writer, levelError, content)
 }
 
 func outputJson(writer io.Writer, info interface{}) {
@@ -481,30 +528,40 @@ func setupWithVolume(c LogConf) error {
 }
 
 func severeSync(msg string) {
-	if shouldLog(SevereLevel) {
-		output(severeLog, levelSevere, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+	if shallLog(SevereLevel) {
+		outputText(severeLog, levelSevere, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
-func shouldLog(level uint32) bool {
+func shallLog(level uint32) bool {
 	return atomic.LoadUint32(&logLevel) <= level
 }
 
-func slowSync(msg string) {
-	if shouldLog(ErrorLevel) {
-		output(slowLog, levelSlow, msg)
+func shallLogStat() bool {
+	return atomic.LoadUint32(&disableStat) == 0
+}
+
+func slowAnySync(v interface{}) {
+	if shallLog(ErrorLevel) {
+		outputAny(slowLog, levelSlow, v)
+	}
+}
+
+func slowTextSync(msg string) {
+	if shallLog(ErrorLevel) {
+		outputText(slowLog, levelSlow, msg)
 	}
 }
 
 func stackSync(msg string) {
-	if shouldLog(ErrorLevel) {
-		output(stackLog, levelError, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+	if shallLog(ErrorLevel) {
+		outputText(stackLog, levelError, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
 func statSync(msg string) {
-	if shouldLog(InfoLevel) {
-		output(statLog, levelStat, msg)
+	if shallLogStat() && shallLog(InfoLevel) {
+		outputText(statLog, levelStat, msg)
 	}
 }
 

core/logx/logs_test.go

@@ -2,6 +2,7 @@ package logx
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -92,6 +93,30 @@ func TestStructedLogAlert(t *testing.T) {
 	})
 }
 
+func TestStructedLogError(t *testing.T) {
+	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
+		errorLog = writer
+	}, func(v ...interface{}) {
+		Error(v...)
+	})
+}
+
+func TestStructedLogErrorf(t *testing.T) {
+	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
+		errorLog = writer
+	}, func(v ...interface{}) {
+		Errorf("%s", fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogErrorv(t *testing.T) {
+	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
+		errorLog = writer
+	}, func(v ...interface{}) {
+		Errorv(fmt.Sprint(v...))
+	})
+}
+
 func TestStructedLogInfo(t *testing.T) {
 	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
 		infoLog = writer
@@ -100,6 +125,22 @@ func TestStructedLogInfo(t *testing.T) {
 	})
 }
 
+func TestStructedLogInfof(t *testing.T) {
+	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
+		infoLog = writer
+	}, func(v ...interface{}) {
+		Infof("%s", fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogInfov(t *testing.T) {
+	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
+		infoLog = writer
+	}, func(v ...interface{}) {
+		Infov(fmt.Sprint(v...))
+	})
+}
+
 func TestStructedLogSlow(t *testing.T) {
 	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
 		slowLog = writer
@@ -116,6 +157,14 @@ func TestStructedLogSlowf(t *testing.T) {
 	})
 }
 
+func TestStructedLogSlowv(t *testing.T) {
+	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
+		slowLog = writer
+	}, func(v ...interface{}) {
+		Slowv(fmt.Sprint(v...))
+	})
+}
+
 func TestStructedLogStat(t *testing.T) {
 	doTestStructedLog(t, levelStat, func(writer io.WriteCloser) {
 		statLog = writer
@@ -194,6 +243,16 @@ func TestSetLevelWithDuration(t *testing.T) {
 	assert.Equal(t, 0, writer.builder.Len())
 }
 
+func TestErrorfWithWrappedError(t *testing.T) {
+	SetLevel(ErrorLevel)
+	const message = "there"
+	writer := new(mockWriter)
+	errorLog = writer
+	atomic.StoreUint32(&initialized, 1)
+	Errorf("hello %w", errors.New(message))
+	assert.True(t, strings.Contains(writer.builder.String(), "hello there"))
+}
+
 func TestMustNil(t *testing.T) {
 	Must(nil)
 }
@@ -246,6 +305,17 @@ func TestDisable(t *testing.T) {
 	assert.Nil(t, Close())
 }
 
+func TestDisableStat(t *testing.T) {
+	DisableStat()
+
+	const message = "hello there"
+	writer := new(mockWriter)
+	statLog = writer
+	atomic.StoreUint32(&initialized, 1)
+	Stat(message)
+	assert.Equal(t, 0, writer.builder.Len())
+}
+
 func TestWithGzip(t *testing.T) {
 	fn := WithGzip()
 	var opt logOptions
@@ -357,7 +427,9 @@ func doTestStructedLog(t *testing.T, level string, setup func(writer io.WriteClo
 		t.Error(err)
 	}
 	assert.Equal(t, level, entry.Level)
-	assert.True(t, strings.Contains(entry.Content, message))
+	val, ok := entry.Content.(string)
+	assert.True(t, ok)
+	assert.True(t, strings.Contains(val, message))
 }
 
 func testSetLevelTwiceWithMode(t *testing.T, mode string) {

core/logx/rotatelogger.go

@@ -47,7 +47,6 @@ type (
 		done     chan lang.PlaceholderType
 		rule     RotateRule
 		compress bool
-		keepDays int
 		// can't use threading.RoutineGroup because of cycle import
 		waitGroup sync.WaitGroup
 		closeOnce sync.Once

core/logx/rotatelogger_test.go

@@ -3,6 +3,7 @@ package logx
 import (
 	"os"
 	"path/filepath"
+	"syscall"
 	"testing"
 	"time"
 
@@ -97,7 +98,13 @@ func TestRotateLoggerRotate(t *testing.T) {
 		}()
 	}
 	err = logger.rotate()
-	assert.Nil(t, err)
+	switch v := err.(type) {
+	case *os.LinkError:
+		// avoid rename error on docker container
+		assert.Equal(t, syscall.EXDEV, v.Err)
+	default:
+		assert.Nil(t, err)
+	}
 }
 
 func TestRotateLoggerWrite(t *testing.T) {

core/logx/syslog_test.go

@@ -44,5 +44,5 @@ func captureOutput(f func()) string {
 func getContent(jsonStr string) string {
 	var entry logEntry
 	json.Unmarshal([]byte(jsonStr), &entry)
-	return entry.Content
+	return entry.Content.(string)
 }

core/logx/tracelogger.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/timex"
-	"github.com/tal-tech/go-zero/core/trace/tracespec"
+	"go.opentelemetry.io/otel/trace"
 )
 
 type traceLogger struct {
@@ -18,53 +18,75 @@ type traceLogger struct {
 }
 
 func (l *traceLogger) Error(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
 	}
 }
 
 func (l *traceLogger) Errorf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
 	}
 }
 
+func (l *traceLogger) Errorv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(errorLog, levelError, v)
+	}
+}
+
 func (l *traceLogger) Info(v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprint(v...))
 	}
 }
 
 func (l *traceLogger) Infof(format string, v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *traceLogger) Infov(v interface{}) {
+	if shallLog(InfoLevel) {
+		l.write(infoLog, levelInfo, v)
+	}
+}
+
 func (l *traceLogger) Slow(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprint(v...))
 	}
 }
 
 func (l *traceLogger) Slowf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *traceLogger) Slowv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(slowLog, levelSlow, v)
+	}
+}
+
 func (l *traceLogger) WithDuration(duration time.Duration) Logger {
 	l.Duration = timex.ReprOfDuration(duration)
 	return l
 }
 
-func (l *traceLogger) write(writer io.Writer, level, content string) {
-	l.Timestamp = getTimestamp()
-	l.Level = level
-	l.Content = content
-	l.Trace = traceIdFromContext(l.ctx)
-	l.Span = spanIdFromContext(l.ctx)
-	outputJson(writer, l)
+func (l *traceLogger) write(writer io.Writer, level string, val interface{}) {
+	outputJson(writer, &traceLogger{
+		logEntry: logEntry{
+			Timestamp: getTimestamp(),
+			Level:     level,
+			Duration:  l.Duration,
+			Content:   val,
+		},
+		Trace: traceIdFromContext(l.ctx),
+		Span:  spanIdFromContext(l.ctx),
+	})
 }
 
 // WithContext sets ctx to log, for keeping tracing information.
@@ -75,19 +97,19 @@ func WithContext(ctx context.Context) Logger {
 }
 
 func spanIdFromContext(ctx context.Context) string {
-	t, ok := ctx.Value(tracespec.TracingKey).(tracespec.Trace)
-	if !ok {
-		return ""
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasSpanID() {
+		return spanCtx.SpanID().String()
 	}
 
-	return t.SpanId()
+	return ""
 }
 
 func traceIdFromContext(ctx context.Context) string {
-	t, ok := ctx.Value(tracespec.TracingKey).(tracespec.Trace)
-	if !ok {
-		return ""
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasTraceID() {
+		return spanCtx.TraceID().String()
 	}
 
-	return t.TraceId()
+	return ""
 }

core/logx/tracelogger_test.go

@@ -9,71 +9,102 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/tal-tech/go-zero/core/trace/tracespec"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 )
 
 const (
-	mockTraceID = "mock-trace-id"
-	mockSpanID  = "mock-span-id"
+	traceKey = "trace"
+	spanKey  = "span"
 )
 
-var mock tracespec.Trace = new(mockTrace)
-
 func TestTraceLog(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	WithContext(ctx).(*traceLogger).write(&buf, levelInfo, testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceError(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	errorLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Error(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Errorf(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
+	buf.Reset()
+	l.WithDuration(time.Second).Errorv(testlog)
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceInfo(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	infoLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Info(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Infof(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
+	buf.Reset()
+	l.WithDuration(time.Second).Infov(testlog)
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceSlow(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	slowLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Slow(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Slowf(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
+	buf.Reset()
+	l.WithDuration(time.Second).Slowv(testlog)
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceWithoutContext(t *testing.T) {
@@ -83,34 +114,10 @@ func TestTraceWithoutContext(t *testing.T) {
 	l := WithContext(context.Background()).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Info(testlog)
-	assert.False(t, strings.Contains(buf.String(), mockTraceID))
-	assert.False(t, strings.Contains(buf.String(), mockSpanID))
+	assert.False(t, strings.Contains(buf.String(), traceKey))
+	assert.False(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Infof(testlog)
-	assert.False(t, strings.Contains(buf.String(), mockTraceID))
-	assert.False(t, strings.Contains(buf.String(), mockSpanID))
-}
-
-type mockTrace struct{}
-
-func (t mockTrace) TraceId() string {
-	return mockTraceID
-}
-
-func (t mockTrace) SpanId() string {
-	return mockSpanID
-}
-
-func (t mockTrace) Finish() {
-}
-
-func (t mockTrace) Fork(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Follow(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Visit(fn func(key, val string) bool) {
+	assert.False(t, strings.Contains(buf.String(), traceKey))
+	assert.False(t, strings.Contains(buf.String(), spanKey))
 }

core/mapping/jsonunmarshaler.go

@@ -15,6 +15,11 @@ func UnmarshalJsonBytes(content []byte, v interface{}) error {
 	return unmarshalJsonBytes(content, v, jsonUnmarshaler)
 }
 
+// UnmarshalJsonMap unmarshals content from m into v.
+func UnmarshalJsonMap(m map[string]interface{}, v interface{}) error {
+	return jsonUnmarshaler.Unmarshal(m, v)
+}
+
 // UnmarshalJsonReader unmarshals content from reader into v.
 func UnmarshalJsonReader(reader io.Reader, v interface{}) error {
 	return unmarshalJsonReader(reader, v, jsonUnmarshaler)

core/mapping/jsonunmarshaler_test.go

@@ -871,3 +871,50 @@ func TestUnmarshalReaderError(t *testing.T) {
 	assert.NotNil(t, err)
 	assert.True(t, strings.Contains(err.Error(), payload))
 }
+
+func TestUnmarshalMap(t *testing.T) {
+	t.Run("nil map and valid", func(t *testing.T) {
+		var m map[string]interface{}
+		var v struct {
+			Any string `json:",optional"`
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.Nil(t, err)
+		assert.True(t, len(v.Any) == 0)
+	})
+
+	t.Run("empty map but not valid", func(t *testing.T) {
+		m := map[string]interface{}{}
+		var v struct {
+			Any string
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.NotNil(t, err)
+	})
+
+	t.Run("empty map and valid", func(t *testing.T) {
+		m := map[string]interface{}{}
+		var v struct {
+			Any string `json:",optional"`
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.Nil(t, err)
+		assert.True(t, len(v.Any) == 0)
+	})
+
+	t.Run("valid map", func(t *testing.T) {
+		m := map[string]interface{}{
+			"Any": "foo",
+		}
+		var v struct {
+			Any string
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.Nil(t, err)
+		assert.Equal(t, "foo", v.Any)
+	})
+}

core/mapping/unmarshaler.go

@@ -7,7 +7,6 @@ import (
 	"reflect"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/tal-tech/go-zero/core/jsonx"
@@ -25,15 +24,17 @@ var (
 	errValueNotSettable = errors.New("value is not settable")
 	errValueNotStruct   = errors.New("value type is not struct")
 	keyUnmarshaler      = NewUnmarshaler(defaultKeyName)
-	cacheKeys           atomic.Value
-	cacheKeysLock       sync.Mutex
 	durationType        = reflect.TypeOf(time.Duration(0))
+	cacheKeys           map[string][]string
+	cacheKeysLock       sync.Mutex
+	defaultCache        map[string]interface{}
+	defaultCacheLock    sync.Mutex
 	emptyMap            = map[string]interface{}{}
 	emptyValue          = reflect.ValueOf(lang.Placeholder)
 )
 
 type (
-	// A Unmarshaler is used to unmarshal with given tag key.
+	// Unmarshaler is used to unmarshal with given tag key.
 	Unmarshaler struct {
 		key  string
 		opts unmarshalOptions
@@ -43,14 +44,14 @@ type (
 	UnmarshalOption func(*unmarshalOptions)
 
 	unmarshalOptions struct {
-		fromString bool
+		fromString   bool
+		canonicalKey func(key string) string
 	}
-
-	keyCache map[string][]string
 )
 
 func init() {
-	cacheKeys.Store(make(keyCache))
+	cacheKeys = make(map[string][]string)
+	defaultCache = make(map[string]interface{})
 }
 
 // NewUnmarshaler returns a Unmarshaler.
@@ -206,6 +207,8 @@ func (u *Unmarshaler) processFieldNotFromString(field reflect.StructField, value
 	switch {
 	case valueKind == reflect.Map && typeKind == reflect.Struct:
 		return u.processFieldStruct(field, value, mapValue, fullName)
+	case valueKind == reflect.Map && typeKind == reflect.Map:
+		return u.fillMap(field, value, mapValue)
 	case valueKind == reflect.String && typeKind == reflect.Slice:
 		return u.fillSliceFromString(fieldType, value, mapValue)
 	case valueKind == reflect.String && derefedFieldType == durationType:
@@ -229,7 +232,7 @@ func (u *Unmarshaler) processFieldPrimitive(field reflect.StructField, value ref
 	default:
 		switch v := mapValue.(type) {
 		case json.Number:
-			return u.processFieldPrimitiveWithJsonNumber(field, value, v, opts, fullName)
+			return u.processFieldPrimitiveWithJSONNumber(field, value, v, opts, fullName)
 		default:
 			if typeKind == valueKind {
 				if err := validateValueInOptions(opts.options(), mapValue); err != nil {
@@ -244,7 +247,7 @@ func (u *Unmarshaler) processFieldPrimitive(field reflect.StructField, value ref
 	return newTypeMismatchError(fullName)
 }
 
-func (u *Unmarshaler) processFieldPrimitiveWithJsonNumber(field reflect.StructField, value reflect.Value,
+func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(field reflect.StructField, value reflect.Value,
 	v json.Number, opts *fieldOptionsWithContext, fullName string) error {
 	fieldType := field.Type
 	fieldKind := fieldType.Kind()
@@ -323,7 +326,11 @@ func (u *Unmarshaler) processNamedField(field reflect.StructField, value reflect
 	}
 
 	fullName = join(fullName, key)
-	mapValue, hasValue := getValue(m, key)
+	canonicalKey := key
+	if u.opts.canonicalKey != nil {
+		canonicalKey = u.opts.canonicalKey(key)
+	}
+	mapValue, hasValue := getValue(m, canonicalKey)
 	if hasValue {
 		return u.processNamedFieldWithValue(field, value, mapValue, key, opts, fullName)
 	}
@@ -381,7 +388,13 @@ func (u *Unmarshaler) processNamedFieldWithoutValue(field reflect.StructField, v
 		if derefedType == durationType {
 			return fillDurationValue(fieldKind, value, defaultValue)
 		}
-		return setValue(fieldKind, value, defaultValue)
+
+		switch fieldKind {
+		case reflect.Array, reflect.Slice:
+			return u.fillSliceWithDefault(derefedType, value, defaultValue)
+		default:
+			return setValue(fieldKind, value, defaultValue)
+		}
 	}
 
 	switch fieldKind {
@@ -495,7 +508,8 @@ func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.
 	return nil
 }
 
-func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int, baseKind reflect.Kind, value interface{}) error {
+func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int,
+	baseKind reflect.Kind, value interface{}) error {
 	ithVal := slice.Index(index)
 	switch v := value.(type) {
 	case json.Number:
@@ -513,17 +527,39 @@ func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int, baseKind re
 			target.Set(reflect.ValueOf(value))
 			ithVal.Set(target.Addr())
 			return nil
-		} else {
-			if ithVal.Kind() != reflect.TypeOf(value).Kind() {
-				return errTypeMismatch
-			}
-
-			ithVal.Set(reflect.ValueOf(value))
-			return nil
 		}
+
+		if ithVal.Kind() != reflect.TypeOf(value).Kind() {
+			return errTypeMismatch
+		}
+
+		ithVal.Set(reflect.ValueOf(value))
+		return nil
 	}
 }
 
+func (u *Unmarshaler) fillSliceWithDefault(derefedType reflect.Type, value reflect.Value,
+	defaultValue string) error {
+	baseFieldType := Deref(derefedType.Elem())
+	baseFieldKind := baseFieldType.Kind()
+	defaultCacheLock.Lock()
+	slice, ok := defaultCache[defaultValue]
+	defaultCacheLock.Unlock()
+	if !ok {
+		if baseFieldKind == reflect.String {
+			slice = parseGroupedSegments(defaultValue)
+		} else if err := jsonx.UnmarshalFromString(defaultValue, &slice); err != nil {
+			return err
+		}
+
+		defaultCacheLock.Lock()
+		defaultCache[defaultValue] = slice
+		defaultCacheLock.Unlock()
+	}
+
+	return u.fillSlice(derefedType, value, slice)
+}
+
 func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue interface{}) (reflect.Value, error) {
 	mapType := reflect.MapOf(keyType, elemType)
 	valueType := reflect.TypeOf(mapValue)
@@ -579,6 +615,8 @@ func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue inter
 			targetValue.SetMapIndex(key, innerValue)
 		default:
 			switch v := keythData.(type) {
+			case bool:
+				targetValue.SetMapIndex(key, reflect.ValueOf(v))
 			case string:
 				targetValue.SetMapIndex(key, reflect.ValueOf(v))
 			case json.Number:
@@ -621,6 +659,13 @@ func WithStringValues() UnmarshalOption {
 	}
 }
 
+// WithCanonicalKeyFunc customizes a Unmarshaler with Canonical Key func
+func WithCanonicalKeyFunc(f func(string) string) UnmarshalOption {
+	return func(opt *unmarshalOptions) {
+		opt.canonicalKey = f
+	}
+}
+
 func fillDurationValue(fieldKind reflect.Kind, value reflect.Value, dur string) error {
 	d, err := time.ParseDuration(dur)
 	if err != nil {
@@ -708,20 +753,6 @@ func getValueWithChainedKeys(m Valuer, keys []string) (interface{}, bool) {
 	return nil, false
 }
 
-func insertKeys(key string, cache []string) {
-	cacheKeysLock.Lock()
-	defer cacheKeysLock.Unlock()
-
-	keys := cacheKeys.Load().(keyCache)
-	// copy the contents into the new map, to guarantee the old map is immutable
-	newKeys := make(keyCache)
-	for k, v := range keys {
-		newKeys[k] = v
-	}
-	newKeys[key] = cache
-	cacheKeys.Store(newKeys)
-}
-
 func join(elem ...string) string {
 	var builder strings.Builder
 
@@ -752,15 +783,19 @@ func newTypeMismatchError(name string) error {
 }
 
 func readKeys(key string) []string {
-	cache := cacheKeys.Load().(keyCache)
-	if keys, ok := cache[key]; ok {
+	cacheKeysLock.Lock()
+	keys, ok := cacheKeys[key]
+	cacheKeysLock.Unlock()
+	if ok {
 		return keys
 	}
 
-	keys := strings.FieldsFunc(key, func(c rune) bool {
+	keys = strings.FieldsFunc(key, func(c rune) bool {
 		return c == delimiter
 	})
-	insertKeys(key, keys)
+	cacheKeysLock.Lock()
+	cacheKeys[key] = keys
+	cacheKeysLock.Unlock()
 
 	return keys
 }

core/mapping/unmarshaler_test.go

@@ -198,6 +198,66 @@ func TestUnmarshalIntWithDefault(t *testing.T) {
 	assert.Equal(t, 1, in.Int)
 }
 
+func TestUnmarshalBoolSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Bools []bool `key:"bools,default=[true,false]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []bool{true, false}, in.Bools)
+}
+
+func TestUnmarshalIntSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Ints []int `key:"ints,default=[1,2,3]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []int{1, 2, 3}, in.Ints)
+}
+
+func TestUnmarshalIntSliceWithDefaultHasSpaces(t *testing.T) {
+	type inner struct {
+		Ints []int `key:"ints,default=[1, 2, 3]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []int{1, 2, 3}, in.Ints)
+}
+
+func TestUnmarshalFloatSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Floats []float32 `key:"floats,default=[1.1,2.2,3.3]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []float32{1.1, 2.2, 3.3}, in.Floats)
+}
+
+func TestUnmarshalStringSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Strs []string `key:"strs,default=[foo,bar,woo]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []string{"foo", "bar", "woo"}, in.Strs)
+}
+
+func TestUnmarshalStringSliceWithDefaultHasSpaces(t *testing.T) {
+	type inner struct {
+		Strs []string `key:"strs,default=[foo, bar, woo]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []string{"foo", "bar", "woo"}, in.Strs)
+}
+
 func TestUnmarshalUint(t *testing.T) {
 	type inner struct {
 		Uint          uint   `key:"uint"`
@@ -861,10 +921,12 @@ func TestUnmarshalSliceOfStruct(t *testing.T) {
 func TestUnmarshalWithStringOptionsCorrect(t *testing.T) {
 	type inner struct {
 		Value   string `key:"value,options=first|second"`
+		Foo     string `key:"foo,options=[bar,baz]"`
 		Correct string `key:"correct,options=1|2"`
 	}
 	m := map[string]interface{}{
 		"value":   "first",
+		"foo":     "bar",
 		"correct": "2",
 	}
 
@@ -872,6 +934,7 @@ func TestUnmarshalWithStringOptionsCorrect(t *testing.T) {
 	ast := assert.New(t)
 	ast.Nil(UnmarshalKey(m, &in))
 	ast.Equal("first", in.Value)
+	ast.Equal("bar", in.Foo)
 	ast.Equal("2", in.Correct)
 }
 
@@ -943,6 +1006,22 @@ func TestUnmarshalStringOptionsWithStringOptionsIncorrect(t *testing.T) {
 	ast.NotNil(unmarshaler.Unmarshal(m, &in))
 }
 
+func TestUnmarshalStringOptionsWithStringOptionsIncorrectGrouped(t *testing.T) {
+	type inner struct {
+		Value   string `key:"value,options=[first,second]"`
+		Correct string `key:"correct,options=1|2"`
+	}
+	m := map[string]interface{}{
+		"value":   "third",
+		"correct": "2",
+	}
+
+	var in inner
+	unmarshaler := NewUnmarshaler(defaultKeyName, WithStringValues())
+	ast := assert.New(t)
+	ast.NotNil(unmarshaler.Unmarshal(m, &in))
+}
+
 func TestUnmarshalWithStringOptionsIncorrect(t *testing.T) {
 	type inner struct {
 		Value     string `key:"value,options=first|second"`
@@ -2518,3 +2597,29 @@ func TestUnmarshalJsonReaderPtrArray(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res.B))
 }
+
+func TestUnmarshalJsonWithoutKey(t *testing.T) {
+	payload := `{"A": "1", "B": "2"}`
+	var res struct {
+		A string `json:""`
+		B string `json:","`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, "1", res.A)
+	assert.Equal(t, "2", res.B)
+}
+
+func BenchmarkDefaultValue(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var a struct {
+			Ints []int    `json:"ints,default=[1,2,3]"`
+			Strs []string `json:"strs,default=[foo,bar,baz]"`
+		}
+		_ = UnmarshalJsonMap(nil, &a)
+		if len(a.Strs) != 3 || len(a.Ints) != 3 {
+			b.Fatal("failed")
+		}
+	}
+}

core/mapping/utils.go

@@ -14,13 +14,19 @@ import (
 )
 
 const (
-	defaultOption   = "default"
-	stringOption    = "string"
-	optionalOption  = "optional"
-	optionsOption   = "options"
-	rangeOption     = "range"
-	optionSeparator = "|"
-	equalToken      = "="
+	defaultOption      = "default"
+	stringOption       = "string"
+	optionalOption     = "optional"
+	optionsOption      = "options"
+	rangeOption        = "range"
+	optionSeparator    = "|"
+	equalToken         = "="
+	escapeChar         = '\\'
+	leftBracket        = '('
+	rightBracket       = ')'
+	leftSquareBracket  = '['
+	rightSquareBracket = ']'
+	segmentSeparator   = ','
 )
 
 var (
@@ -118,7 +124,7 @@ func convertType(kind reflect.Kind, str string) (interface{}, error) {
 }
 
 func doParseKeyAndOptions(field reflect.StructField, value string) (string, *fieldOptions, error) {
-	segments := strings.Split(value, ",")
+	segments := parseSegments(value)
 	key := strings.TrimSpace(segments[0])
 	options := segments[1:]
 
@@ -198,6 +204,16 @@ func maybeNewValue(field reflect.StructField, value reflect.Value) {
 	}
 }
 
+func parseGroupedSegments(val string) []string {
+	val = strings.TrimLeftFunc(val, func(r rune) bool {
+		return r == leftBracket || r == leftSquareBracket
+	})
+	val = strings.TrimRightFunc(val, func(r rune) bool {
+		return r == rightBracket || r == rightSquareBracket
+	})
+	return parseSegments(val)
+}
+
 // don't modify returned fieldOptions, it's cached and shared among different calls.
 func parseKeyAndOptions(tagName string, field reflect.StructField) (string, *fieldOptions, error) {
 	value := field.Tag.Get(tagName)
@@ -309,7 +325,7 @@ func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 			return fmt.Errorf("field %s has wrong options", fieldName)
 		}
 
-		fieldOpts.Options = strings.Split(segs[1], optionSeparator)
+		fieldOpts.Options = parseOptions(segs[1])
 	case strings.HasPrefix(option, defaultOption):
 		segs := strings.Split(option, equalToken)
 		if len(segs) != 2 {
@@ -334,6 +350,69 @@ func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 	return nil
 }
 
+// parseOptions parses the given options in tag.
+// for example: `json:"name,options=foo|bar"` or `json:"name,options=[foo,bar]"`
+func parseOptions(val string) []string {
+	if len(val) == 0 {
+		return nil
+	}
+
+	if val[0] == leftSquareBracket {
+		return parseGroupedSegments(val)
+	}
+
+	return strings.Split(val, optionSeparator)
+}
+
+func parseSegments(val string) []string {
+	var segments []string
+	var escaped, grouped bool
+	var buf strings.Builder
+
+	for _, ch := range val {
+		if escaped {
+			buf.WriteRune(ch)
+			escaped = false
+			continue
+		}
+
+		switch ch {
+		case segmentSeparator:
+			if grouped {
+				buf.WriteRune(ch)
+			} else {
+				// need to trim spaces, but we cannot ignore empty string,
+				// because the first segment stands for the key might be empty.
+				// if ignored, the later tag will be used as the key.
+				segments = append(segments, strings.TrimSpace(buf.String()))
+				buf.Reset()
+			}
+		case escapeChar:
+			if grouped {
+				buf.WriteRune(ch)
+			} else {
+				escaped = true
+			}
+		case leftBracket, leftSquareBracket:
+			buf.WriteRune(ch)
+			grouped = true
+		case rightBracket, rightSquareBracket:
+			buf.WriteRune(ch)
+			grouped = false
+		default:
+			buf.WriteRune(ch)
+		}
+	}
+
+	last := strings.TrimSpace(buf.String())
+	// ignore last empty string
+	if len(last) > 0 {
+		segments = append(segments, last)
+	}
+
+	return segments
+}
+
 func reprOfValue(val reflect.Value) string {
 	switch vt := val.Interface().(type) {
 	case bool:

core/mapping/utils_test.go

@@ -90,6 +90,82 @@ func TestParseKeyAndOptionWithTagAndOption(t *testing.T) {
 	assert.True(t, options.FromString)
 }
 
+func TestParseSegments(t *testing.T) {
+	tests := []struct {
+		input  string
+		expect []string
+	}{
+		{
+			input:  "",
+			expect: []string{},
+		},
+		{
+			input:  ",",
+			expect: []string{""},
+		},
+		{
+			input:  "foo,",
+			expect: []string{"foo"},
+		},
+		{
+			input: ",foo",
+			// the first empty string cannot be ignored, it's the key.
+			expect: []string{"", "foo"},
+		},
+		{
+			input:  "foo",
+			expect: []string{"foo"},
+		},
+		{
+			input:  "foo,bar",
+			expect: []string{"foo", "bar"},
+		},
+		{
+			input:  "foo,bar,baz",
+			expect: []string{"foo", "bar", "baz"},
+		},
+		{
+			input:  "foo,options=a|b",
+			expect: []string{"foo", "options=a|b"},
+		},
+		{
+			input:  "foo,bar,default=[baz,qux]",
+			expect: []string{"foo", "bar", "default=[baz,qux]"},
+		},
+		{
+			input:  "foo,bar,options=[baz,qux]",
+			expect: []string{"foo", "bar", "options=[baz,qux]"},
+		},
+		{
+			input:  `foo\,bar,options=[baz,qux]`,
+			expect: []string{`foo,bar`, "options=[baz,qux]"},
+		},
+		{
+			input:  `foo,bar,options=\[baz,qux]`,
+			expect: []string{"foo", "bar", "options=[baz", "qux]"},
+		},
+		{
+			input:  `foo,bar,options=[baz\,qux]`,
+			expect: []string{"foo", "bar", `options=[baz\,qux]`},
+		},
+		{
+			input:  `foo\,bar,options=[baz,qux],default=baz`,
+			expect: []string{`foo,bar`, "options=[baz,qux]", "default=baz"},
+		},
+		{
+			input:  `foo\,bar,options=[baz,qux, quux],default=[qux, baz]`,
+			expect: []string{`foo,bar`, "options=[baz,qux, quux]", "default=[qux, baz]"},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.input, func(t *testing.T) {
+			assert.ElementsMatch(t, test.expect, parseSegments(test.input))
+		})
+	}
+}
+
 func TestValidatePtrWithNonPtr(t *testing.T) {
 	var foo string
 	rve := reflect.ValueOf(foo)
@@ -209,6 +285,12 @@ func TestRepr(t *testing.T) {
 			newMockPtr(),
 			"mockptr",
 		},
+		{
+			&mockOpacity{
+				val: 1,
+			},
+			"{1}",
+		},
 		{
 			true,
 			"true",

core/mapping/yamlunmarshaler_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"k8s.io/utils/io"
 )
 
 func TestUnmarshalYamlBytes(t *testing.T) {
@@ -18,6 +19,22 @@ func TestUnmarshalYamlBytes(t *testing.T) {
 	assert.Equal(t, "liao", c.Name)
 }
 
+func TestUnmarshalYamlBytesErrorInput(t *testing.T) {
+	var c struct {
+		Name string
+	}
+	content := []byte(`liao`)
+	assert.NotNil(t, UnmarshalYamlBytes(content, &c))
+}
+
+func TestUnmarshalYamlBytesEmptyInput(t *testing.T) {
+	var c struct {
+		Name string
+	}
+	content := []byte(``)
+	assert.NotNil(t, UnmarshalYamlBytes(content, &c))
+}
+
 func TestUnmarshalYamlBytesOptional(t *testing.T) {
 	var c struct {
 		Name string
@@ -918,3 +935,82 @@ func TestUnmarshalYamlReaderError(t *testing.T) {
 	err := UnmarshalYamlReader(reader, &v)
 	assert.NotNil(t, err)
 }
+
+func TestUnmarshalYamlBadReader(t *testing.T) {
+	var v struct {
+		Any string
+	}
+
+	err := UnmarshalYamlReader(new(badReader), &v)
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshalYamlMapBool(t *testing.T) {
+	text := `machine:
+  node1: true
+  node2: true
+  node3: true
+`
+	var v struct {
+		Machine map[string]bool `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.True(t, v.Machine["node1"])
+	assert.True(t, v.Machine["node2"])
+	assert.True(t, v.Machine["node3"])
+}
+
+func TestUnmarshalYamlMapInt(t *testing.T) {
+	text := `machine:
+  node1: 1
+  node2: 2
+  node3: 3
+`
+	var v struct {
+		Machine map[string]int `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.Equal(t, 1, v.Machine["node1"])
+	assert.Equal(t, 2, v.Machine["node2"])
+	assert.Equal(t, 3, v.Machine["node3"])
+}
+
+func TestUnmarshalYamlMapByte(t *testing.T) {
+	text := `machine:
+  node1: 1
+  node2: 2
+  node3: 3
+`
+	var v struct {
+		Machine map[string]byte `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.Equal(t, byte(1), v.Machine["node1"])
+	assert.Equal(t, byte(2), v.Machine["node2"])
+	assert.Equal(t, byte(3), v.Machine["node3"])
+}
+
+func TestUnmarshalYamlMapRune(t *testing.T) {
+	text := `machine:
+  node1: 1
+  node2: 2
+  node3: 3
+`
+	var v struct {
+		Machine map[string]rune `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.Equal(t, rune(1), v.Machine["node1"])
+	assert.Equal(t, rune(2), v.Machine["node2"])
+	assert.Equal(t, rune(3), v.Machine["node3"])
+}
+
+type badReader struct{}
+
+func (b *badReader) Read(p []byte) (n int, err error) {
+	return 0, io.ErrLimitReached
+}

core/mr/mapreduce.go

@@ -1,6 +1,7 @@
 package mr
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sync"
@@ -43,6 +44,7 @@ type (
 	Option func(opts *mapReduceOptions)
 
 	mapReduceOptions struct {
+		ctx     context.Context
 		workers int
 	}
 
@@ -95,14 +97,15 @@ func Map(generate GenerateFunc, mapper MapFunc, opts ...Option) chan interface{}
 	collector := make(chan interface{}, options.workers)
 	done := syncx.NewDoneChan()
 
-	go executeMappers(mapper, source, collector, done.Done(), options.workers)
+	go executeMappers(options.ctx, mapper, source, collector, done.Done(), options.workers)
 
 	return collector
 }
 
 // MapReduce maps all elements generated from given generate func,
 // and reduces the output elements with given reducer.
-func MapReduce(generate GenerateFunc, mapper MapperFunc, reducer ReducerFunc, opts ...Option) (interface{}, error) {
+func MapReduce(generate GenerateFunc, mapper MapperFunc, reducer ReducerFunc,
+	opts ...Option) (interface{}, error) {
 	source := buildSource(generate)
 	return MapReduceWithSource(source, mapper, reducer, opts...)
 }
@@ -112,9 +115,15 @@ func MapReduceWithSource(source <-chan interface{}, mapper MapperFunc, reducer R
 	opts ...Option) (interface{}, error) {
 	options := buildOptions(opts...)
 	output := make(chan interface{})
+	defer func() {
+		for range output {
+			panic("more than one element written in reducer")
+		}
+	}()
+
 	collector := make(chan interface{}, options.workers)
 	done := syncx.NewDoneChan()
-	writer := newGuardedWriter(output, done.Done())
+	writer := newGuardedWriter(options.ctx, output, done.Done())
 	var closeOnce sync.Once
 	var retErr errorx.AtomicError
 	finish := func() {
@@ -148,7 +157,7 @@ func MapReduceWithSource(source <-chan interface{}, mapper MapperFunc, reducer R
 		reducer(collector, writer, cancel)
 	}()
 
-	go executeMappers(func(item interface{}, w Writer) {
+	go executeMappers(options.ctx, func(item interface{}, w Writer) {
 		mapper(item, w, cancel)
 	}, source, collector, done.Done(), options.workers)
 
@@ -181,6 +190,13 @@ func MapVoid(generate GenerateFunc, mapper VoidMapFunc, opts ...Option) {
 	}, opts...))
 }
 
+// WithContext customizes a mapreduce processing accepts a given ctx.
+func WithContext(ctx context.Context) Option {
+	return func(opts *mapReduceOptions) {
+		opts.ctx = ctx
+	}
+}
+
 // WithWorkers customizes a mapreduce processing with given workers.
 func WithWorkers(workers int) Option {
 	return func(opts *mapReduceOptions) {
@@ -218,8 +234,8 @@ func drain(channel <-chan interface{}) {
 	}
 }
 
-func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- interface{},
-	done <-chan lang.PlaceholderType, workers int) {
+func executeMappers(ctx context.Context, mapper MapFunc, input <-chan interface{},
+	collector chan<- interface{}, done <-chan lang.PlaceholderType, workers int) {
 	var wg sync.WaitGroup
 	defer func() {
 		wg.Wait()
@@ -227,9 +243,11 @@ func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- i
 	}()
 
 	pool := make(chan lang.PlaceholderType, workers)
-	writer := newGuardedWriter(collector, done)
+	writer := newGuardedWriter(ctx, collector, done)
 	for {
 		select {
+		case <-ctx.Done():
+			return
 		case <-done:
 			return
 		case pool <- lang.Placeholder:
@@ -255,6 +273,7 @@ func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- i
 
 func newOptions() *mapReduceOptions {
 	return &mapReduceOptions{
+		ctx:     context.Background(),
 		workers: defaultWorkers,
 	}
 }
@@ -269,12 +288,15 @@ func once(fn func(error)) func(error) {
 }
 
 type guardedWriter struct {
+	ctx     context.Context
 	channel chan<- interface{}
 	done    <-chan lang.PlaceholderType
 }
 
-func newGuardedWriter(channel chan<- interface{}, done <-chan lang.PlaceholderType) guardedWriter {
+func newGuardedWriter(ctx context.Context, channel chan<- interface{},
+	done <-chan lang.PlaceholderType) guardedWriter {
 	return guardedWriter{
+		ctx:     ctx,
 		channel: channel,
 		done:    done,
 	}
@@ -282,6 +304,8 @@ func newGuardedWriter(channel chan<- interface{}, done <-chan lang.PlaceholderTy
 
 func (gw guardedWriter) Write(v interface{}) {
 	select {
+	case <-gw.ctx.Done():
+		return
 	case <-gw.done:
 		return
 	default:

core/mr/mapreduce_test.go

@@ -1,6 +1,7 @@
 package mr
 
 import (
+	"context"
 	"errors"
 	"io/ioutil"
 	"log"
@@ -202,6 +203,22 @@ func TestMapReduce(t *testing.T) {
 	}
 }
 
+func TestMapReduceWithReduerWriteMoreThanOnce(t *testing.T) {
+	assert.Panics(t, func() {
+		MapReduce(func(source chan<- interface{}) {
+			for i := 0; i < 10; i++ {
+				source <- i
+			}
+		}, func(item interface{}, writer Writer, cancel func(error)) {
+			writer.Write(item)
+		}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+			drain(pipe)
+			writer.Write("one")
+			writer.Write("two")
+		})
+	})
+}
+
 func TestMapReduceVoid(t *testing.T) {
 	var value uint32
 	tests := []struct {
@@ -394,6 +411,50 @@ func TestMapReduceWithoutReducerWrite(t *testing.T) {
 	assert.Nil(t, res)
 }
 
+func TestMapReduceVoidPanicInReducer(t *testing.T) {
+	const message = "foo"
+	var done syncx.AtomicBool
+	err := MapReduceVoid(func(source chan<- interface{}) {
+		for i := 0; i < defaultWorkers*2; i++ {
+			source <- i
+		}
+		done.Set(true)
+	}, func(item interface{}, writer Writer, cancel func(error)) {
+		i := item.(int)
+		writer.Write(i)
+	}, func(pipe <-chan interface{}, cancel func(error)) {
+		panic(message)
+	}, WithWorkers(1))
+	assert.NotNil(t, err)
+	assert.Equal(t, message, err.Error())
+	assert.True(t, done.True())
+}
+
+func TestMapReduceWithContext(t *testing.T) {
+	var done syncx.AtomicBool
+	var result []int
+	ctx, cancel := context.WithCancel(context.Background())
+	err := MapReduceVoid(func(source chan<- interface{}) {
+		for i := 0; i < defaultWorkers*2; i++ {
+			source <- i
+		}
+		done.Set(true)
+	}, func(item interface{}, writer Writer, c func(error)) {
+		i := item.(int)
+		if i == defaultWorkers/2 {
+			cancel()
+		}
+		writer.Write(i)
+	}, func(pipe <-chan interface{}, cancel func(error)) {
+		for item := range pipe {
+			i := item.(int)
+			result = append(result, i)
+		}
+	}, WithContext(ctx))
+	assert.NotNil(t, err)
+	assert.Equal(t, ErrReduceNoOutput, err)
+}
+
 func BenchmarkMapReduce(b *testing.B) {
 	b.ReportAllocs()
 

core/proc/goroutines+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package proc

core/proc/goroutines.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc

core/proc/profile+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package proc

core/proc/profile.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc

core/proc/shutdown+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package proc
@@ -14,5 +15,5 @@ func AddWrapUpListener(fn func()) func() {
 	return fn
 }
 
-func SetTimeoutToForceQuit(duration time.Duration) {
+func SetTimeToForceQuit(duration time.Duration) {
 }

core/proc/shutdown.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/core/threading"
 )
 
 const (
@@ -45,10 +47,10 @@ func gracefulStop(signals chan os.Signal) {
 	signal.Stop(signals)
 
 	logx.Info("Got signal SIGTERM, shutting down...")
-	wrapUpListeners.notifyListeners()
+	go wrapUpListeners.notifyListeners()
 
 	time.Sleep(wrapUpTime)
-	shutdownListeners.notifyListeners()
+	go shutdownListeners.notifyListeners()
 
 	time.Sleep(delayTimeBeforeForceQuit - wrapUpTime)
 	logx.Infof("Still alive after %v, going to force kill the process...", delayTimeBeforeForceQuit)
@@ -80,7 +82,9 @@ func (lm *listenerManager) notifyListeners() {
 	lm.lock.Lock()
 	defer lm.lock.Unlock()
 
+	group := threading.NewRoutineGroup()
 	for _, listener := range lm.listeners {
-		listener()
+		group.RunSafe(listener)
 	}
+	group.Wait()
 }

core/proc/signals+polyfill.go

@@ -0,0 +1,10 @@
+//go:build windows
+// +build windows
+
+package proc
+
+import "context"
+
+func Done() <-chan struct{} {
+	return context.Background().Done()
+}

core/proc/signals.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc
@@ -12,6 +13,8 @@ import (
 
 const timeFormat = "0102150405"
 
+var done = make(chan struct{})
+
 func init() {
 	go func() {
 		var profiler Stopper
@@ -33,6 +36,13 @@ func init() {
 					profiler = nil
 				}
 			case syscall.SIGTERM:
+				select {
+				case <-done:
+					// already closed
+				default:
+					close(done)
+				}
+
 				gracefulStop(signals)
 			default:
 				logx.Error("Got unregistered signal:", v)
@@ -40,3 +50,8 @@ func init() {
 		}
 	}()
 }
+
+// Done returns the channel that notifies the process quitting.
+func Done() <-chan struct{} {
+	return done
+}

core/proc/signals_test.go

@@ -0,0 +1,16 @@
+package proc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDone(t *testing.T) {
+	select {
+	case <-Done():
+		assert.Fail(t, "should run")
+	default:
+	}
+	assert.NotNil(t, Done())
+}

core/prometheus/agent.go

@@ -23,11 +23,11 @@ func Enabled() bool {
 
 // StartAgent starts a prometheus agent.
 func StartAgent(c Config) {
-	once.Do(func() {
-		if len(c.Host) == 0 {
-			return
-		}
+	if len(c.Host) == 0 {
+		return
+	}
 
+	once.Do(func() {
 		enabled.Set(true)
 		threading.GoSafe(func() {
 			http.Handle(c.Path, promhttp.Handler())

core/search/tree_debug.go

@@ -1,3 +1,4 @@
+//go:build debug
 // +build debug
 
 package search

core/search/tree_test.go

@@ -1,9 +1,12 @@
 package search
 
 import (
+	"math/rand"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/stringx"
 )
 
 type mockedRoute struct {
@@ -139,11 +142,9 @@ func TestStrictSearchSibling(t *testing.T) {
 		tree.Add(r.route, r.value)
 	}
 
-	for i := 0; i < 1000; i++ {
-		result, ok := tree.Search(query)
-		assert.True(t, ok)
-		assert.Equal(t, 3, result.Item.(int))
-	}
+	result, ok := tree.Search(query)
+	assert.True(t, ok)
+	assert.Equal(t, 3, result.Item.(int))
 }
 
 func TestAddDuplicate(t *testing.T) {
@@ -185,3 +186,41 @@ func TestSearchInvalidItem(t *testing.T) {
 	err := tree.Add("/", nil)
 	assert.Equal(t, errEmptyItem, err)
 }
+
+func BenchmarkSearchTree(b *testing.B) {
+	const (
+		avgLen  = 1000
+		entries = 10000
+	)
+
+	tree := NewTree()
+	generate := func() string {
+		var buf strings.Builder
+		size := rand.Intn(avgLen) + avgLen/2
+		val := stringx.Randn(size)
+		prev := 0
+		for j := rand.Intn(9) + 1; j < size; j += rand.Intn(9) + 1 {
+			buf.WriteRune('/')
+			buf.WriteString(val[prev:j])
+			prev = j
+		}
+		if prev < size {
+			buf.WriteRune('/')
+			buf.WriteString(val[prev:])
+		}
+		return buf.String()
+	}
+	index := rand.Intn(entries)
+	var query string
+	for i := 0; i < entries; i++ {
+		val := generate()
+		if i == index {
+			query = val
+		}
+		tree.Add(val, i)
+	}
+
+	for i := 0; i < b.N; i++ {
+		tree.Search(query)
+	}
+}

core/service/serviceconf.go

@@ -7,6 +7,7 @@ import (
 	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/prometheus"
 	"github.com/tal-tech/go-zero/core/stat"
+	"github.com/tal-tech/go-zero/core/trace"
 )
 
 const (
@@ -29,6 +30,7 @@ type ServiceConf struct {
 	Mode       string            `json:",default=pro,options=dev|test|rt|pre|pro"`
 	MetricsUrl string            `json:",optional"`
 	Prometheus prometheus.Config `json:",optional"`
+	Telemetry  trace.Config      `json:",optional"`
 }
 
 // MustSetUp sets up the service, exits on error.
@@ -49,6 +51,12 @@ func (sc ServiceConf) SetUp() error {
 
 	sc.initMode()
 	prometheus.StartAgent(sc.Prometheus)
+
+	if len(sc.Telemetry.Name) == 0 {
+		sc.Telemetry.Name = sc.Name
+	}
+	trace.StartAgent(sc.Telemetry)
+
 	if len(sc.MetricsUrl) > 0 {
 		stat.SetReportWriter(stat.NewRemoteWriter(sc.MetricsUrl))
 	}

core/service/servicegroup.go

@@ -26,6 +26,7 @@ type (
 	}
 
 	// A ServiceGroup is a group of services.
+	// Attention: the starting order of the added services is not guaranteed.
 	ServiceGroup struct {
 		services []Service
 		stopOnce func()
@@ -41,7 +42,8 @@ func NewServiceGroup() *ServiceGroup {
 
 // Add adds service into sg.
 func (sg *ServiceGroup) Add(service Service) {
-	sg.services = append(sg.services, service)
+	// push front, stop with reverse order.
+	sg.services = append([]Service{service}, sg.services...)
 }
 
 // Start starts the ServiceGroup.

core/stat/alert+polyfill.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package stat

core/stat/alert.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package stat

core/stat/alert_test.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package stat

core/stat/internal/cpu_linux_test.go

@@ -7,7 +7,9 @@ import (
 )
 
 func TestRefreshCpu(t *testing.T) {
-	assert.True(t, RefreshCpu() >= 0)
+	assert.NotPanics(t, func() {
+		RefreshCpu()
+	})
 }
 
 func BenchmarkRefreshCpu(b *testing.B) {

core/stat/internal/cpu_other.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package internal

core/stat/usage.go

@@ -38,7 +38,9 @@ func init() {
 					atomic.StoreInt64(&cpuUsage, usage)
 				})
 			case <-allTicker.C:
-				printUsage()
+				if logEnabled.True() {
+					printUsage()
+				}
 			}
 		}
 	}()

core/stores/builder/builder.go

@@ -0,0 +1,63 @@
+package builder
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+)
+
+const dbTag = "db"
+
+// RawFieldNames converts golang struct field into slice string.
+func RawFieldNames(in interface{}, postgresSql ...bool) []string {
+	out := make([]string, 0)
+	v := reflect.ValueOf(in)
+	if v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+
+	var pg bool
+	if len(postgresSql) > 0 {
+		pg = postgresSql[0]
+	}
+
+	// we only accept structs
+	if v.Kind() != reflect.Struct {
+		panic(fmt.Errorf("ToMap only accepts structs; got %T", v))
+	}
+
+	typ := v.Type()
+	for i := 0; i < v.NumField(); i++ {
+		// gets us a StructField
+		fi := typ.Field(i)
+		if tagv := fi.Tag.Get(dbTag); tagv != "" {
+			if pg {
+				out = append(out, tagv)
+			} else {
+				out = append(out, fmt.Sprintf("`%s`", tagv))
+			}
+		} else {
+			if pg {
+				out = append(out, fi.Name)
+			} else {
+				out = append(out, fmt.Sprintf("`%s`", fi.Name))
+			}
+		}
+	}
+
+	return out
+}
+
+// PostgreSqlJoin concatenates the given elements into a string.
+func PostgreSqlJoin(elems []string) string {
+	b := new(strings.Builder)
+	for index, e := range elems {
+		b.WriteString(fmt.Sprintf("%s = $%d, ", e, index+2))
+	}
+
+	if b.Len() == 0 {
+		return b.String()
+	}
+
+	return b.String()[0 : b.Len()-2]
+}

core/stores/builder/builder_test.go

@@ -0,0 +1,24 @@
+package builder
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mockedUser struct {
+	ID       string `db:"id" json:"id,omitempty"`
+	UserName string `db:"user_name" json:"userName,omitempty"`
+	Sex      int    `db:"sex" json:"sex,omitempty"`
+	UUID     string `db:"uuid" uuid:"uuid,omitempty"`
+	Age      int    `db:"age" json:"age"`
+}
+
+func TestFieldNames(t *testing.T) {
+	t.Run("new", func(t *testing.T) {
+		var u mockedUser
+		out := RawFieldNames(&u)
+		expected := []string{"`id`", "`user_name`", "`sex`", "`uuid`", "`age`"}
+		assert.Equal(t, expected, out)
+	})
+}

core/stores/cache/cache.go

@@ -29,7 +29,7 @@ type (
 )
 
 // New returns a Cache.
-func New(c ClusterConf, barrier syncx.SharedCalls, st *Stat, errNotFound error,
+func New(c ClusterConf, barrier syncx.SingleFlight, st *Stat, errNotFound error,
 	opts ...Option) Cache {
 	if len(c) == 0 || TotalWeights(c) <= 0 {
 		log.Fatal("no cache nodes")

core/stores/cache/cache_test.go

@@ -23,6 +23,7 @@ type mockedNode struct {
 
 func (mc *mockedNode) Del(keys ...string) error {
 	var be errorx.BatchError
+
 	for _, key := range keys {
 		if _, ok := mc.vals[key]; !ok {
 			be.Add(mc.errNotFound)
@@ -30,6 +31,7 @@ func (mc *mockedNode) Del(keys ...string) error {
 			delete(mc.vals, key)
 		}
 	}
+
 	return be.Err()
 }
 
@@ -102,7 +104,7 @@ func TestCache_SetDel(t *testing.T) {
 			Weight: 100,
 		},
 	}
-	c := New(conf, syncx.NewSharedCalls(), NewStat("mock"), errPlaceholder)
+	c := New(conf, syncx.NewSingleFlight(), NewStat("mock"), errPlaceholder)
 	for i := 0; i < total; i++ {
 		if i%2 == 0 {
 			assert.Nil(t, c.Set(fmt.Sprintf("key/%d", i), i))
@@ -140,7 +142,7 @@ func TestCache_OneNode(t *testing.T) {
 			Weight: 100,
 		},
 	}
-	c := New(conf, syncx.NewSharedCalls(), NewStat("mock"), errPlaceholder)
+	c := New(conf, syncx.NewSingleFlight(), NewStat("mock"), errPlaceholder)
 	for i := 0; i < total; i++ {
 		if i%2 == 0 {
 			assert.Nil(t, c.Set(fmt.Sprintf("key/%d", i), i))

core/stores/cache/cachenode.go

@@ -29,7 +29,7 @@ type cacheNode struct {
 	rds            *redis.Redis
 	expiry         time.Duration
 	notFoundExpiry time.Duration
-	barrier        syncx.SharedCalls
+	barrier        syncx.SingleFlight
 	r              *rand.Rand
 	lock           *sync.Mutex
 	unstableExpiry mathx.Unstable
@@ -43,7 +43,7 @@ type cacheNode struct {
 // st is used to stat the cache.
 // errNotFound defines the error that returned on cache not found.
 // opts are the options that customize the cacheNode.
-func NewNode(rds *redis.Redis, barrier syncx.SharedCalls, st *Stat,
+func NewNode(rds *redis.Redis, barrier syncx.SingleFlight, st *Stat,
 	errNotFound error, opts ...Option) Cache {
 	o := newOptions(opts...)
 	return cacheNode{
@@ -65,9 +65,18 @@ func (c cacheNode) Del(keys ...string) error {
 		return nil
 	}
 
-	if _, err := c.rds.Del(keys...); err != nil {
-		logx.Errorf("failed to clear cache with keys: %q, error: %v", formatKeys(keys), err)
-		c.asyncRetryDelCache(keys...)
+	if len(keys) > 1 && c.rds.Type == redis.ClusterType {
+		for _, key := range keys {
+			if _, err := c.rds.Del(key); err != nil {
+				logx.Errorf("failed to clear cache with key: %q, error: %v", key, err)
+				c.asyncRetryDelCache(key)
+			}
+		}
+	} else {
+		if _, err := c.rds.Del(keys...); err != nil {
+			logx.Errorf("failed to clear cache with keys: %q, error: %v", formatKeys(keys), err)
+			c.asyncRetryDelCache(keys...)
+		}
 	}
 
 	return nil

core/stores/cache/cachenode_test.go

@@ -29,6 +29,7 @@ func init() {
 func TestCacheNode_DelCache(t *testing.T) {
 	store, clean, err := redistest.CreateRedis()
 	assert.Nil(t, err)
+	store.Type = redis.ClusterType
 	defer clean()
 
 	cn := cacheNode{
@@ -49,13 +50,30 @@ func TestCacheNode_DelCache(t *testing.T) {
 	assert.Nil(t, cn.Del("first", "second"))
 }
 
+func TestCacheNode_DelCacheWithErrors(t *testing.T) {
+	store, clean, err := redistest.CreateRedis()
+	assert.Nil(t, err)
+	defer clean()
+	store.Type = redis.ClusterType
+
+	cn := cacheNode{
+		rds:            store,
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewStat("any"),
+		errNotFound:    errTestNotFound,
+	}
+	assert.Nil(t, cn.Del("third", "fourth"))
+}
+
 func TestCacheNode_InvalidCache(t *testing.T) {
 	s, err := miniredis.Run()
 	assert.Nil(t, err)
 	defer s.Close()
 
 	cn := cacheNode{
-		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		rds:            redis.New(s.Addr()),
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
@@ -78,7 +96,7 @@ func TestCacheNode_Take(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -105,7 +123,7 @@ func TestCacheNode_TakeNotFound(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -144,7 +162,7 @@ func TestCacheNode_TakeWithExpire(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -171,7 +189,7 @@ func TestCacheNode_String(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -188,7 +206,7 @@ func TestCacheValueWithBigInt(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),

core/stores/clickhouse/clickhouse.go

@@ -1,7 +1,7 @@
 package clickhouse
 
 import (
-	// imports the driver.
+	// imports the driver, don't remove this comment, golint requires.
 	_ "github.com/ClickHouse/clickhouse-go"
 	"github.com/tal-tech/go-zero/core/stores/sqlx"
 )

core/stores/kv/store.go

@@ -16,6 +16,8 @@ var ErrNoRedisNode = errors.New("no redis node")
 type (
 	// Store interface represents a KV store.
 	Store interface {
+		Decr(key string) (int64, error)
+		Decrby(key string, increment int64) (int64, error)
 		Del(keys ...string) (int, error)
 		Eval(script, key string, args ...interface{}) (interface{}, error)
 		Exists(key string) (bool, error)
@@ -36,6 +38,7 @@ type (
 		Hvals(key string) ([]string, error)
 		Incr(key string) (int64, error)
 		Incrby(key string, increment int64) (int64, error)
+		Lindex(key string, index int64) (string, error)
 		Llen(key string) (int, error)
 		Lpop(key string) (string, error)
 		Lpush(key string, values ...interface{}) (int, error)
@@ -102,6 +105,24 @@ func NewStore(c KvConf) Store {
 	}
 }
 
+func (cs clusterStore) Decr(key string) (int64, error) {
+	node, err := cs.getRedis(key)
+	if err != nil {
+		return 0, err
+	}
+
+	return node.Decr(key)
+}
+
+func (cs clusterStore) Decrby(key string, increment int64) (int64, error) {
+	node, err := cs.getRedis(key)
+	if err != nil {
+		return 0, err
+	}
+
+	return node.Decrby(key, increment)
+}
+
 func (cs clusterStore) Del(keys ...string) (int, error) {
 	var val int
 	var be errorx.BatchError
@@ -303,6 +324,15 @@ func (cs clusterStore) Llen(key string) (int, error) {
 	return node.Llen(key)
 }
 
+func (cs clusterStore) Lindex(key string, index int64) (string, error) {
+	node, err := cs.getRedis(key)
+	if err != nil {
+		return "", err
+	}
+
+	return node.Lindex(key, index)
+}
+
 func (cs clusterStore) Lpop(key string) (string, error) {
 	node, err := cs.getRedis(key)
 	if err != nil {

core/stores/kv/store_test.go

@@ -17,6 +17,36 @@ var (
 	s2, _ = miniredis.Run()
 )
 
+func TestRedis_Decr(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Decr("a")
+	assert.NotNil(t, err)
+
+	runOnCluster(t, func(client Store) {
+		val, err := client.Decr("a")
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-1), val)
+		val, err = client.Decr("a")
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-2), val)
+	})
+}
+
+func TestRedis_DecrBy(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Incrby("a", 2)
+	assert.NotNil(t, err)
+
+	runOnCluster(t, func(client Store) {
+		val, err := client.Decrby("a", 2)
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-2), val)
+		val, err = client.Decrby("a", 3)
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-5), val)
+	})
+}
+
 func TestRedis_Exists(t *testing.T) {
 	store := clusterStore{dispatcher: hash.NewConsistentHash()}
 	_, err := store.Exists("foo")
@@ -234,6 +264,8 @@ func TestRedis_List(t *testing.T) {
 	assert.NotNil(t, err)
 	_, err = store.Lrem("key", 0, "val")
 	assert.NotNil(t, err)
+	_, err = store.Lindex("key", 0)
+	assert.NotNil(t, err)
 
 	runOnCluster(t, func(client Store) {
 		val, err := client.Lpush("key", "value1", "value2")
@@ -245,6 +277,9 @@ func TestRedis_List(t *testing.T) {
 		val, err = client.Llen("key")
 		assert.Nil(t, err)
 		assert.Equal(t, 4, val)
+		value, err := client.Lindex("key", 0)
+		assert.Nil(t, err)
+		assert.Equal(t, "value2", value)
 		vals, err := client.Lrange("key", 0, 10)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"value2", "value1", "value3", "value4"}, vals)
@@ -667,10 +702,12 @@ func TestRedis_HyperLogLog(t *testing.T) {
 	assert.NotNil(t, err)
 
 	runOnCluster(t, func(cluster Store) {
-		_, err := cluster.Pfadd("key")
-		assert.NotNil(t, err)
-		_, err = cluster.Pfcount("key")
-		assert.NotNil(t, err)
+		ok, err := cluster.Pfadd("key", "value")
+		assert.Nil(t, err)
+		assert.True(t, ok)
+		val, err := cluster.Pfcount("key")
+		assert.Nil(t, err)
+		assert.Equal(t, int64(1), val)
 	})
 }
 

core/stores/mongo/collection.go

@@ -11,7 +11,7 @@ import (
 	"github.com/tal-tech/go-zero/core/timex"
 )
 
-const slowThreshold = time.Millisecond * 500
+const defaultSlowThreshold = time.Millisecond * 500
 
 // ErrNotFound is an alias of mgo.ErrNotFound.
 var ErrNotFound = mgo.ErrNotFound
@@ -203,7 +203,7 @@ func (c *decoratedCollection) logDuration(method string, duration time.Duration,
 	if e != nil {
 		logx.Error(err)
 	} else if err != nil {
-		if duration > slowThreshold {
+		if duration > slowThreshold.Load() {
 			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - fail(%s) - %s",
 				c.name, method, err.Error(), string(content))
 		} else {
@@ -211,7 +211,7 @@ func (c *decoratedCollection) logDuration(method string, duration time.Duration,
 				c.name, method, err.Error(), string(content))
 		}
 	} else {
-		if duration > slowThreshold {
+		if duration > slowThreshold.Load() {
 			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - ok - %s",
 				c.name, method, string(content))
 		} else {

core/stores/mongo/internal/collection_mock.go

@@ -5,9 +5,10 @@
 package internal
 
 import (
+	reflect "reflect"
+
 	mgo "github.com/globalsign/mgo"
 	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
 )
 
 // MockMgoCollection is a mock of MgoCollection interface

core/stores/mongo/iter_mock.go

@@ -5,9 +5,10 @@
 package mongo
 
 import (
+	reflect "reflect"
+
 	bson "github.com/globalsign/mgo/bson"
 	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
 )
 
 // MockIter is a mock of Iter interface

core/stores/mongo/model.go

@@ -8,23 +8,14 @@ import (
 	"github.com/tal-tech/go-zero/core/breaker"
 )
 
-type (
-	options struct {
-		timeout time.Duration
-	}
-
-	// Option defines the method to customize a mongo model.
-	Option func(opts *options)
-
-	// A Model is a mongo model.
-	Model struct {
-		session    *concurrentSession
-		db         *mgo.Database
-		collection string
-		brk        breaker.Breaker
-		opts       []Option
-	}
-)
+// A Model is a mongo model.
+type Model struct {
+	session    *concurrentSession
+	db         *mgo.Database
+	collection string
+	brk        breaker.Breaker
+	opts       []Option
+}
 
 // MustNewModel returns a Model, exits on errors.
 func MustNewModel(url, collection string, opts ...Option) *Model {

core/stores/mongo/model_test.go

@@ -0,0 +1,14 @@
+package mongo
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWithTimeout(t *testing.T) {
+	o := defaultOptions()
+	WithTimeout(time.Second)(o)
+	assert.Equal(t, time.Second, o.timeout)
+}

core/stores/mongo/options.go

@@ -0,0 +1,29 @@
+package mongo
+
+import (
+	"time"
+
+	"github.com/tal-tech/go-zero/core/syncx"
+)
+
+var slowThreshold = syncx.ForAtomicDuration(defaultSlowThreshold)
+
+type (
+	options struct {
+		timeout time.Duration
+	}
+
+	// Option defines the method to customize a mongo model.
+	Option func(opts *options)
+)
+
+// SetSlowThreshold sets the slow threshold.
+func SetSlowThreshold(threshold time.Duration) {
+	slowThreshold.Set(threshold)
+}
+
+func defaultOptions() *options {
+	return &options{
+		timeout: defaultTimeout,
+	}
+}

core/stores/mongo/options_test.go

@@ -0,0 +1,14 @@
+package mongo
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSetSlowThreshold(t *testing.T) {
+	assert.Equal(t, defaultSlowThreshold, slowThreshold.Load())
+	SetSlowThreshold(time.Second)
+	assert.Equal(t, time.Second, slowThreshold.Load())
+}

core/stores/mongo/sessionmanager.go

@@ -57,9 +57,7 @@ func (cs *concurrentSession) putSession(session *mgo.Session) {
 }
 
 func (cs *concurrentSession) takeSession(opts ...Option) (*mgo.Session, error) {
-	o := &options{
-		timeout: defaultTimeout,
-	}
+	o := defaultOptions()
 	for _, opt := range opts {
 		opt(o)
 	}