docs: update roadmap (#1405)

* feat: support tls for etcd client

* chore: fix typo

* refactor: rename TrustedCAFile to CACertFile

* docs: add comments

* fix: missing tls registration

* feat: add InsecureSkipVerify config for testing

.codecov.yml

@@ -1,4 +1,3 @@
+comment: false
 ignore:
-  - "doc"
-  - "example"
-  - "tools"
+  - "tools"

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: https://gitee.com/kevwan/static/raw/master/images/sponsor.jpg

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior, if applicable:
+
+1. The code is
+
+   ```go
+   
+   ```
+
+2. The error is
+
+   ```
+   
+   ```
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environments (please complete the following information):**
+ - OS: [e.g. Linux]
+ - go-zero version [e.g. 1.2.1]
+ - goctl version [e.g. 1.2.1, optional]
+
+**More description**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: Question
+about: Ask a question on using go-zero or goctl
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/workflows/go.yml

@@ -15,7 +15,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.13
+        go-version: ^1.14
       id: go
 
     - name: Check out code into the Go module directory
@@ -25,10 +25,14 @@ jobs:
       run: |
         go get -v -t -d ./...
 
+    - name: Lint
+      run: |
+        go vet -stdmethods=false $(go list ./...)
+        go install mvdan.cc/gofumpt@latest
+        test -z "$(gofumpt -s -l -extra .)" || echo "Please run 'gofumpt -l -w -extra .'"
+
     - name: Test
       run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
 
     - name: Codecov
-      uses: codecov/codecov-action@v1.0.6
-      with:
-        token: ${{secrets.CODECOV_TOKEN}}
+      uses: codecov/codecov-action@v2

.github/workflows/issues.yml

@@ -0,0 +1,19 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v3
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/reviewdog.yml

@@ -0,0 +1,19 @@
+name: reviewdog
+on: [pull_request]
+jobs:
+  staticcheck:
+    name: runner / staticcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: reviewdog/action-staticcheck@v1
+        with:
+          github_token: ${{ secrets.github_token }}
+          # Change reviewdog reporter if you need [github-pr-check,github-check,github-pr-review].
+          reporter: github-pr-review
+          # Report all results.
+          filter_mode: nofilter
+          # Exit with 1 when it find at least one finding.
+          fail_on_error: true
+          # Set staticcheck flags
+          staticcheck_flags: -checks=inherit,-SA1019,-SA1029,-SA5008

.gitignore

@@ -10,10 +10,14 @@
 !*/
 !api
 
+# ignore
 .idea
 **/.DS_Store
 **/logs
 
+# for test purpose
+adhoc
+
 # gitlab ci
 .cache
 

CONTRIBUTING.md

@@ -90,7 +90,7 @@ After that, run these local verifications before submitting pull request to pred
 fail of continuous integration.
 
 * Format the code with `gofmt`
-* Run the test with data race enabled `go test -race ./…`
+* Run the test with data race enabled `go test -race ./...`
 
 ## Code Review
 

ROADMAP.md

@@ -5,17 +5,24 @@ Community and contributor involvement is vital for successfully implementing all
 We hope that the items listed below will inspire further engagement from the community to keep go-zero progressing and shipping exciting and valuable features.
 
 ## 2021 Q2
-- Support TLS in redis connections
-- Support service discovery through K8S watch api
-- Log full sql statements for easier sql problem solving
+- [x] Support service discovery through K8S client api
+- [x] Log full sql statements for easier sql problem solving
 
 ## 2021 Q3
-- Support `goctl mock` command to start a mocking server with given `.api` file
-- Adapt builtin tracing mechanism to opentracing solutions
-- Support `goctl model pg` to support PostgreSQL code generation
+- [x] Support `goctl model pg` to support PostgreSQL code generation
+- [x] Adapt builtin tracing mechanism to opentracing solutions
 
 ## 2021 Q4
-- Support `goctl doctor` command to report potential issues for given service
-- Support `context` in redis related methods for timeout and tracing
-- Support `context` in sql related methods for timeout and tracing
-- Support `context` in mongodb related methods for timeout and tracing
+- [x] Support `username/password` authentication in ETCD
+- [x] Support `SSL/TLS` in ETCD
+- [x] Support `SSL/TLS` in `zRPC`
+- [x] Support `TLS` in redis connections
+- [x] Support `goctl bug` to report bugs conveniently
+
+## 2022
+- [ ] Support `goctl mock` command to start a mocking server with given `.api` file
+- [ ] Add `httpx.Client` with governance, like circuit breaker etc.
+- [ ] Support `goctl doctor` command to report potential issues for given service
+- [ ] Support `context` in redis related methods for timeout and tracing
+- [ ] Support `context` in sql related methods for timeout and tracing
+- [ ] Support `context` in mongodb related methods for timeout and tracing

core/collection/cache.go

@@ -34,7 +34,7 @@ type (
 		expire         time.Duration
 		timingWheel    *TimingWheel
 		lruCache       lru
-		barrier        syncx.SharedCalls
+		barrier        syncx.SingleFlight
 		unstableExpiry mathx.Unstable
 		stats          *cacheStat
 	}
@@ -46,7 +46,7 @@ func NewCache(expire time.Duration, opts ...CacheOption) (*Cache, error) {
 		data:           make(map[string]interface{}),
 		expire:         expire,
 		lruCache:       emptyLruCache,
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 	}
 

core/contextx/unmarshaler_test.go

@@ -47,9 +47,11 @@ func TestUnmarshalContextWithMissing(t *testing.T) {
 		Name string `ctx:"name"`
 		Age  int    `ctx:"age"`
 	}
+	type name string
+	const PersonNameKey name = "name"
 
 	ctx := context.Background()
-	ctx = context.WithValue(ctx, "name", "kevin")
+	ctx = context.WithValue(ctx, PersonNameKey, "kevin")
 
 	var person Person
 	err := For(ctx, &person)

core/contextx/valueonlycontext_test.go

@@ -9,7 +9,9 @@ import (
 )
 
 func TestContextCancel(t *testing.T) {
-	c := context.WithValue(context.Background(), "key", "value")
+	type key string
+	var nameKey key = "name"
+	c := context.WithValue(context.Background(), nameKey, "value")
 	c1, cancel := context.WithCancel(c)
 	o := ValueOnlyFrom(c1)
 	c2, cancel2 := context.WithCancel(o)

core/discov/accountregistry.go

@@ -0,0 +1,14 @@
+package discov
+
+import "github.com/tal-tech/go-zero/core/discov/internal"
+
+// RegisterAccount registers the username/password to the given etcd cluster.
+func RegisterAccount(endpoints []string, user, pass string) {
+	internal.AddAccount(endpoints, user, pass)
+}
+
+// RegisterTLS registers the CertFile/CertKeyFile/CACertFile to the given etcd.
+func RegisterTLS(endpoints []string, certFile, certKeyFile, caFile string,
+	insecureSkipVerify bool) error {
+	return internal.AddTLS(endpoints, certFile, certKeyFile, caFile, insecureSkipVerify)
+}

core/discov/accountregistry_test.go

@@ -0,0 +1,21 @@
+package discov
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/discov/internal"
+	"github.com/tal-tech/go-zero/core/stringx"
+)
+
+func TestRegisterAccount(t *testing.T) {
+	endpoints := []string{
+		"localhost:2379",
+	}
+	user := "foo" + stringx.Rand()
+	RegisterAccount(endpoints, user, "bar")
+	account, ok := internal.GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, user, account.User)
+	assert.Equal(t, "bar", account.Pass)
+}

core/discov/config.go

@@ -4,8 +4,24 @@ import "errors"
 
 // EtcdConf is the config item with the given key on etcd.
 type EtcdConf struct {
-	Hosts []string
-	Key   string
+	Hosts              []string
+	Key                string
+	User               string `json:",optional"`
+	Pass               string `json:",optional"`
+	CertFile           string `json:",optional"`
+	CertKeyFile        string `json:",optional=CertFile"`
+	CACertFile         string `json:",optional=CertFile"`
+	InsecureSkipVerify bool   `json:",optional"`
+}
+
+// HasAccount returns if account provided.
+func (c EtcdConf) HasAccount() bool {
+	return len(c.User) > 0 && len(c.Pass) > 0
+}
+
+// HasTLS returns if TLS CertFile/CertKeyFile/CACertFile are provided.
+func (c EtcdConf) HasTLS() bool {
+	return len(c.CertFile) > 0 && len(c.CertKeyFile) > 0 && len(c.CACertFile) > 0
 }
 
 // Validate validates c.

core/discov/config_test.go

@@ -44,3 +44,39 @@ func TestConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestEtcdConf_HasAccount(t *testing.T) {
+	tests := []struct {
+		EtcdConf
+		hasAccount bool
+	}{
+		{
+			EtcdConf: EtcdConf{
+				Hosts: []string{"any"},
+				Key:   "key",
+			},
+			hasAccount: false,
+		},
+		{
+			EtcdConf: EtcdConf{
+				Hosts: []string{"any"},
+				Key:   "key",
+				User:  "foo",
+			},
+			hasAccount: false,
+		},
+		{
+			EtcdConf: EtcdConf{
+				Hosts: []string{"any"},
+				Key:   "key",
+				User:  "foo",
+				Pass:  "bar",
+			},
+			hasAccount: true,
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.hasAccount, test.EtcdConf.HasAccount())
+	}
+}

core/discov/internal/accountmanager.go

@@ -0,0 +1,75 @@
+package internal
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"sync"
+)
+
+var (
+	accounts   = make(map[string]Account)
+	tlsConfigs = make(map[string]*tls.Config)
+	lock       sync.RWMutex
+)
+
+// Account holds the username/password for an etcd cluster.
+type Account struct {
+	User string
+	Pass string
+}
+
+// AddAccount adds the username/password for the given etcd cluster.
+func AddAccount(endpoints []string, user, pass string) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	accounts[getClusterKey(endpoints)] = Account{
+		User: user,
+		Pass: pass,
+	}
+}
+
+// AddTLS adds the tls cert files for the given etcd cluster.
+func AddTLS(endpoints []string, certFile, certKeyFile, caFile string, insecureSkipVerify bool) error {
+	cert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
+	if err != nil {
+		return err
+	}
+
+	caData, err := ioutil.ReadFile(caFile)
+	if err != nil {
+		return err
+	}
+
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(caData)
+
+	lock.Lock()
+	defer lock.Unlock()
+	tlsConfigs[getClusterKey(endpoints)] = &tls.Config{
+		Certificates:       []tls.Certificate{cert},
+		RootCAs:            pool,
+		InsecureSkipVerify: insecureSkipVerify,
+	}
+
+	return nil
+}
+
+// GetAccount gets the username/password for the given etcd cluster.
+func GetAccount(endpoints []string) (Account, bool) {
+	lock.RLock()
+	defer lock.RUnlock()
+
+	account, ok := accounts[getClusterKey(endpoints)]
+	return account, ok
+}
+
+// GetTLS gets the tls config for the given etcd cluster.
+func GetTLS(endpoints []string) (*tls.Config, bool) {
+	lock.RLock()
+	defer lock.RUnlock()
+
+	cfg, ok := tlsConfigs[getClusterKey(endpoints)]
+	return cfg, ok
+}

core/discov/internal/accountmanager_test.go

@@ -0,0 +1,34 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/stringx"
+)
+
+func TestAccount(t *testing.T) {
+	endpoints := []string{
+		"192.168.0.2:2379",
+		"192.168.0.3:2379",
+		"192.168.0.4:2379",
+	}
+	username := "foo" + stringx.Rand()
+	password := "bar"
+	anotherPassword := "any"
+
+	_, ok := GetAccount(endpoints)
+	assert.False(t, ok)
+
+	AddAccount(endpoints, username, password)
+	account, ok := GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, username, account.User)
+	assert.Equal(t, password, account.Pass)
+
+	AddAccount(endpoints, username, anotherPassword)
+	account, ok = GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, username, account.User)
+	assert.Equal(t, anotherPassword, account.Pass)
+}

core/discov/internal/registry.go

@@ -37,25 +37,35 @@ func GetRegistry() *Registry {
 
 // GetConn returns an etcd client connection associated with given endpoints.
 func (r *Registry) GetConn(endpoints []string) (EtcdClient, error) {
-	return r.getCluster(endpoints).getClient()
+	c, _ := r.getCluster(endpoints)
+	return c.getClient()
 }
 
 // Monitor monitors the key on given etcd endpoints, notify with the given UpdateListener.
 func (r *Registry) Monitor(endpoints []string, key string, l UpdateListener) error {
-	return r.getCluster(endpoints).monitor(key, l)
+	c, exists := r.getCluster(endpoints)
+	// if exists, the existing values should be updated to the listener.
+	if exists {
+		kvs := c.getCurrent(key)
+		for _, kv := range kvs {
+			l.OnAdd(kv)
+		}
+	}
+
+	return c.monitor(key, l)
 }
 
-func (r *Registry) getCluster(endpoints []string) *cluster {
+func (r *Registry) getCluster(endpoints []string) (c *cluster, exists bool) {
 	clusterKey := getClusterKey(endpoints)
 	r.lock.Lock()
 	defer r.lock.Unlock()
-	c, ok := r.clusters[clusterKey]
-	if !ok {
+	c, exists = r.clusters[clusterKey]
+	if !exists {
 		c = newCluster(endpoints)
 		r.clusters[clusterKey] = c
 	}
 
-	return c
+	return
 }
 
 type cluster struct {
@@ -94,6 +104,21 @@ func (c *cluster) getClient() (EtcdClient, error) {
 	return val.(EtcdClient), nil
 }
 
+func (c *cluster) getCurrent(key string) []KV {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	var kvs []KV
+	for k, v := range c.values[key] {
+		kvs = append(kvs, KV{
+			Key: k,
+			Val: v,
+		})
+	}
+
+	return kvs
+}
+
 func (c *cluster) handleChanges(key string, kvs []KV) {
 	var add []KV
 	var remove []KV
@@ -197,14 +222,12 @@ func (c *cluster) load(cli EtcdClient, key string) {
 	}
 
 	var kvs []KV
-	c.lock.Lock()
 	for _, ev := range resp.Kvs {
 		kvs = append(kvs, KV{
 			Key: string(ev.Key),
 			Val: string(ev.Value),
 		})
 	}
-	c.lock.Unlock()
 
 	c.handleChanges(key, kvs)
 }
@@ -302,14 +325,23 @@ func (c *cluster) watchConnState(cli EtcdClient) {
 
 // DialClient dials an etcd cluster with given endpoints.
 func DialClient(endpoints []string) (EtcdClient, error) {
-	return clientv3.New(clientv3.Config{
+	cfg := clientv3.Config{
 		Endpoints:            endpoints,
 		AutoSyncInterval:     autoSyncInterval,
 		DialTimeout:          DialTimeout,
 		DialKeepAliveTime:    dialKeepAliveTime,
 		DialKeepAliveTimeout: DialTimeout,
 		RejectOldCluster:     true,
-	})
+	}
+	if account, ok := GetAccount(endpoints); ok {
+		cfg.Username = account.User
+		cfg.Password = account.Pass
+	}
+	if tlsCfg, ok := GetTLS(endpoints); ok {
+		cfg.TLS = tlsCfg
+	}
+
+	return clientv3.New(cfg)
 }
 
 func getClusterKey(endpoints []string) string {

core/discov/internal/registry_test.go

@@ -33,9 +33,10 @@ func setMockClient(cli EtcdClient) func() {
 }
 
 func TestGetCluster(t *testing.T) {
-	c1 := GetRegistry().getCluster([]string{"first"})
-	c2 := GetRegistry().getCluster([]string{"second"})
-	c3 := GetRegistry().getCluster([]string{"first"})
+	AddAccount([]string{"first"}, "foo", "bar")
+	c1, _ := GetRegistry().getCluster([]string{"first"})
+	c2, _ := GetRegistry().getCluster([]string{"second"})
+	c3, _ := GetRegistry().getCluster([]string{"first"})
 	assert.Equal(t, c1, c3)
 	assert.NotEqual(t, c1, c2)
 }

core/discov/internal/statewatcher_mock.go

@@ -6,9 +6,10 @@ package internal
 
 import (
 	context "context"
+	reflect "reflect"
+
 	gomock "github.com/golang/mock/gomock"
 	connectivity "google.golang.org/grpc/connectivity"
-	reflect "reflect"
 )
 
 // MocketcdConn is a mock of etcdConn interface

core/discov/internal/updatelistener_mock.go

@@ -5,8 +5,9 @@
 package internal
 
 import (
-	gomock "github.com/golang/mock/gomock"
 	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
 )
 
 // MockUpdateListener is a mock of UpdateListener interface

core/discov/publisher.go

@@ -11,8 +11,8 @@ import (
 )
 
 type (
-	// PublisherOption defines the method to customize a Publisher.
-	PublisherOption func(client *Publisher)
+	// PubOption defines the method to customize a Publisher.
+	PubOption func(client *Publisher)
 
 	// A Publisher can be used to publish the value to an etcd cluster on the given key.
 	Publisher struct {
@@ -32,7 +32,7 @@ type (
 // endpoints is the hosts of the etcd cluster.
 // key:value are a pair to be published.
 // opts are used to customize the Publisher.
-func NewPublisher(endpoints []string, key, value string, opts ...PublisherOption) *Publisher {
+func NewPublisher(endpoints []string, key, value string, opts ...PubOption) *Publisher {
 	publisher := &Publisher{
 		endpoints:  endpoints,
 		key:        key,
@@ -146,8 +146,22 @@ func (p *Publisher) revoke(cli internal.EtcdClient) {
 }
 
 // WithId customizes a Publisher with the id.
-func WithId(id int64) PublisherOption {
+func WithId(id int64) PubOption {
 	return func(publisher *Publisher) {
 		publisher.id = id
 	}
 }
+
+// WithPubEtcdAccount provides the etcd username/password.
+func WithPubEtcdAccount(user, pass string) PubOption {
+	return func(pub *Publisher) {
+		RegisterAccount(pub.endpoints, user, pass)
+	}
+}
+
+// WithPubEtcdTLS provides the etcd CertFile/CertKeyFile/CACertFile.
+func WithPubEtcdTLS(certFile, certKeyFile, caFile string, insecureSkipVerify bool) PubOption {
+	return func(pub *Publisher) {
+		logx.Must(RegisterTLS(pub.endpoints, certFile, certKeyFile, caFile, insecureSkipVerify))
+	}
+}

core/discov/publisher_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/tal-tech/go-zero/core/discov/internal"
 	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/core/stringx"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
@@ -30,7 +31,8 @@ func TestPublisher_register(t *testing.T) {
 		ID: id,
 	}, nil)
 	cli.EXPECT().Put(gomock.Any(), makeEtcdKey("thekey", id), "thevalue", gomock.Any())
-	pub := NewPublisher(nil, "thekey", "thevalue")
+	pub := NewPublisher(nil, "thekey", "thevalue",
+		WithPubEtcdAccount(stringx.Rand(), "bar"))
 	_, err := pub.register(cli)
 	assert.Nil(t, err)
 }

core/discov/subscriber.go

@@ -5,20 +5,19 @@ import (
 	"sync/atomic"
 
 	"github.com/tal-tech/go-zero/core/discov/internal"
+	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/syncx"
 )
 
 type (
-	subOptions struct {
-		exclusive bool
-	}
-
 	// SubOption defines the method to customize a Subscriber.
-	SubOption func(opts *subOptions)
+	SubOption func(sub *Subscriber)
 
 	// A Subscriber is used to subscribe the given key on a etcd cluster.
 	Subscriber struct {
-		items *container
+		endpoints []string
+		exclusive bool
+		items     *container
 	}
 )
 
@@ -27,14 +26,14 @@ type (
 // key is the key to subscribe.
 // opts are used to customize the Subscriber.
 func NewSubscriber(endpoints []string, key string, opts ...SubOption) (*Subscriber, error) {
-	var subOpts subOptions
-	for _, opt := range opts {
-		opt(&subOpts)
-	}
-
 	sub := &Subscriber{
-		items: newContainer(subOpts.exclusive),
+		endpoints: endpoints,
 	}
+	for _, opt := range opts {
+		opt(sub)
+	}
+	sub.items = newContainer(sub.exclusive)
+
 	if err := internal.GetRegistry().Monitor(endpoints, key, sub.items); err != nil {
 		return nil, err
 	}
@@ -55,8 +54,22 @@ func (s *Subscriber) Values() []string {
 // Exclusive means that key value can only be 1:1,
 // which means later added value will remove the keys associated with the same value previously.
 func Exclusive() SubOption {
-	return func(opts *subOptions) {
-		opts.exclusive = true
+	return func(sub *Subscriber) {
+		sub.exclusive = true
+	}
+}
+
+// WithSubEtcdAccount provides the etcd username/password.
+func WithSubEtcdAccount(user, pass string) SubOption {
+	return func(sub *Subscriber) {
+		RegisterAccount(sub.endpoints, user, pass)
+	}
+}
+
+// WithSubEtcdTLS provides the etcd CertFile/CertKeyFile/CACertFile.
+func WithSubEtcdTLS(certFile, certKeyFile, caFile string, insecureSkipVerify bool) SubOption {
+	return func(sub *Subscriber) {
+		logx.Must(RegisterTLS(sub.endpoints, certFile, certKeyFile, caFile, insecureSkipVerify))
 	}
 }
 

core/discov/subscriber_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/tal-tech/go-zero/core/discov/internal"
+	"github.com/tal-tech/go-zero/core/stringx"
 )
 
 const (
@@ -201,11 +202,9 @@ func TestContainer(t *testing.T) {
 }
 
 func TestSubscriber(t *testing.T) {
-	var opt subOptions
-	Exclusive()(&opt)
-
 	sub := new(Subscriber)
-	sub.items = newContainer(opt.exclusive)
+	Exclusive()(sub)
+	sub.items = newContainer(sub.exclusive)
 	var count int32
 	sub.AddListener(func() {
 		atomic.AddInt32(&count, 1)
@@ -214,3 +213,15 @@ func TestSubscriber(t *testing.T) {
 	assert.Empty(t, sub.Values())
 	assert.Equal(t, int32(1), atomic.LoadInt32(&count))
 }
+
+func TestWithSubEtcdAccount(t *testing.T) {
+	endpoints := []string{"localhost:2379"}
+	user := stringx.Rand()
+	WithSubEtcdAccount(user, "bar")(&Subscriber{
+		endpoints: endpoints,
+	})
+	account, ok := internal.GetAccount(endpoints)
+	assert.True(t, ok)
+	assert.Equal(t, user, account.User)
+	assert.Equal(t, "bar", account.Pass)
+}

core/errorx/atomicerror.go

@@ -9,7 +9,9 @@ type AtomicError struct {
 
 // Set sets the error.
 func (ae *AtomicError) Set(err error) {
-	ae.err.Store(err)
+	if err != nil {
+		ae.err.Store(err)
+	}
 }
 
 // Load returns the error.

core/errorx/atomicerror_test.go

@@ -17,6 +17,15 @@ func TestAtomicError(t *testing.T) {
 	assert.Equal(t, errDummy, err.Load())
 }
 
+func TestAtomicErrorSetNil(t *testing.T) {
+	var (
+		errNil error
+		err    AtomicError
+	)
+	err.Set(errNil)
+	assert.Equal(t, errNil, err.Load())
+}
+
 func TestAtomicErrorNil(t *testing.T) {
 	var err AtomicError
 	assert.Nil(t, err.Load())

core/fs/files+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package fs

core/fs/files.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package fs

core/fx/stream.go

@@ -90,6 +90,8 @@ func Range(source <-chan interface{}) Stream {
 func (s Stream) AllMach(predicate func(item interface{}) bool) bool {
 	for item := range s.source {
 		if !predicate(item) {
+			// make sure the former goroutine not block, and current func returns fast.
+			go drain(s.source)
 			return false
 		}
 	}
@@ -103,6 +105,8 @@ func (s Stream) AllMach(predicate func(item interface{}) bool) bool {
 func (s Stream) AnyMach(predicate func(item interface{}) bool) bool {
 	for item := range s.source {
 		if predicate(item) {
+			// make sure the former goroutine not block, and current func returns fast.
+			go drain(s.source)
 			return true
 		}
 	}
@@ -186,8 +190,7 @@ func (s Stream) Distinct(fn KeyFunc) Stream {
 
 // Done waits all upstreaming operations to be done.
 func (s Stream) Done() {
-	for range s.source {
-	}
+	drain(s.source)
 }
 
 // Filter filters the items by the given FilterFunc.
@@ -199,9 +202,22 @@ func (s Stream) Filter(fn FilterFunc, opts ...Option) Stream {
 	}, opts...)
 }
 
+// First returns the first item, nil if no items.
+func (s Stream) First() interface{} {
+	for item := range s.source {
+		// make sure the former goroutine not block, and current func returns fast.
+		go drain(s.source)
+		return item
+	}
+
+	return nil
+}
+
 // ForAll handles the streaming elements from the source and no later streams.
 func (s Stream) ForAll(fn ForAllFunc) {
 	fn(s.source)
+	// avoid goroutine leak on fn not consuming all items.
+	go drain(s.source)
 }
 
 // ForEach seals the Stream with the ForEachFunc on each item, no successive operations.
@@ -246,11 +262,14 @@ func (s Stream) Head(n int64) Stream {
 			}
 			if n == 0 {
 				// let successive method go ASAP even we have more items to skip
-				// why we don't just break the loop, because if break,
-				// this former goroutine will block forever, which will cause goroutine leak.
 				close(source)
+				// why we don't just break the loop, and drain to consume all items.
+				// because if breaks, this former goroutine will block forever,
+				// which will cause goroutine leak.
+				drain(s.source)
 			}
 		}
+		// not enough items in s.source, but we need to let successive method to go ASAP.
 		if n > 0 {
 			close(source)
 		}
@@ -259,6 +278,13 @@ func (s Stream) Head(n int64) Stream {
 	return Range(source)
 }
 
+// Last returns the last item, or nil if no items.
+func (s Stream) Last() (item interface{}) {
+	for item = range s.source {
+	}
+	return
+}
+
 // Map converts each item to another corresponding item, which means it's a 1:1 model.
 func (s Stream) Map(fn MapFunc, opts ...Option) Stream {
 	return s.Walk(func(item interface{}, pipe chan<- interface{}) {
@@ -280,6 +306,21 @@ func (s Stream) Merge() Stream {
 	return Range(source)
 }
 
+// NoneMatch returns whether all elements of this stream don't match the provided predicate.
+// May not evaluate the predicate on all elements if not necessary for determining the result.
+// If the stream is empty then true is returned and the predicate is not evaluated.
+func (s Stream) NoneMatch(predicate func(item interface{}) bool) bool {
+	for item := range s.source {
+		if predicate(item) {
+			// make sure the former goroutine not block, and current func returns fast.
+			go drain(s.source)
+			return false
+		}
+	}
+
+	return true
+}
+
 // Parallel applies the given ParallelFunc to each item concurrently with given number of workers.
 func (s Stream) Parallel(fn ParallelFunc, opts ...Option) {
 	s.Walk(func(item interface{}, pipe chan<- interface{}) {
@@ -411,15 +452,12 @@ func (s Stream) walkLimited(fn WalkFunc, option *rxOptions) Stream {
 		var wg sync.WaitGroup
 		pool := make(chan lang.PlaceholderType, option.workers)
 
-		for {
+		for item := range s.source {
+			// important, used in another goroutine
+			val := item
 			pool <- lang.Placeholder
-			item, ok := <-s.source
-			if !ok {
-				<-pool
-				break
-			}
-
 			wg.Add(1)
+
 			// better to safely run caller defined method
 			threading.GoSafe(func() {
 				defer func() {
@@ -427,7 +465,7 @@ func (s Stream) walkLimited(fn WalkFunc, option *rxOptions) Stream {
 					<-pool
 				}()
 
-				fn(item, pipe)
+				fn(val, pipe)
 			})
 		}
 
@@ -439,22 +477,19 @@ func (s Stream) walkLimited(fn WalkFunc, option *rxOptions) Stream {
 }
 
 func (s Stream) walkUnlimited(fn WalkFunc, option *rxOptions) Stream {
-	pipe := make(chan interface{}, defaultWorkers)
+	pipe := make(chan interface{}, option.workers)
 
 	go func() {
 		var wg sync.WaitGroup
 
-		for {
-			item, ok := <-s.source
-			if !ok {
-				break
-			}
-
+		for item := range s.source {
+			// important, used in another goroutine
+			val := item
 			wg.Add(1)
 			// better to safely run caller defined method
 			threading.GoSafe(func() {
 				defer wg.Done()
-				fn(item, pipe)
+				fn(val, pipe)
 			})
 		}
 
@@ -465,14 +500,14 @@ func (s Stream) walkUnlimited(fn WalkFunc, option *rxOptions) Stream {
 	return Range(pipe)
 }
 
-// UnlimitedWorkers lets the caller to use as many workers as the tasks.
+// UnlimitedWorkers lets the caller use as many workers as the tasks.
 func UnlimitedWorkers() Option {
 	return func(opts *rxOptions) {
 		opts.unlimitedWorkers = true
 	}
 }
 
-// WithWorkers lets the caller to customize the concurrent workers.
+// WithWorkers lets the caller customize the concurrent workers.
 func WithWorkers(workers int) Option {
 	return func(opts *rxOptions) {
 		if workers < minWorkers {
@@ -483,6 +518,7 @@ func WithWorkers(workers int) Option {
 	}
 }
 
+// buildOptions returns a rxOptions with given customizations.
 func buildOptions(opts ...Option) *rxOptions {
 	options := newOptions()
 	for _, opt := range opts {
@@ -492,6 +528,13 @@ func buildOptions(opts ...Option) *rxOptions {
 	return options
 }
 
+// drain drains the given channel.
+func drain(channel <-chan interface{}) {
+	for range channel {
+	}
+}
+
+// newOptions returns a default rxOptions.
 func newOptions() *rxOptions {
 	return &rxOptions{
 		workers: defaultWorkers,

core/fx/stream_test.go

@@ -17,320 +17,489 @@ import (
 )
 
 func TestBuffer(t *testing.T) {
-	const N = 5
-	var count int32
-	var wait sync.WaitGroup
-	wait.Add(1)
-	From(func(source chan<- interface{}) {
-		ticker := time.NewTicker(10 * time.Millisecond)
-		defer ticker.Stop()
+	runCheckedTest(t, func(t *testing.T) {
+		const N = 5
+		var count int32
+		var wait sync.WaitGroup
+		wait.Add(1)
+		From(func(source chan<- interface{}) {
+			ticker := time.NewTicker(10 * time.Millisecond)
+			defer ticker.Stop()
 
-		for i := 0; i < 2*N; i++ {
-			select {
-			case source <- i:
-				atomic.AddInt32(&count, 1)
-			case <-ticker.C:
-				wait.Done()
-				return
+			for i := 0; i < 2*N; i++ {
+				select {
+				case source <- i:
+					atomic.AddInt32(&count, 1)
+				case <-ticker.C:
+					wait.Done()
+					return
+				}
 			}
-		}
-	}).Buffer(N).ForAll(func(pipe <-chan interface{}) {
-		wait.Wait()
-		// why N+1, because take one more to wait for sending into the channel
-		assert.Equal(t, int32(N+1), atomic.LoadInt32(&count))
+		}).Buffer(N).ForAll(func(pipe <-chan interface{}) {
+			wait.Wait()
+			// why N+1, because take one more to wait for sending into the channel
+			assert.Equal(t, int32(N+1), atomic.LoadInt32(&count))
+		})
 	})
 }
 
 func TestBufferNegative(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4).Buffer(-1).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-		for item := range pipe {
-			result += item.(int)
-		}
-		return result, nil
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4).Buffer(-1).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+			for item := range pipe {
+				result += item.(int)
+			}
+			return result, nil
+		})
+		assert.Equal(t, 10, result)
 	})
-	assert.Equal(t, 10, result)
 }
 
 func TestCount(t *testing.T) {
-	tests := []struct {
-		name     string
-		elements []interface{}
-	}{
-		{
-			name: "no elements with nil",
-		},
-		{
-			name:     "no elements",
-			elements: []interface{}{},
-		},
-		{
-			name:     "1 element",
-			elements: []interface{}{1},
-		},
-		{
-			name:     "multiple elements",
-			elements: []interface{}{1, 2, 3},
-		},
-	}
+	runCheckedTest(t, func(t *testing.T) {
+		tests := []struct {
+			name     string
+			elements []interface{}
+		}{
+			{
+				name: "no elements with nil",
+			},
+			{
+				name:     "no elements",
+				elements: []interface{}{},
+			},
+			{
+				name:     "1 element",
+				elements: []interface{}{1},
+			},
+			{
+				name:     "multiple elements",
+				elements: []interface{}{1, 2, 3},
+			},
+		}
 
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			val := Just(test.elements...).Count()
-			assert.Equal(t, len(test.elements), val)
-		})
-	}
+		for _, test := range tests {
+			t.Run(test.name, func(t *testing.T) {
+				val := Just(test.elements...).Count()
+				assert.Equal(t, len(test.elements), val)
+			})
+		}
+	})
 }
 
 func TestDone(t *testing.T) {
-	var count int32
-	Just(1, 2, 3).Walk(func(item interface{}, pipe chan<- interface{}) {
-		time.Sleep(time.Millisecond * 100)
-		atomic.AddInt32(&count, int32(item.(int)))
-	}).Done()
-	assert.Equal(t, int32(6), count)
+	runCheckedTest(t, func(t *testing.T) {
+		var count int32
+		Just(1, 2, 3).Walk(func(item interface{}, pipe chan<- interface{}) {
+			time.Sleep(time.Millisecond * 100)
+			atomic.AddInt32(&count, int32(item.(int)))
+		}).Done()
+		assert.Equal(t, int32(6), count)
+	})
 }
 
 func TestJust(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-		for item := range pipe {
-			result += item.(int)
-		}
-		return result, nil
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+			for item := range pipe {
+				result += item.(int)
+			}
+			return result, nil
+		})
+		assert.Equal(t, 10, result)
 	})
-	assert.Equal(t, 10, result)
 }
 
 func TestDistinct(t *testing.T) {
-	var result int
-	Just(4, 1, 3, 2, 3, 4).Distinct(func(item interface{}) interface{} {
-		return item
-	}).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-		for item := range pipe {
-			result += item.(int)
-		}
-		return result, nil
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(4, 1, 3, 2, 3, 4).Distinct(func(item interface{}) interface{} {
+			return item
+		}).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+			for item := range pipe {
+				result += item.(int)
+			}
+			return result, nil
+		})
+		assert.Equal(t, 10, result)
 	})
-	assert.Equal(t, 10, result)
 }
 
 func TestFilter(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4).Filter(func(item interface{}) bool {
-		return item.(int)%2 == 0
-	}).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-		for item := range pipe {
-			result += item.(int)
-		}
-		return result, nil
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4).Filter(func(item interface{}) bool {
+			return item.(int)%2 == 0
+		}).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+			for item := range pipe {
+				result += item.(int)
+			}
+			return result, nil
+		})
+		assert.Equal(t, 6, result)
+	})
+}
+
+func TestFirst(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		assert.Nil(t, Just().First())
+		assert.Equal(t, "foo", Just("foo").First())
+		assert.Equal(t, "foo", Just("foo", "bar").First())
 	})
-	assert.Equal(t, 6, result)
 }
 
 func TestForAll(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4).Filter(func(item interface{}) bool {
-		return item.(int)%2 == 0
-	}).ForAll(func(pipe <-chan interface{}) {
-		for item := range pipe {
-			result += item.(int)
-		}
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4).Filter(func(item interface{}) bool {
+			return item.(int)%2 == 0
+		}).ForAll(func(pipe <-chan interface{}) {
+			for item := range pipe {
+				result += item.(int)
+			}
+		})
+		assert.Equal(t, 6, result)
 	})
-	assert.Equal(t, 6, result)
 }
 
 func TestGroup(t *testing.T) {
-	var groups [][]int
-	Just(10, 11, 20, 21).Group(func(item interface{}) interface{} {
-		v := item.(int)
-		return v / 10
-	}).ForEach(func(item interface{}) {
-		v := item.([]interface{})
-		var group []int
-		for _, each := range v {
-			group = append(group, each.(int))
-		}
-		groups = append(groups, group)
-	})
+	runCheckedTest(t, func(t *testing.T) {
+		var groups [][]int
+		Just(10, 11, 20, 21).Group(func(item interface{}) interface{} {
+			v := item.(int)
+			return v / 10
+		}).ForEach(func(item interface{}) {
+			v := item.([]interface{})
+			var group []int
+			for _, each := range v {
+				group = append(group, each.(int))
+			}
+			groups = append(groups, group)
+		})
 
-	assert.Equal(t, 2, len(groups))
-	for _, group := range groups {
-		assert.Equal(t, 2, len(group))
-		assert.True(t, group[0]/10 == group[1]/10)
-	}
+		assert.Equal(t, 2, len(groups))
+		for _, group := range groups {
+			assert.Equal(t, 2, len(group))
+			assert.True(t, group[0]/10 == group[1]/10)
+		}
+	})
 }
 
 func TestHead(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4).Head(2).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-		for item := range pipe {
-			result += item.(int)
-		}
-		return result, nil
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4).Head(2).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+			for item := range pipe {
+				result += item.(int)
+			}
+			return result, nil
+		})
+		assert.Equal(t, 3, result)
 	})
-	assert.Equal(t, 3, result)
 }
 
 func TestHeadZero(t *testing.T) {
-	assert.Panics(t, func() {
-		Just(1, 2, 3, 4).Head(0).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-			return nil, nil
+	runCheckedTest(t, func(t *testing.T) {
+		assert.Panics(t, func() {
+			Just(1, 2, 3, 4).Head(0).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+				return nil, nil
+			})
 		})
 	})
 }
 
 func TestHeadMore(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4).Head(6).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-		for item := range pipe {
-			result += item.(int)
-		}
-		return result, nil
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4).Head(6).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+			for item := range pipe {
+				result += item.(int)
+			}
+			return result, nil
+		})
+		assert.Equal(t, 10, result)
+	})
+}
+
+func TestLast(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		goroutines := runtime.NumGoroutine()
+		assert.Nil(t, Just().Last())
+		assert.Equal(t, "foo", Just("foo").Last())
+		assert.Equal(t, "bar", Just("foo", "bar").Last())
+		// let scheduler schedule first
+		runtime.Gosched()
+		assert.Equal(t, goroutines, runtime.NumGoroutine())
 	})
-	assert.Equal(t, 10, result)
 }
 
 func TestMap(t *testing.T) {
-	log.SetOutput(ioutil.Discard)
+	runCheckedTest(t, func(t *testing.T) {
+		log.SetOutput(ioutil.Discard)
 
-	tests := []struct {
-		mapper MapFunc
-		expect int
-	}{
-		{
-			mapper: func(item interface{}) interface{} {
-				v := item.(int)
-				return v * v
+		tests := []struct {
+			mapper MapFunc
+			expect int
+		}{
+			{
+				mapper: func(item interface{}) interface{} {
+					v := item.(int)
+					return v * v
+				},
+				expect: 30,
 			},
-			expect: 30,
-		},
-		{
-			mapper: func(item interface{}) interface{} {
-				v := item.(int)
-				if v%2 == 0 {
-					return 0
-				}
-				return v * v
-			},
-			expect: 10,
-		},
-		{
-			mapper: func(item interface{}) interface{} {
-				v := item.(int)
-				if v%2 == 0 {
-					panic(v)
-				}
-				return v * v
-			},
-			expect: 10,
-		},
-	}
-
-	// Map(...) works even WithWorkers(0)
-	for i, test := range tests {
-		t.Run(stringx.Rand(), func(t *testing.T) {
-			var result int
-			var workers int
-			if i%2 == 0 {
-				workers = 0
-			} else {
-				workers = runtime.NumCPU()
-			}
-			From(func(source chan<- interface{}) {
-				for i := 1; i < 5; i++ {
-					source <- i
-				}
-			}).Map(test.mapper, WithWorkers(workers)).Reduce(
-				func(pipe <-chan interface{}) (interface{}, error) {
-					for item := range pipe {
-						result += item.(int)
+			{
+				mapper: func(item interface{}) interface{} {
+					v := item.(int)
+					if v%2 == 0 {
+						return 0
 					}
-					return result, nil
-				})
+					return v * v
+				},
+				expect: 10,
+			},
+			{
+				mapper: func(item interface{}) interface{} {
+					v := item.(int)
+					if v%2 == 0 {
+						panic(v)
+					}
+					return v * v
+				},
+				expect: 10,
+			},
+		}
 
-			assert.Equal(t, test.expect, result)
-		})
-	}
+		// Map(...) works even WithWorkers(0)
+		for i, test := range tests {
+			t.Run(stringx.Rand(), func(t *testing.T) {
+				var result int
+				var workers int
+				if i%2 == 0 {
+					workers = 0
+				} else {
+					workers = runtime.NumCPU()
+				}
+				From(func(source chan<- interface{}) {
+					for i := 1; i < 5; i++ {
+						source <- i
+					}
+				}).Map(test.mapper, WithWorkers(workers)).Reduce(
+					func(pipe <-chan interface{}) (interface{}, error) {
+						for item := range pipe {
+							result += item.(int)
+						}
+						return result, nil
+					})
+
+				assert.Equal(t, test.expect, result)
+			})
+		}
+	})
 }
 
 func TestMerge(t *testing.T) {
-	Just(1, 2, 3, 4).Merge().ForEach(func(item interface{}) {
-		assert.ElementsMatch(t, []interface{}{1, 2, 3, 4}, item.([]interface{}))
+	runCheckedTest(t, func(t *testing.T) {
+		Just(1, 2, 3, 4).Merge().ForEach(func(item interface{}) {
+			assert.ElementsMatch(t, []interface{}{1, 2, 3, 4}, item.([]interface{}))
+		})
 	})
 }
 
 func TestParallelJust(t *testing.T) {
-	var count int32
-	Just(1, 2, 3).Parallel(func(item interface{}) {
-		time.Sleep(time.Millisecond * 100)
-		atomic.AddInt32(&count, int32(item.(int)))
-	}, UnlimitedWorkers())
-	assert.Equal(t, int32(6), count)
+	runCheckedTest(t, func(t *testing.T) {
+		var count int32
+		Just(1, 2, 3).Parallel(func(item interface{}) {
+			time.Sleep(time.Millisecond * 100)
+			atomic.AddInt32(&count, int32(item.(int)))
+		}, UnlimitedWorkers())
+		assert.Equal(t, int32(6), count)
+	})
 }
 
 func TestReverse(t *testing.T) {
-	Just(1, 2, 3, 4).Reverse().Merge().ForEach(func(item interface{}) {
-		assert.ElementsMatch(t, []interface{}{4, 3, 2, 1}, item.([]interface{}))
+	runCheckedTest(t, func(t *testing.T) {
+		Just(1, 2, 3, 4).Reverse().Merge().ForEach(func(item interface{}) {
+			assert.ElementsMatch(t, []interface{}{4, 3, 2, 1}, item.([]interface{}))
+		})
 	})
 }
 
 func TestSort(t *testing.T) {
-	var prev int
-	Just(5, 3, 7, 1, 9, 6, 4, 8, 2).Sort(func(a, b interface{}) bool {
-		return a.(int) < b.(int)
-	}).ForEach(func(item interface{}) {
-		next := item.(int)
-		assert.True(t, prev < next)
-		prev = next
+	runCheckedTest(t, func(t *testing.T) {
+		var prev int
+		Just(5, 3, 7, 1, 9, 6, 4, 8, 2).Sort(func(a, b interface{}) bool {
+			return a.(int) < b.(int)
+		}).ForEach(func(item interface{}) {
+			next := item.(int)
+			assert.True(t, prev < next)
+			prev = next
+		})
 	})
 }
 
 func TestSplit(t *testing.T) {
-	assert.Panics(t, func() {
-		Just(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).Split(0).Done()
+	runCheckedTest(t, func(t *testing.T) {
+		assert.Panics(t, func() {
+			Just(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).Split(0).Done()
+		})
+		var chunks [][]interface{}
+		Just(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).Split(4).ForEach(func(item interface{}) {
+			chunk := item.([]interface{})
+			chunks = append(chunks, chunk)
+		})
+		assert.EqualValues(t, [][]interface{}{
+			{1, 2, 3, 4},
+			{5, 6, 7, 8},
+			{9, 10},
+		}, chunks)
 	})
-	var chunks [][]interface{}
-	Just(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).Split(4).ForEach(func(item interface{}) {
-		chunk := item.([]interface{})
-		chunks = append(chunks, chunk)
-	})
-	assert.EqualValues(t, [][]interface{}{
-		{1, 2, 3, 4},
-		{5, 6, 7, 8},
-		{9, 10},
-	}, chunks)
 }
 
 func TestTail(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4).Tail(2).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-		for item := range pipe {
-			result += item.(int)
-		}
-		return result, nil
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4).Tail(2).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+			for item := range pipe {
+				result += item.(int)
+			}
+			return result, nil
+		})
+		assert.Equal(t, 7, result)
 	})
-	assert.Equal(t, 7, result)
 }
 
 func TestTailZero(t *testing.T) {
-	assert.Panics(t, func() {
-		Just(1, 2, 3, 4).Tail(0).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
-			return nil, nil
+	runCheckedTest(t, func(t *testing.T) {
+		assert.Panics(t, func() {
+			Just(1, 2, 3, 4).Tail(0).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
+				return nil, nil
+			})
 		})
 	})
 }
 
 func TestWalk(t *testing.T) {
-	var result int
-	Just(1, 2, 3, 4, 5).Walk(func(item interface{}, pipe chan<- interface{}) {
-		if item.(int)%2 != 0 {
-			pipe <- item
-		}
-	}, UnlimitedWorkers()).ForEach(func(item interface{}) {
-		result += item.(int)
+	runCheckedTest(t, func(t *testing.T) {
+		var result int
+		Just(1, 2, 3, 4, 5).Walk(func(item interface{}, pipe chan<- interface{}) {
+			if item.(int)%2 != 0 {
+				pipe <- item
+			}
+		}, UnlimitedWorkers()).ForEach(func(item interface{}) {
+			result += item.(int)
+		})
+		assert.Equal(t, 9, result)
+	})
+}
+
+func TestStream_AnyMach(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
+			return item.(int) == 4
+		}))
+		assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
+			return item.(int) == 0
+		}))
+		assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
+			return item.(int) == 2
+		}))
+		assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
+			return item.(int) == 2
+		}))
+	})
+}
+
+func TestStream_AllMach(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		assetEqual(
+			t, true, Just(1, 2, 3).AllMach(func(item interface{}) bool {
+				return true
+			}),
+		)
+		assetEqual(
+			t, false, Just(1, 2, 3).AllMach(func(item interface{}) bool {
+				return false
+			}),
+		)
+		assetEqual(
+			t, false, Just(1, 2, 3).AllMach(func(item interface{}) bool {
+				return item.(int) == 1
+			}),
+		)
+	})
+}
+
+func TestStream_NoneMatch(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		assetEqual(
+			t, true, Just(1, 2, 3).NoneMatch(func(item interface{}) bool {
+				return false
+			}),
+		)
+		assetEqual(
+			t, false, Just(1, 2, 3).NoneMatch(func(item interface{}) bool {
+				return true
+			}),
+		)
+		assetEqual(
+			t, true, Just(1, 2, 3).NoneMatch(func(item interface{}) bool {
+				return item.(int) == 4
+			}),
+		)
+	})
+}
+
+func TestConcat(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		a1 := []interface{}{1, 2, 3}
+		a2 := []interface{}{4, 5, 6}
+		s1 := Just(a1...)
+		s2 := Just(a2...)
+		stream := Concat(s1, s2)
+		var items []interface{}
+		for item := range stream.source {
+			items = append(items, item)
+		}
+		sort.Slice(items, func(i, j int) bool {
+			return items[i].(int) < items[j].(int)
+		})
+		ints := make([]interface{}, 0)
+		ints = append(ints, a1...)
+		ints = append(ints, a2...)
+		assetEqual(t, ints, items)
+	})
+}
+
+func TestStream_Skip(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		assetEqual(t, 3, Just(1, 2, 3, 4).Skip(1).Count())
+		assetEqual(t, 1, Just(1, 2, 3, 4).Skip(3).Count())
+		assetEqual(t, 4, Just(1, 2, 3, 4).Skip(0).Count())
+		equal(t, Just(1, 2, 3, 4).Skip(3), []interface{}{4})
+		assert.Panics(t, func() {
+			Just(1, 2, 3, 4).Skip(-1)
+		})
+	})
+}
+
+func TestStream_Concat(t *testing.T) {
+	runCheckedTest(t, func(t *testing.T) {
+		stream := Just(1).Concat(Just(2), Just(3))
+		var items []interface{}
+		for item := range stream.source {
+			items = append(items, item)
+		}
+		sort.Slice(items, func(i, j int) bool {
+			return items[i].(int) < items[j].(int)
+		})
+		assetEqual(t, []interface{}{1, 2, 3}, items)
+
+		just := Just(1)
+		equal(t, just.Concat(just), []interface{}{1})
 	})
-	assert.Equal(t, 9, result)
 }
 
 func BenchmarkParallelMapReduce(b *testing.B) {
@@ -377,6 +546,12 @@ func BenchmarkMapReduce(b *testing.B) {
 	}).Map(mapper).Reduce(reducer)
 }
 
+func assetEqual(t *testing.T, except, data interface{}) {
+	if !reflect.DeepEqual(except, data) {
+		t.Errorf(" %v, want %v", data, except)
+	}
+}
+
 func equal(t *testing.T, stream Stream, data []interface{}) {
 	items := make([]interface{}, 0)
 	for item := range stream.source {
@@ -387,85 +562,10 @@ func equal(t *testing.T, stream Stream, data []interface{}) {
 	}
 }
 
-func assetEqual(t *testing.T, except, data interface{}) {
-	if !reflect.DeepEqual(except, data) {
-		t.Errorf(" %v, want %v", data, except)
-	}
-}
-
-func TestStream_AnyMach(t *testing.T) {
-	assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 4 == item.(int)
-	}))
-	assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 0 == item.(int)
-	}))
-	assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 2 == item.(int)
-	}))
-	assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
-		return 2 == item.(int)
-	}))
-}
-
-func TestStream_AllMach(t *testing.T) {
-	assetEqual(
-		t, true, Just(1, 2, 3).AllMach(func(item interface{}) bool {
-			return true
-		}),
-	)
-	assetEqual(
-		t, false, Just(1, 2, 3).AllMach(func(item interface{}) bool {
-			return false
-		}),
-	)
-	assetEqual(
-		t, false, Just(1, 2, 3).AllMach(func(item interface{}) bool {
-			return item.(int) == 1
-		}),
-	)
-}
-
-func TestConcat(t *testing.T) {
-	a1 := []interface{}{1, 2, 3}
-	a2 := []interface{}{4, 5, 6}
-	s1 := Just(a1...)
-	s2 := Just(a2...)
-	stream := Concat(s1, s2)
-	var items []interface{}
-	for item := range stream.source {
-		items = append(items, item)
-	}
-	sort.Slice(items, func(i, j int) bool {
-		return items[i].(int) < items[j].(int)
-	})
-	ints := make([]interface{}, 0)
-	ints = append(ints, a1...)
-	ints = append(ints, a2...)
-	assetEqual(t, ints, items)
-}
-
-func TestStream_Skip(t *testing.T) {
-	assetEqual(t, 3, Just(1, 2, 3, 4).Skip(1).Count())
-	assetEqual(t, 1, Just(1, 2, 3, 4).Skip(3).Count())
-	assetEqual(t, 4, Just(1, 2, 3, 4).Skip(0).Count())
-	equal(t, Just(1, 2, 3, 4).Skip(3), []interface{}{4})
-	assert.Panics(t, func() {
-		Just(1, 2, 3, 4).Skip(-1)
-	})
-}
-
-func TestStream_Concat(t *testing.T) {
-	stream := Just(1).Concat(Just(2), Just(3))
-	var items []interface{}
-	for item := range stream.source {
-		items = append(items, item)
-	}
-	sort.Slice(items, func(i, j int) bool {
-		return items[i].(int) < items[j].(int)
-	})
-	assetEqual(t, []interface{}{1, 2, 3}, items)
-
-	just := Just(1)
-	equal(t, just.Concat(just), []interface{}{1})
+func runCheckedTest(t *testing.T, fn func(t *testing.T)) {
+	goroutines := runtime.NumGoroutine()
+	fn(t)
+	// let scheduler schedule first
+	time.Sleep(time.Millisecond)
+	assert.True(t, runtime.NumGoroutine() <= goroutines)
 }

core/fx/timeout.go

@@ -2,6 +2,9 @@ package fx
 
 import (
 	"context"
+	"fmt"
+	"runtime/debug"
+	"strings"
 	"time"
 )
 
@@ -30,7 +33,8 @@ func DoWithTimeout(fn func() error, timeout time.Duration, opts ...DoOption) err
 	go func() {
 		defer func() {
 			if p := recover(); p != nil {
-				panicChan <- p
+				// attach call stack to avoid missing in different goroutine
+				panicChan <- fmt.Sprintf("%+v\n\n%s", p, strings.TrimSpace(string(debug.Stack())))
 			}
 		}()
 		done <- fn()

core/hash/consistenthash_test.go

@@ -74,12 +74,12 @@ func TestConsistentHashIncrementalTransfer(t *testing.T) {
 		laterCh := create()
 		laterCh.AddWithWeight(node, 10*(i+1))
 
-		for i := 0; i < requestSize; i++ {
-			key, ok := laterCh.Get(requestSize + i)
+		for j := 0; j < requestSize; j++ {
+			key, ok := laterCh.Get(requestSize + j)
 			assert.True(t, ok)
 			assert.NotNil(t, key)
 			value := key.(string)
-			assert.True(t, value == keys[i] || value == node)
+			assert.True(t, value == keys[j] || value == node)
 		}
 	}
 }

core/hash/hash_test.go

@@ -20,6 +20,11 @@ func TestMd5(t *testing.T) {
 	assert.Equal(t, md5Digest, actual)
 }
 
+func TestMd5Hex(t *testing.T) {
+	actual := Md5Hex([]byte(text))
+	assert.Equal(t, md5Digest, actual)
+}
+
 func BenchmarkHashFnv(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		h := fnv.New32()

core/jsontype/time_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/globalsign/mgo/bson"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -106,3 +107,20 @@ func TestMilliTime_UnmarshalJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestUnmarshalWithError(t *testing.T) {
+	var mt MilliTime
+	assert.NotNil(t, mt.UnmarshalJSON([]byte("hello")))
+}
+
+func TestSetBSON(t *testing.T) {
+	data, err := bson.Marshal(time.Now())
+	assert.Nil(t, err)
+
+	var raw bson.Raw
+	assert.Nil(t, bson.Unmarshal(data, &raw))
+
+	var mt MilliTime
+	assert.Nil(t, mt.SetBSON(raw))
+	assert.NotNil(t, mt.SetBSON(bson.Raw{}))
+}

core/lang/lang.go

@@ -4,8 +4,8 @@ package lang
 var Placeholder PlaceholderType
 
 type (
-	// GenericType can be used to hold any type.
-	GenericType = interface{}
+	// AnyType can be used to hold any type.
+	AnyType = interface{}
 	// PlaceholderType represents a placeholder type.
 	PlaceholderType = struct{}
 )

core/limit/tokenlimit_test.go

@@ -24,7 +24,7 @@ func TestTokenLimit_Rescue(t *testing.T) {
 		rate  = 5
 		burst = 10
 	)
-	l := NewTokenLimiter(rate, burst, redis.NewRedis(s.Addr(), redis.NodeType), "tokenlimit")
+	l := NewTokenLimiter(rate, burst, redis.New(s.Addr()), "tokenlimit")
 	s.Close()
 
 	var allowed int

core/load/adaptiveshedder.go

@@ -31,6 +31,8 @@ var (
 
 	// default to be enabled
 	enabled = syncx.ForAtomicBool(true)
+	// default to be enabled
+	logEnabled = syncx.ForAtomicBool(true)
 	// make it a variable for unit test
 	systemOverloadChecker = func(cpuThreshold int64) bool {
 		return stat.CpuUsage() >= cpuThreshold
@@ -80,6 +82,11 @@ func Disable() {
 	enabled.Set(false)
 }
 
+// DisableLog disables the stat logs for load shedding.
+func DisableLog() {
+	logEnabled.Set(false)
+}
+
 // NewAdaptiveShedder returns an adaptive shedder.
 // opts can be used to customize the Shedder.
 func NewAdaptiveShedder(opts ...ShedderOption) Shedder {

core/load/adaptiveshedder_test.go

@@ -25,6 +25,7 @@ func init() {
 }
 
 func TestAdaptiveShedder(t *testing.T) {
+	DisableLog()
 	shedder := NewAdaptiveShedder(WithWindow(bucketDuration), WithBuckets(buckets), WithCpuThreshold(100))
 	var wg sync.WaitGroup
 	var drop int64

core/load/sheddingstat.go

@@ -48,6 +48,25 @@ func (s *SheddingStat) IncrementDrop() {
 	atomic.AddInt64(&s.drop, 1)
 }
 
+func (s *SheddingStat) loop(c <-chan time.Time) {
+	for range c {
+		st := s.reset()
+
+		if !logEnabled.True() {
+			continue
+		}
+
+		c := stat.CpuUsage()
+		if st.Drop == 0 {
+			logx.Statf("(%s) shedding_stat [1m], cpu: %d, total: %d, pass: %d, drop: %d",
+				s.name, c, st.Total, st.Pass, st.Drop)
+		} else {
+			logx.Statf("(%s) shedding_stat_drop [1m], cpu: %d, total: %d, pass: %d, drop: %d",
+				s.name, c, st.Total, st.Pass, st.Drop)
+		}
+	}
+}
+
 func (s *SheddingStat) reset() snapshot {
 	return snapshot{
 		Total: atomic.SwapInt64(&s.total, 0),
@@ -59,15 +78,6 @@ func (s *SheddingStat) reset() snapshot {
 func (s *SheddingStat) run() {
 	ticker := time.NewTicker(time.Minute)
 	defer ticker.Stop()
-	for range ticker.C {
-		c := stat.CpuUsage()
-		st := s.reset()
-		if st.Drop == 0 {
-			logx.Statf("(%s) shedding_stat [1m], cpu: %d, total: %d, pass: %d, drop: %d",
-				s.name, c, st.Total, st.Pass, st.Drop)
-		} else {
-			logx.Statf("(%s) shedding_stat_drop [1m], cpu: %d, total: %d, pass: %d, drop: %d",
-				s.name, c, st.Total, st.Pass, st.Drop)
-		}
-	}
+
+	s.loop(ticker.C)
 }

core/load/sheddingstat_test.go

@@ -2,6 +2,7 @@ package load
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -22,3 +23,32 @@ func TestSheddingStat(t *testing.T) {
 	assert.Equal(t, int64(5), result.Pass)
 	assert.Equal(t, int64(7), result.Drop)
 }
+
+func TestLoopTrue(t *testing.T) {
+	ch := make(chan time.Time, 1)
+	ch <- time.Now()
+	close(ch)
+	st := new(SheddingStat)
+	logEnabled.Set(true)
+	st.loop(ch)
+}
+
+func TestLoopTrueAndDrop(t *testing.T) {
+	ch := make(chan time.Time, 1)
+	ch <- time.Now()
+	close(ch)
+	st := new(SheddingStat)
+	st.IncrementDrop()
+	logEnabled.Set(true)
+	st.loop(ch)
+}
+
+func TestLoopFalseAndDrop(t *testing.T) {
+	ch := make(chan time.Time, 1)
+	ch <- time.Now()
+	close(ch)
+	st := new(SheddingStat)
+	st.IncrementDrop()
+	logEnabled.Set(false)
+	st.loop(ch)
+}

core/logx/durationlogger.go

@@ -20,49 +20,69 @@ func WithDuration(d time.Duration) Logger {
 }
 
 func (l *durationLogger) Error(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
 	}
 }
 
 func (l *durationLogger) Errorf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
 	}
 }
 
+func (l *durationLogger) Errorv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(errorLog, levelError, v)
+	}
+}
+
 func (l *durationLogger) Info(v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprint(v...))
 	}
 }
 
 func (l *durationLogger) Infof(format string, v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *durationLogger) Infov(v interface{}) {
+	if shallLog(InfoLevel) {
+		l.write(infoLog, levelInfo, v)
+	}
+}
+
 func (l *durationLogger) Slow(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprint(v...))
 	}
 }
 
 func (l *durationLogger) Slowf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *durationLogger) Slowv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(slowLog, levelSlow, v)
+	}
+}
+
 func (l *durationLogger) WithDuration(duration time.Duration) Logger {
 	l.Duration = timex.ReprOfDuration(duration)
 	return l
 }
 
-func (l *durationLogger) write(writer io.Writer, level, content string) {
-	l.Timestamp = getTimestamp()
-	l.Level = level
-	l.Content = content
-	outputJson(writer, logEntry(*l))
+func (l *durationLogger) write(writer io.Writer, level string, val interface{}) {
+	outputJson(writer, &durationLogger{
+		Timestamp: getTimestamp(),
+		Level:     level,
+		Content:   val,
+		Duration:  l.Duration,
+	})
 }

core/logx/durationlogger_test.go

@@ -23,6 +23,13 @@ func TestWithDurationErrorf(t *testing.T) {
 	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
 }
 
+func TestWithDurationErrorv(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Errorv("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
 func TestWithDurationInfo(t *testing.T) {
 	var builder strings.Builder
 	log.SetOutput(&builder)
@@ -37,6 +44,13 @@ func TestWithDurationInfof(t *testing.T) {
 	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
 }
 
+func TestWithDurationInfov(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Infov("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
 func TestWithDurationSlow(t *testing.T) {
 	var builder strings.Builder
 	log.SetOutput(&builder)
@@ -50,3 +64,10 @@ func TestWithDurationSlowf(t *testing.T) {
 	WithDuration(time.Second).WithDuration(time.Hour).Slowf("foo")
 	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
 }
+
+func TestWithDurationSlowv(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).WithDuration(time.Hour).Slowv("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}

core/logx/limitedexecutor_test.go

@@ -0,0 +1,62 @@
+package logx
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/timex"
+)
+
+func TestLimitedExecutor_logOrDiscard(t *testing.T) {
+	tests := []struct {
+		name      string
+		threshold time.Duration
+		lastTime  time.Duration
+		discarded uint32
+		executed  bool
+	}{
+		{
+			name:     "nil executor",
+			executed: true,
+		},
+		{
+			name:      "regular",
+			threshold: time.Hour,
+			lastTime:  timex.Now(),
+			discarded: 10,
+			executed:  false,
+		},
+		{
+			name:      "slow",
+			threshold: time.Duration(1),
+			lastTime:  -1000,
+			discarded: 10,
+			executed:  true,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			executor := newLimitedExecutor(0)
+			executor.threshold = test.threshold
+			executor.discarded = test.discarded
+			executor.lastTime.Set(test.lastTime)
+
+			var run int32
+			executor.logOrDiscard(func() {
+				atomic.AddInt32(&run, 1)
+			})
+			if test.executed {
+				assert.Equal(t, int32(1), atomic.LoadInt32(&run))
+			} else {
+				assert.Equal(t, int32(0), atomic.LoadInt32(&run))
+				assert.Equal(t, test.discarded+1, atomic.LoadUint32(&executor.discarded))
+			}
+		})
+	}
+}

core/logx/logs.go

@@ -65,12 +65,14 @@ var (
 	timeFormat   = "2006-01-02T15:04:05.000Z07"
 	writeConsole bool
 	logLevel     uint32
-	infoLog      io.WriteCloser
-	errorLog     io.WriteCloser
-	severeLog    io.WriteCloser
-	slowLog      io.WriteCloser
-	statLog      io.WriteCloser
-	stackLog     io.Writer
+	// use uint32 for atomic operations
+	disableStat uint32
+	infoLog     io.WriteCloser
+	errorLog    io.WriteCloser
+	severeLog   io.WriteCloser
+	slowLog     io.WriteCloser
+	statLog     io.WriteCloser
+	stackLog    io.Writer
 
 	once        sync.Once
 	initialized uint32
@@ -79,10 +81,10 @@ var (
 
 type (
 	logEntry struct {
-		Timestamp string `json:"@timestamp"`
-		Level     string `json:"level"`
-		Duration  string `json:"duration,omitempty"`
-		Content   string `json:"content"`
+		Timestamp string      `json:"@timestamp"`
+		Level     string      `json:"level"`
+		Duration  string      `json:"duration,omitempty"`
+		Content   interface{} `json:"content"`
 	}
 
 	logOptions struct {
@@ -98,10 +100,13 @@ type (
 	Logger interface {
 		Error(...interface{})
 		Errorf(string, ...interface{})
+		Errorv(interface{})
 		Info(...interface{})
 		Infof(string, ...interface{})
+		Infov(interface{})
 		Slow(...interface{})
 		Slowf(string, ...interface{})
+		Slowv(interface{})
 		WithDuration(time.Duration) Logger
 	}
 )
@@ -133,7 +138,7 @@ func SetUp(c LogConf) error {
 
 // Alert alerts v in alert level, and the message is written to error log.
 func Alert(v string) {
-	output(errorLog, levelAlert, v)
+	outputText(errorLog, levelAlert, v)
 }
 
 // Close closes the logging.
@@ -195,24 +200,29 @@ func Disable() {
 	})
 }
 
+// DisableStat disables the stat logs.
+func DisableStat() {
+	atomic.StoreUint32(&disableStat, 1)
+}
+
 // Error writes v into error log.
 func Error(v ...interface{}) {
 	ErrorCaller(1, v...)
 }
 
-// Errorf writes v with format into error log.
-func Errorf(format string, v ...interface{}) {
-	ErrorCallerf(1, format, v...)
-}
-
 // ErrorCaller writes v with context into error log.
 func ErrorCaller(callDepth int, v ...interface{}) {
-	errorSync(fmt.Sprint(v...), callDepth+callerInnerDepth)
+	errorTextSync(fmt.Sprint(v...), callDepth+callerInnerDepth)
 }
 
 // ErrorCallerf writes v with context in format into error log.
 func ErrorCallerf(callDepth int, format string, v ...interface{}) {
-	errorSync(fmt.Sprintf(format, v...), callDepth+callerInnerDepth)
+	errorTextSync(fmt.Errorf(format, v...).Error(), callDepth+callerInnerDepth)
+}
+
+// Errorf writes v with format into error log.
+func Errorf(format string, v ...interface{}) {
+	ErrorCallerf(1, format, v...)
 }
 
 // ErrorStack writes v along with call stack into error log.
@@ -227,14 +237,25 @@ func ErrorStackf(format string, v ...interface{}) {
 	stackSync(fmt.Sprintf(format, v...))
 }
 
+// Errorv writes v into error log with json content.
+// No call stack attached, because not elegant to pack the messages.
+func Errorv(v interface{}) {
+	errorAnySync(v)
+}
+
 // Info writes v into access log.
 func Info(v ...interface{}) {
-	infoSync(fmt.Sprint(v...))
+	infoTextSync(fmt.Sprint(v...))
 }
 
 // Infof writes v with format into access log.
 func Infof(format string, v ...interface{}) {
-	infoSync(fmt.Sprintf(format, v...))
+	infoTextSync(fmt.Sprintf(format, v...))
+}
+
+// Infov writes v into access log with json content.
+func Infov(v interface{}) {
+	infoAnySync(v)
 }
 
 // Must checks if err is nil, otherwise logs the err and exits.
@@ -242,7 +263,7 @@ func Must(err error) {
 	if err != nil {
 		msg := formatWithCaller(err.Error(), 3)
 		log.Print(msg)
-		output(severeLog, levelFatal, msg)
+		outputText(severeLog, levelFatal, msg)
 		os.Exit(1)
 	}
 }
@@ -264,12 +285,17 @@ func Severef(format string, v ...interface{}) {
 
 // Slow writes v into slow log.
 func Slow(v ...interface{}) {
-	slowSync(fmt.Sprint(v...))
+	slowTextSync(fmt.Sprint(v...))
 }
 
 // Slowf writes v with format into slow log.
 func Slowf(format string, v ...interface{}) {
-	slowSync(fmt.Sprintf(format, v...))
+	slowTextSync(fmt.Sprintf(format, v...))
+}
+
+// Slowv writes v into slow log with json content.
+func Slowv(v interface{}) {
+	slowAnySync(v)
 }
 
 // Stat writes v into stat log.
@@ -312,8 +338,14 @@ func createOutput(path string) (io.WriteCloser, error) {
 		options.gzipEnabled), options.gzipEnabled)
 }
 
-func errorSync(msg string, callDepth int) {
-	if shouldLog(ErrorLevel) {
+func errorAnySync(v interface{}) {
+	if shallLog(ErrorLevel) {
+		outputAny(errorLog, levelError, v)
+	}
+}
+
+func errorTextSync(msg string, callDepth int) {
+	if shallLog(ErrorLevel) {
 		outputError(errorLog, msg, callDepth)
 	}
 }
@@ -362,13 +394,28 @@ func handleOptions(opts []LogOption) {
 	}
 }
 
-func infoSync(msg string) {
-	if shouldLog(InfoLevel) {
-		output(infoLog, levelInfo, msg)
+func infoAnySync(val interface{}) {
+	if shallLog(InfoLevel) {
+		outputAny(infoLog, levelInfo, val)
 	}
 }
 
-func output(writer io.Writer, level, msg string) {
+func infoTextSync(msg string) {
+	if shallLog(InfoLevel) {
+		outputText(infoLog, levelInfo, msg)
+	}
+}
+
+func outputAny(writer io.Writer, level string, val interface{}) {
+	info := logEntry{
+		Timestamp: getTimestamp(),
+		Level:     level,
+		Content:   val,
+	}
+	outputJson(writer, info)
+}
+
+func outputText(writer io.Writer, level, msg string) {
 	info := logEntry{
 		Timestamp: getTimestamp(),
 		Level:     level,
@@ -379,7 +426,7 @@ func output(writer io.Writer, level, msg string) {
 
 func outputError(writer io.Writer, msg string, callDepth int) {
 	content := formatWithCaller(msg, callDepth)
-	output(writer, levelError, content)
+	outputText(writer, levelError, content)
 }
 
 func outputJson(writer io.Writer, info interface{}) {
@@ -481,30 +528,40 @@ func setupWithVolume(c LogConf) error {
 }
 
 func severeSync(msg string) {
-	if shouldLog(SevereLevel) {
-		output(severeLog, levelSevere, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+	if shallLog(SevereLevel) {
+		outputText(severeLog, levelSevere, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
-func shouldLog(level uint32) bool {
+func shallLog(level uint32) bool {
 	return atomic.LoadUint32(&logLevel) <= level
 }
 
-func slowSync(msg string) {
-	if shouldLog(ErrorLevel) {
-		output(slowLog, levelSlow, msg)
+func shallLogStat() bool {
+	return atomic.LoadUint32(&disableStat) == 0
+}
+
+func slowAnySync(v interface{}) {
+	if shallLog(ErrorLevel) {
+		outputAny(slowLog, levelSlow, v)
+	}
+}
+
+func slowTextSync(msg string) {
+	if shallLog(ErrorLevel) {
+		outputText(slowLog, levelSlow, msg)
 	}
 }
 
 func stackSync(msg string) {
-	if shouldLog(ErrorLevel) {
-		output(stackLog, levelError, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
+	if shallLog(ErrorLevel) {
+		outputText(stackLog, levelError, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 	}
 }
 
 func statSync(msg string) {
-	if shouldLog(InfoLevel) {
-		output(statLog, levelStat, msg)
+	if shallLogStat() && shallLog(InfoLevel) {
+		outputText(statLog, levelStat, msg)
 	}
 }
 

core/logx/logs_test.go

@@ -2,6 +2,7 @@ package logx
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -92,6 +93,30 @@ func TestStructedLogAlert(t *testing.T) {
 	})
 }
 
+func TestStructedLogError(t *testing.T) {
+	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
+		errorLog = writer
+	}, func(v ...interface{}) {
+		Error(v...)
+	})
+}
+
+func TestStructedLogErrorf(t *testing.T) {
+	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
+		errorLog = writer
+	}, func(v ...interface{}) {
+		Errorf("%s", fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogErrorv(t *testing.T) {
+	doTestStructedLog(t, levelError, func(writer io.WriteCloser) {
+		errorLog = writer
+	}, func(v ...interface{}) {
+		Errorv(fmt.Sprint(v...))
+	})
+}
+
 func TestStructedLogInfo(t *testing.T) {
 	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
 		infoLog = writer
@@ -100,6 +125,22 @@ func TestStructedLogInfo(t *testing.T) {
 	})
 }
 
+func TestStructedLogInfof(t *testing.T) {
+	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
+		infoLog = writer
+	}, func(v ...interface{}) {
+		Infof("%s", fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogInfov(t *testing.T) {
+	doTestStructedLog(t, levelInfo, func(writer io.WriteCloser) {
+		infoLog = writer
+	}, func(v ...interface{}) {
+		Infov(fmt.Sprint(v...))
+	})
+}
+
 func TestStructedLogSlow(t *testing.T) {
 	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
 		slowLog = writer
@@ -116,6 +157,14 @@ func TestStructedLogSlowf(t *testing.T) {
 	})
 }
 
+func TestStructedLogSlowv(t *testing.T) {
+	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
+		slowLog = writer
+	}, func(v ...interface{}) {
+		Slowv(fmt.Sprint(v...))
+	})
+}
+
 func TestStructedLogStat(t *testing.T) {
 	doTestStructedLog(t, levelStat, func(writer io.WriteCloser) {
 		statLog = writer
@@ -194,6 +243,16 @@ func TestSetLevelWithDuration(t *testing.T) {
 	assert.Equal(t, 0, writer.builder.Len())
 }
 
+func TestErrorfWithWrappedError(t *testing.T) {
+	SetLevel(ErrorLevel)
+	const message = "there"
+	writer := new(mockWriter)
+	errorLog = writer
+	atomic.StoreUint32(&initialized, 1)
+	Errorf("hello %w", errors.New(message))
+	assert.True(t, strings.Contains(writer.builder.String(), "hello there"))
+}
+
 func TestMustNil(t *testing.T) {
 	Must(nil)
 }
@@ -246,6 +305,17 @@ func TestDisable(t *testing.T) {
 	assert.Nil(t, Close())
 }
 
+func TestDisableStat(t *testing.T) {
+	DisableStat()
+
+	const message = "hello there"
+	writer := new(mockWriter)
+	statLog = writer
+	atomic.StoreUint32(&initialized, 1)
+	Stat(message)
+	assert.Equal(t, 0, writer.builder.Len())
+}
+
 func TestWithGzip(t *testing.T) {
 	fn := WithGzip()
 	var opt logOptions
@@ -357,7 +427,9 @@ func doTestStructedLog(t *testing.T, level string, setup func(writer io.WriteClo
 		t.Error(err)
 	}
 	assert.Equal(t, level, entry.Level)
-	assert.True(t, strings.Contains(entry.Content, message))
+	val, ok := entry.Content.(string)
+	assert.True(t, ok)
+	assert.True(t, strings.Contains(val, message))
 }
 
 func testSetLevelTwiceWithMode(t *testing.T, mode string) {

core/logx/rotatelogger.go

@@ -47,7 +47,6 @@ type (
 		done     chan lang.PlaceholderType
 		rule     RotateRule
 		compress bool
-		keepDays int
 		// can't use threading.RoutineGroup because of cycle import
 		waitGroup sync.WaitGroup
 		closeOnce sync.Once

core/logx/rotatelogger_test.go

@@ -3,6 +3,7 @@ package logx
 import (
 	"os"
 	"path/filepath"
+	"syscall"
 	"testing"
 	"time"
 
@@ -97,7 +98,13 @@ func TestRotateLoggerRotate(t *testing.T) {
 		}()
 	}
 	err = logger.rotate()
-	assert.Nil(t, err)
+	switch v := err.(type) {
+	case *os.LinkError:
+		// avoid rename error on docker container
+		assert.Equal(t, syscall.EXDEV, v.Err)
+	default:
+		assert.Nil(t, err)
+	}
 }
 
 func TestRotateLoggerWrite(t *testing.T) {

core/logx/syslog_test.go

@@ -44,5 +44,5 @@ func captureOutput(f func()) string {
 func getContent(jsonStr string) string {
 	var entry logEntry
 	json.Unmarshal([]byte(jsonStr), &entry)
-	return entry.Content
+	return entry.Content.(string)
 }

core/logx/tracelogger.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/timex"
-	"github.com/tal-tech/go-zero/core/trace/tracespec"
+	"go.opentelemetry.io/otel/trace"
 )
 
 type traceLogger struct {
@@ -18,53 +18,75 @@ type traceLogger struct {
 }
 
 func (l *traceLogger) Error(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
 	}
 }
 
 func (l *traceLogger) Errorf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
 	}
 }
 
+func (l *traceLogger) Errorv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(errorLog, levelError, v)
+	}
+}
+
 func (l *traceLogger) Info(v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprint(v...))
 	}
 }
 
 func (l *traceLogger) Infof(format string, v ...interface{}) {
-	if shouldLog(InfoLevel) {
+	if shallLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *traceLogger) Infov(v interface{}) {
+	if shallLog(InfoLevel) {
+		l.write(infoLog, levelInfo, v)
+	}
+}
+
 func (l *traceLogger) Slow(v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprint(v...))
 	}
 }
 
 func (l *traceLogger) Slowf(format string, v ...interface{}) {
-	if shouldLog(ErrorLevel) {
+	if shallLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
 	}
 }
 
+func (l *traceLogger) Slowv(v interface{}) {
+	if shallLog(ErrorLevel) {
+		l.write(slowLog, levelSlow, v)
+	}
+}
+
 func (l *traceLogger) WithDuration(duration time.Duration) Logger {
 	l.Duration = timex.ReprOfDuration(duration)
 	return l
 }
 
-func (l *traceLogger) write(writer io.Writer, level, content string) {
-	l.Timestamp = getTimestamp()
-	l.Level = level
-	l.Content = content
-	l.Trace = traceIdFromContext(l.ctx)
-	l.Span = spanIdFromContext(l.ctx)
-	outputJson(writer, l)
+func (l *traceLogger) write(writer io.Writer, level string, val interface{}) {
+	outputJson(writer, &traceLogger{
+		logEntry: logEntry{
+			Timestamp: getTimestamp(),
+			Level:     level,
+			Duration:  l.Duration,
+			Content:   val,
+		},
+		Trace: traceIdFromContext(l.ctx),
+		Span:  spanIdFromContext(l.ctx),
+	})
 }
 
 // WithContext sets ctx to log, for keeping tracing information.
@@ -75,19 +97,19 @@ func WithContext(ctx context.Context) Logger {
 }
 
 func spanIdFromContext(ctx context.Context) string {
-	t, ok := ctx.Value(tracespec.TracingKey).(tracespec.Trace)
-	if !ok {
-		return ""
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasSpanID() {
+		return spanCtx.SpanID().String()
 	}
 
-	return t.SpanId()
+	return ""
 }
 
 func traceIdFromContext(ctx context.Context) string {
-	t, ok := ctx.Value(tracespec.TracingKey).(tracespec.Trace)
-	if !ok {
-		return ""
+	spanCtx := trace.SpanContextFromContext(ctx)
+	if spanCtx.HasTraceID() {
+		return spanCtx.TraceID().String()
 	}
 
-	return t.TraceId()
+	return ""
 }

core/logx/tracelogger_test.go

@@ -9,71 +9,102 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/tal-tech/go-zero/core/trace/tracespec"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 )
 
 const (
-	mockTraceID = "mock-trace-id"
-	mockSpanID  = "mock-span-id"
+	traceKey = "trace"
+	spanKey  = "span"
 )
 
-var mock tracespec.Trace = new(mockTrace)
-
 func TestTraceLog(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	WithContext(ctx).(*traceLogger).write(&buf, levelInfo, testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceError(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	errorLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Error(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Errorf(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
+	buf.Reset()
+	l.WithDuration(time.Second).Errorv(testlog)
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceInfo(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	infoLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Info(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Infof(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
+	buf.Reset()
+	l.WithDuration(time.Second).Infov(testlog)
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceSlow(t *testing.T) {
 	var buf mockWriter
 	atomic.StoreUint32(&initialized, 1)
 	slowLog = newLogWriter(log.New(&buf, "", flags))
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	otp := otel.GetTracerProvider()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
+	otel.SetTracerProvider(tp)
+	defer otel.SetTracerProvider(otp)
+
+	ctx, _ := tp.Tracer("foo").Start(context.Background(), "bar")
 	l := WithContext(ctx).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Slow(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Slowf(testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceID))
-	assert.True(t, strings.Contains(buf.String(), mockSpanID))
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
+	buf.Reset()
+	l.WithDuration(time.Second).Slowv(testlog)
+	assert.True(t, strings.Contains(buf.String(), traceKey))
+	assert.True(t, strings.Contains(buf.String(), spanKey))
 }
 
 func TestTraceWithoutContext(t *testing.T) {
@@ -83,34 +114,10 @@ func TestTraceWithoutContext(t *testing.T) {
 	l := WithContext(context.Background()).(*traceLogger)
 	SetLevel(InfoLevel)
 	l.WithDuration(time.Second).Info(testlog)
-	assert.False(t, strings.Contains(buf.String(), mockTraceID))
-	assert.False(t, strings.Contains(buf.String(), mockSpanID))
+	assert.False(t, strings.Contains(buf.String(), traceKey))
+	assert.False(t, strings.Contains(buf.String(), spanKey))
 	buf.Reset()
 	l.WithDuration(time.Second).Infof(testlog)
-	assert.False(t, strings.Contains(buf.String(), mockTraceID))
-	assert.False(t, strings.Contains(buf.String(), mockSpanID))
-}
-
-type mockTrace struct{}
-
-func (t mockTrace) TraceId() string {
-	return mockTraceID
-}
-
-func (t mockTrace) SpanId() string {
-	return mockSpanID
-}
-
-func (t mockTrace) Finish() {
-}
-
-func (t mockTrace) Fork(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Follow(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Visit(fn func(key, val string) bool) {
+	assert.False(t, strings.Contains(buf.String(), traceKey))
+	assert.False(t, strings.Contains(buf.String(), spanKey))
 }

core/mapping/jsonunmarshaler.go

@@ -15,6 +15,11 @@ func UnmarshalJsonBytes(content []byte, v interface{}) error {
 	return unmarshalJsonBytes(content, v, jsonUnmarshaler)
 }
 
+// UnmarshalJsonMap unmarshals content from m into v.
+func UnmarshalJsonMap(m map[string]interface{}, v interface{}) error {
+	return jsonUnmarshaler.Unmarshal(m, v)
+}
+
 // UnmarshalJsonReader unmarshals content from reader into v.
 func UnmarshalJsonReader(reader io.Reader, v interface{}) error {
 	return unmarshalJsonReader(reader, v, jsonUnmarshaler)

core/mapping/jsonunmarshaler_test.go

@@ -871,3 +871,50 @@ func TestUnmarshalReaderError(t *testing.T) {
 	assert.NotNil(t, err)
 	assert.True(t, strings.Contains(err.Error(), payload))
 }
+
+func TestUnmarshalMap(t *testing.T) {
+	t.Run("nil map and valid", func(t *testing.T) {
+		var m map[string]interface{}
+		var v struct {
+			Any string `json:",optional"`
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.Nil(t, err)
+		assert.True(t, len(v.Any) == 0)
+	})
+
+	t.Run("empty map but not valid", func(t *testing.T) {
+		m := map[string]interface{}{}
+		var v struct {
+			Any string
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.NotNil(t, err)
+	})
+
+	t.Run("empty map and valid", func(t *testing.T) {
+		m := map[string]interface{}{}
+		var v struct {
+			Any string `json:",optional"`
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.Nil(t, err)
+		assert.True(t, len(v.Any) == 0)
+	})
+
+	t.Run("valid map", func(t *testing.T) {
+		m := map[string]interface{}{
+			"Any": "foo",
+		}
+		var v struct {
+			Any string
+		}
+
+		err := UnmarshalJsonMap(m, &v)
+		assert.Nil(t, err)
+		assert.Equal(t, "foo", v.Any)
+	})
+}

core/mapping/unmarshaler.go

@@ -7,7 +7,6 @@ import (
 	"reflect"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/tal-tech/go-zero/core/jsonx"
@@ -25,15 +24,17 @@ var (
 	errValueNotSettable = errors.New("value is not settable")
 	errValueNotStruct   = errors.New("value type is not struct")
 	keyUnmarshaler      = NewUnmarshaler(defaultKeyName)
-	cacheKeys           atomic.Value
-	cacheKeysLock       sync.Mutex
 	durationType        = reflect.TypeOf(time.Duration(0))
+	cacheKeys           map[string][]string
+	cacheKeysLock       sync.Mutex
+	defaultCache        map[string]interface{}
+	defaultCacheLock    sync.Mutex
 	emptyMap            = map[string]interface{}{}
 	emptyValue          = reflect.ValueOf(lang.Placeholder)
 )
 
 type (
-	// A Unmarshaler is used to unmarshal with given tag key.
+	// Unmarshaler is used to unmarshal with given tag key.
 	Unmarshaler struct {
 		key  string
 		opts unmarshalOptions
@@ -43,14 +44,14 @@ type (
 	UnmarshalOption func(*unmarshalOptions)
 
 	unmarshalOptions struct {
-		fromString bool
+		fromString   bool
+		canonicalKey func(key string) string
 	}
-
-	keyCache map[string][]string
 )
 
 func init() {
-	cacheKeys.Store(make(keyCache))
+	cacheKeys = make(map[string][]string)
+	defaultCache = make(map[string]interface{})
 }
 
 // NewUnmarshaler returns a Unmarshaler.
@@ -206,6 +207,8 @@ func (u *Unmarshaler) processFieldNotFromString(field reflect.StructField, value
 	switch {
 	case valueKind == reflect.Map && typeKind == reflect.Struct:
 		return u.processFieldStruct(field, value, mapValue, fullName)
+	case valueKind == reflect.Map && typeKind == reflect.Map:
+		return u.fillMap(field, value, mapValue)
 	case valueKind == reflect.String && typeKind == reflect.Slice:
 		return u.fillSliceFromString(fieldType, value, mapValue)
 	case valueKind == reflect.String && derefedFieldType == durationType:
@@ -229,7 +232,7 @@ func (u *Unmarshaler) processFieldPrimitive(field reflect.StructField, value ref
 	default:
 		switch v := mapValue.(type) {
 		case json.Number:
-			return u.processFieldPrimitiveWithJsonNumber(field, value, v, opts, fullName)
+			return u.processFieldPrimitiveWithJSONNumber(field, value, v, opts, fullName)
 		default:
 			if typeKind == valueKind {
 				if err := validateValueInOptions(opts.options(), mapValue); err != nil {
@@ -244,7 +247,7 @@ func (u *Unmarshaler) processFieldPrimitive(field reflect.StructField, value ref
 	return newTypeMismatchError(fullName)
 }
 
-func (u *Unmarshaler) processFieldPrimitiveWithJsonNumber(field reflect.StructField, value reflect.Value,
+func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(field reflect.StructField, value reflect.Value,
 	v json.Number, opts *fieldOptionsWithContext, fullName string) error {
 	fieldType := field.Type
 	fieldKind := fieldType.Kind()
@@ -323,7 +326,11 @@ func (u *Unmarshaler) processNamedField(field reflect.StructField, value reflect
 	}
 
 	fullName = join(fullName, key)
-	mapValue, hasValue := getValue(m, key)
+	canonicalKey := key
+	if u.opts.canonicalKey != nil {
+		canonicalKey = u.opts.canonicalKey(key)
+	}
+	mapValue, hasValue := getValue(m, canonicalKey)
 	if hasValue {
 		return u.processNamedFieldWithValue(field, value, mapValue, key, opts, fullName)
 	}
@@ -381,7 +388,13 @@ func (u *Unmarshaler) processNamedFieldWithoutValue(field reflect.StructField, v
 		if derefedType == durationType {
 			return fillDurationValue(fieldKind, value, defaultValue)
 		}
-		return setValue(fieldKind, value, defaultValue)
+
+		switch fieldKind {
+		case reflect.Array, reflect.Slice:
+			return u.fillSliceWithDefault(derefedType, value, defaultValue)
+		default:
+			return setValue(fieldKind, value, defaultValue)
+		}
 	}
 
 	switch fieldKind {
@@ -495,7 +508,8 @@ func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.
 	return nil
 }
 
-func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int, baseKind reflect.Kind, value interface{}) error {
+func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int,
+	baseKind reflect.Kind, value interface{}) error {
 	ithVal := slice.Index(index)
 	switch v := value.(type) {
 	case json.Number:
@@ -513,17 +527,39 @@ func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int, baseKind re
 			target.Set(reflect.ValueOf(value))
 			ithVal.Set(target.Addr())
 			return nil
-		} else {
-			if ithVal.Kind() != reflect.TypeOf(value).Kind() {
-				return errTypeMismatch
-			}
-
-			ithVal.Set(reflect.ValueOf(value))
-			return nil
 		}
+
+		if ithVal.Kind() != reflect.TypeOf(value).Kind() {
+			return errTypeMismatch
+		}
+
+		ithVal.Set(reflect.ValueOf(value))
+		return nil
 	}
 }
 
+func (u *Unmarshaler) fillSliceWithDefault(derefedType reflect.Type, value reflect.Value,
+	defaultValue string) error {
+	baseFieldType := Deref(derefedType.Elem())
+	baseFieldKind := baseFieldType.Kind()
+	defaultCacheLock.Lock()
+	slice, ok := defaultCache[defaultValue]
+	defaultCacheLock.Unlock()
+	if !ok {
+		if baseFieldKind == reflect.String {
+			slice = parseGroupedSegments(defaultValue)
+		} else if err := jsonx.UnmarshalFromString(defaultValue, &slice); err != nil {
+			return err
+		}
+
+		defaultCacheLock.Lock()
+		defaultCache[defaultValue] = slice
+		defaultCacheLock.Unlock()
+	}
+
+	return u.fillSlice(derefedType, value, slice)
+}
+
 func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue interface{}) (reflect.Value, error) {
 	mapType := reflect.MapOf(keyType, elemType)
 	valueType := reflect.TypeOf(mapValue)
@@ -579,6 +615,8 @@ func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue inter
 			targetValue.SetMapIndex(key, innerValue)
 		default:
 			switch v := keythData.(type) {
+			case bool:
+				targetValue.SetMapIndex(key, reflect.ValueOf(v))
 			case string:
 				targetValue.SetMapIndex(key, reflect.ValueOf(v))
 			case json.Number:
@@ -621,6 +659,13 @@ func WithStringValues() UnmarshalOption {
 	}
 }
 
+// WithCanonicalKeyFunc customizes a Unmarshaler with Canonical Key func
+func WithCanonicalKeyFunc(f func(string) string) UnmarshalOption {
+	return func(opt *unmarshalOptions) {
+		opt.canonicalKey = f
+	}
+}
+
 func fillDurationValue(fieldKind reflect.Kind, value reflect.Value, dur string) error {
 	d, err := time.ParseDuration(dur)
 	if err != nil {
@@ -708,20 +753,6 @@ func getValueWithChainedKeys(m Valuer, keys []string) (interface{}, bool) {
 	return nil, false
 }
 
-func insertKeys(key string, cache []string) {
-	cacheKeysLock.Lock()
-	defer cacheKeysLock.Unlock()
-
-	keys := cacheKeys.Load().(keyCache)
-	// copy the contents into the new map, to guarantee the old map is immutable
-	newKeys := make(keyCache)
-	for k, v := range keys {
-		newKeys[k] = v
-	}
-	newKeys[key] = cache
-	cacheKeys.Store(newKeys)
-}
-
 func join(elem ...string) string {
 	var builder strings.Builder
 
@@ -752,15 +783,19 @@ func newTypeMismatchError(name string) error {
 }
 
 func readKeys(key string) []string {
-	cache := cacheKeys.Load().(keyCache)
-	if keys, ok := cache[key]; ok {
+	cacheKeysLock.Lock()
+	keys, ok := cacheKeys[key]
+	cacheKeysLock.Unlock()
+	if ok {
 		return keys
 	}
 
-	keys := strings.FieldsFunc(key, func(c rune) bool {
+	keys = strings.FieldsFunc(key, func(c rune) bool {
 		return c == delimiter
 	})
-	insertKeys(key, keys)
+	cacheKeysLock.Lock()
+	cacheKeys[key] = keys
+	cacheKeysLock.Unlock()
 
 	return keys
 }

core/mapping/unmarshaler_test.go

@@ -198,6 +198,66 @@ func TestUnmarshalIntWithDefault(t *testing.T) {
 	assert.Equal(t, 1, in.Int)
 }
 
+func TestUnmarshalBoolSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Bools []bool `key:"bools,default=[true,false]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []bool{true, false}, in.Bools)
+}
+
+func TestUnmarshalIntSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Ints []int `key:"ints,default=[1,2,3]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []int{1, 2, 3}, in.Ints)
+}
+
+func TestUnmarshalIntSliceWithDefaultHasSpaces(t *testing.T) {
+	type inner struct {
+		Ints []int `key:"ints,default=[1, 2, 3]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []int{1, 2, 3}, in.Ints)
+}
+
+func TestUnmarshalFloatSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Floats []float32 `key:"floats,default=[1.1,2.2,3.3]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []float32{1.1, 2.2, 3.3}, in.Floats)
+}
+
+func TestUnmarshalStringSliceWithDefault(t *testing.T) {
+	type inner struct {
+		Strs []string `key:"strs,default=[foo,bar,woo]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []string{"foo", "bar", "woo"}, in.Strs)
+}
+
+func TestUnmarshalStringSliceWithDefaultHasSpaces(t *testing.T) {
+	type inner struct {
+		Strs []string `key:"strs,default=[foo, bar, woo]"`
+	}
+
+	var in inner
+	assert.Nil(t, UnmarshalKey(nil, &in))
+	assert.ElementsMatch(t, []string{"foo", "bar", "woo"}, in.Strs)
+}
+
 func TestUnmarshalUint(t *testing.T) {
 	type inner struct {
 		Uint          uint   `key:"uint"`
@@ -861,10 +921,12 @@ func TestUnmarshalSliceOfStruct(t *testing.T) {
 func TestUnmarshalWithStringOptionsCorrect(t *testing.T) {
 	type inner struct {
 		Value   string `key:"value,options=first|second"`
+		Foo     string `key:"foo,options=[bar,baz]"`
 		Correct string `key:"correct,options=1|2"`
 	}
 	m := map[string]interface{}{
 		"value":   "first",
+		"foo":     "bar",
 		"correct": "2",
 	}
 
@@ -872,6 +934,7 @@ func TestUnmarshalWithStringOptionsCorrect(t *testing.T) {
 	ast := assert.New(t)
 	ast.Nil(UnmarshalKey(m, &in))
 	ast.Equal("first", in.Value)
+	ast.Equal("bar", in.Foo)
 	ast.Equal("2", in.Correct)
 }
 
@@ -943,6 +1006,22 @@ func TestUnmarshalStringOptionsWithStringOptionsIncorrect(t *testing.T) {
 	ast.NotNil(unmarshaler.Unmarshal(m, &in))
 }
 
+func TestUnmarshalStringOptionsWithStringOptionsIncorrectGrouped(t *testing.T) {
+	type inner struct {
+		Value   string `key:"value,options=[first,second]"`
+		Correct string `key:"correct,options=1|2"`
+	}
+	m := map[string]interface{}{
+		"value":   "third",
+		"correct": "2",
+	}
+
+	var in inner
+	unmarshaler := NewUnmarshaler(defaultKeyName, WithStringValues())
+	ast := assert.New(t)
+	ast.NotNil(unmarshaler.Unmarshal(m, &in))
+}
+
 func TestUnmarshalWithStringOptionsIncorrect(t *testing.T) {
 	type inner struct {
 		Value     string `key:"value,options=first|second"`
@@ -2518,3 +2597,29 @@ func TestUnmarshalJsonReaderPtrArray(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res.B))
 }
+
+func TestUnmarshalJsonWithoutKey(t *testing.T) {
+	payload := `{"A": "1", "B": "2"}`
+	var res struct {
+		A string `json:""`
+		B string `json:","`
+	}
+	reader := strings.NewReader(payload)
+	err := UnmarshalJsonReader(reader, &res)
+	assert.Nil(t, err)
+	assert.Equal(t, "1", res.A)
+	assert.Equal(t, "2", res.B)
+}
+
+func BenchmarkDefaultValue(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var a struct {
+			Ints []int    `json:"ints,default=[1,2,3]"`
+			Strs []string `json:"strs,default=[foo,bar,baz]"`
+		}
+		_ = UnmarshalJsonMap(nil, &a)
+		if len(a.Strs) != 3 || len(a.Ints) != 3 {
+			b.Fatal("failed")
+		}
+	}
+}

core/mapping/utils.go

@@ -14,13 +14,19 @@ import (
 )
 
 const (
-	defaultOption   = "default"
-	stringOption    = "string"
-	optionalOption  = "optional"
-	optionsOption   = "options"
-	rangeOption     = "range"
-	optionSeparator = "|"
-	equalToken      = "="
+	defaultOption      = "default"
+	stringOption       = "string"
+	optionalOption     = "optional"
+	optionsOption      = "options"
+	rangeOption        = "range"
+	optionSeparator    = "|"
+	equalToken         = "="
+	escapeChar         = '\\'
+	leftBracket        = '('
+	rightBracket       = ')'
+	leftSquareBracket  = '['
+	rightSquareBracket = ']'
+	segmentSeparator   = ','
 )
 
 var (
@@ -118,7 +124,7 @@ func convertType(kind reflect.Kind, str string) (interface{}, error) {
 }
 
 func doParseKeyAndOptions(field reflect.StructField, value string) (string, *fieldOptions, error) {
-	segments := strings.Split(value, ",")
+	segments := parseSegments(value)
 	key := strings.TrimSpace(segments[0])
 	options := segments[1:]
 
@@ -198,6 +204,16 @@ func maybeNewValue(field reflect.StructField, value reflect.Value) {
 	}
 }
 
+func parseGroupedSegments(val string) []string {
+	val = strings.TrimLeftFunc(val, func(r rune) bool {
+		return r == leftBracket || r == leftSquareBracket
+	})
+	val = strings.TrimRightFunc(val, func(r rune) bool {
+		return r == rightBracket || r == rightSquareBracket
+	})
+	return parseSegments(val)
+}
+
 // don't modify returned fieldOptions, it's cached and shared among different calls.
 func parseKeyAndOptions(tagName string, field reflect.StructField) (string, *fieldOptions, error) {
 	value := field.Tag.Get(tagName)
@@ -309,7 +325,7 @@ func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 			return fmt.Errorf("field %s has wrong options", fieldName)
 		}
 
-		fieldOpts.Options = strings.Split(segs[1], optionSeparator)
+		fieldOpts.Options = parseOptions(segs[1])
 	case strings.HasPrefix(option, defaultOption):
 		segs := strings.Split(option, equalToken)
 		if len(segs) != 2 {
@@ -334,6 +350,69 @@ func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 	return nil
 }
 
+// parseOptions parses the given options in tag.
+// for example: `json:"name,options=foo|bar"` or `json:"name,options=[foo,bar]"`
+func parseOptions(val string) []string {
+	if len(val) == 0 {
+		return nil
+	}
+
+	if val[0] == leftSquareBracket {
+		return parseGroupedSegments(val)
+	}
+
+	return strings.Split(val, optionSeparator)
+}
+
+func parseSegments(val string) []string {
+	var segments []string
+	var escaped, grouped bool
+	var buf strings.Builder
+
+	for _, ch := range val {
+		if escaped {
+			buf.WriteRune(ch)
+			escaped = false
+			continue
+		}
+
+		switch ch {
+		case segmentSeparator:
+			if grouped {
+				buf.WriteRune(ch)
+			} else {
+				// need to trim spaces, but we cannot ignore empty string,
+				// because the first segment stands for the key might be empty.
+				// if ignored, the later tag will be used as the key.
+				segments = append(segments, strings.TrimSpace(buf.String()))
+				buf.Reset()
+			}
+		case escapeChar:
+			if grouped {
+				buf.WriteRune(ch)
+			} else {
+				escaped = true
+			}
+		case leftBracket, leftSquareBracket:
+			buf.WriteRune(ch)
+			grouped = true
+		case rightBracket, rightSquareBracket:
+			buf.WriteRune(ch)
+			grouped = false
+		default:
+			buf.WriteRune(ch)
+		}
+	}
+
+	last := strings.TrimSpace(buf.String())
+	// ignore last empty string
+	if len(last) > 0 {
+		segments = append(segments, last)
+	}
+
+	return segments
+}
+
 func reprOfValue(val reflect.Value) string {
 	switch vt := val.Interface().(type) {
 	case bool:

core/mapping/utils_test.go

@@ -90,6 +90,82 @@ func TestParseKeyAndOptionWithTagAndOption(t *testing.T) {
 	assert.True(t, options.FromString)
 }
 
+func TestParseSegments(t *testing.T) {
+	tests := []struct {
+		input  string
+		expect []string
+	}{
+		{
+			input:  "",
+			expect: []string{},
+		},
+		{
+			input:  ",",
+			expect: []string{""},
+		},
+		{
+			input:  "foo,",
+			expect: []string{"foo"},
+		},
+		{
+			input: ",foo",
+			// the first empty string cannot be ignored, it's the key.
+			expect: []string{"", "foo"},
+		},
+		{
+			input:  "foo",
+			expect: []string{"foo"},
+		},
+		{
+			input:  "foo,bar",
+			expect: []string{"foo", "bar"},
+		},
+		{
+			input:  "foo,bar,baz",
+			expect: []string{"foo", "bar", "baz"},
+		},
+		{
+			input:  "foo,options=a|b",
+			expect: []string{"foo", "options=a|b"},
+		},
+		{
+			input:  "foo,bar,default=[baz,qux]",
+			expect: []string{"foo", "bar", "default=[baz,qux]"},
+		},
+		{
+			input:  "foo,bar,options=[baz,qux]",
+			expect: []string{"foo", "bar", "options=[baz,qux]"},
+		},
+		{
+			input:  `foo\,bar,options=[baz,qux]`,
+			expect: []string{`foo,bar`, "options=[baz,qux]"},
+		},
+		{
+			input:  `foo,bar,options=\[baz,qux]`,
+			expect: []string{"foo", "bar", "options=[baz", "qux]"},
+		},
+		{
+			input:  `foo,bar,options=[baz\,qux]`,
+			expect: []string{"foo", "bar", `options=[baz\,qux]`},
+		},
+		{
+			input:  `foo\,bar,options=[baz,qux],default=baz`,
+			expect: []string{`foo,bar`, "options=[baz,qux]", "default=baz"},
+		},
+		{
+			input:  `foo\,bar,options=[baz,qux, quux],default=[qux, baz]`,
+			expect: []string{`foo,bar`, "options=[baz,qux, quux]", "default=[qux, baz]"},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.input, func(t *testing.T) {
+			assert.ElementsMatch(t, test.expect, parseSegments(test.input))
+		})
+	}
+}
+
 func TestValidatePtrWithNonPtr(t *testing.T) {
 	var foo string
 	rve := reflect.ValueOf(foo)
@@ -209,6 +285,12 @@ func TestRepr(t *testing.T) {
 			newMockPtr(),
 			"mockptr",
 		},
+		{
+			&mockOpacity{
+				val: 1,
+			},
+			"{1}",
+		},
 		{
 			true,
 			"true",

core/mapping/yamlunmarshaler_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"k8s.io/utils/io"
 )
 
 func TestUnmarshalYamlBytes(t *testing.T) {
@@ -18,6 +19,22 @@ func TestUnmarshalYamlBytes(t *testing.T) {
 	assert.Equal(t, "liao", c.Name)
 }
 
+func TestUnmarshalYamlBytesErrorInput(t *testing.T) {
+	var c struct {
+		Name string
+	}
+	content := []byte(`liao`)
+	assert.NotNil(t, UnmarshalYamlBytes(content, &c))
+}
+
+func TestUnmarshalYamlBytesEmptyInput(t *testing.T) {
+	var c struct {
+		Name string
+	}
+	content := []byte(``)
+	assert.NotNil(t, UnmarshalYamlBytes(content, &c))
+}
+
 func TestUnmarshalYamlBytesOptional(t *testing.T) {
 	var c struct {
 		Name string
@@ -918,3 +935,82 @@ func TestUnmarshalYamlReaderError(t *testing.T) {
 	err := UnmarshalYamlReader(reader, &v)
 	assert.NotNil(t, err)
 }
+
+func TestUnmarshalYamlBadReader(t *testing.T) {
+	var v struct {
+		Any string
+	}
+
+	err := UnmarshalYamlReader(new(badReader), &v)
+	assert.NotNil(t, err)
+}
+
+func TestUnmarshalYamlMapBool(t *testing.T) {
+	text := `machine:
+  node1: true
+  node2: true
+  node3: true
+`
+	var v struct {
+		Machine map[string]bool `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.True(t, v.Machine["node1"])
+	assert.True(t, v.Machine["node2"])
+	assert.True(t, v.Machine["node3"])
+}
+
+func TestUnmarshalYamlMapInt(t *testing.T) {
+	text := `machine:
+  node1: 1
+  node2: 2
+  node3: 3
+`
+	var v struct {
+		Machine map[string]int `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.Equal(t, 1, v.Machine["node1"])
+	assert.Equal(t, 2, v.Machine["node2"])
+	assert.Equal(t, 3, v.Machine["node3"])
+}
+
+func TestUnmarshalYamlMapByte(t *testing.T) {
+	text := `machine:
+  node1: 1
+  node2: 2
+  node3: 3
+`
+	var v struct {
+		Machine map[string]byte `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.Equal(t, byte(1), v.Machine["node1"])
+	assert.Equal(t, byte(2), v.Machine["node2"])
+	assert.Equal(t, byte(3), v.Machine["node3"])
+}
+
+func TestUnmarshalYamlMapRune(t *testing.T) {
+	text := `machine:
+  node1: 1
+  node2: 2
+  node3: 3
+`
+	var v struct {
+		Machine map[string]rune `json:"machine,optional"`
+	}
+	reader := strings.NewReader(text)
+	assert.Nil(t, UnmarshalYamlReader(reader, &v))
+	assert.Equal(t, rune(1), v.Machine["node1"])
+	assert.Equal(t, rune(2), v.Machine["node2"])
+	assert.Equal(t, rune(3), v.Machine["node3"])
+}
+
+type badReader struct{}
+
+func (b *badReader) Read(p []byte) (n int, err error) {
+	return 0, io.ErrLimitReached
+}

core/mr/mapreduce.go

@@ -1,13 +1,13 @@
 package mr
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sync"
 
 	"github.com/tal-tech/go-zero/core/errorx"
 	"github.com/tal-tech/go-zero/core/lang"
-	"github.com/tal-tech/go-zero/core/syncx"
 	"github.com/tal-tech/go-zero/core/threading"
 )
 
@@ -43,6 +43,7 @@ type (
 	Option func(opts *mapReduceOptions)
 
 	mapReduceOptions struct {
+		ctx     context.Context
 		workers int
 	}
 
@@ -93,16 +94,17 @@ func Map(generate GenerateFunc, mapper MapFunc, opts ...Option) chan interface{}
 	options := buildOptions(opts...)
 	source := buildSource(generate)
 	collector := make(chan interface{}, options.workers)
-	done := syncx.NewDoneChan()
+	done := make(chan lang.PlaceholderType)
 
-	go executeMappers(mapper, source, collector, done.Done(), options.workers)
+	go executeMappers(options.ctx, mapper, source, collector, done, options.workers)
 
 	return collector
 }
 
 // MapReduce maps all elements generated from given generate func,
 // and reduces the output elements with given reducer.
-func MapReduce(generate GenerateFunc, mapper MapperFunc, reducer ReducerFunc, opts ...Option) (interface{}, error) {
+func MapReduce(generate GenerateFunc, mapper MapperFunc, reducer ReducerFunc,
+	opts ...Option) (interface{}, error) {
 	source := buildSource(generate)
 	return MapReduceWithSource(source, mapper, reducer, opts...)
 }
@@ -112,14 +114,20 @@ func MapReduceWithSource(source <-chan interface{}, mapper MapperFunc, reducer R
 	opts ...Option) (interface{}, error) {
 	options := buildOptions(opts...)
 	output := make(chan interface{})
+	defer func() {
+		for range output {
+			panic("more than one element written in reducer")
+		}
+	}()
+
 	collector := make(chan interface{}, options.workers)
-	done := syncx.NewDoneChan()
-	writer := newGuardedWriter(output, done.Done())
+	done := make(chan lang.PlaceholderType)
+	writer := newGuardedWriter(options.ctx, output, done)
 	var closeOnce sync.Once
 	var retErr errorx.AtomicError
 	finish := func() {
 		closeOnce.Do(func() {
-			done.Close()
+			close(done)
 			close(output)
 		})
 	}
@@ -148,9 +156,9 @@ func MapReduceWithSource(source <-chan interface{}, mapper MapperFunc, reducer R
 		reducer(collector, writer, cancel)
 	}()
 
-	go executeMappers(func(item interface{}, w Writer) {
+	go executeMappers(options.ctx, func(item interface{}, w Writer) {
 		mapper(item, w, cancel)
-	}, source, collector, done.Done(), options.workers)
+	}, source, collector, done, options.workers)
 
 	value, ok := <-output
 	if err := retErr.Load(); err != nil {
@@ -181,6 +189,13 @@ func MapVoid(generate GenerateFunc, mapper VoidMapFunc, opts ...Option) {
 	}, opts...))
 }
 
+// WithContext customizes a mapreduce processing accepts a given ctx.
+func WithContext(ctx context.Context) Option {
+	return func(opts *mapReduceOptions) {
+		opts.ctx = ctx
+	}
+}
+
 // WithWorkers customizes a mapreduce processing with given workers.
 func WithWorkers(workers int) Option {
 	return func(opts *mapReduceOptions) {
@@ -218,8 +233,8 @@ func drain(channel <-chan interface{}) {
 	}
 }
 
-func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- interface{},
-	done <-chan lang.PlaceholderType, workers int) {
+func executeMappers(ctx context.Context, mapper MapFunc, input <-chan interface{},
+	collector chan<- interface{}, done <-chan lang.PlaceholderType, workers int) {
 	var wg sync.WaitGroup
 	defer func() {
 		wg.Wait()
@@ -227,9 +242,11 @@ func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- i
 	}()
 
 	pool := make(chan lang.PlaceholderType, workers)
-	writer := newGuardedWriter(collector, done)
+	writer := newGuardedWriter(ctx, collector, done)
 	for {
 		select {
+		case <-ctx.Done():
+			return
 		case <-done:
 			return
 		case pool <- lang.Placeholder:
@@ -255,6 +272,7 @@ func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- i
 
 func newOptions() *mapReduceOptions {
 	return &mapReduceOptions{
+		ctx:     context.Background(),
 		workers: defaultWorkers,
 	}
 }
@@ -269,12 +287,15 @@ func once(fn func(error)) func(error) {
 }
 
 type guardedWriter struct {
+	ctx     context.Context
 	channel chan<- interface{}
 	done    <-chan lang.PlaceholderType
 }
 
-func newGuardedWriter(channel chan<- interface{}, done <-chan lang.PlaceholderType) guardedWriter {
+func newGuardedWriter(ctx context.Context, channel chan<- interface{},
+	done <-chan lang.PlaceholderType) guardedWriter {
 	return guardedWriter{
+		ctx:     ctx,
 		channel: channel,
 		done:    done,
 	}
@@ -282,6 +303,8 @@ func newGuardedWriter(channel chan<- interface{}, done <-chan lang.PlaceholderTy
 
 func (gw guardedWriter) Write(v interface{}) {
 	select {
+	case <-gw.ctx.Done():
+		return
 	case <-gw.done:
 		return
 	default:

core/mr/mapreduce_test.go

@@ -1,6 +1,7 @@
 package mr
 
 import (
+	"context"
 	"errors"
 	"io/ioutil"
 	"log"
@@ -202,6 +203,22 @@ func TestMapReduce(t *testing.T) {
 	}
 }
 
+func TestMapReduceWithReduerWriteMoreThanOnce(t *testing.T) {
+	assert.Panics(t, func() {
+		MapReduce(func(source chan<- interface{}) {
+			for i := 0; i < 10; i++ {
+				source <- i
+			}
+		}, func(item interface{}, writer Writer, cancel func(error)) {
+			writer.Write(item)
+		}, func(pipe <-chan interface{}, writer Writer, cancel func(error)) {
+			drain(pipe)
+			writer.Write("one")
+			writer.Write("two")
+		})
+	})
+}
+
 func TestMapReduceVoid(t *testing.T) {
 	var value uint32
 	tests := []struct {
@@ -394,6 +411,50 @@ func TestMapReduceWithoutReducerWrite(t *testing.T) {
 	assert.Nil(t, res)
 }
 
+func TestMapReduceVoidPanicInReducer(t *testing.T) {
+	const message = "foo"
+	var done syncx.AtomicBool
+	err := MapReduceVoid(func(source chan<- interface{}) {
+		for i := 0; i < defaultWorkers*2; i++ {
+			source <- i
+		}
+		done.Set(true)
+	}, func(item interface{}, writer Writer, cancel func(error)) {
+		i := item.(int)
+		writer.Write(i)
+	}, func(pipe <-chan interface{}, cancel func(error)) {
+		panic(message)
+	}, WithWorkers(1))
+	assert.NotNil(t, err)
+	assert.Equal(t, message, err.Error())
+	assert.True(t, done.True())
+}
+
+func TestMapReduceWithContext(t *testing.T) {
+	var done syncx.AtomicBool
+	var result []int
+	ctx, cancel := context.WithCancel(context.Background())
+	err := MapReduceVoid(func(source chan<- interface{}) {
+		for i := 0; i < defaultWorkers*2; i++ {
+			source <- i
+		}
+		done.Set(true)
+	}, func(item interface{}, writer Writer, c func(error)) {
+		i := item.(int)
+		if i == defaultWorkers/2 {
+			cancel()
+		}
+		writer.Write(i)
+	}, func(pipe <-chan interface{}, cancel func(error)) {
+		for item := range pipe {
+			i := item.(int)
+			result = append(result, i)
+		}
+	}, WithContext(ctx))
+	assert.NotNil(t, err)
+	assert.Equal(t, ErrReduceNoOutput, err)
+}
+
 func BenchmarkMapReduce(b *testing.B) {
 	b.ReportAllocs()
 

core/mr/readme-cn.md

@@ -0,0 +1,89 @@
+# mapreduce
+
+[English](readme.md) | 简体中文
+
+## 为什么需要 MapReduce
+
+在实际的业务场景中我们常常需要从不同的 rpc 服务中获取相应属性来组装成复杂对象。
+
+比如要查询商品详情：
+
+1. 商品服务-查询商品属性
+2. 库存服务-查询库存属性
+3. 价格服务-查询价格属性
+4. 营销服务-查询营销属性
+
+如果是串行调用的话响应时间会随着 rpc 调用次数呈线性增长，所以我们要优化性能一般会将串行改并行。
+
+简单的场景下使用 `WaitGroup` 也能够满足需求，但是如果我们需要对 rpc 调用返回的数据进行校验、数据加工转换、数据汇总呢？继续使用 `WaitGroup` 就有点力不从心了，go 的官方库中并没有这种工具（java 中提供了 CompleteFuture），我们依据 MapReduce 架构思想实现了进程内的数据批处理 MapReduce 并发工具类。
+
+## 设计思路
+
+我们尝试把自己代入到作者的角色梳理一下并发工具可能的业务场景：
+
+1. 查询商品详情：支持并发调用多个服务来组合产品属性，支持调用错误可以立即结束。
+2. 商品详情页自动推荐用户卡券：支持并发校验卡券，校验失败自动剔除，返回全部卡券。
+
+以上实际都是在进行对输入数据进行处理最后输出清洗后的数据，针对数据处理有个非常经典的异步模式：生产者消费者模式。于是我们可以抽象一下数据批处理的生命周期，大致可以分为三个阶段：
+
+<img src="https://raw.githubusercontent.com/zeromicro/zero-doc/main/doc/images/mapreduce-serial-cn.png" width="500">
+
+1. 数据生产 generate
+2. 数据加工 mapper
+3. 数据聚合 reducer
+
+其中数据生产是不可或缺的阶段，数据加工、数据聚合是可选阶段，数据生产与加工支持并发调用，数据聚合基本属于纯内存操作单协程即可。
+
+再来思考一下不同阶段之间数据应该如何流转，既然不同阶段的数据处理都是由不同 goroutine 执行的，那么很自然的可以考虑采用 channel 来实现 goroutine 之间的通信。
+
+<img src="https://raw.githubusercontent.com/zeromicro/zero-doc/main/doc/images/mapreduce-cn.png" width="500">
+
+
+如何实现随时终止流程呢？
+
+`goroutine` 中监听一个全局的结束 `channel` 和调用方提供的 `ctx` 就行。
+
+## 简单示例
+
+并行求平方和（不要嫌弃示例简单，只是模拟并发）
+
+```go
+package main
+
+import (
+    "fmt"
+    "log"
+
+    "github.com/tal-tech/go-zero/core/mr"
+)
+
+func main() {
+    val, err := mr.MapReduce(func(source chan<- interface{}) {
+        // generator
+        for i := 0; i < 10; i++ {
+            source <- i
+        }
+    }, func(item interface{}, writer mr.Writer, cancel func(error)) {
+        // mapper
+        i := item.(int)
+        writer.Write(i * i)
+    }, func(pipe <-chan interface{}, writer mr.Writer, cancel func(error)) {
+        // reducer
+        var sum int
+        for i := range pipe {
+            sum += i.(int)
+        }
+        writer.Write(sum)
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+    fmt.Println("result:", val)
+}
+```
+
+更多示例：[https://github.com/zeromicro/zero-examples/tree/main/mapreduce](https://github.com/zeromicro/zero-examples/tree/main/mapreduce)
+
+## 欢迎 star！⭐
+
+如果你正在使用或者觉得这个项目对你有帮助，请 **star** 支持，感谢！

core/mr/readme.md

@@ -0,0 +1,90 @@
+<img align="right" width="150px" src="https://raw.githubusercontent.com/zeromicro/zero-doc/main/doc/images/go-zero.png">
+
+# mapreduce
+
+English | [简体中文](readme-cn.md)
+
+## Why MapReduce is needed
+
+In practical business scenarios we often need to get the corresponding properties from different rpc services to assemble complex objects.
+
+For example, to query product details.
+
+1. product service - query product attributes
+2. inventory service - query inventory properties
+3. price service - query price attributes
+4. marketing service - query marketing properties
+
+If it is a serial call, the response time will increase linearly with the number of rpc calls, so we will generally change serial to parallel to optimize response time.
+
+Simple scenarios using `WaitGroup` can also meet the needs, but what if we need to check the data returned by the rpc call, data processing, data aggregation? The official go library does not have such a tool (CompleteFuture is provided in java), so we implemented an in-process data batching MapReduce concurrent tool based on the MapReduce architecture.
+
+## Design ideas
+
+Let's try to put ourselves in the author's shoes and sort out the possible business scenarios for the concurrency tool:
+
+1. querying product details: supporting concurrent calls to multiple services to combine product attributes, and supporting call errors that can be ended immediately.
+2. automatic recommendation of user card coupons on product details page: support concurrently verifying card coupons, automatically rejecting them if they fail, and returning all of them.
+
+The above is actually processing the input data and finally outputting the cleaned data. There is a very classic asynchronous pattern for data processing: the producer-consumer pattern. So we can abstract the life cycle of data batch processing, which can be roughly divided into three phases.
+
+<img src="https://raw.githubusercontent.com/zeromicro/zero-doc/main/doc/images/mapreduce-serial-en.png" width="500">
+
+1. data production generate
+2. data processing mapper
+3. data aggregation reducer
+
+Data producing is an indispensable stage, data processing and data aggregation are optional stages, data producing and processing support concurrent calls, data aggregation is basically a pure memory operation, so a single concurrent process can do it.
+
+Since different stages of data processing are performed by different goroutines, it is natural to consider the use of channel to achieve communication between goroutines.
+
+<img src="https://raw.githubusercontent.com/zeromicro/zero-doc/main/doc/images/mapreduce-en.png" width="500">
+
+How can I terminate the process at any time?
+
+It's simple, just receive from a  channel or the given context in the goroutine.
+
+## A simple example
+
+Calculate the sum of squares, simulating the concurrency.
+
+```go
+package main
+
+import (
+    "fmt"
+    "log"
+
+    "github.com/tal-tech/go-zero/core/mr"
+)
+
+func main() {
+    val, err := mr.MapReduce(func(source chan<- interface{}) {
+        // generator
+        for i := 0; i < 10; i++ {
+            source <- i
+        }
+    }, func(item interface{}, writer mr.Writer, cancel func(error)) {
+        // mapper
+        i := item.(int)
+        writer.Write(i * i)
+    }, func(pipe <-chan interface{}, writer mr.Writer, cancel func(error)) {
+        // reducer
+        var sum int
+        for i := range pipe {
+            sum += i.(int)
+        }
+        writer.Write(sum)
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+    fmt.Println("result:", val)
+}
+```
+
+More examples: [https://github.com/zeromicro/zero-examples/tree/main/mapreduce](https://github.com/zeromicro/zero-examples/tree/main/mapreduce)
+
+## Give a Star! ⭐
+
+If you like or are using this project to learn or start your solution, please give it a star. Thanks!

core/proc/goroutines+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package proc

core/proc/goroutines.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc

core/proc/profile+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package proc

core/proc/profile.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc

core/proc/shutdown+polyfill.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package proc
@@ -14,5 +15,5 @@ func AddWrapUpListener(fn func()) func() {
 	return fn
 }
 
-func SetTimeoutToForceQuit(duration time.Duration) {
+func SetTimeToForceQuit(duration time.Duration) {
 }

core/proc/shutdown.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/core/threading"
 )
 
 const (
@@ -45,10 +47,10 @@ func gracefulStop(signals chan os.Signal) {
 	signal.Stop(signals)
 
 	logx.Info("Got signal SIGTERM, shutting down...")
-	wrapUpListeners.notifyListeners()
+	go wrapUpListeners.notifyListeners()
 
 	time.Sleep(wrapUpTime)
-	shutdownListeners.notifyListeners()
+	go shutdownListeners.notifyListeners()
 
 	time.Sleep(delayTimeBeforeForceQuit - wrapUpTime)
 	logx.Infof("Still alive after %v, going to force kill the process...", delayTimeBeforeForceQuit)
@@ -80,7 +82,9 @@ func (lm *listenerManager) notifyListeners() {
 	lm.lock.Lock()
 	defer lm.lock.Unlock()
 
+	group := threading.NewRoutineGroup()
 	for _, listener := range lm.listeners {
-		listener()
+		group.RunSafe(listener)
 	}
+	group.Wait()
 }

core/proc/signals+polyfill.go

@@ -0,0 +1,10 @@
+//go:build windows
+// +build windows
+
+package proc
+
+import "context"
+
+func Done() <-chan struct{} {
+	return context.Background().Done()
+}

core/proc/signals.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package proc
@@ -12,6 +13,8 @@ import (
 
 const timeFormat = "0102150405"
 
+var done = make(chan struct{})
+
 func init() {
 	go func() {
 		var profiler Stopper
@@ -33,6 +36,13 @@ func init() {
 					profiler = nil
 				}
 			case syscall.SIGTERM:
+				select {
+				case <-done:
+					// already closed
+				default:
+					close(done)
+				}
+
 				gracefulStop(signals)
 			default:
 				logx.Error("Got unregistered signal:", v)
@@ -40,3 +50,8 @@ func init() {
 		}
 	}()
 }
+
+// Done returns the channel that notifies the process quitting.
+func Done() <-chan struct{} {
+	return done
+}

core/proc/signals_test.go

@@ -0,0 +1,16 @@
+package proc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDone(t *testing.T) {
+	select {
+	case <-Done():
+		assert.Fail(t, "should run")
+	default:
+	}
+	assert.NotNil(t, Done())
+}

core/prometheus/agent.go

@@ -23,11 +23,11 @@ func Enabled() bool {
 
 // StartAgent starts a prometheus agent.
 func StartAgent(c Config) {
-	once.Do(func() {
-		if len(c.Host) == 0 {
-			return
-		}
+	if len(c.Host) == 0 {
+		return
+	}
 
+	once.Do(func() {
 		enabled.Set(true)
 		threading.GoSafe(func() {
 			http.Handle(c.Path, promhttp.Handler())

core/search/tree_debug.go

@@ -1,3 +1,4 @@
+//go:build debug
 // +build debug
 
 package search

core/search/tree_test.go

@@ -1,9 +1,12 @@
 package search
 
 import (
+	"math/rand"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/stringx"
 )
 
 type mockedRoute struct {
@@ -139,11 +142,9 @@ func TestStrictSearchSibling(t *testing.T) {
 		tree.Add(r.route, r.value)
 	}
 
-	for i := 0; i < 1000; i++ {
-		result, ok := tree.Search(query)
-		assert.True(t, ok)
-		assert.Equal(t, 3, result.Item.(int))
-	}
+	result, ok := tree.Search(query)
+	assert.True(t, ok)
+	assert.Equal(t, 3, result.Item.(int))
 }
 
 func TestAddDuplicate(t *testing.T) {
@@ -185,3 +186,41 @@ func TestSearchInvalidItem(t *testing.T) {
 	err := tree.Add("/", nil)
 	assert.Equal(t, errEmptyItem, err)
 }
+
+func BenchmarkSearchTree(b *testing.B) {
+	const (
+		avgLen  = 1000
+		entries = 10000
+	)
+
+	tree := NewTree()
+	generate := func() string {
+		var buf strings.Builder
+		size := rand.Intn(avgLen) + avgLen/2
+		val := stringx.Randn(size)
+		prev := 0
+		for j := rand.Intn(9) + 1; j < size; j += rand.Intn(9) + 1 {
+			buf.WriteRune('/')
+			buf.WriteString(val[prev:j])
+			prev = j
+		}
+		if prev < size {
+			buf.WriteRune('/')
+			buf.WriteString(val[prev:])
+		}
+		return buf.String()
+	}
+	index := rand.Intn(entries)
+	var query string
+	for i := 0; i < entries; i++ {
+		val := generate()
+		if i == index {
+			query = val
+		}
+		tree.Add(val, i)
+	}
+
+	for i := 0; i < b.N; i++ {
+		tree.Search(query)
+	}
+}

core/service/serviceconf.go

@@ -7,6 +7,7 @@ import (
 	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/prometheus"
 	"github.com/tal-tech/go-zero/core/stat"
+	"github.com/tal-tech/go-zero/core/trace"
 )
 
 const (
@@ -29,6 +30,7 @@ type ServiceConf struct {
 	Mode       string            `json:",default=pro,options=dev|test|rt|pre|pro"`
 	MetricsUrl string            `json:",optional"`
 	Prometheus prometheus.Config `json:",optional"`
+	Telemetry  trace.Config      `json:",optional"`
 }
 
 // MustSetUp sets up the service, exits on error.
@@ -49,6 +51,12 @@ func (sc ServiceConf) SetUp() error {
 
 	sc.initMode()
 	prometheus.StartAgent(sc.Prometheus)
+
+	if len(sc.Telemetry.Name) == 0 {
+		sc.Telemetry.Name = sc.Name
+	}
+	trace.StartAgent(sc.Telemetry)
+
 	if len(sc.MetricsUrl) > 0 {
 		stat.SetReportWriter(stat.NewRemoteWriter(sc.MetricsUrl))
 	}

core/service/servicegroup.go

@@ -26,6 +26,7 @@ type (
 	}
 
 	// A ServiceGroup is a group of services.
+	// Attention: the starting order of the added services is not guaranteed.
 	ServiceGroup struct {
 		services []Service
 		stopOnce func()
@@ -41,7 +42,8 @@ func NewServiceGroup() *ServiceGroup {
 
 // Add adds service into sg.
 func (sg *ServiceGroup) Add(service Service) {
-	sg.services = append(sg.services, service)
+	// push front, stop with reverse order.
+	sg.services = append([]Service{service}, sg.services...)
 }
 
 // Start starts the ServiceGroup.

core/stat/alert+polyfill.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package stat

core/stat/alert.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package stat

core/stat/alert_test.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package stat

core/stat/internal/cpu_linux_test.go

@@ -7,7 +7,9 @@ import (
 )
 
 func TestRefreshCpu(t *testing.T) {
-	assert.True(t, RefreshCpu() >= 0)
+	assert.NotPanics(t, func() {
+		RefreshCpu()
+	})
 }
 
 func BenchmarkRefreshCpu(b *testing.B) {

core/stat/internal/cpu_other.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package internal

core/stat/usage.go

@@ -38,7 +38,9 @@ func init() {
 					atomic.StoreInt64(&cpuUsage, usage)
 				})
 			case <-allTicker.C:
-				printUsage()
+				if logEnabled.True() {
+					printUsage()
+				}
 			}
 		}
 	}()

core/stores/builder/builder.go

@@ -0,0 +1,63 @@
+package builder
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+)
+
+const dbTag = "db"
+
+// RawFieldNames converts golang struct field into slice string.
+func RawFieldNames(in interface{}, postgresSql ...bool) []string {
+	out := make([]string, 0)
+	v := reflect.ValueOf(in)
+	if v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+
+	var pg bool
+	if len(postgresSql) > 0 {
+		pg = postgresSql[0]
+	}
+
+	// we only accept structs
+	if v.Kind() != reflect.Struct {
+		panic(fmt.Errorf("ToMap only accepts structs; got %T", v))
+	}
+
+	typ := v.Type()
+	for i := 0; i < v.NumField(); i++ {
+		// gets us a StructField
+		fi := typ.Field(i)
+		if tagv := fi.Tag.Get(dbTag); tagv != "" {
+			if pg {
+				out = append(out, tagv)
+			} else {
+				out = append(out, fmt.Sprintf("`%s`", tagv))
+			}
+		} else {
+			if pg {
+				out = append(out, fi.Name)
+			} else {
+				out = append(out, fmt.Sprintf("`%s`", fi.Name))
+			}
+		}
+	}
+
+	return out
+}
+
+// PostgreSqlJoin concatenates the given elements into a string.
+func PostgreSqlJoin(elems []string) string {
+	b := new(strings.Builder)
+	for index, e := range elems {
+		b.WriteString(fmt.Sprintf("%s = $%d, ", e, index+2))
+	}
+
+	if b.Len() == 0 {
+		return b.String()
+	}
+
+	return b.String()[0 : b.Len()-2]
+}

core/stores/builder/builder_test.go

@@ -0,0 +1,24 @@
+package builder
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mockedUser struct {
+	ID       string `db:"id" json:"id,omitempty"`
+	UserName string `db:"user_name" json:"userName,omitempty"`
+	Sex      int    `db:"sex" json:"sex,omitempty"`
+	UUID     string `db:"uuid" uuid:"uuid,omitempty"`
+	Age      int    `db:"age" json:"age"`
+}
+
+func TestFieldNames(t *testing.T) {
+	t.Run("new", func(t *testing.T) {
+		var u mockedUser
+		out := RawFieldNames(&u)
+		expected := []string{"`id`", "`user_name`", "`sex`", "`uuid`", "`age`"}
+		assert.Equal(t, expected, out)
+	})
+}

core/stores/cache/cache.go

@@ -29,7 +29,7 @@ type (
 )
 
 // New returns a Cache.
-func New(c ClusterConf, barrier syncx.SharedCalls, st *Stat, errNotFound error,
+func New(c ClusterConf, barrier syncx.SingleFlight, st *Stat, errNotFound error,
 	opts ...Option) Cache {
 	if len(c) == 0 || TotalWeights(c) <= 0 {
 		log.Fatal("no cache nodes")

core/stores/cache/cache_test.go

@@ -23,6 +23,7 @@ type mockedNode struct {
 
 func (mc *mockedNode) Del(keys ...string) error {
 	var be errorx.BatchError
+
 	for _, key := range keys {
 		if _, ok := mc.vals[key]; !ok {
 			be.Add(mc.errNotFound)
@@ -30,6 +31,7 @@ func (mc *mockedNode) Del(keys ...string) error {
 			delete(mc.vals, key)
 		}
 	}
+
 	return be.Err()
 }
 
@@ -102,7 +104,7 @@ func TestCache_SetDel(t *testing.T) {
 			Weight: 100,
 		},
 	}
-	c := New(conf, syncx.NewSharedCalls(), NewStat("mock"), errPlaceholder)
+	c := New(conf, syncx.NewSingleFlight(), NewStat("mock"), errPlaceholder)
 	for i := 0; i < total; i++ {
 		if i%2 == 0 {
 			assert.Nil(t, c.Set(fmt.Sprintf("key/%d", i), i))
@@ -140,7 +142,7 @@ func TestCache_OneNode(t *testing.T) {
 			Weight: 100,
 		},
 	}
-	c := New(conf, syncx.NewSharedCalls(), NewStat("mock"), errPlaceholder)
+	c := New(conf, syncx.NewSingleFlight(), NewStat("mock"), errPlaceholder)
 	for i := 0; i < total; i++ {
 		if i%2 == 0 {
 			assert.Nil(t, c.Set(fmt.Sprintf("key/%d", i), i))

core/stores/cache/cachenode.go

@@ -29,7 +29,7 @@ type cacheNode struct {
 	rds            *redis.Redis
 	expiry         time.Duration
 	notFoundExpiry time.Duration
-	barrier        syncx.SharedCalls
+	barrier        syncx.SingleFlight
 	r              *rand.Rand
 	lock           *sync.Mutex
 	unstableExpiry mathx.Unstable
@@ -43,7 +43,7 @@ type cacheNode struct {
 // st is used to stat the cache.
 // errNotFound defines the error that returned on cache not found.
 // opts are the options that customize the cacheNode.
-func NewNode(rds *redis.Redis, barrier syncx.SharedCalls, st *Stat,
+func NewNode(rds *redis.Redis, barrier syncx.SingleFlight, st *Stat,
 	errNotFound error, opts ...Option) Cache {
 	o := newOptions(opts...)
 	return cacheNode{
@@ -65,9 +65,18 @@ func (c cacheNode) Del(keys ...string) error {
 		return nil
 	}
 
-	if _, err := c.rds.Del(keys...); err != nil {
-		logx.Errorf("failed to clear cache with keys: %q, error: %v", formatKeys(keys), err)
-		c.asyncRetryDelCache(keys...)
+	if len(keys) > 1 && c.rds.Type == redis.ClusterType {
+		for _, key := range keys {
+			if _, err := c.rds.Del(key); err != nil {
+				logx.Errorf("failed to clear cache with key: %q, error: %v", key, err)
+				c.asyncRetryDelCache(key)
+			}
+		}
+	} else {
+		if _, err := c.rds.Del(keys...); err != nil {
+			logx.Errorf("failed to clear cache with keys: %q, error: %v", formatKeys(keys), err)
+			c.asyncRetryDelCache(keys...)
+		}
 	}
 
 	return nil

core/stores/cache/cachenode_test.go

@@ -29,6 +29,7 @@ func init() {
 func TestCacheNode_DelCache(t *testing.T) {
 	store, clean, err := redistest.CreateRedis()
 	assert.Nil(t, err)
+	store.Type = redis.ClusterType
 	defer clean()
 
 	cn := cacheNode{
@@ -49,13 +50,30 @@ func TestCacheNode_DelCache(t *testing.T) {
 	assert.Nil(t, cn.Del("first", "second"))
 }
 
+func TestCacheNode_DelCacheWithErrors(t *testing.T) {
+	store, clean, err := redistest.CreateRedis()
+	assert.Nil(t, err)
+	defer clean()
+	store.Type = redis.ClusterType
+
+	cn := cacheNode{
+		rds:            store,
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewStat("any"),
+		errNotFound:    errTestNotFound,
+	}
+	assert.Nil(t, cn.Del("third", "fourth"))
+}
+
 func TestCacheNode_InvalidCache(t *testing.T) {
 	s, err := miniredis.Run()
 	assert.Nil(t, err)
 	defer s.Close()
 
 	cn := cacheNode{
-		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		rds:            redis.New(s.Addr()),
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
@@ -78,7 +96,7 @@ func TestCacheNode_Take(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -105,7 +123,7 @@ func TestCacheNode_TakeNotFound(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -144,7 +162,7 @@ func TestCacheNode_TakeWithExpire(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -171,7 +189,7 @@ func TestCacheNode_String(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),
@@ -188,7 +206,7 @@ func TestCacheValueWithBigInt(t *testing.T) {
 	cn := cacheNode{
 		rds:            store,
 		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		barrier:        syncx.NewSharedCalls(),
+		barrier:        syncx.NewSingleFlight(),
 		lock:           new(sync.Mutex),
 		unstableExpiry: mathx.NewUnstable(expiryDeviation),
 		stat:           NewStat("any"),

core/stores/clickhouse/clickhouse.go

@@ -1,7 +1,7 @@
 package clickhouse
 
 import (
-	// imports the driver.
+	// imports the driver, don't remove this comment, golint requires.
 	_ "github.com/ClickHouse/clickhouse-go"
 	"github.com/tal-tech/go-zero/core/stores/sqlx"
 )

core/stores/kv/store.go

@@ -16,6 +16,8 @@ var ErrNoRedisNode = errors.New("no redis node")
 type (
 	// Store interface represents a KV store.
 	Store interface {
+		Decr(key string) (int64, error)
+		Decrby(key string, increment int64) (int64, error)
 		Del(keys ...string) (int, error)
 		Eval(script, key string, args ...interface{}) (interface{}, error)
 		Exists(key string) (bool, error)
@@ -36,6 +38,7 @@ type (
 		Hvals(key string) ([]string, error)
 		Incr(key string) (int64, error)
 		Incrby(key string, increment int64) (int64, error)
+		Lindex(key string, index int64) (string, error)
 		Llen(key string) (int, error)
 		Lpop(key string) (string, error)
 		Lpush(key string, values ...interface{}) (int, error)
@@ -102,6 +105,24 @@ func NewStore(c KvConf) Store {
 	}
 }
 
+func (cs clusterStore) Decr(key string) (int64, error) {
+	node, err := cs.getRedis(key)
+	if err != nil {
+		return 0, err
+	}
+
+	return node.Decr(key)
+}
+
+func (cs clusterStore) Decrby(key string, increment int64) (int64, error) {
+	node, err := cs.getRedis(key)
+	if err != nil {
+		return 0, err
+	}
+
+	return node.Decrby(key, increment)
+}
+
 func (cs clusterStore) Del(keys ...string) (int, error) {
 	var val int
 	var be errorx.BatchError
@@ -303,6 +324,15 @@ func (cs clusterStore) Llen(key string) (int, error) {
 	return node.Llen(key)
 }
 
+func (cs clusterStore) Lindex(key string, index int64) (string, error) {
+	node, err := cs.getRedis(key)
+	if err != nil {
+		return "", err
+	}
+
+	return node.Lindex(key, index)
+}
+
 func (cs clusterStore) Lpop(key string) (string, error) {
 	node, err := cs.getRedis(key)
 	if err != nil {

core/stores/kv/store_test.go

@@ -17,6 +17,36 @@ var (
 	s2, _ = miniredis.Run()
 )
 
+func TestRedis_Decr(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Decr("a")
+	assert.NotNil(t, err)
+
+	runOnCluster(t, func(client Store) {
+		val, err := client.Decr("a")
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-1), val)
+		val, err = client.Decr("a")
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-2), val)
+	})
+}
+
+func TestRedis_DecrBy(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Incrby("a", 2)
+	assert.NotNil(t, err)
+
+	runOnCluster(t, func(client Store) {
+		val, err := client.Decrby("a", 2)
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-2), val)
+		val, err = client.Decrby("a", 3)
+		assert.Nil(t, err)
+		assert.Equal(t, int64(-5), val)
+	})
+}
+
 func TestRedis_Exists(t *testing.T) {
 	store := clusterStore{dispatcher: hash.NewConsistentHash()}
 	_, err := store.Exists("foo")
@@ -234,6 +264,8 @@ func TestRedis_List(t *testing.T) {
 	assert.NotNil(t, err)
 	_, err = store.Lrem("key", 0, "val")
 	assert.NotNil(t, err)
+	_, err = store.Lindex("key", 0)
+	assert.NotNil(t, err)
 
 	runOnCluster(t, func(client Store) {
 		val, err := client.Lpush("key", "value1", "value2")
@@ -245,6 +277,9 @@ func TestRedis_List(t *testing.T) {
 		val, err = client.Llen("key")
 		assert.Nil(t, err)
 		assert.Equal(t, 4, val)
+		value, err := client.Lindex("key", 0)
+		assert.Nil(t, err)
+		assert.Equal(t, "value2", value)
 		vals, err := client.Lrange("key", 0, 10)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"value2", "value1", "value3", "value4"}, vals)
@@ -667,10 +702,12 @@ func TestRedis_HyperLogLog(t *testing.T) {
 	assert.NotNil(t, err)
 
 	runOnCluster(t, func(cluster Store) {
-		_, err := cluster.Pfadd("key")
-		assert.NotNil(t, err)
-		_, err = cluster.Pfcount("key")
-		assert.NotNil(t, err)
+		ok, err := cluster.Pfadd("key", "value")
+		assert.Nil(t, err)
+		assert.True(t, ok)
+		val, err := cluster.Pfcount("key")
+		assert.Nil(t, err)
+		assert.Equal(t, int64(1), val)
 	})
 }
 

core/stores/mongo/collection.go

@@ -11,7 +11,7 @@ import (
 	"github.com/tal-tech/go-zero/core/timex"
 )
 
-const slowThreshold = time.Millisecond * 500
+const defaultSlowThreshold = time.Millisecond * 500
 
 // ErrNotFound is an alias of mgo.ErrNotFound.
 var ErrNotFound = mgo.ErrNotFound
@@ -203,7 +203,7 @@ func (c *decoratedCollection) logDuration(method string, duration time.Duration,
 	if e != nil {
 		logx.Error(err)
 	} else if err != nil {
-		if duration > slowThreshold {
+		if duration > slowThreshold.Load() {
 			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - fail(%s) - %s",
 				c.name, method, err.Error(), string(content))
 		} else {
@@ -211,7 +211,7 @@ func (c *decoratedCollection) logDuration(method string, duration time.Duration,
 				c.name, method, err.Error(), string(content))
 		}
 	} else {
-		if duration > slowThreshold {
+		if duration > slowThreshold.Load() {
 			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - ok - %s",
 				c.name, method, string(content))
 		} else {

core/stores/mongo/internal/collection_mock.go

@@ -5,9 +5,10 @@
 package internal
 
 import (
+	reflect "reflect"
+
 	mgo "github.com/globalsign/mgo"
 	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
 )
 
 // MockMgoCollection is a mock of MgoCollection interface

core/stores/mongo/iter_mock.go

@@ -5,9 +5,10 @@
 package mongo
 
 import (
+	reflect "reflect"
+
 	bson "github.com/globalsign/mgo/bson"
 	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
 )
 
 // MockIter is a mock of Iter interface

core/stores/mongo/model.go

@@ -8,23 +8,14 @@ import (
 	"github.com/tal-tech/go-zero/core/breaker"
 )
 
-type (
-	options struct {
-		timeout time.Duration
-	}
-
-	// Option defines the method to customize a mongo model.
-	Option func(opts *options)
-
-	// A Model is a mongo model.
-	Model struct {
-		session    *concurrentSession
-		db         *mgo.Database
-		collection string
-		brk        breaker.Breaker
-		opts       []Option
-	}
-)
+// A Model is a mongo model.
+type Model struct {
+	session    *concurrentSession
+	db         *mgo.Database
+	collection string
+	brk        breaker.Breaker
+	opts       []Option
+}
 
 // MustNewModel returns a Model, exits on errors.
 func MustNewModel(url, collection string, opts ...Option) *Model {

core/stores/mongo/model_test.go

@@ -0,0 +1,14 @@
+package mongo
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWithTimeout(t *testing.T) {
+	o := defaultOptions()
+	WithTimeout(time.Second)(o)
+	assert.Equal(t, time.Second, o.timeout)
+}

core/stores/mongo/options.go

@@ -0,0 +1,29 @@
+package mongo
+
+import (
+	"time"
+
+	"github.com/tal-tech/go-zero/core/syncx"
+)
+
+var slowThreshold = syncx.ForAtomicDuration(defaultSlowThreshold)
+
+type (
+	options struct {
+		timeout time.Duration
+	}
+
+	// Option defines the method to customize a mongo model.
+	Option func(opts *options)
+)
+
+// SetSlowThreshold sets the slow threshold.
+func SetSlowThreshold(threshold time.Duration) {
+	slowThreshold.Set(threshold)
+}
+
+func defaultOptions() *options {
+	return &options{
+		timeout: defaultTimeout,
+	}
+}