feat: support http stream response (#4055)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.codecov.yml

@@ -1,3 +1,7 @@
+coverage:
+  status:
+    patch: true
+    project: false # disabled because project coverage is not stable
 comment:
   layout: "flags, files"
   behavior: once
@@ -6,3 +10,4 @@ ignore:
   - "tools"
   - "**/mock"
   - "**/*_mock.go"
+  - "**/*test"

.dockerignore

@@ -1 +1,7 @@
 **/.git
+.dockerignore
+Dockerfile
+goctl
+Makefile
+readme.md
+readme-cn.md

.github/dependabot.yml

@@ -5,6 +5,14 @@
 
 version: 2
 updates:
+  - package-ecosystem: "docker" # Update image tags in Dockerfile
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions" # Update GitHub Actions
+    directory: "/"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "gomod" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:

.github/workflows/codeql-analysis.yml

@@ -35,11 +35,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/go.yml

@@ -12,12 +12,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Go 1.x
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.18
+          go-version: '1.19'
           check-latest: true
           cache: true
         id: go
@@ -40,20 +40,20 @@ jobs:
         run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
 
       - name: Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
 
   test-win:
     name: Windows
     runs-on: windows-latest
     steps:
       - name: Checkout codebase
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Go 1.x
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          # use 1.18 to guarantee Go 1.18 compatibility
-          go-version: 1.18
+          # use 1.19 to guarantee Go 1.19 compatibility
+          go-version: '1.19'
           check-latest: true
           cache: true
 
@@ -61,5 +61,5 @@ jobs:
         run: |
           go mod verify
           go mod download
-          go test -v -race ./...
+          go test ./...
           cd tools/goctl && go build -v goctl.go

.github/workflows/issues.yml

@@ -7,7 +7,7 @@ jobs:
   close-issues:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v6
+      - uses: actions/stale@v9
         with:
           days-before-issue-stale: 365
           days-before-issue-close: 90

.github/workflows/release.yaml

@@ -16,13 +16,13 @@ jobs:
           - goarch: "386"
             goos: darwin
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: zeromicro/go-zero-release-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
-          goversion: "https://dl.google.com/go/go1.18.10.linux-amd64.tar.gz"
+          goversion: "https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz"
           project_path: "tools/goctl"
           binary_name: "goctl"
           extra_files: tools/goctl/readme.md tools/goctl/readme-cn.md

.github/workflows/reviewdog.yml

@@ -5,7 +5,7 @@ jobs:
     name: runner / staticcheck
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: reviewdog/action-staticcheck@v1
         with:
           github_token: ${{ secrets.github_token }}

.gitignore

@@ -12,11 +12,13 @@
 
 # ignore
 **/.idea
+**/.vscode
 **/.DS_Store
 **/logs
+**/adhoc
+**/coverage.txt
 
 # for test purpose
-**/adhoc
 go.work
 go.work.sum
 

CONTRIBUTING.md

@@ -1,102 +1,76 @@
-# Contributing
+# 🚀 Contributing to go-zero
 
-Welcome to go-zero!
+Welcome to the go-zero community! We're thrilled to have you here. Contributing to our project is a fantastic way to be a part of the go-zero journey. Let's make this guide exciting and fun!
 
--   [Before you get started](#before-you-get-started)
-    -   [Code of Conduct](#code-of-conduct)
-    -   [Community Expectations](#community-expectations)
--   [Getting started](#getting-started)
--   [Your First Contribution](#your-first-contribution)
-    -   [Find something to work on](#find-something-to-work-on)
-        -   [Find a good first topic](#find-a-good-first-topic)
-        -   [Work on an Issue](#work-on-an-issue)
-        -   [File an Issue](#file-an-issue)
--   [Contributor Workflow](#contributor-workflow)
-    -   [Creating Pull Requests](#creating-pull-requests)
-    -   [Code Review](#code-review)
-    -   [Testing](#testing)
+## 📜 Before You Dive In
 
-# Before you get started
+### 🤝 Code of Conduct
 
-## Code of Conduct
+Let's start on the right foot. Please take a moment to read and embrace our [Code of Conduct](/code-of-conduct.md). We're all about creating a welcoming and respectful environment.
 
-Please make sure to read and observe our [Code of Conduct](/code-of-conduct.md).
+### 🌟 Community Expectations
 
-## Community Expectations
+At go-zero, we're like a close-knit family, and we believe in creating a healthy, friendly, and productive atmosphere. It's all about sharing knowledge and building amazing things together.
 
-go-zero is a community project driven by its community which strives to promote a healthy, friendly and productive environment.
-go-zero is a web and rpc framework written in Go. It's born to ensure the stability of the busy sites with resilient design. Builtin goctl greatly improves the development productivity.
+## 🚀 Getting Started
 
-# Getting started
+Get your adventure rolling! Here's how to begin:
 
-- Fork the repository on GitHub.
-- Make your changes on your fork repository.
-- Submit a PR.
+1. 🍴 **Fork the Repository**: Head over to the GitHub repository and fork it to your own space.
 
+2. 🛠️ **Make Your Magic**: Work your magic in your forked repository. Create new features, squash bugs, or improve documentation - it's your world to conquer!
 
-# Your First Contribution
+3. 🚀 **Submit a PR (Pull Request)**: When you're ready to unveil your creation, submit a Pull Request. We can't wait to see your awesome work!
 
-We will help you to contribute in different areas like filing issues, developing features, fixing critical bugs and
-getting your work reviewed and merged.
+## 🌟 Your First Contribution
 
-If you have questions about the development process,
-feel free to [file an issue](https://github.com/zeromicro/go-zero/issues/new/choose).
+We're here to guide you on your quest to become a go-zero contributor. Whether you want to file issues, develop features, or tame some critical bugs, we've got you covered.
 
-## Find something to work on
+If you have questions or need guidance at any stage, don't hesitate to [open an issue](https://github.com/zeromicro/go-zero/issues/new/choose).
 
-We are always in need of help, be it fixing documentation, reporting bugs or writing some code.
-Look at places where you feel best coding practices aren't followed, code refactoring is needed or tests are missing.
-Here is how you get started.
+## 🔍 Find Something to Work On
 
-### Find a good first topic
+Ready to dive into the action? There are several ways to contribute:
 
-[go-zero](https://github.com/zeromicro/go-zero) has beginner-friendly issues that provide a good first issue.
-For example, [go-zero](https://github.com/zeromicro/go-zero) has
-[help wanted](https://github.com/zeromicro/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) and
-[good first issue](https://github.com/zeromicro/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-labels for issues that should not need deep knowledge of the system.
-We can help new contributors who wish to work on such issues.
+### 💼 Find a Good First Topic
 
-Another good way to contribute is to find a documentation improvement, such as a missing/broken link.
-Please see [Contributing](#contributing) below for the workflow.
+Discover easy-entry issues labeled as [help wanted](https://github.com/zeromicro/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) or [good first issue](https://github.com/zeromicro/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). These issues are perfect for newcomers and don't require deep knowledge of the system. We're here to assist you with these tasks.
 
-#### Work on an issue
+### 🪄 Work on an Issue
 
-When you are willing to take on an issue, just reply on the issue. The maintainer will assign it to you.
+Once you've picked an issue that excites you, let us know by commenting on it. Our maintainers will assign it to you, and you can embark on your mission!
 
-### File an Issue
+### 📢 File an Issue
 
-While we encourage everyone to contribute code, it is also appreciated when someone reports an issue.
+Reporting an issue is just as valuable as code contributions. If you discover a problem, don't hesitate to [open an issue](https://github.com/zeromicro/go-zero/issues/new/choose). Be sure to follow our guidelines when submitting an issue.
 
-Please follow the prompted submission guidelines while opening an issue.
+## 🎯 Contributor Workflow
 
-# Contributor Workflow
+Here's a rough guide to your contributor journey:
 
-Please do not ever hesitate to ask a question or send a pull request.
+1. 🌱 Create a New Branch: Start by creating a topic branch, usually based on the 'master' branch. This is where your contribution will grow.
 
-This is a rough outline of what a contributor's workflow looks like:
+2. 💡 Make Commits: Commit your work in logical units. Each commit should tell a story.
 
-- Create a topic branch from where to base the contribution. This is usually master.
-- Make commits of logical units.
-- Push changes in a topic branch to a personal fork of the repository.
-- Submit a pull request to [go-zero](https://github.com/zeromicro/go-zero).
+3. 🚀 Push Changes: Push the changes in your topic branch to your personal fork of the repository.
 
-## Creating Pull Requests
+4. 📦 Submit a Pull Request: When your creation is complete, submit a Pull Request to the [go-zero repository](https://github.com/zeromicro/go-zero).
 
-Pull requests are often called simply "PR".
-go-zero generally follows the standard [github pull request](https://help.github.com/articles/about-pull-requests/) process.
-To submit a proposed change, please develop the code/fix and add new test cases.
-After that, run these local verifications before submitting pull request to predict the pass or
-fail of continuous integration.
+## 🌠 Creating Pull Requests
 
-* Format the code with `gofmt`
-* Run the test with data race enabled `go test -race ./...`
+Pull Requests (PRs) are your way of making a grand entrance with your contribution. Here's how to do it:
 
-## Code Review
+- 💼 Format Your Code: Ensure your code is beautifully formatted with `gofmt`.
+- 🏃 Run Tests: Verify that your changes pass all the tests, including data race tests. Run `go test -race ./...` for the ultimate validation.
 
-To make it easier for your PR to receive reviews, consider the reviewers will need you to:
+## 👁️‍🗨️ Code Review
 
-* follow [good coding guidelines](https://github.com/golang/go/wiki/CodeReviewComments).
-* write [good commit messages](https://chris.beams.io/posts/git-commit/).
-* break large changes into a logical series of smaller patches which individually make easily understandable changes, and in aggregate solve a broader issue.
+Getting your PR reviewed is the final step before your contribution becomes part of go-zero's magical world. To make the process smooth, keep these things in mind:
 
+- 🧙‍♀️ Follow Good Coding Practices: Stick to [good coding guidelines](https://github.com/golang/go/wiki/CodeReviewComments).
+- 📝 Write Awesome Commit Messages: Craft [impressive commit messages](https://chris.beams.io/posts/git-commit/) - they're like spells in the wizard's book!
+- 🔍 Break It Down: For larger changes, consider breaking them into a series of smaller, logical patches. Each patch should make an understandable and meaningful improvement.
+
+Congratulations on your contribution journey! We're thrilled to have you as part of our go-zero community. Let's make amazing things together! 🌟
+
+Now, go out there and start your adventure! If you have any more magical ideas to enhance this guide, please share them. 🔥

SECURITY.md

@@ -0,0 +1,16 @@
+# Security Policy
+
+## Supported Versions
+
+We publish releases monthly.
+
+| Version  | Supported          |
+| -------  | ------------------ |
+| >= 1.4.4 | :white_check_mark: |
+| < 1.4.4  | :x:                |
+
+## Reporting a Vulnerability
+
+https://github.com/zeromicro/go-zero/security/advisories
+
+Accepted vulnerabilities are expected to be fixed within a month.

code-of-conduct.md

@@ -1,76 +1,127 @@
+
 # Contributor Covenant Code of Conduct
 
 ## Our Pledge
 
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
 
 ## Our Standards
 
-Examples of behavior that contributes to creating a positive environment
-include:
+Examples of behavior that contributes to a positive environment for our
+community include:
 
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
 
-Examples of unacceptable behavior by participants include:
+Examples of unacceptable behavior include:
 
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
 * Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
-## Our Responsibilities
+## Enforcement Responsibilities
 
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
 
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
 
 ## Scope
 
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
 
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
+reported to the community leaders responsible for enforcement at
+[INSERT CONTACT METHOD].
+All complaints will be reviewed and investigated promptly and fairly.
 
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
 
-[homepage]: https://www.contributor-covenant.org
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
 
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].

core/bloom/bloom.go

@@ -130,7 +130,7 @@ func (r *redisBitSet) check(ctx context.Context, offsets []uint) (bool, error) {
 	}
 
 	resp, err := r.store.ScriptRunCtx(ctx, testScript, []string{r.key}, args)
-	if err == redis.Nil {
+	if errors.Is(err, redis.Nil) {
 		return false, nil
 	} else if err != nil {
 		return false, err
@@ -162,7 +162,7 @@ func (r *redisBitSet) set(ctx context.Context, offsets []uint) error {
 	}
 
 	_, err = r.store.ScriptRunCtx(ctx, setScript, []string{r.key}, args)
-	if err == redis.Nil {
+	if errors.Is(err, redis.Nil) {
 		return nil
 	}
 

core/breaker/breaker.go

@@ -31,9 +31,10 @@ type (
 		Name() string
 
 		// Allow checks if the request is allowed.
-		// If allowed, a promise will be returned, the caller needs to call promise.Accept()
-		// on success, or call promise.Reject() on failure.
-		// If not allow, ErrServiceUnavailable will be returned.
+		// If allowed, a promise will be returned,
+		// otherwise ErrServiceUnavailable will be returned as the error.
+		// The caller needs to call promise.Accept() on success,
+		// or call promise.Reject() on failure.
 		Allow() (Promise, error)
 
 		// Do runs the given request if the Breaker accepts it.
@@ -46,23 +47,26 @@ type (
 		// DoWithAcceptable returns an error instantly if the Breaker rejects the request.
 		// If a panic occurs in the request, the Breaker handles it as an error
 		// and causes the same panic again.
-		// acceptable checks if it's a successful call, even if the err is not nil.
+		// acceptable checks if it's a successful call, even if the error is not nil.
 		DoWithAcceptable(req func() error, acceptable Acceptable) error
 
 		// DoWithFallback runs the given request if the Breaker accepts it.
 		// DoWithFallback runs the fallback if the Breaker rejects the request.
 		// If a panic occurs in the request, the Breaker handles it as an error
 		// and causes the same panic again.
-		DoWithFallback(req func() error, fallback func(err error) error) error
+		DoWithFallback(req func() error, fallback Fallback) error
 
 		// DoWithFallbackAcceptable runs the given request if the Breaker accepts it.
 		// DoWithFallbackAcceptable runs the fallback if the Breaker rejects the request.
 		// If a panic occurs in the request, the Breaker handles it as an error
 		// and causes the same panic again.
-		// acceptable checks if it's a successful call, even if the err is not nil.
-		DoWithFallbackAcceptable(req func() error, fallback func(err error) error, acceptable Acceptable) error
+		// acceptable checks if it's a successful call, even if the error is not nil.
+		DoWithFallbackAcceptable(req func() error, fallback Fallback, acceptable Acceptable) error
 	}
 
+	// Fallback is the func to be called if the request is rejected.
+	Fallback func(err error) error
+
 	// Option defines the method to customize a Breaker.
 	Option func(breaker *circuitBreaker)
 
@@ -86,12 +90,12 @@ type (
 
 	internalThrottle interface {
 		allow() (internalPromise, error)
-		doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error
+		doReq(req func() error, fallback Fallback, acceptable Acceptable) error
 	}
 
 	throttle interface {
 		allow() (Promise, error)
-		doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error
+		doReq(req func() error, fallback Fallback, acceptable Acceptable) error
 	}
 )
 
@@ -122,11 +126,11 @@ func (cb *circuitBreaker) DoWithAcceptable(req func() error, acceptable Acceptab
 	return cb.throttle.doReq(req, nil, acceptable)
 }
 
-func (cb *circuitBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
+func (cb *circuitBreaker) DoWithFallback(req func() error, fallback Fallback) error {
 	return cb.throttle.doReq(req, fallback, defaultAcceptable)
 }
 
-func (cb *circuitBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
+func (cb *circuitBreaker) DoWithFallbackAcceptable(req func() error, fallback Fallback,
 	acceptable Acceptable) error {
 	return cb.throttle.doReq(req, fallback, acceptable)
 }
@@ -168,7 +172,7 @@ func (lt loggedThrottle) allow() (Promise, error) {
 	}, lt.logError(err)
 }
 
-func (lt loggedThrottle) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
+func (lt loggedThrottle) doReq(req func() error, fallback Fallback, acceptable Acceptable) error {
 	return lt.logError(lt.internalThrottle.doReq(req, fallback, func(err error) bool {
 		accept := acceptable(err)
 		if !accept && err != nil {
@@ -179,7 +183,7 @@ func (lt loggedThrottle) doReq(req func() error, fallback func(err error) error,
 }
 
 func (lt loggedThrottle) logError(err error) error {
-	if err == ErrServiceUnavailable {
+	if errors.Is(err, ErrServiceUnavailable) {
 		// if circuit open, not possible to have empty error window
 		stat.Report(fmt.Sprintf(
 			"proc(%s/%d), callee: %s, breaker is open and requests dropped\nlast errors:\n%s",

core/breaker/breakers.go

@@ -22,14 +22,14 @@ func DoWithAcceptable(name string, req func() error, acceptable Acceptable) erro
 }
 
 // DoWithFallback calls Breaker.DoWithFallback on the Breaker with given name.
-func DoWithFallback(name string, req func() error, fallback func(err error) error) error {
+func DoWithFallback(name string, req func() error, fallback Fallback) error {
 	return do(name, func(b Breaker) error {
 		return b.DoWithFallback(req, fallback)
 	})
 }
 
 // DoWithFallbackAcceptable calls Breaker.DoWithFallbackAcceptable on the Breaker with given name.
-func DoWithFallbackAcceptable(name string, req func() error, fallback func(err error) error,
+func DoWithFallbackAcceptable(name string, req func() error, fallback Fallback,
 	acceptable Acceptable) error {
 	return do(name, func(b Breaker) error {
 		return b.DoWithFallbackAcceptable(req, fallback, acceptable)
@@ -59,7 +59,7 @@ func GetBreaker(name string) Breaker {
 // NoBreakerFor disables the circuit breaker for the given name.
 func NoBreakerFor(name string) {
 	lock.Lock()
-	breakers[name] = newNoOpBreaker()
+	breakers[name] = NopBreaker()
 	lock.Unlock()
 }
 

core/breaker/breakers_test.go

@@ -30,7 +30,7 @@ func TestBreakersDoWithAcceptable(t *testing.T) {
 		assert.Equal(t, errDummy, GetBreaker("anyone").DoWithAcceptable(func() error {
 			return errDummy
 		}, func(err error) bool {
-			return err == nil || err == errDummy
+			return err == nil || errors.Is(err, errDummy)
 		}))
 	}
 	verify(t, func() bool {
@@ -45,12 +45,12 @@ func TestBreakersDoWithAcceptable(t *testing.T) {
 		}, func(err error) bool {
 			return err == nil
 		})
-		assert.True(t, err == errDummy || err == ErrServiceUnavailable)
+		assert.True(t, errors.Is(err, errDummy) || errors.Is(err, ErrServiceUnavailable))
 	}
 	verify(t, func() bool {
-		return ErrServiceUnavailable == Do("another", func() error {
+		return errors.Is(Do("another", func() error {
 			return nil
-		})
+		}), ErrServiceUnavailable)
 	})
 }
 
@@ -75,12 +75,12 @@ func TestBreakersFallback(t *testing.T) {
 		}, func(err error) error {
 			return nil
 		})
-		assert.True(t, err == nil || err == errDummy)
+		assert.True(t, err == nil || errors.Is(err, errDummy))
 	}
 	verify(t, func() bool {
-		return ErrServiceUnavailable == Do("fallback", func() error {
+		return errors.Is(Do("fallback", func() error {
 			return nil
-		})
+		}), ErrServiceUnavailable)
 	})
 }
 
@@ -94,12 +94,12 @@ func TestBreakersAcceptableFallback(t *testing.T) {
 		}, func(err error) bool {
 			return err == nil
 		})
-		assert.True(t, err == nil || err == errDummy)
+		assert.True(t, err == nil || errors.Is(err, errDummy))
 	}
 	verify(t, func() bool {
-		return ErrServiceUnavailable == Do("acceptablefallback", func() error {
+		return errors.Is(Do("acceptablefallback", func() error {
 			return nil
-		})
+		}), ErrServiceUnavailable)
 	})
 }
 

core/breaker/googlebreaker.go

@@ -1,7 +1,6 @@
 package breaker
 
 import (
-	"math"
 	"time"
 
 	"github.com/zeromicro/go-zero/core/collection"
@@ -38,7 +37,8 @@ func (b *googleBreaker) accept() error {
 	accepts, total := b.history()
 	weightedAccepts := b.k * float64(accepts)
 	// https://landing.google.com/sre/sre-book/chapters/handling-overload/#eq2101
-	dropRatio := math.Max(0, (float64(total-protection)-weightedAccepts)/float64(total+1))
+	// for better performance, no need to care about negative ratio
+	dropRatio := (float64(total-protection) - weightedAccepts) / float64(total+1)
 	if dropRatio <= 0 {
 		return nil
 	}
@@ -52,6 +52,7 @@ func (b *googleBreaker) accept() error {
 
 func (b *googleBreaker) allow() (internalPromise, error) {
 	if err := b.accept(); err != nil {
+		b.markFailure()
 		return nil, err
 	}
 
@@ -60,8 +61,9 @@ func (b *googleBreaker) allow() (internalPromise, error) {
 	}, nil
 }
 
-func (b *googleBreaker) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
+func (b *googleBreaker) doReq(req func() error, fallback Fallback, acceptable Acceptable) error {
 	if err := b.accept(); err != nil {
+		b.markFailure()
 		if fallback != nil {
 			return fallback(err)
 		}
@@ -69,18 +71,19 @@ func (b *googleBreaker) doReq(req func() error, fallback func(err error) error,
 		return err
 	}
 
+	var success bool
 	defer func() {
-		if e := recover(); e != nil {
+		// if req() panic, success is false, mark as failure
+		if success {
+			b.markSuccess()
+		} else {
 			b.markFailure()
-			panic(e)
 		}
 	}()
 
 	err := req()
 	if acceptable(err) {
-		b.markSuccess()
-	} else {
-		b.markFailure()
+		success = true
 	}
 
 	return err

core/breaker/googlebreaker_test.go

@@ -95,7 +95,7 @@ func TestGoogleBreakerAcceptable(t *testing.T) {
 	assert.Equal(t, errAcceptable, b.doReq(func() error {
 		return errAcceptable
 	}, nil, func(err error) bool {
-		return err == errAcceptable
+		return errors.Is(err, errAcceptable)
 	}))
 }
 
@@ -105,7 +105,7 @@ func TestGoogleBreakerNotAcceptable(t *testing.T) {
 	assert.Equal(t, errAcceptable, b.doReq(func() error {
 		return errAcceptable
 	}, nil, func(err error) bool {
-		return err != errAcceptable
+		return !errors.Is(err, errAcceptable)
 	}))
 }
 
@@ -206,7 +206,7 @@ func BenchmarkGoogleBreakerAllow(b *testing.B) {
 	breaker := getGoogleBreaker()
 	b.ResetTimer()
 	for i := 0; i <= b.N; i++ {
-		breaker.accept()
+		_ = breaker.accept()
 		if i%2 == 0 {
 			breaker.markSuccess()
 		} else {
@@ -215,6 +215,16 @@ func BenchmarkGoogleBreakerAllow(b *testing.B) {
 	}
 }
 
+func BenchmarkGoogleBreakerDoReq(b *testing.B) {
+	breaker := getGoogleBreaker()
+	b.ResetTimer()
+	for i := 0; i <= b.N; i++ {
+		_ = breaker.doReq(func() error {
+			return nil
+		}, nil, defaultAcceptable)
+	}
+}
+
 func markSuccess(b *googleBreaker, count int) {
 	for i := 0; i < count; i++ {
 		p, err := b.allow()

core/breaker/nopbreaker.go

@@ -1,35 +1,35 @@
 package breaker
 
-const noOpBreakerName = "nopBreaker"
+const nopBreakerName = "nopBreaker"
 
-type noOpBreaker struct{}
+type nopBreaker struct{}
 
-func newNoOpBreaker() Breaker {
-	return noOpBreaker{}
+// NopBreaker returns a breaker that never trigger breaker circuit.
+func NopBreaker() Breaker {
+	return nopBreaker{}
 }
 
-func (b noOpBreaker) Name() string {
-	return noOpBreakerName
+func (b nopBreaker) Name() string {
+	return nopBreakerName
 }
 
-func (b noOpBreaker) Allow() (Promise, error) {
+func (b nopBreaker) Allow() (Promise, error) {
 	return nopPromise{}, nil
 }
 
-func (b noOpBreaker) Do(req func() error) error {
+func (b nopBreaker) Do(req func() error) error {
 	return req()
 }
 
-func (b noOpBreaker) DoWithAcceptable(req func() error, _ Acceptable) error {
+func (b nopBreaker) DoWithAcceptable(req func() error, _ Acceptable) error {
 	return req()
 }
 
-func (b noOpBreaker) DoWithFallback(req func() error, _ func(err error) error) error {
+func (b nopBreaker) DoWithFallback(req func() error, _ Fallback) error {
 	return req()
 }
 
-func (b noOpBreaker) DoWithFallbackAcceptable(req func() error, _ func(err error) error,
-	_ Acceptable) error {
+func (b nopBreaker) DoWithFallbackAcceptable(req func() error, _ Fallback, _ Acceptable) error {
 	return req()
 }
 

core/breaker/nopbreaker_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestNopBreaker(t *testing.T) {
-	b := newNoOpBreaker()
-	assert.Equal(t, noOpBreakerName, b.Name())
+	b := NopBreaker()
+	assert.Equal(t, nopBreakerName, b.Name())
 	p, err := b.Allow()
 	assert.Nil(t, err)
 	p.Accept()

core/codec/dh.go

@@ -23,7 +23,7 @@ var (
 	zero = big.NewInt(0)
 )
 
-// DhKey defines the Diffie Hellman key.
+// DhKey defines the Diffie-Hellman key.
 type DhKey struct {
 	PriKey *big.Int
 	PubKey *big.Int
@@ -46,7 +46,7 @@ func ComputeKey(pubKey, priKey *big.Int) (*big.Int, error) {
 	return new(big.Int).Exp(pubKey, priKey, p), nil
 }
 
-// GenerateKey returns a Diffie Hellman key.
+// GenerateKey returns a Diffie-Hellman key.
 func GenerateKey() (*DhKey, error) {
 	var err error
 	var x *big.Int

core/collection/cache.go

@@ -128,8 +128,8 @@ func (c *Cache) Take(key string, fetch func() (any, error)) (any, error) {
 
 	var fresh bool
 	val, err := c.barrier.Do(key, func() (any, error) {
-		// because O(1) on map search in memory, and fetch is an IO query
-		// so we do double check, cache might be taken by another call
+		// because O(1) on map search in memory, and fetch is an IO query,
+		// so we do double-check, cache might be taken by another call
 		if val, ok := c.doGet(key); ok {
 			return val, nil
 		}

core/collection/ring.go

@@ -25,8 +25,14 @@ func (r *Ring) Add(v any) {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	r.elements[r.index%len(r.elements)] = v
+	rlen := len(r.elements)
+	r.elements[r.index%rlen] = v
 	r.index++
+
+	// prevent ring index overflow
+	if r.index >= rlen<<1 {
+		r.index -= rlen
+	}
 }
 
 // Take takes all items from r.
@@ -36,16 +42,18 @@ func (r *Ring) Take() []any {
 
 	var size int
 	var start int
-	if r.index > len(r.elements) {
-		size = len(r.elements)
-		start = r.index % len(r.elements)
+	rlen := len(r.elements)
+
+	if r.index > rlen {
+		size = rlen
+		start = r.index % rlen
 	} else {
 		size = r.index
 	}
 
 	elements := make([]any, size)
 	for i := 0; i < size; i++ {
-		elements[i] = r.elements[(start+i)%len(r.elements)]
+		elements[i] = r.elements[(start+i)%rlen]
 	}
 
 	return elements

core/collection/safemap.go

@@ -29,6 +29,8 @@ func NewSafeMap() *SafeMap {
 // Del deletes the value with the given key from m.
 func (m *SafeMap) Del(key any) {
 	m.lock.Lock()
+	defer m.lock.Unlock()
+
 	if _, ok := m.dirtyOld[key]; ok {
 		delete(m.dirtyOld, key)
 		m.deletionOld++
@@ -52,7 +54,6 @@ func (m *SafeMap) Del(key any) {
 		m.dirtyNew = make(map[any]any)
 		m.deletionNew = 0
 	}
-	m.lock.Unlock()
 }
 
 // Get gets the value with the given key from m.
@@ -89,6 +90,8 @@ func (m *SafeMap) Range(f func(key, val any) bool) {
 // Set sets the value into m with the given key.
 func (m *SafeMap) Set(key, value any) {
 	m.lock.Lock()
+	defer m.lock.Unlock()
+
 	if m.deletionOld <= maxDeletion {
 		if _, ok := m.dirtyNew[key]; ok {
 			delete(m.dirtyNew, key)
@@ -102,7 +105,6 @@ func (m *SafeMap) Set(key, value any) {
 		}
 		m.dirtyNew[key] = value
 	}
-	m.lock.Unlock()
 }
 
 // Size returns the size of m.

core/collection/safemap_test.go

@@ -147,3 +147,65 @@ func TestSafeMap_Range(t *testing.T) {
 	assert.Equal(t, m.dirtyNew, newMap.dirtyNew)
 	assert.Equal(t, m.dirtyOld, newMap.dirtyOld)
 }
+
+func TestSetManyTimes(t *testing.T) {
+	const iteration = maxDeletion * 2
+	m := NewSafeMap()
+	for i := 0; i < iteration; i++ {
+		m.Set(i, i)
+		if i%3 == 0 {
+			m.Del(i / 2)
+		}
+	}
+	var count int
+	m.Range(func(k, v any) bool {
+		count++
+		return count < maxDeletion/2
+	})
+	assert.Equal(t, maxDeletion/2, count)
+	for i := 0; i < iteration; i++ {
+		m.Set(i, i)
+		if i%3 == 0 {
+			m.Del(i / 2)
+		}
+	}
+	for i := 0; i < iteration; i++ {
+		m.Set(i, i)
+		if i%3 == 0 {
+			m.Del(i / 2)
+		}
+	}
+	for i := 0; i < iteration; i++ {
+		m.Set(i, i)
+		if i%3 == 0 {
+			m.Del(i / 2)
+		}
+	}
+
+	count = 0
+	m.Range(func(k, v any) bool {
+		count++
+		return count < maxDeletion
+	})
+	assert.Equal(t, maxDeletion, count)
+}
+
+func TestSetManyTimesNew(t *testing.T) {
+	m := NewSafeMap()
+	for i := 0; i < maxDeletion*3; i++ {
+		m.Set(i, i)
+	}
+	for i := 0; i < maxDeletion*2; i++ {
+		m.Del(i)
+	}
+	for i := 0; i < maxDeletion*3; i++ {
+		m.Set(i+maxDeletion*3, i+maxDeletion*3)
+	}
+	for i := 0; i < maxDeletion*2; i++ {
+		m.Del(i + maxDeletion*2)
+	}
+	for i := 0; i < maxDeletion-copyThreshold+1; i++ {
+		m.Del(i + maxDeletion*2)
+	}
+	assert.Equal(t, 0, len(m.dirtyNew))
+}

core/conf/config.go

@@ -133,7 +133,7 @@ func addOrMergeFields(info *fieldInfo, key string, child *fieldInfo, fullName st
 			return newConflictKeyError(fullName)
 		}
 
-		if err := mergeFields(prev, key, child.children, fullName); err != nil {
+		if err := mergeFields(prev, child.children, fullName); err != nil {
 			return err
 		}
 	} else {
@@ -281,7 +281,7 @@ func getTagName(field reflect.StructField) string {
 	return field.Name
 }
 
-func mergeFields(prev *fieldInfo, key string, children map[string]*fieldInfo, fullName string) error {
+func mergeFields(prev *fieldInfo, children map[string]*fieldInfo, fullName string) error {
 	if len(prev.children) == 0 || len(children) == 0 {
 		return newConflictKeyError(fullName)
 	}

core/conf/config_test.go

@@ -35,11 +35,11 @@ func TestConfigJson(t *testing.T) {
 	"c": "${FOO}",
 	"d": "abcd!@#$112"
 }`
+	t.Setenv("FOO", "2")
+
 	for _, test := range tests {
 		test := test
 		t.Run(test, func(t *testing.T) {
-			os.Setenv("FOO", "2")
-			defer os.Unsetenv("FOO")
 			tmpfile, err := createTempFile(test, text)
 			assert.Nil(t, err)
 			defer os.Remove(tmpfile)
@@ -81,8 +81,7 @@ b = 1
 c = "${FOO}"
 d = "abcd!@#$112"
 `
-	os.Setenv("FOO", "2")
-	defer os.Unsetenv("FOO")
+	t.Setenv("FOO", "2")
 	tmpfile, err := createTempFile(".toml", text)
 	assert.Nil(t, err)
 	defer os.Remove(tmpfile)
@@ -207,8 +206,7 @@ b = 1
 c = "${FOO}"
 d = "abcd!@#112"
 `
-	os.Setenv("FOO", "2")
-	defer os.Unsetenv("FOO")
+	t.Setenv("FOO", "2")
 	tmpfile, err := createTempFile(".toml", text)
 	assert.Nil(t, err)
 	defer os.Remove(tmpfile)
@@ -239,11 +237,10 @@ func TestConfigJsonEnv(t *testing.T) {
 	"c": "${FOO}",
 	"d": "abcd!@#$a12 3"
 }`
+	t.Setenv("FOO", "2")
 	for _, test := range tests {
 		test := test
 		t.Run(test, func(t *testing.T) {
-			os.Setenv("FOO", "2")
-			defer os.Unsetenv("FOO")
 			tmpfile, err := createTempFile(test, text)
 			assert.Nil(t, err)
 			defer os.Remove(tmpfile)

core/conf/properties_test.go

@@ -45,8 +45,7 @@ func TestPropertiesEnv(t *testing.T) {
 	assert.Nil(t, err)
 	defer os.Remove(tmpfile)
 
-	os.Setenv("FOO", "2")
-	defer os.Unsetenv("FOO")
+	t.Setenv("FOO", "2")
 
 	props, err := LoadProperties(tmpfile, UseEnv())
 	assert.Nil(t, err)

core/conf/readme.md

@@ -12,7 +12,7 @@ type RestfulConf struct {
 	MaxConns     int           `json:",default=10000"`
 	MaxBytes     int64         `json:",default=1048576"`
 	Timeout      time.Duration `json:",default=3s"`
-	CpuThreshold int64         `json:",default=900,range=[0:1000]"`
+	CpuThreshold int64         `json:",default=900,range=[0:1000)"`
 }
 ```
 

core/discov/internal/registry.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"sort"
@@ -14,6 +15,7 @@ import (
 	"github.com/zeromicro/go-zero/core/logx"
 	"github.com/zeromicro/go-zero/core/syncx"
 	"github.com/zeromicro/go-zero/core/threading"
+	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
@@ -22,6 +24,7 @@ var (
 		clusters: make(map[string]*cluster),
 	}
 	connManager = syncx.NewResourceManager()
+	errClosed   = errors.New("etcd monitor chan has been closed")
 )
 
 // A Registry is a registry that manages the etcd client connections.
@@ -219,7 +222,7 @@ func (c *cluster) load(cli EtcdClient, key string) int64 {
 			break
 		}
 
-		logx.Error(err)
+		logx.Errorf("%s, key is %s", err.Error(), key)
 		time.Sleep(coolDownInterval)
 	}
 
@@ -288,40 +291,47 @@ func (c *cluster) reload(cli EtcdClient) {
 
 func (c *cluster) watch(cli EtcdClient, key string, rev int64) {
 	for {
-		if c.watchStream(cli, key, rev) {
+		err := c.watchStream(cli, key, rev)
+		if err == nil {
 			return
 		}
+
+		if rev != 0 && errors.Is(err, rpctypes.ErrCompacted) {
+			logx.Errorf("etcd watch stream has been compacted, try to reload, rev %d", rev)
+			rev = c.load(cli, key)
+		}
+
+		// log the error and retry
+		logx.Error(err)
 	}
 }
 
-func (c *cluster) watchStream(cli EtcdClient, key string, rev int64) bool {
+func (c *cluster) watchStream(cli EtcdClient, key string, rev int64) error {
 	var rch clientv3.WatchChan
 	if rev != 0 {
-		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix(),
-			clientv3.WithRev(rev+1))
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key),
+			clientv3.WithPrefix(), clientv3.WithRev(rev+1))
 	} else {
-		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
+		rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key),
+			clientv3.WithPrefix())
 	}
 
 	for {
 		select {
 		case wresp, ok := <-rch:
 			if !ok {
-				logx.Error("etcd monitor chan has been closed")
-				return false
+				return errClosed
 			}
 			if wresp.Canceled {
-				logx.Errorf("etcd monitor chan has been canceled, error: %v", wresp.Err())
-				return false
+				return fmt.Errorf("etcd monitor chan has been canceled, error: %w", wresp.Err())
 			}
 			if wresp.Err() != nil {
-				logx.Error(fmt.Sprintf("etcd monitor chan error: %v", wresp.Err()))
-				return false
+				return fmt.Errorf("etcd monitor chan error: %w", wresp.Err())
 			}
 
 			c.handleWatchEvents(key, wresp.Events)
 		case <-c.done:
-			return true
+			return nil
 		}
 	}
 }
@@ -337,13 +347,11 @@ func (c *cluster) watchConnState(cli EtcdClient) {
 // DialClient dials an etcd cluster with given endpoints.
 func DialClient(endpoints []string) (EtcdClient, error) {
 	cfg := clientv3.Config{
-		Endpoints:            endpoints,
-		AutoSyncInterval:     autoSyncInterval,
-		DialTimeout:          DialTimeout,
-		DialKeepAliveTime:    dialKeepAliveTime,
-		DialKeepAliveTimeout: DialTimeout,
-		RejectOldCluster:     true,
-		PermitWithoutStream:  true,
+		Endpoints:           endpoints,
+		AutoSyncInterval:    autoSyncInterval,
+		DialTimeout:         DialTimeout,
+		RejectOldCluster:    true,
+		PermitWithoutStream: true,
 	}
 	if account, ok := GetAccount(endpoints); ok {
 		cfg.Username = account.User

core/discov/internal/vars.go

@@ -9,7 +9,6 @@ const (
 	autoSyncInterval   = time.Minute
 	coolDownInterval   = time.Second
 	dialTimeout        = 5 * time.Second
-	dialKeepAliveTime  = 5 * time.Second
 	requestTimeout     = 3 * time.Second
 	endpointsSeparator = ","
 )

core/errorx/batcherror.go

@@ -1,11 +1,15 @@
 package errorx
 
-import "bytes"
+import (
+	"bytes"
+	"sync"
+)
 
 type (
 	// A BatchError is an error that can hold multiple errors.
 	BatchError struct {
 		errs errorArray
+		lock sync.Mutex
 	}
 
 	errorArray []error
@@ -13,6 +17,9 @@ type (
 
 // Add adds errs to be, nil errors are ignored.
 func (be *BatchError) Add(errs ...error) {
+	be.lock.Lock()
+	defer be.lock.Unlock()
+
 	for _, err := range errs {
 		if err != nil {
 			be.errs = append(be.errs, err)
@@ -22,6 +29,9 @@ func (be *BatchError) Add(errs ...error) {
 
 // Err returns an error that represents all errors.
 func (be *BatchError) Err() error {
+	be.lock.Lock()
+	defer be.lock.Unlock()
+
 	switch len(be.errs) {
 	case 0:
 		return nil
@@ -34,6 +44,9 @@ func (be *BatchError) Err() error {
 
 // NotNil checks if any error inside.
 func (be *BatchError) NotNil() bool {
+	be.lock.Lock()
+	defer be.lock.Unlock()
+
 	return len(be.errs) > 0
 }
 

core/errorx/batcherror_test.go

@@ -3,6 +3,7 @@ package errorx
 import (
 	"errors"
 	"fmt"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -33,7 +34,7 @@ func TestBatchErrorNilFromFunc(t *testing.T) {
 func TestBatchErrorOneError(t *testing.T) {
 	var batch BatchError
 	batch.Add(errors.New(err1))
-	assert.NotNil(t, batch)
+	assert.NotNil(t, batch.Err())
 	assert.Equal(t, err1, batch.Err().Error())
 	assert.True(t, batch.NotNil())
 }
@@ -42,7 +43,26 @@ func TestBatchErrorWithErrors(t *testing.T) {
 	var batch BatchError
 	batch.Add(errors.New(err1))
 	batch.Add(errors.New(err2))
-	assert.NotNil(t, batch)
+	assert.NotNil(t, batch.Err())
 	assert.Equal(t, fmt.Sprintf("%s\n%s", err1, err2), batch.Err().Error())
 	assert.True(t, batch.NotNil())
 }
+
+func TestBatchErrorConcurrentAdd(t *testing.T) {
+	const count = 10000
+	var batch BatchError
+	var wg sync.WaitGroup
+
+	wg.Add(count)
+	for i := 0; i < count; i++ {
+		go func() {
+			defer wg.Done()
+			batch.Add(errors.New(err1))
+		}()
+	}
+	wg.Wait()
+
+	assert.NotNil(t, batch.Err())
+	assert.Equal(t, count, len(batch.errs))
+	assert.True(t, batch.NotNil())
+}

core/executors/bulkexecutor_test.go

@@ -78,7 +78,7 @@ func TestBulkExecutorFlush(t *testing.T) {
 	wait.Wait()
 }
 
-func TestBuldExecutorFlushSlowTasks(t *testing.T) {
+func TestBulkExecutorFlushSlowTasks(t *testing.T) {
 	const total = 1500
 	lock := new(sync.Mutex)
 	result := make([]any, 0, 10000)

core/executors/periodicalexecutor_test.go

@@ -168,23 +168,23 @@ func TestPeriodicalExecutor_FlushPanic(t *testing.T) {
 
 func TestPeriodicalExecutor_Wait(t *testing.T) {
 	var lock sync.Mutex
-	executer := NewBulkExecutor(func(tasks []any) {
+	executor := NewBulkExecutor(func(tasks []any) {
 		lock.Lock()
 		defer lock.Unlock()
 		time.Sleep(10 * time.Millisecond)
 	}, WithBulkTasks(1), WithBulkInterval(time.Second))
 	for i := 0; i < 10; i++ {
-		executer.Add(1)
+		executor.Add(1)
 	}
-	executer.Flush()
-	executer.Wait()
+	executor.Flush()
+	executor.Wait()
 }
 
 func TestPeriodicalExecutor_WaitFast(t *testing.T) {
 	const total = 3
 	var cnt int
 	var lock sync.Mutex
-	executer := NewBulkExecutor(func(tasks []any) {
+	executor := NewBulkExecutor(func(tasks []any) {
 		defer func() {
 			cnt++
 		}()
@@ -193,10 +193,10 @@ func TestPeriodicalExecutor_WaitFast(t *testing.T) {
 		time.Sleep(10 * time.Millisecond)
 	}, WithBulkTasks(1), WithBulkInterval(10*time.Millisecond))
 	for i := 0; i < total; i++ {
-		executer.Add(2)
+		executor.Add(2)
 	}
-	executer.Flush()
-	executer.Wait()
+	executor.Flush()
+	executor.Wait()
 	assert.Equal(t, total, cnt)
 }
 

core/filex/file_test.go

@@ -74,6 +74,11 @@ func TestFirstLineShort(t *testing.T) {
 	assert.Equal(t, "first line", val)
 }
 
+func TestFirstLineError(t *testing.T) {
+	_, err := FirstLine("/tmp/does-not-exist")
+	assert.Error(t, err)
+}
+
 func TestLastLine(t *testing.T) {
 	filename, err := fs.TempFilenameWithText(text)
 	assert.Nil(t, err)
@@ -113,3 +118,8 @@ func TestLastLineWithLastNewlineShort(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "last line", val)
 }
+
+func TestLastLineError(t *testing.T) {
+	_, err := LastLine("/tmp/does-not-exist")
+	assert.Error(t, err)
+}

core/filex/progressscanner.go

@@ -5,7 +5,7 @@ import "gopkg.in/cheggaaa/pb.v1"
 type (
 	// A Scanner is used to read lines.
 	Scanner interface {
-		// Scan checks if has remaining to read.
+		// Scan checks if it has remaining to read.
 		Scan() bool
 		// Text returns next line.
 		Text() string

core/fs/temps.go

@@ -11,29 +11,29 @@ import (
 // The file is kept as open, the caller should close the file handle,
 // and remove the file by name.
 func TempFileWithText(text string) (*os.File, error) {
-	tmpfile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text)))
+	tmpFile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text)))
 	if err != nil {
 		return nil, err
 	}
 
-	if err := os.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+	if err := os.WriteFile(tmpFile.Name(), []byte(text), os.ModeTemporary); err != nil {
 		return nil, err
 	}
 
-	return tmpfile, nil
+	return tmpFile, nil
 }
 
 // TempFilenameWithText creates the file with the given content,
 // and returns the filename (full path).
 // The caller should remove the file after use.
 func TempFilenameWithText(text string) (string, error) {
-	tmpfile, err := TempFileWithText(text)
+	tmpFile, err := TempFileWithText(text)
 	if err != nil {
 		return "", err
 	}
 
-	filename := tmpfile.Name()
-	if err = tmpfile.Close(); err != nil {
+	filename := tmpFile.Name()
+	if err = tmpFile.Close(); err != nil {
 		return "", err
 	}
 

core/fx/retry.go

@@ -1,6 +1,12 @@
 package fx
 
-import "github.com/zeromicro/go-zero/core/errorx"
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/zeromicro/go-zero/core/errorx"
+)
 
 const defaultRetryTimes = 3
 
@@ -9,36 +15,110 @@ type (
 	RetryOption func(*retryOptions)
 
 	retryOptions struct {
-		times int
+		times        int
+		interval     time.Duration
+		timeout      time.Duration
+		ignoreErrors []error
 	}
 )
 
 // DoWithRetry runs fn, and retries if failed. Default to retry 3 times.
+// Note that if the fn function accesses global variables outside the function
+// and performs modification operations, it is best to lock them,
+// otherwise there may be data race issues
 func DoWithRetry(fn func() error, opts ...RetryOption) error {
+	return retry(context.Background(), func(errChan chan error, retryCount int) {
+		errChan <- fn()
+	}, opts...)
+}
+
+// DoWithRetryCtx runs fn, and retries if failed. Default to retry 3 times.
+// fn retryCount indicates the current number of retries, starting from 0
+// Note that if the fn function accesses global variables outside the function
+// and performs modification operations, it is best to lock them,
+// otherwise there may be data race issues
+func DoWithRetryCtx(ctx context.Context, fn func(ctx context.Context, retryCount int) error,
+	opts ...RetryOption) error {
+	return retry(ctx, func(errChan chan error, retryCount int) {
+		errChan <- fn(ctx, retryCount)
+	}, opts...)
+}
+
+func retry(ctx context.Context, fn func(errChan chan error, retryCount int), opts ...RetryOption) error {
 	options := newRetryOptions()
 	for _, opt := range opts {
 		opt(options)
 	}
 
 	var berr errorx.BatchError
+	var cancelFunc context.CancelFunc
+	if options.timeout > 0 {
+		ctx, cancelFunc = context.WithTimeout(ctx, options.timeout)
+		defer cancelFunc()
+	}
+
+	errChan := make(chan error, 1)
 	for i := 0; i < options.times; i++ {
-		if err := fn(); err != nil {
-			berr.Add(err)
-		} else {
-			return nil
+		go fn(errChan, i)
+
+		select {
+		case err := <-errChan:
+			if err != nil {
+				for _, ignoreErr := range options.ignoreErrors {
+					if errors.Is(err, ignoreErr) {
+						return nil
+					}
+				}
+				berr.Add(err)
+			} else {
+				return nil
+			}
+		case <-ctx.Done():
+			berr.Add(ctx.Err())
+			return berr.Err()
+		}
+
+		if options.interval > 0 {
+			select {
+			case <-ctx.Done():
+				berr.Add(ctx.Err())
+				return berr.Err()
+			case <-time.After(options.interval):
+			}
 		}
 	}
 
 	return berr.Err()
 }
 
-// WithRetry customize a DoWithRetry call with given retry times.
+// WithIgnoreErrors Ignore the specified errors
+func WithIgnoreErrors(ignoreErrors []error) RetryOption {
+	return func(options *retryOptions) {
+		options.ignoreErrors = ignoreErrors
+	}
+}
+
+// WithInterval customizes a DoWithRetry call with given interval.
+func WithInterval(interval time.Duration) RetryOption {
+	return func(options *retryOptions) {
+		options.interval = interval
+	}
+}
+
+// WithRetry customizes a DoWithRetry call with given retry times.
 func WithRetry(times int) RetryOption {
 	return func(options *retryOptions) {
 		options.times = times
 	}
 }
 
+// WithTimeout customizes a DoWithRetry call with given timeout.
+func WithTimeout(timeout time.Duration) RetryOption {
+	return func(options *retryOptions) {
+		options.timeout = timeout
+	}
+}
+
 func newRetryOptions() *retryOptions {
 	return &retryOptions{
 		times: defaultRetryTimes,

core/fx/retry_test.go

@@ -1,8 +1,10 @@
 package fx
 
 import (
+	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -12,31 +14,153 @@ func TestRetry(t *testing.T) {
 		return errors.New("any")
 	}))
 
-	var times int
+	times1 := 0
 	assert.Nil(t, DoWithRetry(func() error {
-		times++
-		if times == defaultRetryTimes {
+		times1++
+		if times1 == defaultRetryTimes {
 			return nil
 		}
 		return errors.New("any")
 	}))
 
-	times = 0
+	times2 := 0
 	assert.NotNil(t, DoWithRetry(func() error {
-		times++
-		if times == defaultRetryTimes+1 {
+		times2++
+		if times2 == defaultRetryTimes+1 {
 			return nil
 		}
 		return errors.New("any")
 	}))
 
 	total := 2 * defaultRetryTimes
-	times = 0
+	times3 := 0
 	assert.Nil(t, DoWithRetry(func() error {
-		times++
-		if times == total {
+		times3++
+		if times3 == total {
 			return nil
 		}
 		return errors.New("any")
 	}, WithRetry(total)))
 }
+
+func TestRetryWithTimeout(t *testing.T) {
+	assert.Nil(t, DoWithRetry(func() error {
+		return nil
+	}, WithTimeout(time.Millisecond*500)))
+
+	times1 := 0
+	assert.Nil(t, DoWithRetry(func() error {
+		times1++
+		if times1 == 1 {
+			return errors.New("any ")
+		}
+		time.Sleep(time.Millisecond * 150)
+		return nil
+	}, WithTimeout(time.Millisecond*250)))
+
+	total := defaultRetryTimes
+	times2 := 0
+	assert.Nil(t, DoWithRetry(func() error {
+		times2++
+		if times2 == total {
+			return nil
+		}
+		time.Sleep(time.Millisecond * 50)
+		return errors.New("any")
+	}, WithTimeout(time.Millisecond*50*(time.Duration(total)+2))))
+
+	assert.NotNil(t, DoWithRetry(func() error {
+		return errors.New("any")
+	}, WithTimeout(time.Millisecond*250)))
+}
+
+func TestRetryWithInterval(t *testing.T) {
+	times1 := 0
+	assert.NotNil(t, DoWithRetry(func() error {
+		times1++
+		if times1 == 1 {
+			return errors.New("any")
+		}
+		time.Sleep(time.Millisecond * 150)
+		return nil
+	}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
+
+	times2 := 0
+	assert.NotNil(t, DoWithRetry(func() error {
+		times2++
+		if times2 == 2 {
+			return nil
+		}
+		time.Sleep(time.Millisecond * 150)
+		return errors.New("any ")
+	}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
+
+}
+
+func TestRetryWithWithIgnoreErrors(t *testing.T) {
+	ignoreErr1 := errors.New("ignore error1")
+	ignoreErr2 := errors.New("ignore error2")
+	ignoreErrs := []error{ignoreErr1, ignoreErr2}
+
+	assert.Nil(t, DoWithRetry(func() error {
+		return ignoreErr1
+	}, WithIgnoreErrors(ignoreErrs)))
+
+	assert.Nil(t, DoWithRetry(func() error {
+		return ignoreErr2
+	}, WithIgnoreErrors(ignoreErrs)))
+
+	assert.NotNil(t, DoWithRetry(func() error {
+		return errors.New("any")
+	}))
+}
+
+func TestRetryCtx(t *testing.T) {
+	t.Run("with timeout", func(t *testing.T) {
+		assert.NotNil(t, DoWithRetryCtx(context.Background(), func(ctx context.Context, retryCount int) error {
+			if retryCount == 0 {
+				return errors.New("any")
+			}
+			time.Sleep(time.Millisecond * 150)
+			return nil
+		}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
+
+		assert.NotNil(t, DoWithRetryCtx(context.Background(), func(ctx context.Context, retryCount int) error {
+			if retryCount == 1 {
+				return nil
+			}
+			time.Sleep(time.Millisecond * 150)
+			return errors.New("any ")
+		}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
+	})
+
+	t.Run("with deadline exceeded", func(t *testing.T) {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond*250))
+		defer cancel()
+
+		var times int
+		assert.Error(t, DoWithRetryCtx(ctx, func(ctx context.Context, retryCount int) error {
+			times++
+			time.Sleep(time.Millisecond * 150)
+			return errors.New("any")
+		}, WithInterval(time.Millisecond*150)))
+		assert.Equal(t, 1, times)
+	})
+
+	t.Run("with deadline not exceeded", func(t *testing.T) {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond*250))
+		defer cancel()
+
+		var times int
+		assert.NoError(t, DoWithRetryCtx(ctx, func(ctx context.Context, retryCount int) error {
+			times++
+			if times == defaultRetryTimes {
+				return nil
+			}
+
+			time.Sleep(time.Millisecond * 50)
+			return errors.New("any")
+		}))
+		assert.Equal(t, defaultRetryTimes, times)
+	})
+}

core/fx/stream.go

@@ -352,7 +352,7 @@ func (s Stream) Parallel(fn ParallelFunc, opts ...Option) {
 	}, opts...).Done()
 }
 
-// Reduce is an utility method to let the caller deal with the underlying channel.
+// Reduce is a utility method to let the caller deal with the underlying channel.
 func (s Stream) Reduce(fn ReduceFunc) (any, error) {
 	return fn(s.source)
 }

core/iox/bufferpool.go

@@ -32,6 +32,10 @@ func (bp *BufferPool) Get() *bytes.Buffer {
 
 // Put returns buf into bp.
 func (bp *BufferPool) Put(buf *bytes.Buffer) {
+	if buf == nil {
+		return
+	}
+
 	if buf.Cap() < bp.capability {
 		bp.pool.Put(buf)
 	}

core/iox/bufferpool_test.go

@@ -13,3 +13,26 @@ func TestBufferPool(t *testing.T) {
 	pool.Put(bytes.NewBuffer(make([]byte, 0, 2*capacity)))
 	assert.True(t, pool.Get().Cap() <= capacity)
 }
+
+func TestBufferPool_Put(t *testing.T) {
+	t.Run("with nil buf", func(t *testing.T) {
+		pool := NewBufferPool(1024)
+		pool.Put(nil)
+		val := pool.Get()
+		assert.IsType(t, new(bytes.Buffer), val)
+	})
+
+	t.Run("with less-cap buf", func(t *testing.T) {
+		pool := NewBufferPool(1024)
+		pool.Put(bytes.NewBuffer(make([]byte, 0, 512)))
+		val := pool.Get()
+		assert.IsType(t, new(bytes.Buffer), val)
+	})
+
+	t.Run("with more-cap buf", func(t *testing.T) {
+		pool := NewBufferPool(1024)
+		pool.Put(bytes.NewBuffer(make([]byte, 0, 1024<<1)))
+		val := pool.Get()
+		assert.IsType(t, new(bytes.Buffer), val)
+	})
+}

core/iox/nopcloser_test.go

@@ -0,0 +1,12 @@
+package iox
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNopCloser(t *testing.T) {
+	closer := NopCloser(nil)
+	assert.NoError(t, closer.Close())
+}

core/iox/pipe.go

@@ -2,7 +2,7 @@ package iox
 
 import "os"
 
-// RedirectInOut redirects stdin to r, stdout to w, and callers need to call restore afterwards.
+// RedirectInOut redirects stdin to r, stdout to w, and callers need to call restore afterward.
 func RedirectInOut() (restore func(), err error) {
 	var r, w *os.File
 	r, w, err = os.Pipe()

core/iox/read.go

@@ -35,6 +35,16 @@ func KeepSpace() TextReadOption {
 	}
 }
 
+// LimitDupReadCloser returns two io.ReadCloser that read from the first will be written to the second.
+// But the second io.ReadCloser is limited to up to n bytes.
+// The first returned reader needs to be read first, because the content
+// read from it will be written to the underlying buffer of the second reader.
+func LimitDupReadCloser(reader io.ReadCloser, n int64) (io.ReadCloser, io.ReadCloser) {
+	var buf bytes.Buffer
+	tee := LimitTeeReader(reader, &buf, n)
+	return io.NopCloser(tee), io.NopCloser(&buf)
+}
+
 // ReadBytes reads exactly the bytes with the length of len(buf)
 func ReadBytes(reader io.Reader, buf []byte) error {
 	var got int

core/iox/read_test.go

@@ -40,17 +40,22 @@ b`,
 
 	for _, test := range tests {
 		t.Run(test.input, func(t *testing.T) {
-			tmpfile, err := fs.TempFilenameWithText(test.input)
+			tmpFile, err := fs.TempFilenameWithText(test.input)
 			assert.Nil(t, err)
-			defer os.Remove(tmpfile)
+			defer os.Remove(tmpFile)
 
-			content, err := ReadText(tmpfile)
+			content, err := ReadText(tmpFile)
 			assert.Nil(t, err)
 			assert.Equal(t, test.expect, content)
 		})
 	}
 }
 
+func TestReadTextError(t *testing.T) {
+	_, err := ReadText("not-exist")
+	assert.NotNil(t, err)
+}
+
 func TestReadTextLines(t *testing.T) {
 	text := `1
 
@@ -59,9 +64,9 @@ func TestReadTextLines(t *testing.T) {
     #a
     3`
 
-	tmpfile, err := fs.TempFilenameWithText(text)
+	tmpFile, err := fs.TempFilenameWithText(text)
 	assert.Nil(t, err)
-	defer os.Remove(tmpfile)
+	defer os.Remove(tmpFile)
 
 	tests := []struct {
 		options     []TextReadOption
@@ -87,13 +92,18 @@ func TestReadTextLines(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(stringx.Rand(), func(t *testing.T) {
-			lines, err := ReadTextLines(tmpfile, test.options...)
+			lines, err := ReadTextLines(tmpFile, test.options...)
 			assert.Nil(t, err)
 			assert.Equal(t, test.expectLines, len(lines))
 		})
 	}
 }
 
+func TestReadTextLinesError(t *testing.T) {
+	_, err := ReadTextLines("not-exist")
+	assert.NotNil(t, err)
+}
+
 func TestDupReadCloser(t *testing.T) {
 	input := "hello"
 	reader := io.NopCloser(bytes.NewBufferString(input))
@@ -108,6 +118,29 @@ func TestDupReadCloser(t *testing.T) {
 	verify(r2)
 }
 
+func TestLimitDupReadCloser(t *testing.T) {
+	input := "hello world"
+	limitBytes := int64(4)
+	reader := io.NopCloser(bytes.NewBufferString(input))
+	r1, r2 := LimitDupReadCloser(reader, limitBytes)
+	verify := func(r io.Reader) {
+		output, err := io.ReadAll(r)
+		assert.Nil(t, err)
+		assert.Equal(t, input, string(output))
+	}
+	verifyLimit := func(r io.Reader, limit int64) {
+		output, err := io.ReadAll(r)
+		if limit < int64(len(input)) {
+			input = input[:limit]
+		}
+		assert.Nil(t, err)
+		assert.Equal(t, input, string(output))
+	}
+
+	verify(r1)
+	verifyLimit(r2, limitBytes)
+}
+
 func TestReadBytes(t *testing.T) {
 	reader := io.NopCloser(bytes.NewBufferString("helloworld"))
 	buf := make([]byte, 5)

core/iox/tee.go

@@ -0,0 +1,35 @@
+package iox
+
+import "io"
+
+// LimitTeeReader returns a Reader that writes up to n bytes to w what it reads from r.
+// First n bytes reads from r performed through it are matched with
+// corresponding writes to w. There is no internal buffering -
+// the write must complete before the first n bytes read completes.
+// Any error encountered while writing is reported as a read error.
+func LimitTeeReader(r io.Reader, w io.Writer, n int64) io.Reader {
+	return &limitTeeReader{r, w, n}
+}
+
+type limitTeeReader struct {
+	r io.Reader
+	w io.Writer
+	n int64 // limit bytes remaining
+}
+
+func (t *limitTeeReader) Read(p []byte) (n int, err error) {
+	n, err = t.r.Read(p)
+	if n > 0 && t.n > 0 {
+		limit := int64(n)
+		if limit > t.n {
+			limit = t.n
+		}
+		if n, err := t.w.Write(p[:limit]); err != nil {
+			return n, err
+		}
+
+		t.n -= limit
+	}
+
+	return
+}

core/iox/tee_test.go

@@ -0,0 +1,40 @@
+package iox
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLimitTeeReader(t *testing.T) {
+	limit := int64(4)
+	src := []byte("hello, world")
+	dst := make([]byte, len(src))
+	rb := bytes.NewBuffer(src)
+	wb := new(bytes.Buffer)
+	r := LimitTeeReader(rb, wb, limit)
+	if n, err := io.ReadFull(r, dst); err != nil || n != len(src) {
+		t.Fatalf("ReadFull(r, dst) = %d, %v; want %d, nil", n, err, len(src))
+	}
+	if !bytes.Equal(dst, src) {
+		t.Errorf("bytes read = %q want %q", dst, src)
+	}
+	if !bytes.Equal(wb.Bytes(), src[:limit]) {
+		t.Errorf("bytes written = %q want %q", wb.Bytes(), src)
+	}
+
+	n, err := r.Read(dst)
+	assert.Equal(t, 0, n)
+	assert.Equal(t, io.EOF, err)
+
+	rb = bytes.NewBuffer(src)
+	pr, pw := io.Pipe()
+	if assert.NoError(t, pr.Close()) {
+		r = LimitTeeReader(rb, pw, limit)
+		n, err := io.ReadFull(r, dst)
+		assert.Equal(t, 0, n)
+		assert.Equal(t, io.ErrClosedPipe, err)
+	}
+}

core/iox/textfile.go

@@ -2,13 +2,14 @@ package iox
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"os"
 )
 
 const bufSize = 32 * 1024
 
-// CountLines returns the number of lines in file.
+// CountLines returns the number of lines in the file.
 func CountLines(file string) (int, error) {
 	f, err := os.Open(file)
 	if err != nil {
@@ -26,7 +27,7 @@ func CountLines(file string) (int, error) {
 		count += bytes.Count(buf[:c], lineSep)
 
 		switch {
-		case err == io.EOF:
+		case errors.Is(err, io.EOF):
 			if noEol {
 				count++
 			}

core/iox/textfile_test.go

@@ -24,3 +24,8 @@ func TestCountLines(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 4, lines)
 }
+
+func TestCountLinesError(t *testing.T) {
+	_, err := CountLines("not-exist")
+	assert.NotNil(t, err)
+}

core/iox/textlinescanner.go

@@ -2,11 +2,12 @@ package iox
 
 import (
 	"bufio"
+	"errors"
 	"io"
 	"strings"
 )
 
-// A TextLineScanner is a scanner that can scan lines from given reader.
+// A TextLineScanner is a scanner that can scan lines from the given reader.
 type TextLineScanner struct {
 	reader  *bufio.Reader
 	hasNext bool
@@ -14,7 +15,7 @@ type TextLineScanner struct {
 	err     error
 }
 
-// NewTextLineScanner returns a TextLineScanner with given reader.
+// NewTextLineScanner returns a TextLineScanner with the given reader.
 func NewTextLineScanner(reader io.Reader) *TextLineScanner {
 	return &TextLineScanner{
 		reader:  bufio.NewReader(reader),
@@ -30,7 +31,7 @@ func (scanner *TextLineScanner) Scan() bool {
 
 	line, err := scanner.reader.ReadString('\n')
 	scanner.line = strings.TrimRight(line, "\n")
-	if err == io.EOF {
+	if errors.Is(err, io.EOF) {
 		scanner.hasNext = false
 		return true
 	} else if err != nil {

core/iox/textlinescanner_test.go

@@ -3,6 +3,7 @@ package iox
 import (
 	"strings"
 	"testing"
+	"testing/iotest"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -22,3 +23,10 @@ func TestScanner(t *testing.T) {
 	}
 	assert.EqualValues(t, []string{"1", "2", "3", "4"}, lines)
 }
+
+func TestBadScanner(t *testing.T) {
+	scanner := NewTextLineScanner(iotest.ErrReader(iotest.ErrTimeout))
+	assert.False(t, scanner.Scan())
+	_, err := scanner.Line()
+	assert.ErrorIs(t, err, iotest.ErrTimeout)
+}

core/limit/tokenlimit.go

@@ -125,7 +125,7 @@ func (lim *TokenLimiter) reserveN(ctx context.Context, now time.Time, n int) boo
 		})
 	// redis allowed == false
 	// Lua boolean false -> r Nil bulk reply
-	if err == redis.Nil {
+	if errors.Is(err, redis.Nil) {
 		return false
 	}
 	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {

core/load/adaptiveshedder.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/zeromicro/go-zero/core/collection"
 	"github.com/zeromicro/go-zero/core/logx"
+	"github.com/zeromicro/go-zero/core/mathx"
 	"github.com/zeromicro/go-zero/core/stat"
 	"github.com/zeromicro/go-zero/core/syncx"
 	"github.com/zeromicro/go-zero/core/timex"
@@ -21,8 +22,11 @@ const (
 	defaultCpuThreshold = 900
 	defaultMinRt        = float64(time.Second / time.Millisecond)
 	// moving average hyperparameter beta for calculating requests on the fly
-	flyingBeta      = 0.9
-	coolOffDuration = time.Second
+	flyingBeta               = 0.9
+	coolOffDuration          = time.Second
+	cpuMax                   = 1000 // millicpu
+	millisecondsPerSecond    = 1000
+	overloadFactorLowerBound = 0.1
 )
 
 var (
@@ -66,7 +70,7 @@ type (
 
 	adaptiveShedder struct {
 		cpuThreshold    int64
-		windows         int64
+		windowScale     float64
 		flying          int64
 		avgFlying       float64
 		avgFlyingLock   syncx.SpinLock
@@ -105,7 +109,7 @@ func NewAdaptiveShedder(opts ...ShedderOption) Shedder {
 	bucketDuration := options.window / time.Duration(options.buckets)
 	return &adaptiveShedder{
 		cpuThreshold:    options.cpuThreshold,
-		windows:         int64(time.Second / bucketDuration),
+		windowScale:     float64(time.Second) / float64(bucketDuration) / millisecondsPerSecond,
 		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.NewAtomicBool(),
 		passCounter: collection.NewRollingWindow(options.buckets, bucketDuration,
@@ -134,10 +138,10 @@ func (as *adaptiveShedder) Allow() (Promise, error) {
 func (as *adaptiveShedder) addFlying(delta int64) {
 	flying := atomic.AddInt64(&as.flying, delta)
 	// update avgFlying when the request is finished.
-	// this strategy makes avgFlying have a little bit lag against flying, and smoother.
+	// this strategy makes avgFlying have a little bit of lag against flying, and smoother.
 	// when the flying requests increase rapidly, avgFlying increase slower, accept more requests.
-	// when the flying requests drop rapidly, avgFlying drop slower, accept less requests.
-	// it makes the service to serve as more requests as possible.
+	// when the flying requests drop rapidly, avgFlying drop slower, accept fewer requests.
+	// it makes the service to serve as many requests as possible.
 	if delta < 0 {
 		as.avgFlyingLock.Lock()
 		as.avgFlying = as.avgFlying*flyingBeta + float64(flying)*(1-flyingBeta)
@@ -149,16 +153,17 @@ func (as *adaptiveShedder) highThru() bool {
 	as.avgFlyingLock.Lock()
 	avgFlying := as.avgFlying
 	as.avgFlyingLock.Unlock()
-	maxFlight := as.maxFlight()
-	return int64(avgFlying) > maxFlight && atomic.LoadInt64(&as.flying) > maxFlight
+	maxFlight := as.maxFlight() * as.overloadFactor()
+	return avgFlying > maxFlight && float64(atomic.LoadInt64(&as.flying)) > maxFlight
 }
 
-func (as *adaptiveShedder) maxFlight() int64 {
+func (as *adaptiveShedder) maxFlight() float64 {
 	// windows = buckets per second
 	// maxQPS = maxPASS * windows
 	// minRT = min average response time in milliseconds
-	// maxQPS * minRT / milliseconds_per_second
-	return int64(math.Max(1, float64(as.maxPass()*as.windows)*(as.minRt()/1e3)))
+	// allowedFlying = maxQPS * minRT / milliseconds_per_second
+	maxFlight := float64(as.maxPass()) * as.minRt() * as.windowScale
+	return mathx.AtLeast(maxFlight, 1)
 }
 
 func (as *adaptiveShedder) maxPass() int64 {
@@ -174,6 +179,8 @@ func (as *adaptiveShedder) maxPass() int64 {
 }
 
 func (as *adaptiveShedder) minRt() float64 {
+	// if no requests in previous windows, return defaultMinRt,
+	// its a reasonable large value to avoid dropping requests.
 	result := defaultMinRt
 
 	as.rtCounter.Reduce(func(b *collection.Bucket) {
@@ -190,6 +197,13 @@ func (as *adaptiveShedder) minRt() float64 {
 	return result
 }
 
+func (as *adaptiveShedder) overloadFactor() float64 {
+	// as.cpuThreshold must be less than cpuMax
+	factor := (cpuMax - float64(stat.CpuUsage())) / (cpuMax - float64(as.cpuThreshold))
+	// at least accept 10% of acceptable requests, even cpu is highly overloaded.
+	return mathx.Between(factor, overloadFactorLowerBound, 1)
+}
+
 func (as *adaptiveShedder) shouldDrop() bool {
 	if as.systemOverloaded() || as.stillHot() {
 		if as.highThru() {
@@ -236,14 +250,14 @@ func (as *adaptiveShedder) systemOverloaded() bool {
 	return true
 }
 
-// WithBuckets customizes the Shedder with given number of buckets.
+// WithBuckets customizes the Shedder with the given number of buckets.
 func WithBuckets(buckets int) ShedderOption {
 	return func(opts *shedderOptions) {
 		opts.buckets = buckets
 	}
 }
 
-// WithCpuThreshold customizes the Shedder with given cpu threshold.
+// WithCpuThreshold customizes the Shedder with the given cpu threshold.
 func WithCpuThreshold(threshold int64) ShedderOption {
 	return func(opts *shedderOptions) {
 		opts.cpuThreshold = threshold

core/load/adaptiveshedder_test.go

@@ -19,6 +19,7 @@ import (
 const (
 	buckets        = 10
 	bucketDuration = time.Millisecond * 50
+	windowFactor   = 0.01
 )
 
 func init() {
@@ -114,10 +115,10 @@ func TestAdaptiveShedderMaxFlight(t *testing.T) {
 	shedder := &adaptiveShedder{
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
-		windows:         buckets,
+		windowScale:     windowFactor,
 		droppedRecently: syncx.NewAtomicBool(),
 	}
-	assert.Equal(t, int64(54), shedder.maxFlight())
+	assert.Equal(t, float64(54), shedder.maxFlight())
 }
 
 func TestAdaptiveShedderShouldDrop(t *testing.T) {
@@ -136,7 +137,7 @@ func TestAdaptiveShedderShouldDrop(t *testing.T) {
 	shedder := &adaptiveShedder{
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
-		windows:         buckets,
+		windowScale:     windowFactor,
 		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.NewAtomicBool(),
 	}
@@ -149,7 +150,8 @@ func TestAdaptiveShedderShouldDrop(t *testing.T) {
 
 	// cpu >=  800, inflight > maxPass
 	shedder.avgFlying = 80
-	shedder.flying = 50
+	// because of the overloadFactor, so we need to make sure maxFlight is greater than flying
+	shedder.flying = int64(shedder.maxFlight()*shedder.overloadFactor()) - 5
 	assert.False(t, shedder.shouldDrop())
 
 	// cpu >=  800, inflight > maxPass
@@ -190,7 +192,7 @@ func TestAdaptiveShedderStillHot(t *testing.T) {
 	shedder := &adaptiveShedder{
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
-		windows:         buckets,
+		windowScale:     windowFactor,
 		overloadTime:    syncx.NewAtomicDuration(),
 		droppedRecently: syncx.ForAtomicBool(true),
 	}
@@ -239,6 +241,30 @@ func BenchmarkAdaptiveShedder_Allow(b *testing.B) {
 	b.Run("low load", bench)
 }
 
+func BenchmarkMaxFlight(b *testing.B) {
+	passCounter := newRollingWindow()
+	rtCounter := newRollingWindow()
+	for i := 0; i < 10; i++ {
+		if i > 0 {
+			time.Sleep(bucketDuration)
+		}
+		passCounter.Add(float64((i + 1) * 100))
+		for j := i*10 + 1; j <= i*10+10; j++ {
+			rtCounter.Add(float64(j))
+		}
+	}
+	shedder := &adaptiveShedder{
+		passCounter:     passCounter,
+		rtCounter:       rtCounter,
+		windowScale:     windowFactor,
+		droppedRecently: syncx.NewAtomicBool(),
+	}
+
+	for i := 0; i < b.N; i++ {
+		_ = shedder.maxFlight()
+	}
+}
+
 func newRollingWindow() *collection.RollingWindow {
 	return collection.NewRollingWindow(buckets, bucketDuration, collection.IgnoreCurrentBucket())
 }

core/load/sheddergroup.go

@@ -6,7 +6,7 @@ import (
 	"github.com/zeromicro/go-zero/core/syncx"
 )
 
-// A ShedderGroup is a manager to manage key based shedders.
+// A ShedderGroup is a manager to manage key-based shedders.
 type ShedderGroup struct {
 	options []ShedderOption
 	manager *syncx.ResourceManager

core/logc/logs.go

@@ -42,7 +42,7 @@ func Debugv(ctx context.Context, v interface{}) {
 	getLogger(ctx).Debugv(v)
 }
 
-// Debugw writes msg along with fields into access log.
+// Debugw writes msg along with fields into the access log.
 func Debugw(ctx context.Context, msg string, fields ...LogField) {
 	getLogger(ctx).Debugw(msg, fields...)
 }
@@ -63,7 +63,7 @@ func Errorv(ctx context.Context, v any) {
 	getLogger(ctx).Errorv(v)
 }
 
-// Errorw writes msg along with fields into error log.
+// Errorw writes msg along with fields into the error log.
 func Errorw(ctx context.Context, msg string, fields ...LogField) {
 	getLogger(ctx).Errorw(msg, fields...)
 }
@@ -88,7 +88,7 @@ func Infov(ctx context.Context, v any) {
 	getLogger(ctx).Infov(v)
 }
 
-// Infow writes msg along with fields into access log.
+// Infow writes msg along with fields into the access log.
 func Infow(ctx context.Context, msg string, fields ...LogField) {
 	getLogger(ctx).Infow(msg, fields...)
 }
@@ -108,10 +108,11 @@ func SetLevel(level uint32) {
 	logx.SetLevel(level)
 }
 
-// SetUp sets up the logx. If already set up, just return nil.
-// we allow SetUp to be called multiple times, because for example
+// SetUp sets up the logx.
+// If already set up, return nil.
+// We allow SetUp to be called multiple times, because, for example,
 // we need to allow different service frameworks to initialize logx respectively.
-// the same logic for SetUp
+// The same logic for SetUp
 func SetUp(c LogConf) error {
 	return logx.SetUp(c)
 }

core/logc/logs_test.go

@@ -1,7 +1,6 @@
 package logc
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -11,14 +10,11 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/zeromicro/go-zero/core/logx"
+	"github.com/zeromicro/go-zero/core/logx/logtest"
 )
 
 func TestAddGlobalFields(t *testing.T) {
-	var buf bytes.Buffer
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
+	buf := logtest.NewCollector(t)
 
 	Info(context.Background(), "hello")
 	buf.Reset()
@@ -34,155 +30,90 @@ func TestAddGlobalFields(t *testing.T) {
 }
 
 func TestAlert(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	Alert(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), "foo"), buf.String())
 }
 
 func TestError(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Error(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestErrorf(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Errorf(context.Background(), "foo %s", "bar")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestErrorv(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Errorv(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestErrorw(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Errorw(context.Background(), "foo", Field("a", "b"))
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestInfo(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Info(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestInfof(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Infof(context.Background(), "foo %s", "bar")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestInfov(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Infov(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestInfow(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Infow(context.Background(), "foo", Field("a", "b"))
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestDebug(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Debug(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestDebugf(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Debugf(context.Background(), "foo %s", "bar")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestDebugv(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Debugv(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
 }
 
 func TestDebugw(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Debugw(context.Background(), "foo", Field("a", "b"))
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)))
@@ -204,48 +135,28 @@ func TestMisc(t *testing.T) {
 }
 
 func TestSlow(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Slow(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
 }
 
 func TestSlowf(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Slowf(context.Background(), "foo %s", "bar")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
 }
 
 func TestSlowv(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Slowv(context.Background(), "foo")
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())
 }
 
 func TestSloww(t *testing.T) {
-	var buf strings.Builder
-	writer := logx.NewWriter(&buf)
-	old := logx.Reset()
-	logx.SetWriter(writer)
-	defer logx.SetWriter(old)
-
+	buf := logtest.NewCollector(t)
 	file, line := getFileLine()
 	Sloww(context.Background(), "foo", Field("a", "b"))
 	assert.True(t, strings.Contains(buf.String(), fmt.Sprintf("%s:%d", file, line+1)), buf.String())

core/logx/config.go

@@ -32,7 +32,7 @@ type LogConf struct {
 	StackCooldownMillis int `json:",default=100"`
 	// MaxBackups represents how many backup log files will be kept. 0 means all files will be kept forever.
 	// Only take effect when RotationRuleType is `size`.
-	// Even thougth `MaxBackups` sets 0, log files will still be removed
+	// Even though `MaxBackups` sets 0, log files will still be removed
 	// if the `KeepDays` limitation is reached.
 	MaxBackups int `json:",default=0"`
 	// MaxSize represents how much space the writing log file takes up. 0 means no limit. The unit is `MB`.

core/logx/fs.go

@@ -0,0 +1,40 @@
+package logx
+
+import (
+	"io"
+	"os"
+)
+
+var fileSys realFileSystem
+
+type (
+	fileSystem interface {
+		Close(closer io.Closer) error
+		Copy(writer io.Writer, reader io.Reader) (int64, error)
+		Create(name string) (*os.File, error)
+		Open(name string) (*os.File, error)
+		Remove(name string) error
+	}
+
+	realFileSystem struct{}
+)
+
+func (fs realFileSystem) Close(closer io.Closer) error {
+	return closer.Close()
+}
+
+func (fs realFileSystem) Copy(writer io.Writer, reader io.Reader) (int64, error) {
+	return io.Copy(writer, reader)
+}
+
+func (fs realFileSystem) Create(name string) (*os.File, error) {
+	return os.Create(name)
+}
+
+func (fs realFileSystem) Open(name string) (*os.File, error) {
+	return os.Open(name)
+}
+
+func (fs realFileSystem) Remove(name string) error {
+	return os.Remove(name)
+}

core/logx/lesslogger.go

@@ -1,6 +1,6 @@
 package logx
 
-// A LessLogger is a logger that control to log once during the given duration.
+// A LessLogger is a logger that controls to log once during the given duration.
 type LessLogger struct {
 	*limitedExecutor
 }

core/logx/logs.go

@@ -17,14 +17,13 @@ import (
 const callerDepth = 4
 
 var (
-	timeFormat = "2006-01-02T15:04:05.000Z07:00"
-	logLevel   uint32
+	timeFormat        = "2006-01-02T15:04:05.000Z07:00"
 	encoding   uint32 = jsonEncodingType
 	// maxContentLength is used to truncate the log content, 0 for not truncating.
 	maxContentLength uint32
 	// use uint32 for atomic operations
-	disableLog  uint32
 	disableStat uint32
+	logLevel    uint32
 	options     logOptions
 	writer      = new(atomicWriter)
 	setupOnce   sync.Once
@@ -68,27 +67,35 @@ func Close() error {
 
 // Debug writes v into access log.
 func Debug(v ...any) {
-	writeDebug(fmt.Sprint(v...))
+	if shallLog(DebugLevel) {
+		writeDebug(fmt.Sprint(v...))
+	}
 }
 
 // Debugf writes v with format into access log.
 func Debugf(format string, v ...any) {
-	writeDebug(fmt.Sprintf(format, v...))
+	if shallLog(DebugLevel) {
+		writeDebug(fmt.Sprintf(format, v...))
+	}
 }
 
 // Debugv writes v into access log with json content.
 func Debugv(v any) {
-	writeDebug(v)
+	if shallLog(DebugLevel) {
+		writeDebug(v)
+	}
 }
 
-// Debugw writes msg along with fields into access log.
+// Debugw writes msg along with fields into the access log.
 func Debugw(msg string, fields ...LogField) {
-	writeDebug(msg, fields...)
+	if shallLog(DebugLevel) {
+		writeDebug(msg, fields...)
+	}
 }
 
 // Disable disables the logging.
 func Disable() {
-	atomic.StoreUint32(&disableLog, 1)
+	atomic.StoreUint32(&logLevel, disableLevel)
 	writer.Store(nopWriter{})
 }
 
@@ -99,35 +106,47 @@ func DisableStat() {
 
 // Error writes v into error log.
 func Error(v ...any) {
-	writeError(fmt.Sprint(v...))
+	if shallLog(ErrorLevel) {
+		writeError(fmt.Sprint(v...))
+	}
 }
 
 // Errorf writes v with format into error log.
 func Errorf(format string, v ...any) {
-	writeError(fmt.Errorf(format, v...).Error())
+	if shallLog(ErrorLevel) {
+		writeError(fmt.Errorf(format, v...).Error())
+	}
 }
 
 // ErrorStack writes v along with call stack into error log.
 func ErrorStack(v ...any) {
-	// there is newline in stack string
-	writeStack(fmt.Sprint(v...))
+	if shallLog(ErrorLevel) {
+		// there is newline in stack string
+		writeStack(fmt.Sprint(v...))
+	}
 }
 
 // ErrorStackf writes v along with call stack in format into error log.
 func ErrorStackf(format string, v ...any) {
-	// there is newline in stack string
-	writeStack(fmt.Sprintf(format, v...))
+	if shallLog(ErrorLevel) {
+		// there is newline in stack string
+		writeStack(fmt.Sprintf(format, v...))
+	}
 }
 
 // Errorv writes v into error log with json content.
 // No call stack attached, because not elegant to pack the messages.
 func Errorv(v any) {
-	writeError(v)
+	if shallLog(ErrorLevel) {
+		writeError(v)
+	}
 }
 
-// Errorw writes msg along with fields into error log.
+// Errorw writes msg along with fields into the error log.
 func Errorw(msg string, fields ...LogField) {
-	writeError(msg, fields...)
+	if shallLog(ErrorLevel) {
+		writeError(msg, fields...)
+	}
 }
 
 // Field returns a LogField for the given key and value.
@@ -170,22 +189,30 @@ func Field(key string, value any) LogField {
 
 // Info writes v into access log.
 func Info(v ...any) {
-	writeInfo(fmt.Sprint(v...))
+	if shallLog(InfoLevel) {
+		writeInfo(fmt.Sprint(v...))
+	}
 }
 
 // Infof writes v with format into access log.
 func Infof(format string, v ...any) {
-	writeInfo(fmt.Sprintf(format, v...))
+	if shallLog(InfoLevel) {
+		writeInfo(fmt.Sprintf(format, v...))
+	}
 }
 
 // Infov writes v into access log with json content.
 func Infov(v any) {
-	writeInfo(v)
+	if shallLog(InfoLevel) {
+		writeInfo(v)
+	}
 }
 
-// Infow writes msg along with fields into access log.
+// Infow writes msg along with fields into the access log.
 func Infow(msg string, fields ...LogField) {
-	writeInfo(msg, fields...)
+	if shallLog(InfoLevel) {
+		writeInfo(msg, fields...)
+	}
 }
 
 // Must checks if err is nil, otherwise logs the error and exits.
@@ -194,7 +221,7 @@ func Must(err error) {
 		return
 	}
 
-	msg := err.Error()
+	msg := fmt.Sprintf("%+v\n\n%s", err.Error(), debug.Stack())
 	log.Print(msg)
 	getWriter().Severe(msg)
 
@@ -222,16 +249,17 @@ func SetLevel(level uint32) {
 
 // SetWriter sets the logging writer. It can be used to customize the logging.
 func SetWriter(w Writer) {
-	if atomic.LoadUint32(&disableLog) == 0 {
+	if atomic.LoadUint32(&logLevel) != disableLevel {
 		writer.Store(w)
 	}
 }
 
-// SetUp sets up the logx. If already set up, just return nil.
-// we allow SetUp to be called multiple times, because for example
+// SetUp sets up the logx.
+// If already set up, return nil.
+// We allow SetUp to be called multiple times, because, for example,
 // we need to allow different service frameworks to initialize logx respectively.
 func SetUp(c LogConf) (err error) {
-	// Just ignore the subsequent SetUp calls.
+	// Ignore the later SetUp calls.
 	// Because multiple services in one process might call SetUp respectively.
 	// Need to wait for the first caller to complete the execution.
 	setupOnce.Do(func() {
@@ -269,42 +297,58 @@ func SetUp(c LogConf) (err error) {
 
 // Severe writes v into severe log.
 func Severe(v ...any) {
-	writeSevere(fmt.Sprint(v...))
+	if shallLog(SevereLevel) {
+		writeSevere(fmt.Sprint(v...))
+	}
 }
 
 // Severef writes v with format into severe log.
 func Severef(format string, v ...any) {
-	writeSevere(fmt.Sprintf(format, v...))
+	if shallLog(SevereLevel) {
+		writeSevere(fmt.Sprintf(format, v...))
+	}
 }
 
 // Slow writes v into slow log.
 func Slow(v ...any) {
-	writeSlow(fmt.Sprint(v...))
+	if shallLog(ErrorLevel) {
+		writeSlow(fmt.Sprint(v...))
+	}
 }
 
 // Slowf writes v with format into slow log.
 func Slowf(format string, v ...any) {
-	writeSlow(fmt.Sprintf(format, v...))
+	if shallLog(ErrorLevel) {
+		writeSlow(fmt.Sprintf(format, v...))
+	}
 }
 
 // Slowv writes v into slow log with json content.
 func Slowv(v any) {
-	writeSlow(v)
+	if shallLog(ErrorLevel) {
+		writeSlow(v)
+	}
 }
 
 // Sloww writes msg along with fields into slow log.
 func Sloww(msg string, fields ...LogField) {
-	writeSlow(msg, fields...)
+	if shallLog(ErrorLevel) {
+		writeSlow(msg, fields...)
+	}
 }
 
 // Stat writes v into stat log.
 func Stat(v ...any) {
-	writeStat(fmt.Sprint(v...))
+	if shallLogStat() && shallLog(InfoLevel) {
+		writeStat(fmt.Sprint(v...))
+	}
 }
 
 // Statf writes v with format into stat log.
 func Statf(format string, v ...any) {
-	writeStat(fmt.Sprintf(format, v...))
+	if shallLogStat() && shallLog(InfoLevel) {
+		writeStat(fmt.Sprintf(format, v...))
+	}
 }
 
 // WithCooldownMillis customizes logging on writing call stack interval.
@@ -358,14 +402,16 @@ func createOutput(path string) (io.WriteCloser, error) {
 		return nil, ErrLogPathNotSet
 	}
 
+	var rule RotateRule
 	switch options.rotationRule {
 	case sizeRotationRule:
-		return NewLogger(path, NewSizeLimitRotateRule(path, backupFileDelimiter, options.keepDays,
-			options.maxSize, options.maxBackups, options.gzipEnabled), options.gzipEnabled)
+		rule = NewSizeLimitRotateRule(path, backupFileDelimiter, options.keepDays, options.maxSize,
+			options.maxBackups, options.gzipEnabled)
 	default:
-		return NewLogger(path, DefaultRotateRule(path, backupFileDelimiter, options.keepDays,
-			options.gzipEnabled), options.gzipEnabled)
+		rule = DefaultRotateRule(path, backupFileDelimiter, options.keepDays, options.gzipEnabled)
 	}
+
+	return NewLogger(path, rule, options.gzipEnabled)
 }
 
 func getWriter() Writer {
@@ -427,44 +473,58 @@ func shallLogStat() bool {
 	return atomic.LoadUint32(&disableStat) == 0
 }
 
+// writeDebug writes v into debug log.
+// Not checking shallLog here is for performance consideration.
+// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.
+// The caller should check shallLog before calling this function.
 func writeDebug(val any, fields ...LogField) {
-	if shallLog(DebugLevel) {
-		getWriter().Debug(val, addCaller(fields...)...)
-	}
+	getWriter().Debug(val, addCaller(fields...)...)
 }
 
+// writeError writes v into the error log.
+// Not checking shallLog here is for performance consideration.
+// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.
+// The caller should check shallLog before calling this function.
 func writeError(val any, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Error(val, addCaller(fields...)...)
-	}
+	getWriter().Error(val, addCaller(fields...)...)
 }
 
+// writeInfo writes v into info log.
+// Not checking shallLog here is for performance consideration.
+// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.
+// The caller should check shallLog before calling this function.
 func writeInfo(val any, fields ...LogField) {
-	if shallLog(InfoLevel) {
-		getWriter().Info(val, addCaller(fields...)...)
-	}
+	getWriter().Info(val, addCaller(fields...)...)
 }
 
+// writeSevere writes v into severe log.
+// Not checking shallLog here is for performance consideration.
+// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.
+// The caller should check shallLog before calling this function.
 func writeSevere(msg string) {
-	if shallLog(SevereLevel) {
-		getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
-	}
+	getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 }
 
+// writeSlow writes v into slow log.
+// Not checking shallLog here is for performance consideration.
+// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.
+// The caller should check shallLog before calling this function.
 func writeSlow(val any, fields ...LogField) {
-	if shallLog(ErrorLevel) {
-		getWriter().Slow(val, addCaller(fields...)...)
-	}
+	getWriter().Slow(val, addCaller(fields...)...)
 }
 
+// writeStack writes v into stack log.
+// Not checking shallLog here is for performance consideration.
+// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.
+// The caller should check shallLog before calling this function.
 func writeStack(msg string) {
-	if shallLog(ErrorLevel) {
-		getWriter().Stack(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
-	}
+	getWriter().Stack(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
 }
 
+// writeStat writes v into the stat log.
+// Not checking shallLog here is for performance consideration.
+// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.
+// The caller should check shallLog before calling this function.
 func writeStat(msg string) {
-	if shallLogStat() && shallLog(InfoLevel) {
-		getWriter().Stat(msg, addCaller()...)
-	}
+	getWriter().Stat(msg, addCaller()...)
 }

core/logx/logs_test.go

@@ -570,7 +570,7 @@ func TestErrorfWithWrappedError(t *testing.T) {
 	old := writer.Swap(w)
 	defer writer.Store(old)
 
-	Errorf("hello %w", errors.New(message))
+	Errorf("hello %s", errors.New(message))
 	assert.True(t, strings.Contains(w.String(), "hello there"))
 }
 
@@ -666,6 +666,7 @@ func TestDisable(t *testing.T) {
 	WithMaxSize(1024)(&opt)
 	assert.Nil(t, Close())
 	assert.Nil(t, Close())
+	assert.Equal(t, uint32(disableLevel), atomic.LoadUint32(&logLevel))
 }
 
 func TestDisableStat(t *testing.T) {
@@ -680,7 +681,7 @@ func TestDisableStat(t *testing.T) {
 }
 
 func TestSetWriter(t *testing.T) {
-	atomic.StoreUint32(&disableLog, 0)
+	atomic.StoreUint32(&logLevel, 0)
 	Reset()
 	SetWriter(nopWriter{})
 	assert.NotNil(t, writer.Load())

core/logx/logtest/logtest.go

@@ -0,0 +1,84 @@
+package logtest
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"testing"
+
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+type Buffer struct {
+	buf *bytes.Buffer
+	t   *testing.T
+}
+
+func Discard(t *testing.T) {
+	prev := logx.Reset()
+	logx.SetWriter(logx.NewWriter(io.Discard))
+
+	t.Cleanup(func() {
+		logx.SetWriter(prev)
+	})
+}
+
+func NewCollector(t *testing.T) *Buffer {
+	var buf bytes.Buffer
+	writer := logx.NewWriter(&buf)
+	prev := logx.Reset()
+	logx.SetWriter(writer)
+
+	t.Cleanup(func() {
+		logx.SetWriter(prev)
+	})
+
+	return &Buffer{
+		buf: &buf,
+		t:   t,
+	}
+}
+
+func (b *Buffer) Bytes() []byte {
+	return b.buf.Bytes()
+}
+
+func (b *Buffer) Content() string {
+	var m map[string]interface{}
+	if err := json.Unmarshal(b.buf.Bytes(), &m); err != nil {
+		return ""
+	}
+
+	content, ok := m["content"]
+	if !ok {
+		return ""
+	}
+
+	switch val := content.(type) {
+	case string:
+		return val
+	default:
+		// err is impossible to be not nil, unmarshaled from b.buf.Bytes()
+		bs, _ := json.Marshal(content)
+		return string(bs)
+	}
+}
+
+func (b *Buffer) Reset() {
+	b.buf.Reset()
+}
+
+func (b *Buffer) String() string {
+	return b.buf.String()
+}
+
+func PanicOnFatal(t *testing.T) {
+	ok := logx.ExitOnFatal.CompareAndSwap(true, false)
+	if !ok {
+		return
+	}
+
+	t.Cleanup(func() {
+		logx.ExitOnFatal.CompareAndSwap(false, true)
+	})
+}

core/logx/logtest/logtest_test.go

@@ -0,0 +1,44 @@
+package logtest
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+func TestCollector(t *testing.T) {
+	const input = "hello"
+	c := NewCollector(t)
+	logx.Info(input)
+	assert.Equal(t, input, c.Content())
+	assert.Contains(t, c.String(), input)
+	c.Reset()
+	assert.Empty(t, c.Bytes())
+}
+
+func TestPanicOnFatal(t *testing.T) {
+	const input = "hello"
+	Discard(t)
+	logx.Info(input)
+
+	PanicOnFatal(t)
+	PanicOnFatal(t)
+	assert.Panics(t, func() {
+		logx.Must(errors.New("foo"))
+	})
+}
+
+func TestCollectorContent(t *testing.T) {
+	const input = "hello"
+	c := NewCollector(t)
+	c.buf.WriteString(input)
+	assert.Empty(t, c.Content())
+	c.Reset()
+	c.buf.WriteString(`{}`)
+	assert.Empty(t, c.Content())
+	c.Reset()
+	c.buf.WriteString(`{"content":1}`)
+	assert.Equal(t, "1", c.Content())
+}

core/logx/readme.md

@@ -40,7 +40,7 @@ type LogConf struct {
 - `Compress`: whether or not to compress log files, only works with `file` mode.
 - `KeepDays`: how many days that the log files are kept, after the given days, the outdated files will be deleted automatically. It has no effect on `console` mode.
 - `StackCooldownMillis`: how many milliseconds to rewrite stacktrace again. It’s used to avoid stacktrace flooding.
-- `MaxBackups`: represents how many backup log files will be kept. 0 means all files will be kept forever. Only take effect when `Rotation` is `size`. NOTE: the level of option `KeepDays` will be higher. Even thougth `MaxBackups` sets 0, log files will still be removed if the `KeepDays` limitation is reached.
+- `MaxBackups`: represents how many backup log files will be kept. 0 means all files will be kept forever. Only take effect when `Rotation` is `size`. NOTE: the level of option `KeepDays` will be higher. Even though `MaxBackups` sets 0, log files will still be removed if the `KeepDays` limitation is reached.
 - `MaxSize`: represents how much space the writing log file takes up. 0 means no limit. The unit is `MB`. Only take effect when `Rotation` is `size`.
 - `Rotation`: represents the type of log rotation rule. Default is `daily`.
   - `daily` rotate the logs by day.

core/logx/richlogger.go

@@ -41,67 +41,99 @@ type richLogger struct {
 }
 
 func (l *richLogger) Debug(v ...any) {
-	l.debug(fmt.Sprint(v...))
+	if shallLog(DebugLevel) {
+		l.debug(fmt.Sprint(v...))
+	}
 }
 
 func (l *richLogger) Debugf(format string, v ...any) {
-	l.debug(fmt.Sprintf(format, v...))
+	if shallLog(DebugLevel) {
+		l.debug(fmt.Sprintf(format, v...))
+	}
 }
 
 func (l *richLogger) Debugv(v any) {
-	l.debug(v)
+	if shallLog(DebugLevel) {
+		l.debug(v)
+	}
 }
 
 func (l *richLogger) Debugw(msg string, fields ...LogField) {
-	l.debug(msg, fields...)
+	if shallLog(DebugLevel) {
+		l.debug(msg, fields...)
+	}
 }
 
 func (l *richLogger) Error(v ...any) {
-	l.err(fmt.Sprint(v...))
+	if shallLog(ErrorLevel) {
+		l.err(fmt.Sprint(v...))
+	}
 }
 
 func (l *richLogger) Errorf(format string, v ...any) {
-	l.err(fmt.Sprintf(format, v...))
+	if shallLog(ErrorLevel) {
+		l.err(fmt.Sprintf(format, v...))
+	}
 }
 
 func (l *richLogger) Errorv(v any) {
-	l.err(fmt.Sprint(v))
+	if shallLog(ErrorLevel) {
+		l.err(v)
+	}
 }
 
 func (l *richLogger) Errorw(msg string, fields ...LogField) {
-	l.err(msg, fields...)
+	if shallLog(ErrorLevel) {
+		l.err(msg, fields...)
+	}
 }
 
 func (l *richLogger) Info(v ...any) {
-	l.info(fmt.Sprint(v...))
+	if shallLog(InfoLevel) {
+		l.info(fmt.Sprint(v...))
+	}
 }
 
 func (l *richLogger) Infof(format string, v ...any) {
-	l.info(fmt.Sprintf(format, v...))
+	if shallLog(InfoLevel) {
+		l.info(fmt.Sprintf(format, v...))
+	}
 }
 
 func (l *richLogger) Infov(v any) {
-	l.info(v)
+	if shallLog(InfoLevel) {
+		l.info(v)
+	}
 }
 
 func (l *richLogger) Infow(msg string, fields ...LogField) {
-	l.info(msg, fields...)
+	if shallLog(InfoLevel) {
+		l.info(msg, fields...)
+	}
 }
 
 func (l *richLogger) Slow(v ...any) {
-	l.slow(fmt.Sprint(v...))
+	if shallLog(ErrorLevel) {
+		l.slow(fmt.Sprint(v...))
+	}
 }
 
 func (l *richLogger) Slowf(format string, v ...any) {
-	l.slow(fmt.Sprintf(format, v...))
+	if shallLog(ErrorLevel) {
+		l.slow(fmt.Sprintf(format, v...))
+	}
 }
 
 func (l *richLogger) Slowv(v any) {
-	l.slow(v)
+	if shallLog(ErrorLevel) {
+		l.slow(v)
+	}
 }
 
 func (l *richLogger) Sloww(msg string, fields ...LogField) {
-	l.slow(msg, fields...)
+	if shallLog(ErrorLevel) {
+		l.slow(msg, fields...)
+	}
 }
 
 func (l *richLogger) WithCallerSkip(skip int) Logger {

core/logx/richlogger_test.go

@@ -66,6 +66,9 @@ func TestTraceDebug(t *testing.T) {
 	l.WithDuration(time.Second).Debugv(testlog)
 	validate(t, w.String(), true, true)
 	w.Reset()
+	l.WithDuration(time.Second).Debugv(testobj)
+	validateContentType(t, w.String(), map[string]any{}, true, true)
+	w.Reset()
 	l.WithDuration(time.Second).Debugw(testlog, Field("foo", "bar"))
 	validate(t, w.String(), true, true)
 	assert.True(t, strings.Contains(w.String(), "foo"), w.String())
@@ -103,6 +106,9 @@ func TestTraceError(t *testing.T) {
 	l.WithDuration(time.Second).Errorv(testlog)
 	validate(t, w.String(), true, true)
 	w.Reset()
+	l.WithDuration(time.Second).Errorv(testobj)
+	validateContentType(t, w.String(), map[string]any{}, true, true)
+	w.Reset()
 	l.WithDuration(time.Second).Errorw(testlog, Field("basket", "ball"))
 	validate(t, w.String(), true, true)
 	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
@@ -137,6 +143,9 @@ func TestTraceInfo(t *testing.T) {
 	l.WithDuration(time.Second).Infov(testlog)
 	validate(t, w.String(), true, true)
 	w.Reset()
+	l.WithDuration(time.Second).Infov(testobj)
+	validateContentType(t, w.String(), map[string]any{}, true, true)
+	w.Reset()
 	l.WithDuration(time.Second).Infow(testlog, Field("basket", "ball"))
 	validate(t, w.String(), true, true)
 	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
@@ -173,6 +182,9 @@ func TestTraceInfoConsole(t *testing.T) {
 	w.Reset()
 	l.WithDuration(time.Second).Infov(testlog)
 	validate(t, w.String(), true, true)
+	w.Reset()
+	l.WithDuration(time.Second).Infov(testobj)
+	validateContentType(t, w.String(), map[string]any{}, true, true)
 }
 
 func TestTraceSlow(t *testing.T) {
@@ -204,6 +216,9 @@ func TestTraceSlow(t *testing.T) {
 	l.WithDuration(time.Second).Slowv(testlog)
 	validate(t, w.String(), true, true)
 	w.Reset()
+	l.WithDuration(time.Second).Slowv(testobj)
+	validateContentType(t, w.String(), map[string]any{}, true, true)
+	w.Reset()
 	l.WithDuration(time.Second).Sloww(testlog, Field("basket", "ball"))
 	validate(t, w.String(), true, true)
 	assert.True(t, strings.Contains(w.String(), "basket"), w.String())
@@ -311,8 +326,32 @@ func validate(t *testing.T, body string, expectedTrace, expectedSpan bool) {
 	assert.Equal(t, expectedSpan, len(val.Span) > 0, body)
 }
 
-type mockValue struct {
-	Trace string `json:"trace"`
-	Span  string `json:"span"`
-	Foo   string `json:"foo"`
+func validateContentType(t *testing.T, body string, expectedType any, expectedTrace, expectedSpan bool) {
+	var val mockValue
+	dec := json.NewDecoder(strings.NewReader(body))
+
+	for {
+		var doc mockValue
+		err := dec.Decode(&doc)
+		if err == io.EOF {
+			// all done
+			break
+		}
+		if err != nil {
+			continue
+		}
+
+		val = doc
+	}
+
+	assert.IsType(t, expectedType, val.Content, body)
+	assert.Equal(t, expectedTrace, len(val.Trace) > 0, body)
+	assert.Equal(t, expectedSpan, len(val.Span) > 0, body)
+}
+
+type mockValue struct {
+	Trace   string `json:"trace"`
+	Span    string `json:"span"`
+	Foo     string `json:"foo"`
+	Content any    `json:"content"`
 }

core/logx/rotatelogger.go

@@ -4,7 +4,6 @@ import (
 	"compress/gzip"
 	"errors"
 	"fmt"
-	"io"
 	"log"
 	"os"
 	"path"
@@ -299,6 +298,7 @@ func (l *RotateLogger) initialize() error {
 		if l.fp, err = os.OpenFile(l.filename, os.O_APPEND|os.O_WRONLY, defaultFileMode); err != nil {
 			return err
 		}
+
 		l.currentSize = fileInfo.Size()
 	}
 
@@ -319,7 +319,7 @@ func (l *RotateLogger) maybeCompressFile(file string) {
 	}()
 
 	if _, err := os.Stat(file); err != nil {
-		// file not exists or other error, ignore compression
+		// file doesn't exist or another error, ignore compression
 		return
 	}
 
@@ -382,7 +382,15 @@ func (l *RotateLogger) startWorker() {
 			case event := <-l.channel:
 				l.write(event)
 			case <-l.done:
-				return
+				// avoid losing logs before closing.
+				for {
+					select {
+					case event := <-l.channel:
+						l.write(event)
+					default:
+						return
+					}
+				}
 			}
 		}
 	}()
@@ -406,7 +414,7 @@ func (l *RotateLogger) write(v []byte) {
 func compressLogFile(file string) {
 	start := time.Now()
 	Infof("compressing log file: %s", file)
-	if err := gzipFile(file); err != nil {
+	if err := gzipFile(file, fileSys); err != nil {
 		Errorf("compress error: %s", err)
 	} else {
 		Infof("compressed log file: %s, took %s", file, time.Since(start))
@@ -421,25 +429,37 @@ func getNowDateInRFC3339Format() string {
 	return time.Now().Format(fileTimeFormat)
 }
 
-func gzipFile(file string) error {
-	in, err := os.Open(file)
+func gzipFile(file string, fsys fileSystem) (err error) {
+	in, err := fsys.Open(file)
 	if err != nil {
 		return err
 	}
-	defer in.Close()
+	defer func() {
+		if e := fsys.Close(in); e != nil {
+			Errorf("failed to close file: %s, error: %v", file, e)
+		}
+		if err == nil {
+			// only remove the original file when compression is successful
+			err = fsys.Remove(file)
+		}
+	}()
 
-	out, err := os.Create(fmt.Sprintf("%s%s", file, gzipExt))
+	out, err := fsys.Create(fmt.Sprintf("%s%s", file, gzipExt))
 	if err != nil {
 		return err
 	}
-	defer out.Close()
+	defer func() {
+		e := fsys.Close(out)
+		if err == nil {
+			err = e
+		}
+	}()
 
 	w := gzip.NewWriter(out)
-	if _, err = io.Copy(w, in); err != nil {
-		return err
-	} else if err = w.Close(); err != nil {
+	if _, err = fsys.Copy(w, in); err != nil {
+		// failed to copy, no need to close w
 		return err
 	}
 
-	return os.Remove(file)
+	return fsys.Close(w)
 }

core/logx/rotatelogger_test.go

@@ -1,8 +1,12 @@
 package logx
 
 import (
+	"errors"
+	"io"
 	"os"
+	"path"
 	"path/filepath"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
@@ -13,18 +17,58 @@ import (
 )
 
 func TestDailyRotateRuleMarkRotated(t *testing.T) {
-	var rule DailyRotateRule
-	rule.MarkRotated()
-	assert.Equal(t, getNowDate(), rule.rotatedTime)
+	t.Run("daily rule", func(t *testing.T) {
+		var rule DailyRotateRule
+		rule.MarkRotated()
+		assert.Equal(t, getNowDate(), rule.rotatedTime)
+	})
+
+	t.Run("daily rule", func(t *testing.T) {
+		rule := DefaultRotateRule("test", "-", 1, false)
+		_, ok := rule.(*DailyRotateRule)
+		assert.True(t, ok)
+	})
 }
 
 func TestDailyRotateRuleOutdatedFiles(t *testing.T) {
-	var rule DailyRotateRule
-	assert.Empty(t, rule.OutdatedFiles())
-	rule.days = 1
-	assert.Empty(t, rule.OutdatedFiles())
-	rule.gzip = true
-	assert.Empty(t, rule.OutdatedFiles())
+	t.Run("no files", func(t *testing.T) {
+		var rule DailyRotateRule
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.days = 1
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.gzip = true
+		assert.Empty(t, rule.OutdatedFiles())
+	})
+
+	t.Run("bad files", func(t *testing.T) {
+		rule := DailyRotateRule{
+			filename: "[a-z",
+		}
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.days = 1
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.gzip = true
+		assert.Empty(t, rule.OutdatedFiles())
+	})
+
+	t.Run("temp files", func(t *testing.T) {
+		boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay) * 2).Format(dateFormat)
+		f1, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary)
+		assert.NoError(t, err)
+		_ = f1.Close()
+		f2, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary)
+		assert.NoError(t, err)
+		_ = f2.Close()
+		t.Cleanup(func() {
+			_ = os.Remove(f1.Name())
+			_ = os.Remove(f2.Name())
+		})
+		rule := DailyRotateRule{
+			filename: path.Join(os.TempDir(), "go-zero-test-"),
+			days:     1,
+		}
+		assert.NotEmpty(t, rule.OutdatedFiles())
+	})
 }
 
 func TestDailyRotateRuleShallRotate(t *testing.T) {
@@ -34,20 +78,101 @@ func TestDailyRotateRuleShallRotate(t *testing.T) {
 }
 
 func TestSizeLimitRotateRuleMarkRotated(t *testing.T) {
-	var rule SizeLimitRotateRule
-	rule.MarkRotated()
-	assert.Equal(t, getNowDateInRFC3339Format(), rule.rotatedTime)
+	t.Run("size limit rule", func(t *testing.T) {
+		var rule SizeLimitRotateRule
+		rule.MarkRotated()
+		assert.Equal(t, getNowDateInRFC3339Format(), rule.rotatedTime)
+	})
+
+	t.Run("size limit rule", func(t *testing.T) {
+		rule := NewSizeLimitRotateRule("foo", "-", 1, 1, 1, false)
+		rule.MarkRotated()
+		assert.Equal(t, getNowDateInRFC3339Format(), rule.(*SizeLimitRotateRule).rotatedTime)
+	})
 }
 
 func TestSizeLimitRotateRuleOutdatedFiles(t *testing.T) {
-	var rule SizeLimitRotateRule
-	assert.Empty(t, rule.OutdatedFiles())
-	rule.days = 1
-	assert.Empty(t, rule.OutdatedFiles())
-	rule.gzip = true
-	assert.Empty(t, rule.OutdatedFiles())
-	rule.maxBackups = 0
-	assert.Empty(t, rule.OutdatedFiles())
+	t.Run("no files", func(t *testing.T) {
+		var rule SizeLimitRotateRule
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.days = 1
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.gzip = true
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.maxBackups = 0
+		assert.Empty(t, rule.OutdatedFiles())
+	})
+
+	t.Run("bad files", func(t *testing.T) {
+		rule := SizeLimitRotateRule{
+			DailyRotateRule: DailyRotateRule{
+				filename: "[a-z",
+			},
+		}
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.days = 1
+		assert.Empty(t, rule.OutdatedFiles())
+		rule.gzip = true
+		assert.Empty(t, rule.OutdatedFiles())
+	})
+
+	t.Run("temp files", func(t *testing.T) {
+		boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay) * 2).Format(dateFormat)
+		f1, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary)
+		assert.NoError(t, err)
+		f2, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary)
+		assert.NoError(t, err)
+		boundary1 := time.Now().Add(time.Hour * time.Duration(hoursPerDay) * 2).Format(dateFormat)
+		f3, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary1)
+		assert.NoError(t, err)
+		t.Cleanup(func() {
+			_ = f1.Close()
+			_ = os.Remove(f1.Name())
+			_ = f2.Close()
+			_ = os.Remove(f2.Name())
+			_ = f3.Close()
+			_ = os.Remove(f3.Name())
+		})
+		rule := SizeLimitRotateRule{
+			DailyRotateRule: DailyRotateRule{
+				filename: path.Join(os.TempDir(), "go-zero-test-"),
+				days:     1,
+			},
+			maxBackups: 3,
+		}
+		assert.NotEmpty(t, rule.OutdatedFiles())
+	})
+
+	t.Run("no backups", func(t *testing.T) {
+		boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay) * 2).Format(dateFormat)
+		f1, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary)
+		assert.NoError(t, err)
+		f2, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary)
+		assert.NoError(t, err)
+		boundary1 := time.Now().Add(time.Hour * time.Duration(hoursPerDay) * 2).Format(dateFormat)
+		f3, err := os.CreateTemp(os.TempDir(), "go-zero-test-"+boundary1)
+		assert.NoError(t, err)
+		t.Cleanup(func() {
+			_ = f1.Close()
+			_ = os.Remove(f1.Name())
+			_ = f2.Close()
+			_ = os.Remove(f2.Name())
+			_ = f3.Close()
+			_ = os.Remove(f3.Name())
+		})
+		rule := SizeLimitRotateRule{
+			DailyRotateRule: DailyRotateRule{
+				filename: path.Join(os.TempDir(), "go-zero-test-"),
+				days:     1,
+			},
+		}
+		assert.NotEmpty(t, rule.OutdatedFiles())
+
+		logger := new(RotateLogger)
+		logger.rule = &rule
+		logger.maybeDeleteOutdatedFiles()
+		assert.Empty(t, rule.OutdatedFiles())
+	})
 }
 
 func TestSizeLimitRotateRuleShallRotate(t *testing.T) {
@@ -61,14 +186,47 @@ func TestSizeLimitRotateRuleShallRotate(t *testing.T) {
 }
 
 func TestRotateLoggerClose(t *testing.T) {
-	filename, err := fs.TempFilenameWithText("foo")
-	assert.Nil(t, err)
-	if len(filename) > 0 {
-		defer os.Remove(filename)
-	}
-	logger, err := NewLogger(filename, new(DailyRotateRule), false)
-	assert.Nil(t, err)
-	assert.Nil(t, logger.Close())
+	t.Run("close", func(t *testing.T) {
+		filename, err := fs.TempFilenameWithText("foo")
+		assert.Nil(t, err)
+		if len(filename) > 0 {
+			defer os.Remove(filename)
+		}
+		logger, err := NewLogger(filename, new(DailyRotateRule), false)
+		assert.Nil(t, err)
+		_, err = logger.Write([]byte("foo"))
+		assert.Nil(t, err)
+		assert.Nil(t, logger.Close())
+	})
+
+	t.Run("close and write", func(t *testing.T) {
+		logger := new(RotateLogger)
+		logger.done = make(chan struct{})
+		close(logger.done)
+		_, err := logger.Write([]byte("foo"))
+		assert.ErrorIs(t, err, ErrLogFileClosed)
+	})
+
+	t.Run("close without losing logs", func(t *testing.T) {
+		text := "foo"
+		filename, err := fs.TempFilenameWithText(text)
+		assert.Nil(t, err)
+		if len(filename) > 0 {
+			defer os.Remove(filename)
+		}
+		logger, err := NewLogger(filename, new(DailyRotateRule), false)
+		assert.Nil(t, err)
+		msg := []byte("foo")
+		n := 100
+		for i := 0; i < n; i++ {
+			_, err = logger.Write(msg)
+			assert.Nil(t, err)
+		}
+		assert.Nil(t, logger.Close())
+		bs, err := os.ReadFile(filename)
+		assert.Nil(t, err)
+		assert.Equal(t, len(msg)*n+len(text), len(bs))
+	})
 }
 
 func TestRotateLoggerGetBackupFilename(t *testing.T) {
@@ -179,7 +337,7 @@ func TestRotateLoggerWithSizeLimitRotateRuleClose(t *testing.T) {
 	}
 	logger, err := NewLogger(filename, new(SizeLimitRotateRule), false)
 	assert.Nil(t, err)
-	assert.Nil(t, logger.Close())
+	_ = logger.Close()
 }
 
 func TestRotateLoggerGetBackupWithSizeLimitRotateRuleFilename(t *testing.T) {
@@ -295,6 +453,85 @@ func TestRotateLoggerWithSizeLimitRotateRuleWrite(t *testing.T) {
 	logger.write([]byte(`baz`))
 }
 
+func TestGzipFile(t *testing.T) {
+	err := errors.New("any error")
+
+	t.Run("gzip file open failed", func(t *testing.T) {
+		fsys := &fakeFileSystem{
+			openFn: func(name string) (*os.File, error) {
+				return nil, err
+			},
+		}
+		assert.ErrorIs(t, err, gzipFile("any", fsys))
+		assert.False(t, fsys.Removed())
+	})
+
+	t.Run("gzip file create failed", func(t *testing.T) {
+		fsys := &fakeFileSystem{
+			createFn: func(name string) (*os.File, error) {
+				return nil, err
+			},
+		}
+		assert.ErrorIs(t, err, gzipFile("any", fsys))
+		assert.False(t, fsys.Removed())
+	})
+
+	t.Run("gzip file copy failed", func(t *testing.T) {
+		fsys := &fakeFileSystem{
+			copyFn: func(writer io.Writer, reader io.Reader) (int64, error) {
+				return 0, err
+			},
+		}
+		assert.ErrorIs(t, err, gzipFile("any", fsys))
+		assert.False(t, fsys.Removed())
+	})
+
+	t.Run("gzip file last close failed", func(t *testing.T) {
+		var called int32
+		fsys := &fakeFileSystem{
+			closeFn: func(closer io.Closer) error {
+				if atomic.AddInt32(&called, 1) > 2 {
+					return err
+				}
+				return nil
+			},
+		}
+		assert.NoError(t, gzipFile("any", fsys))
+		assert.True(t, fsys.Removed())
+	})
+
+	t.Run("gzip file remove failed", func(t *testing.T) {
+		fsys := &fakeFileSystem{
+			removeFn: func(name string) error {
+				return err
+			},
+		}
+		assert.Error(t, err, gzipFile("any", fsys))
+		assert.True(t, fsys.Removed())
+	})
+
+	t.Run("gzip file everything ok", func(t *testing.T) {
+		fsys := &fakeFileSystem{}
+		assert.NoError(t, gzipFile("any", fsys))
+		assert.True(t, fsys.Removed())
+	})
+}
+
+func TestRotateLogger_WithExistingFile(t *testing.T) {
+	const body = "foo"
+	filename, err := fs.TempFilenameWithText(body)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+
+	rule := NewSizeLimitRotateRule(filename, "-", 1, 100, 3, false)
+	logger, err := NewLogger(filename, rule, false)
+	assert.Nil(t, err)
+	assert.Equal(t, int64(len(body)), logger.currentSize)
+	assert.Nil(t, logger.Close())
+}
+
 func BenchmarkRotateLogger(b *testing.B) {
 	filename := "./test.log"
 	filename2 := "./test2.log"
@@ -346,3 +583,53 @@ func BenchmarkRotateLogger(b *testing.B) {
 		}
 	})
 }
+
+type fakeFileSystem struct {
+	removed  int32
+	closeFn  func(closer io.Closer) error
+	copyFn   func(writer io.Writer, reader io.Reader) (int64, error)
+	createFn func(name string) (*os.File, error)
+	openFn   func(name string) (*os.File, error)
+	removeFn func(name string) error
+}
+
+func (f *fakeFileSystem) Close(closer io.Closer) error {
+	if f.closeFn != nil {
+		return f.closeFn(closer)
+	}
+	return nil
+}
+
+func (f *fakeFileSystem) Copy(writer io.Writer, reader io.Reader) (int64, error) {
+	if f.copyFn != nil {
+		return f.copyFn(writer, reader)
+	}
+	return 0, nil
+}
+
+func (f *fakeFileSystem) Create(name string) (*os.File, error) {
+	if f.createFn != nil {
+		return f.createFn(name)
+	}
+	return nil, nil
+}
+
+func (f *fakeFileSystem) Open(name string) (*os.File, error) {
+	if f.openFn != nil {
+		return f.openFn(name)
+	}
+	return nil, nil
+}
+
+func (f *fakeFileSystem) Remove(name string) error {
+	atomic.AddInt32(&f.removed, 1)
+
+	if f.removeFn != nil {
+		return f.removeFn(name)
+	}
+	return nil
+}
+
+func (f *fakeFileSystem) Removed() bool {
+	return atomic.LoadInt32(&f.removed) > 0
+}

core/logx/syslog_test.go

@@ -12,6 +12,8 @@ import (
 
 const testlog = "Stay hungry, stay foolish."
 
+var testobj = map[string]any{"foo": "bar"}
+
 func TestCollectSysLog(t *testing.T) {
 	CollectSysLog()
 	content := getContent(captureOutput(func() {

core/logx/vars.go

@@ -15,6 +15,8 @@ const (
 	ErrorLevel
 	// SevereLevel only log severe messages
 	SevereLevel
+	// disableLevel doesn't log any messages
+	disableLevel = 0xff
 )
 
 const (

core/logx/writer.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"log"
 	"path"
+	"runtime/debug"
 	"sync"
 	"sync/atomic"
 
@@ -332,11 +333,13 @@ func wrapLevelWithColor(level string) string {
 
 func writeJson(writer io.Writer, info any) {
 	if content, err := json.Marshal(info); err != nil {
-		log.Println(err.Error())
+		log.Printf("err: %s\n\n%s", err.Error(), debug.Stack())
 	} else if writer == nil {
 		log.Println(string(content))
 	} else {
-		writer.Write(append(content, '\n'))
+		if _, err := writer.Write(append(content, '\n')); err != nil {
+			log.Println(err.Error())
+		}
 	}
 }
 
@@ -384,7 +387,7 @@ func writePlainValue(writer io.Writer, level string, val any, fields ...string)
 	buf.WriteString(level)
 	buf.WriteByte(plainEncodingSep)
 	if err := json.NewEncoder(&buf).Encode(val); err != nil {
-		log.Println(err.Error())
+		log.Printf("err: %s\n\n%s", err.Error(), debug.Stack())
 		return
 	}
 

core/logx/writer_test.go

@@ -97,6 +97,15 @@ func TestConsoleWriter(t *testing.T) {
 	w.(*concreteWriter).statLog = easyToCloseWriter{}
 }
 
+func TestNewFileWriter(t *testing.T) {
+	t.Run("access", func(t *testing.T) {
+		_, err := newFileWriter(LogConf{
+			Path: "/not-exists",
+		})
+		assert.Error(t, err)
+	})
+}
+
 func TestNopWriter(t *testing.T) {
 	assert.NotPanics(t, func() {
 		var w nopWriter
@@ -117,9 +126,23 @@ func TestWriteJson(t *testing.T) {
 	log.SetOutput(&buf)
 	writeJson(nil, "foo")
 	assert.Contains(t, buf.String(), "foo")
+
+	buf.Reset()
+	writeJson(hardToWriteWriter{}, "foo")
+	assert.Contains(t, buf.String(), "write error")
+
 	buf.Reset()
 	writeJson(nil, make(chan int))
 	assert.Contains(t, buf.String(), "unsupported type")
+
+	buf.Reset()
+	type C struct {
+		RC func()
+	}
+	writeJson(nil, C{
+		RC: func() {},
+	})
+	assert.Contains(t, buf.String(), "runtime/debug.Stack")
 }
 
 func TestWritePlainAny(t *testing.T) {
@@ -156,6 +179,14 @@ func TestWritePlainAny(t *testing.T) {
 	writePlainAny(hardToWriteWriter{}, levelFatal, "foo")
 	assert.Contains(t, buf.String(), "write error")
 
+	buf.Reset()
+	type C struct {
+		RC func()
+	}
+	writePlainAny(nil, levelError, C{
+		RC: func() {},
+	})
+	assert.Contains(t, buf.String(), "runtime/debug.Stack")
 }
 
 func TestLogWithLimitContentLength(t *testing.T) {

core/mapping/marshaler.go

@@ -12,7 +12,7 @@ const (
 )
 
 // Marshal marshals the given val and returns the map that contains the fields.
-// optional=another is not implemented, and it's hard to implement and not common used.
+// optional=another is not implemented, and it's hard to implement and not commonly used.
 func Marshal(val any) (map[string]map[string]any, error) {
 	ret := make(map[string]map[string]any)
 	tp := reflect.TypeOf(val)

core/mapping/unmarshaler.go

@@ -18,8 +18,10 @@ import (
 )
 
 const (
-	defaultKeyName = "key"
-	delimiter      = '.'
+	defaultKeyName   = "key"
+	delimiter        = '.'
+	ignoreKey        = "-"
+	numberTypeString = "number"
 )
 
 var (
@@ -37,7 +39,7 @@ var (
 )
 
 type (
-	// Unmarshaler is used to unmarshal with given tag key.
+	// Unmarshaler is used to unmarshal with the given tag key.
 	Unmarshaler struct {
 		key  string
 		opts unmarshalOptions
@@ -49,6 +51,7 @@ type (
 	unmarshalOptions struct {
 		fillDefault  bool
 		fromString   bool
+		opaqueKeys   bool
 		canonicalKey func(key string) string
 	}
 )
@@ -66,13 +69,17 @@ func NewUnmarshaler(key string, opts ...UnmarshalOption) *Unmarshaler {
 	return &unmarshaler
 }
 
-// UnmarshalKey unmarshals m into v with tag key.
+// UnmarshalKey unmarshals m into v with the tag key.
 func UnmarshalKey(m map[string]any, v any) error {
 	return keyUnmarshaler.Unmarshal(m, v)
 }
 
 // Unmarshal unmarshals m into v.
-func (u *Unmarshaler) Unmarshal(i any, v any) error {
+func (u *Unmarshaler) Unmarshal(i, v any) error {
+	return u.unmarshal(i, v, "")
+}
+
+func (u *Unmarshaler) unmarshal(i, v any, fullName string) error {
 	valueType := reflect.TypeOf(v)
 	if valueType.Kind() != reflect.Ptr {
 		return errValueNotSettable
@@ -85,13 +92,13 @@ func (u *Unmarshaler) Unmarshal(i any, v any) error {
 			return errTypeMismatch
 		}
 
-		return u.UnmarshalValuer(mapValuer(iv), v)
+		return u.unmarshalValuer(mapValuer(iv), v, fullName)
 	case []any:
 		if elemType.Kind() != reflect.Slice {
 			return errTypeMismatch
 		}
 
-		return u.fillSlice(elemType, reflect.ValueOf(v).Elem(), iv)
+		return u.fillSlice(elemType, reflect.ValueOf(v).Elem(), iv, fullName)
 	default:
 		return errUnsupportedType
 	}
@@ -99,17 +106,21 @@ func (u *Unmarshaler) Unmarshal(i any, v any) error {
 
 // UnmarshalValuer unmarshals m into v.
 func (u *Unmarshaler) UnmarshalValuer(m Valuer, v any) error {
-	return u.unmarshalWithFullName(simpleValuer{current: m}, v, "")
+	return u.unmarshalValuer(simpleValuer{current: m}, v, "")
 }
 
-func (u *Unmarshaler) fillMap(fieldType reflect.Type, value reflect.Value, mapValue any) error {
+func (u *Unmarshaler) unmarshalValuer(m Valuer, v any, fullName string) error {
+	return u.unmarshalWithFullName(simpleValuer{current: m}, v, fullName)
+}
+
+func (u *Unmarshaler) fillMap(fieldType reflect.Type, value reflect.Value, mapValue any, fullName string) error {
 	if !value.CanSet() {
 		return errValueNotSettable
 	}
 
 	fieldKeyType := fieldType.Key()
 	fieldElemType := fieldType.Elem()
-	targetValue, err := u.generateMap(fieldKeyType, fieldElemType, mapValue)
+	targetValue, err := u.generateMap(fieldKeyType, fieldElemType, mapValue, fullName)
 	if err != nil {
 		return err
 	}
@@ -143,19 +154,22 @@ func (u *Unmarshaler) fillMapFromString(value reflect.Value, mapValue any) error
 	return nil
 }
 
-func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, mapValue any) error {
+func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, mapValue any, fullName string) error {
 	if !value.CanSet() {
 		return errValueNotSettable
 	}
 
+	refValue := reflect.ValueOf(mapValue)
+	if refValue.Kind() != reflect.Slice {
+		return newTypeMismatchErrorWithHint(fullName, reflect.Slice.String(), refValue.Type().String())
+	}
+	if refValue.IsNil() {
+		return nil
+	}
+
 	baseType := fieldType.Elem()
 	dereffedBaseType := Deref(baseType)
 	dereffedBaseKind := dereffedBaseType.Kind()
-	refValue := reflect.ValueOf(mapValue)
-	if refValue.IsNil() {
-		return nil
-	}
-
 	conv := reflect.MakeSlice(reflect.SliceOf(baseType), refValue.Len(), refValue.Cap())
 	if refValue.Len() == 0 {
 		value.Set(conv)
@@ -170,20 +184,27 @@ func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, map
 		}
 
 		valid = true
+		sliceFullName := fmt.Sprintf("%s[%d]", fullName, i)
+
 		switch dereffedBaseKind {
 		case reflect.Struct:
 			target := reflect.New(dereffedBaseType)
-			if err := u.Unmarshal(ithValue.(map[string]any), target.Interface()); err != nil {
+			val, ok := ithValue.(map[string]any)
+			if !ok {
+				return errTypeMismatch
+			}
+
+			if err := u.unmarshal(val, target.Interface(), sliceFullName); err != nil {
 				return err
 			}
 
 			SetValue(fieldType.Elem(), conv.Index(i), target.Elem())
 		case reflect.Slice:
-			if err := u.fillSlice(dereffedBaseType, conv.Index(i), ithValue); err != nil {
+			if err := u.fillSlice(dereffedBaseType, conv.Index(i), ithValue, sliceFullName); err != nil {
 				return err
 			}
 		default:
-			if err := u.fillSliceValue(conv, i, dereffedBaseKind, ithValue); err != nil {
+			if err := u.fillSliceValue(conv, i, dereffedBaseKind, ithValue, sliceFullName); err != nil {
 				return err
 			}
 		}
@@ -197,16 +218,16 @@ func (u *Unmarshaler) fillSlice(fieldType reflect.Type, value reflect.Value, map
 }
 
 func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.Value,
-	mapValue any) error {
+	mapValue any, fullName string) error {
 	var slice []any
 	switch v := mapValue.(type) {
 	case fmt.Stringer:
 		if err := jsonx.UnmarshalFromString(v.String(), &slice); err != nil {
-			return err
+			return fmt.Errorf("fullName: `%s`, error: `%w`", fullName, err)
 		}
 	case string:
 		if err := jsonx.UnmarshalFromString(v, &slice); err != nil {
-			return err
+			return fmt.Errorf("fullName: `%s`, error: `%w`", fullName, err)
 		}
 	default:
 		return errUnsupportedType
@@ -217,7 +238,7 @@ func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.
 	conv := reflect.MakeSlice(reflect.SliceOf(baseFieldType), len(slice), cap(slice))
 
 	for i := 0; i < len(slice); i++ {
-		if err := u.fillSliceValue(conv, i, baseFieldKind, slice[i]); err != nil {
+		if err := u.fillSliceValue(conv, i, baseFieldKind, slice[i], fullName); err != nil {
 			return err
 		}
 	}
@@ -227,7 +248,11 @@ func (u *Unmarshaler) fillSliceFromString(fieldType reflect.Type, value reflect.
 }
 
 func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int,
-	baseKind reflect.Kind, value any) error {
+	baseKind reflect.Kind, value any, fullName string) error {
+	if value == nil {
+		return errNilSliceElement
+	}
+
 	ithVal := slice.Index(index)
 	switch v := value.(type) {
 	case fmt.Stringer:
@@ -235,7 +260,7 @@ func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int,
 	case string:
 		return setValueFromString(baseKind, ithVal, v)
 	case map[string]any:
-		return u.fillMap(ithVal.Type(), ithVal, value)
+		return u.fillMap(ithVal.Type(), ithVal, value, fullName)
 	default:
 		// don't need to consider the difference between int, int8, int16, int32, int64,
 		// uint, uint8, uint16, uint32, uint64, because they're handled as json.Number.
@@ -261,7 +286,7 @@ func (u *Unmarshaler) fillSliceValue(slice reflect.Value, index int,
 }
 
 func (u *Unmarshaler) fillSliceWithDefault(derefedType reflect.Type, value reflect.Value,
-	defaultValue string) error {
+	defaultValue, fullName string) error {
 	baseFieldType := Deref(derefedType.Elem())
 	baseFieldKind := baseFieldType.Kind()
 	defaultCacheLock.Lock()
@@ -279,10 +304,10 @@ func (u *Unmarshaler) fillSliceWithDefault(derefedType reflect.Type, value refle
 		defaultCacheLock.Unlock()
 	}
 
-	return u.fillSlice(derefedType, value, slice)
+	return u.fillSlice(derefedType, value, slice, fullName)
 }
 
-func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue any) (reflect.Value, error) {
+func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue any, fullName string) (reflect.Value, error) {
 	mapType := reflect.MapOf(keyType, elemType)
 	valueType := reflect.TypeOf(mapValue)
 	if mapType == valueType {
@@ -301,11 +326,12 @@ func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue any)
 	for _, key := range refValue.MapKeys() {
 		keythValue := refValue.MapIndex(key)
 		keythData := keythValue.Interface()
+		mapFullName := fmt.Sprintf("%s[%s]", fullName, key.String())
 
 		switch dereffedElemKind {
 		case reflect.Slice:
 			target := reflect.New(dereffedElemType)
-			if err := u.fillSlice(elemType, target.Elem(), keythData); err != nil {
+			if err := u.fillSlice(elemType, target.Elem(), keythData, mapFullName); err != nil {
 				return emptyValue, err
 			}
 
@@ -317,7 +343,7 @@ func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue any)
 			}
 
 			target := reflect.New(dereffedElemType)
-			if err := u.Unmarshal(keythMap, target.Interface()); err != nil {
+			if err := u.unmarshal(keythMap, target.Interface(), mapFullName); err != nil {
 				return emptyValue, err
 			}
 
@@ -328,7 +354,7 @@ func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue any)
 				return emptyValue, errTypeMismatch
 			}
 
-			innerValue, err := u.generateMap(elemType.Key(), elemType.Elem(), keythMap)
+			innerValue, err := u.generateMap(elemType.Key(), elemType.Elem(), keythMap, mapFullName)
 			if err != nil {
 				return emptyValue, err
 			}
@@ -347,7 +373,12 @@ func (u *Unmarshaler) generateMap(keyType, elemType reflect.Type, mapValue any)
 					return emptyValue, errTypeMismatch
 				}
 
-				targetValue.SetMapIndex(key, reflect.ValueOf(v))
+				val := reflect.ValueOf(v)
+				if !val.Type().AssignableTo(dereffedElemType) {
+					return emptyValue, errTypeMismatch
+				}
+
+				targetValue.SetMapIndex(key, val)
 			case json.Number:
 				target := reflect.New(dereffedElemType)
 				if err := setValueFromString(dereffedElemKind, target.Elem(), v.String()); err != nil {
@@ -397,6 +428,10 @@ func (u *Unmarshaler) parseOptionsWithContext(field reflect.StructField, m Value
 		}
 	}
 
+	if u.opts.fillDefault {
+		return key, &options.fieldOptionsWithContext, nil
+	}
+
 	optsWithContext, err := options.toOptionsWithContext(key, m, fullName)
 	if err != nil {
 		return "", nil, err
@@ -412,6 +447,10 @@ func (u *Unmarshaler) processAnonymousField(field reflect.StructField, value ref
 		return err
 	}
 
+	if key == ignoreKey {
+		return nil
+	}
+
 	if options.optional() {
 		return u.processAnonymousFieldOptional(field, value, key, m, fullName)
 	}
@@ -470,7 +509,7 @@ func (u *Unmarshaler) processAnonymousStructFieldOptional(fieldType reflect.Type
 			return err
 		}
 
-		_, hasValue := getValue(m, fieldKey)
+		_, hasValue := getValue(m, fieldKey, u.opts.opaqueKeys)
 		if hasValue {
 			if !filled {
 				filled = true
@@ -513,8 +552,8 @@ func (u *Unmarshaler) processFieldNotFromString(fieldType reflect.Type, value re
 	vp valueWithParent, opts *fieldOptionsWithContext, fullName string) error {
 	derefedFieldType := Deref(fieldType)
 	typeKind := derefedFieldType.Kind()
-	valueKind := reflect.TypeOf(vp.value).Kind()
 	mapValue := vp.value
+	valueKind := reflect.TypeOf(mapValue).Kind()
 
 	switch {
 	case valueKind == reflect.Map && typeKind == reflect.Struct:
@@ -527,12 +566,14 @@ func (u *Unmarshaler) processFieldNotFromString(fieldType reflect.Type, value re
 			current: mapValuer(mv),
 			parent:  vp.parent,
 		}, fullName)
+	case typeKind == reflect.Slice && valueKind == reflect.Slice:
+		return u.fillSlice(fieldType, value, mapValue, fullName)
 	case valueKind == reflect.Map && typeKind == reflect.Map:
-		return u.fillMap(fieldType, value, mapValue)
+		return u.fillMap(fieldType, value, mapValue, fullName)
 	case valueKind == reflect.String && typeKind == reflect.Map:
 		return u.fillMapFromString(value, mapValue)
 	case valueKind == reflect.String && typeKind == reflect.Slice:
-		return u.fillSliceFromString(fieldType, value, mapValue)
+		return u.fillSliceFromString(fieldType, value, mapValue, fullName)
 	case valueKind == reflect.String && derefedFieldType == durationType:
 		return fillDurationValue(fieldType, value, mapValue.(string))
 	default:
@@ -545,23 +586,16 @@ func (u *Unmarshaler) processFieldPrimitive(fieldType reflect.Type, value reflec
 	typeKind := Deref(fieldType).Kind()
 	valueKind := reflect.TypeOf(mapValue).Kind()
 
-	switch {
-	case typeKind == reflect.Slice && valueKind == reflect.Slice:
-		return u.fillSlice(fieldType, value, mapValue)
-	case typeKind == reflect.Map && valueKind == reflect.Map:
-		return u.fillMap(fieldType, value, mapValue)
+	switch v := mapValue.(type) {
+	case json.Number:
+		return u.processFieldPrimitiveWithJSONNumber(fieldType, value, v, opts, fullName)
 	default:
-		switch v := mapValue.(type) {
-		case json.Number:
-			return u.processFieldPrimitiveWithJSONNumber(fieldType, value, v, opts, fullName)
-		default:
-			if typeKind == valueKind {
-				if err := validateValueInOptions(mapValue, opts.options()); err != nil {
-					return err
-				}
-
-				return fillWithSameType(fieldType, value, mapValue, opts)
+		if typeKind == valueKind {
+			if err := validateValueInOptions(mapValue, opts.options()); err != nil {
+				return err
 			}
+
+			return fillWithSameType(fieldType, value, mapValue, opts)
 		}
 	}
 
@@ -584,25 +618,28 @@ func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(fieldType reflect.Type
 	target := reflect.New(Deref(fieldType)).Elem()
 
 	switch typeKind {
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		iValue, err := v.Int64()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		if err := setValueFromString(typeKind, target, v.String()); err != nil {
+			return err
+		}
+	case reflect.Float32:
+		fValue, err := v.Float64()
 		if err != nil {
 			return err
 		}
 
-		target.SetInt(iValue)
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		iValue, err := v.Int64()
-		if err != nil {
-			return err
+		// if the value is a pointer, we need to check overflow with the pointer's value.
+		derefedValue := value
+		for derefedValue.Type().Kind() == reflect.Ptr {
+			derefedValue = derefedValue.Elem()
+		}
+		if derefedValue.CanFloat() && derefedValue.OverflowFloat(fValue) {
+			return fmt.Errorf("parsing %q as float32: value out of range", v.String())
 		}
 
-		if iValue < 0 {
-			return fmt.Errorf("unmarshal %q with bad value %q", fullName, v.String())
-		}
-
-		target.SetUint(uint64(iValue))
-	case reflect.Float32, reflect.Float64:
+		target.SetFloat(fValue)
+	case reflect.Float64:
 		fValue, err := v.Float64()
 		if err != nil {
 			return err
@@ -610,7 +647,7 @@ func (u *Unmarshaler) processFieldPrimitiveWithJSONNumber(fieldType reflect.Type
 
 		target.SetFloat(fValue)
 	default:
-		return newTypeMismatchError(fullName)
+		return newTypeMismatchErrorWithHint(fullName, typeKind.String(), numberTypeString)
 	}
 
 	SetValue(fieldType, value, target)
@@ -704,6 +741,10 @@ func (u *Unmarshaler) processNamedField(field reflect.StructField, value reflect
 		return err
 	}
 
+	if key == ignoreKey {
+		return nil
+	}
+
 	fullName = join(fullName, key)
 	if opts != nil && len(opts.EnvVar) > 0 {
 		envVal := proc.Env(opts.EnvVar)
@@ -718,7 +759,7 @@ func (u *Unmarshaler) processNamedField(field reflect.StructField, value reflect
 	}
 
 	valuer := createValuer(m, opts)
-	mapValue, hasValue := getValue(valuer, canonicalKey)
+	mapValue, hasValue := getValue(valuer, canonicalKey, u.opts.opaqueKeys)
 
 	// When fillDefault is used, m is a null value, hasValue must be false, all priority judgments fillDefault.
 	if u.opts.fillDefault {
@@ -811,7 +852,7 @@ func (u *Unmarshaler) processNamedFieldWithoutValue(fieldType reflect.Type, valu
 
 		switch fieldKind {
 		case reflect.Array, reflect.Slice:
-			return u.fillSliceWithDefault(derefedType, value, defaultValue)
+			return u.fillSliceWithDefault(derefedType, value, defaultValue, fullName)
 		default:
 			return setValueFromString(fieldKind, value, defaultValue)
 		}
@@ -859,7 +900,7 @@ func (u *Unmarshaler) processNamedFieldWithoutValue(fieldType reflect.Type, valu
 
 func (u *Unmarshaler) unmarshalWithFullName(m valuerWithParent, v any, fullName string) error {
 	rv := reflect.ValueOf(v)
-	if err := ValidatePtr(&rv); err != nil {
+	if err := ValidatePtr(rv); err != nil {
 		return err
 	}
 
@@ -881,11 +922,6 @@ func (u *Unmarshaler) unmarshalWithFullName(m valuerWithParent, v any, fullName
 		typeField := baseType.Field(i)
 		valueField := valElem.Field(i)
 		if err := u.processField(typeField, valueField, m, fullName); err != nil {
-			if len(fullName) > 0 {
-				err = fmt.Errorf("%w, fullName: %s, field: %s, type: %s",
-					err, fullName, typeField.Name, valueField.Type().Name())
-			}
-
 			return err
 		}
 	}
@@ -914,6 +950,14 @@ func WithDefault() UnmarshalOption {
 	}
 }
 
+// WithOpaqueKeys customizes an Unmarshaler with opaque keys.
+// Opaque keys are keys that are not processed by the unmarshaler.
+func WithOpaqueKeys() UnmarshalOption {
+	return func(opt *unmarshalOptions) {
+		opt.opaqueKeys = true
+	}
+}
+
 func createValuer(v valuerWithParent, opts *fieldOptionsWithContext) valuerWithParent {
 	if opts.inherit() {
 		return recursiveValuer{
@@ -991,8 +1035,8 @@ func fillWithSameType(fieldType reflect.Type, value reflect.Value, mapValue any,
 }
 
 // getValue gets the value for the specific key, the key can be in the format of parentKey.childKey
-func getValue(m valuerWithParent, key string) (any, bool) {
-	keys := readKeys(key)
+func getValue(m valuerWithParent, key string, opaque bool) (any, bool) {
+	keys := readKeys(key, opaque)
 	return getValueWithChainedKeys(m, keys)
 }
 
@@ -1046,7 +1090,16 @@ func newTypeMismatchError(name string) error {
 	return fmt.Errorf("type mismatch for field %q", name)
 }
 
-func readKeys(key string) []string {
+func newTypeMismatchErrorWithHint(name, expectType, actualType string) error {
+	return fmt.Errorf("type mismatch for field %q, expect %q, actual %q",
+		name, expectType, actualType)
+}
+
+func readKeys(key string, opaque bool) []string {
+	if opaque {
+		return []string{key}
+	}
+
 	cacheKeysLock.Lock()
 	keys, ok := cacheKeys[key]
 	cacheKeysLock.Unlock()

core/mapping/utils.go

@@ -30,11 +30,13 @@ const (
 	leftSquareBracket  = '['
 	rightSquareBracket = ']'
 	segmentSeparator   = ','
+	intSize            = 32 << (^uint(0) >> 63) // 32 or 64
 )
 
 var (
 	errUnsupportedType  = errors.New("unsupported type on setting field value")
 	errNumberRange      = errors.New("wrong number range setting")
+	errNilSliceElement  = errors.New("null element for slice")
 	optionsCache        = make(map[string]optionsCacheValue)
 	cacheLock           sync.RWMutex
 	structRequiredCache = make(map[reflect.Type]requiredCacheValue)
@@ -79,7 +81,7 @@ func SetMapIndexValue(tp reflect.Type, value, key, target reflect.Value) {
 }
 
 // ValidatePtr validates v if it's a valid pointer.
-func ValidatePtr(v *reflect.Value) error {
+func ValidatePtr(v reflect.Value) error {
 	// sequence is very important, IsNil must be called after checking Kind() with reflect.Ptr,
 	// panic otherwise
 	if !v.IsValid() || v.Kind() != reflect.Ptr || v.IsNil() {
@@ -100,27 +102,30 @@ func convertTypeFromString(kind reflect.Kind, str string) (any, error) {
 		default:
 			return false, errTypeMismatch
 		}
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		intValue, err := strconv.ParseInt(str, 10, 64)
-		if err != nil {
-			return 0, fmt.Errorf("the value %q cannot parsed as int", str)
-		}
-
-		return intValue, nil
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		uintValue, err := strconv.ParseUint(str, 10, 64)
-		if err != nil {
-			return 0, fmt.Errorf("the value %q cannot parsed as uint", str)
-		}
-
-		return uintValue, nil
-	case reflect.Float32, reflect.Float64:
-		floatValue, err := strconv.ParseFloat(str, 64)
-		if err != nil {
-			return 0, fmt.Errorf("the value %q cannot parsed as float", str)
-		}
-
-		return floatValue, nil
+	case reflect.Int:
+		return strconv.ParseInt(str, 10, intSize)
+	case reflect.Int8:
+		return strconv.ParseInt(str, 10, 8)
+	case reflect.Int16:
+		return strconv.ParseInt(str, 10, 16)
+	case reflect.Int32:
+		return strconv.ParseInt(str, 10, 32)
+	case reflect.Int64:
+		return strconv.ParseInt(str, 10, 64)
+	case reflect.Uint:
+		return strconv.ParseUint(str, 10, intSize)
+	case reflect.Uint8:
+		return strconv.ParseUint(str, 10, 8)
+	case reflect.Uint16:
+		return strconv.ParseUint(str, 10, 16)
+	case reflect.Uint32:
+		return strconv.ParseUint(str, 10, 32)
+	case reflect.Uint64:
+		return strconv.ParseUint(str, 10, 64)
+	case reflect.Float32:
+		return strconv.ParseFloat(str, 32)
+	case reflect.Float64:
+		return strconv.ParseFloat(str, 64)
 	case reflect.String:
 		return str, nil
 	default:
@@ -372,8 +377,6 @@ func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 		default:
 			return fmt.Errorf("field %q has wrong optional", fieldName)
 		}
-	case option == optionalOption:
-		fieldOpts.Optional = true
 	case strings.HasPrefix(option, optionsOption):
 		val, err := parseProperty(fieldName, optionsOption, option)
 		if err != nil {
@@ -413,7 +416,7 @@ func parseOption(fieldOpts *fieldOptions, fieldName, option string) error {
 }
 
 // parseOptions parses the given options in tag.
-// for example: `json:"name,options=foo|bar"` or `json:"name,options=[foo,bar]"`
+// for example, `json:"name,options=foo|bar"` or `json:"name,options=[foo,bar]"`
 func parseOptions(val string) []string {
 	if len(val) == 0 {
 		return nil
@@ -488,19 +491,22 @@ func setMatchedPrimitiveValue(kind reflect.Kind, value reflect.Value, v any) err
 	switch kind {
 	case reflect.Bool:
 		value.SetBool(v.(bool))
+		return nil
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 		value.SetInt(v.(int64))
+		return nil
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
 		value.SetUint(v.(uint64))
+		return nil
 	case reflect.Float32, reflect.Float64:
 		value.SetFloat(v.(float64))
+		return nil
 	case reflect.String:
 		value.SetString(v.(string))
+		return nil
 	default:
 		return errUnsupportedType
 	}
-
-	return nil
 }
 
 func setValueFromString(kind reflect.Kind, value reflect.Value, str string) error {
@@ -577,7 +583,8 @@ func usingDifferentKeys(key string, field reflect.StructField) bool {
 	return false
 }
 
-func validateAndSetValue(kind reflect.Kind, value reflect.Value, str string, opts *fieldOptionsWithContext) error {
+func validateAndSetValue(kind reflect.Kind, value reflect.Value, str string,
+	opts *fieldOptionsWithContext) error {
 	if !value.CanSet() {
 		return errValueNotSettable
 	}

core/mapping/utils_test.go

@@ -218,30 +218,31 @@ func TestParseSegments(t *testing.T) {
 func TestValidatePtrWithNonPtr(t *testing.T) {
 	var foo string
 	rve := reflect.ValueOf(foo)
-	assert.NotNil(t, ValidatePtr(&rve))
+	assert.NotNil(t, ValidatePtr(rve))
 }
 
 func TestValidatePtrWithPtr(t *testing.T) {
 	var foo string
 	rve := reflect.ValueOf(&foo)
-	assert.Nil(t, ValidatePtr(&rve))
+	assert.Nil(t, ValidatePtr(rve))
 }
 
 func TestValidatePtrWithNilPtr(t *testing.T) {
 	var foo *string
 	rve := reflect.ValueOf(foo)
-	assert.NotNil(t, ValidatePtr(&rve))
+	assert.NotNil(t, ValidatePtr(rve))
 }
 
 func TestValidatePtrWithZeroValue(t *testing.T) {
 	var s string
 	e := reflect.Zero(reflect.TypeOf(s))
-	assert.NotNil(t, ValidatePtr(&e))
+	assert.NotNil(t, ValidatePtr(e))
 }
 
 func TestSetValueNotSettable(t *testing.T) {
 	var i int
-	assert.NotNil(t, setValueFromString(reflect.Int, reflect.ValueOf(i), "1"))
+	assert.Error(t, setValueFromString(reflect.Int, reflect.ValueOf(i), "1"))
+	assert.Error(t, validateAndSetValue(reflect.Int, reflect.ValueOf(i), "1", nil))
 }
 
 func TestParseKeyAndOptionsErrors(t *testing.T) {
@@ -300,3 +301,36 @@ func TestSetValueFormatErrors(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateValueRange(t *testing.T) {
+	t.Run("float", func(t *testing.T) {
+		assert.NoError(t, validateValueRange(1.2, nil))
+	})
+
+	t.Run("float number range", func(t *testing.T) {
+		assert.NoError(t, validateNumberRange(1.2, nil))
+	})
+
+	t.Run("bad float", func(t *testing.T) {
+		assert.Error(t, validateValueRange("a", &fieldOptionsWithContext{
+			Range: &numberRange{},
+		}))
+	})
+
+	t.Run("bad float validate", func(t *testing.T) {
+		var v struct {
+			Foo float32
+		}
+		assert.Error(t, validateAndSetValue(reflect.Int, reflect.ValueOf(&v).Elem().Field(0),
+			"1", &fieldOptionsWithContext{
+				Range: &numberRange{
+					left:  2,
+					right: 3,
+				},
+			}))
+	})
+}
+
+func TestSetMatchedPrimitiveValue(t *testing.T) {
+	assert.Error(t, setMatchedPrimitiveValue(reflect.Func, reflect.ValueOf(2), "1"))
+}

core/mapping/valuer.go

@@ -26,9 +26,9 @@ type (
 		parent valuerWithParent
 	}
 
-	// mapValuer is a type for map to meet the Valuer interface.
+	// mapValuer is a type for the map to meet the Valuer interface.
 	mapValuer map[string]any
-	// simpleValuer is a type to get value from current node.
+	// simpleValuer is a type to get value from the current node.
 	simpleValuer node
 	// recursiveValuer is a type to get the value recursively from current and parent nodes.
 	recursiveValuer node

core/mapping/yamlunmarshaler_test.go

@@ -1011,6 +1011,15 @@ func TestUnmarshalYamlMapRune(t *testing.T) {
 	assert.Equal(t, rune(3), v.Machine["node3"])
 }
 
+func TestUnmarshalYamlStringOfInt(t *testing.T) {
+	text := `password: 123456`
+	var v struct {
+		Password string `json:"password"`
+	}
+	reader := strings.NewReader(text)
+	assert.Error(t, UnmarshalYamlReader(reader, &v))
+}
+
 func TestUnmarshalYamlBadInput(t *testing.T) {
 	var v struct {
 		Any string

core/mathx/entropy_test.go

@@ -15,3 +15,17 @@ func TestCalcEntropy(t *testing.T) {
 	}
 	assert.True(t, CalcEntropy(m) > .99)
 }
+
+func TestCalcEmptyEntropy(t *testing.T) {
+	m := make(map[any]int)
+	assert.Equal(t, float64(1), CalcEntropy(m))
+}
+
+func TestCalcDiffEntropy(t *testing.T) {
+	const total = 1000
+	m := make(map[any]int, total)
+	for i := 0; i < total; i++ {
+		m[i] = i
+	}
+	assert.True(t, CalcEntropy(m) < .99)
+}

core/mathx/range.go

@@ -0,0 +1,34 @@
+package mathx
+
+type numerical interface {
+	~int | ~int8 | ~int16 | ~int32 | ~int64 |
+		~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
+		~float32 | ~float64
+}
+
+// AtLeast returns the greater of x or lower.
+func AtLeast[T numerical](x, lower T) T {
+	if x < lower {
+		return lower
+	}
+	return x
+}
+
+// AtMost returns the smaller of x or upper.
+func AtMost[T numerical](x, upper T) T {
+	if x > upper {
+		return upper
+	}
+	return x
+}
+
+// Between returns the value of x clamped to the range [lower, upper].
+func Between[T numerical](x, lower, upper T) T {
+	if x < lower {
+		return lower
+	}
+	if x > upper {
+		return upper
+	}
+	return x
+}

core/mathx/range_test.go

@@ -0,0 +1,513 @@
+package mathx
+
+import "testing"
+
+func TestAtLeast(t *testing.T) {
+	t.Run("test int", func(t *testing.T) {
+		if got := AtLeast(10, 5); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(3, 5); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(5, 5); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int8", func(t *testing.T) {
+		if got := AtLeast(int8(10), int8(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(int8(3), int8(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(int8(5), int8(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int16", func(t *testing.T) {
+		if got := AtLeast(int16(10), int16(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(int16(3), int16(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(int16(5), int16(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int32", func(t *testing.T) {
+		if got := AtLeast(int32(10), int32(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(int32(3), int32(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(int32(5), int32(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int64", func(t *testing.T) {
+		if got := AtLeast(int64(10), int64(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(int64(3), int64(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(int64(5), int64(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint", func(t *testing.T) {
+		if got := AtLeast(uint(10), uint(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(uint(3), uint(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(uint(5), uint(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint8", func(t *testing.T) {
+		if got := AtLeast(uint8(10), uint8(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(uint8(3), uint8(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(uint8(5), uint8(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint16", func(t *testing.T) {
+		if got := AtLeast(uint16(10), uint16(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(uint16(3), uint16(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(uint16(5), uint16(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint32", func(t *testing.T) {
+		if got := AtLeast(uint32(10), uint32(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(uint32(3), uint32(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(uint32(5), uint32(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint64", func(t *testing.T) {
+		if got := AtLeast(uint64(10), uint64(5)); got != 10 {
+			t.Errorf("AtLeast() = %v, want 10", got)
+		}
+		if got := AtLeast(uint64(3), uint64(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+		if got := AtLeast(uint64(5), uint64(5)); got != 5 {
+			t.Errorf("AtLeast() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test float32", func(t *testing.T) {
+		if got := AtLeast(float32(10.0), float32(5.0)); got != 10.0 {
+			t.Errorf("AtLeast() = %v, want 10.0", got)
+		}
+		if got := AtLeast(float32(3.0), float32(5.0)); got != 5.0 {
+			t.Errorf("AtLeast() = %v, want 5.0", got)
+		}
+		if got := AtLeast(float32(5.0), float32(5.0)); got != 5.0 {
+			t.Errorf("AtLeast() = %v, want 5.0", got)
+		}
+	})
+
+	t.Run("test float64", func(t *testing.T) {
+		if got := AtLeast(10.0, 5.0); got != 10.0 {
+			t.Errorf("AtLeast() = %v, want 10.0", got)
+		}
+		if got := AtLeast(3.0, 5.0); got != 5.0 {
+			t.Errorf("AtLeast() = %v, want 5.0", got)
+		}
+		if got := AtLeast(5.0, 5.0); got != 5.0 {
+			t.Errorf("AtLeast() = %v, want 5.0", got)
+		}
+	})
+}
+
+func TestAtMost(t *testing.T) {
+	t.Run("test int", func(t *testing.T) {
+		if got := AtMost(10, 5); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(3, 5); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(5, 5); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int8", func(t *testing.T) {
+		if got := AtMost(int8(10), int8(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(int8(3), int8(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(int8(5), int8(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int16", func(t *testing.T) {
+		if got := AtMost(int16(10), int16(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(int16(3), int16(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(int16(5), int16(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int32", func(t *testing.T) {
+		if got := AtMost(int32(10), int32(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(int32(3), int32(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(int32(5), int32(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test int64", func(t *testing.T) {
+		if got := AtMost(int64(10), int64(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(int64(3), int64(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(int64(5), int64(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint", func(t *testing.T) {
+		if got := AtMost(uint(10), uint(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(uint(3), uint(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(uint(5), uint(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint8", func(t *testing.T) {
+		if got := AtMost(uint8(10), uint8(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(uint8(3), uint8(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(uint8(5), uint8(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint16", func(t *testing.T) {
+		if got := AtMost(uint16(10), uint16(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(uint16(3), uint16(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(uint16(5), uint16(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint32", func(t *testing.T) {
+		if got := AtMost(uint32(10), uint32(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(uint32(3), uint32(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(uint32(5), uint32(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test uint64", func(t *testing.T) {
+		if got := AtMost(uint64(10), uint64(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+		if got := AtMost(uint64(3), uint64(5)); got != 3 {
+			t.Errorf("AtMost() = %v, want 3", got)
+		}
+		if got := AtMost(uint64(5), uint64(5)); got != 5 {
+			t.Errorf("AtMost() = %v, want 5", got)
+		}
+	})
+
+	t.Run("test float32", func(t *testing.T) {
+		if got := AtMost(float32(10.0), float32(5.0)); got != 5.0 {
+			t.Errorf("AtMost() = %v, want 5.0", got)
+		}
+		if got := AtMost(float32(3.0), float32(5.0)); got != 3.0 {
+			t.Errorf("AtMost() = %v, want 3.0", got)
+		}
+		if got := AtMost(float32(5.0), float32(5.0)); got != 5.0 {
+			t.Errorf("AtMost() = %v, want 5.0", got)
+		}
+	})
+
+	t.Run("test float64", func(t *testing.T) {
+		if got := AtMost(10.0, 5.0); got != 5.0 {
+			t.Errorf("AtMost() = %v, want 5.0", got)
+		}
+		if got := AtMost(3.0, 5.0); got != 3.0 {
+			t.Errorf("AtMost() = %v, want 3.0", got)
+		}
+		if got := AtMost(5.0, 5.0); got != 5.0 {
+			t.Errorf("AtMost() = %v, want 5.0", got)
+		}
+	})
+}
+
+func TestBetween(t *testing.T) {
+	t.Run("test int", func(t *testing.T) {
+		if got := Between(10, 5, 15); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(3, 5, 15); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(20, 5, 15); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(5, 5, 15); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(15, 5, 15); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test int8", func(t *testing.T) {
+		if got := Between(int8(10), int8(5), int8(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(int8(3), int8(5), int8(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int8(20), int8(5), int8(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(int8(5), int8(5), int8(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int8(15), int8(5), int8(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test int16", func(t *testing.T) {
+		if got := Between(int16(10), int16(5), int16(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(int16(3), int16(5), int16(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int16(20), int16(5), int16(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(int16(5), int16(5), int16(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int16(15), int16(5), int16(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test int32", func(t *testing.T) {
+		if got := Between(int32(10), int32(5), int32(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(int32(3), int32(5), int32(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int32(20), int32(5), int32(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(int32(5), int32(5), int32(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int32(15), int32(5), int32(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test int64", func(t *testing.T) {
+		if got := Between(int64(10), int64(5), int64(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(int64(3), int64(5), int64(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int64(20), int64(5), int64(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(int64(5), int64(5), int64(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(int64(15), int64(5), int64(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test uint", func(t *testing.T) {
+		if got := Between(uint(10), uint(5), uint(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(uint(3), uint(5), uint(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint(20), uint(5), uint(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(uint(5), uint(5), uint(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint(15), uint(5), uint(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test uint8", func(t *testing.T) {
+		if got := Between(uint8(10), uint8(5), uint8(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(uint8(3), uint8(5), uint8(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint8(20), uint8(5), uint8(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(uint8(5), uint8(5), uint8(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint8(15), uint8(5), uint8(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test uint16", func(t *testing.T) {
+		if got := Between(uint16(10), uint16(5), uint16(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(uint16(3), uint16(5), uint16(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint16(20), uint16(5), uint16(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(uint16(5), uint16(5), uint16(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint16(15), uint16(5), uint16(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test uint32", func(t *testing.T) {
+		if got := Between(uint32(10), uint32(5), uint32(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(uint32(3), uint32(5), uint32(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint32(20), uint32(5), uint32(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(uint32(5), uint32(5), uint32(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint32(15), uint32(5), uint32(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test uint64", func(t *testing.T) {
+		if got := Between(uint64(10), uint64(5), uint64(15)); got != 10 {
+			t.Errorf("Between() = %v, want 10", got)
+		}
+		if got := Between(uint64(3), uint64(5), uint64(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint64(20), uint64(5), uint64(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+		if got := Between(uint64(5), uint64(5), uint64(15)); got != 5 {
+			t.Errorf("Between() = %v, want 5", got)
+		}
+		if got := Between(uint64(15), uint64(5), uint64(15)); got != 15 {
+			t.Errorf("Between() = %v, want 15", got)
+		}
+	})
+
+	t.Run("test float32", func(t *testing.T) {
+		if got := Between(float32(10.0), float32(5.0), float32(15.0)); got != 10.0 {
+			t.Errorf("Between() = %v, want 10.0", got)
+		}
+		if got := Between(float32(3.0), float32(5.0), float32(15.0)); got != 5.0 {
+			t.Errorf("Between() = %v, want 5.0", got)
+		}
+		if got := Between(float32(20.0), float32(5.0), float32(15.0)); got != 15.0 {
+			t.Errorf("Between() = %v, want 15.0", got)
+		}
+		if got := Between(float32(5.0), float32(5.0), float32(15.0)); got != 5.0 {
+			t.Errorf("Between() = %v, want 5.0", got)
+		}
+		if got := Between(float32(15.0), float32(5.0), float32(15.0)); got != 15.0 {
+			t.Errorf("Between() = %v, want 15.0", got)
+		}
+	})
+
+	t.Run("test float64", func(t *testing.T) {
+		if got := Between(10.0, 5.0, 15.0); got != 10.0 {
+			t.Errorf("Between() = %v, want 10.0", got)
+		}
+		if got := Between(3.0, 5.0, 15.0); got != 5.0 {
+			t.Errorf("Between() = %v, want 5.0", got)
+		}
+		if got := Between(20.0, 5.0, 15.0); got != 15.0 {
+			t.Errorf("Between() = %v, want 15.0", got)
+		}
+		if got := Between(5.0, 5.0, 15.0); got != 5.0 {
+			t.Errorf("Between() = %v, want 5.0", got)
+		}
+		if got := Between(15.0, 5.0, 15.0); got != 15.0 {
+			t.Errorf("Between() = %v, want 15.0", got)
+		}
+	})
+}

core/metric/counter.go

@@ -3,7 +3,6 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
-	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -47,20 +46,16 @@ func NewCounterVec(cfg *CounterVecOpts) CounterVec {
 	return cv
 }
 
-func (cv *promCounterVec) Inc(labels ...string) {
-	if !prometheus.Enabled() {
-		return
-	}
-
-	cv.counter.WithLabelValues(labels...).Inc()
+func (cv *promCounterVec) Add(v float64, labels ...string) {
+	update(func() {
+		cv.counter.WithLabelValues(labels...).Add(v)
+	})
 }
 
-func (cv *promCounterVec) Add(v float64, labels ...string) {
-	if !prometheus.Enabled() {
-		return
-	}
-
-	cv.counter.WithLabelValues(labels...).Add(v)
+func (cv *promCounterVec) Inc(labels ...string) {
+	update(func() {
+		cv.counter.WithLabelValues(labels...).Inc()
+	})
 }
 
 func (cv *promCounterVec) close() bool {

core/metric/gauge.go

@@ -3,7 +3,6 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
-	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
@@ -16,8 +15,12 @@ type (
 		Set(v float64, labels ...string)
 		// Inc increments labels.
 		Inc(labels ...string)
+		// Dec decrements labels.
+		Dec(labels ...string)
 		// Add adds v to labels.
 		Add(v float64, labels ...string)
+		// Sub subtracts v to labels.
+		Sub(v float64, labels ...string)
 		close() bool
 	}
 
@@ -32,13 +35,12 @@ func NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {
 		return nil
 	}
 
-	vec := prom.NewGaugeVec(
-		prom.GaugeOpts{
-			Namespace: cfg.Namespace,
-			Subsystem: cfg.Subsystem,
-			Name:      cfg.Name,
-			Help:      cfg.Help,
-		}, cfg.Labels)
+	vec := prom.NewGaugeVec(prom.GaugeOpts{
+		Namespace: cfg.Namespace,
+		Subsystem: cfg.Subsystem,
+		Name:      cfg.Name,
+		Help:      cfg.Help,
+	}, cfg.Labels)
 	prom.MustRegister(vec)
 	gv := &promGaugeVec{
 		gauge: vec,
@@ -50,28 +52,34 @@ func NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {
 	return gv
 }
 
-func (gv *promGaugeVec) Inc(labels ...string) {
-	if !prometheus.Enabled() {
-		return
-	}
-
-	gv.gauge.WithLabelValues(labels...).Inc()
+func (gv *promGaugeVec) Add(v float64, labels ...string) {
+	update(func() {
+		gv.gauge.WithLabelValues(labels...).Add(v)
+	})
 }
 
-func (gv *promGaugeVec) Add(v float64, labels ...string) {
-	if !prometheus.Enabled() {
-		return
-	}
+func (gv *promGaugeVec) Dec(labels ...string) {
+	update(func() {
+		gv.gauge.WithLabelValues(labels...).Dec()
+	})
+}
 
-	gv.gauge.WithLabelValues(labels...).Add(v)
+func (gv *promGaugeVec) Inc(labels ...string) {
+	update(func() {
+		gv.gauge.WithLabelValues(labels...).Inc()
+	})
 }
 
 func (gv *promGaugeVec) Set(v float64, labels ...string) {
-	if !prometheus.Enabled() {
-		return
-	}
+	update(func() {
+		gv.gauge.WithLabelValues(labels...).Set(v)
+	})
+}
 
-	gv.gauge.WithLabelValues(labels...).Set(v)
+func (gv *promGaugeVec) Sub(v float64, labels ...string) {
+	update(func() {
+		gv.gauge.WithLabelValues(labels...).Sub(v)
+	})
 }
 
 func (gv *promGaugeVec) close() bool {

core/metric/gauge_test.go

@@ -40,6 +40,23 @@ func TestGaugeInc(t *testing.T) {
 	assert.Equal(t, float64(2), r)
 }
 
+func TestGaugeDec(t *testing.T) {
+	startAgent()
+	gaugeVec := NewGaugeVec(&GaugeVecOpts{
+		Namespace: "rpc_client",
+		Subsystem: "requests",
+		Name:      "duration_ms",
+		Help:      "rpc server requests duration(ms).",
+		Labels:    []string{"path"},
+	})
+	defer gaugeVec.close()
+	gv, _ := gaugeVec.(*promGaugeVec)
+	gv.Dec("/users")
+	gv.Dec("/users")
+	r := testutil.ToFloat64(gv.gauge)
+	assert.Equal(t, float64(-2), r)
+}
+
 func TestGaugeAdd(t *testing.T) {
 	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{
@@ -57,6 +74,23 @@ func TestGaugeAdd(t *testing.T) {
 	assert.Equal(t, float64(20), r)
 }
 
+func TestGaugeSub(t *testing.T) {
+	startAgent()
+	gaugeVec := NewGaugeVec(&GaugeVecOpts{
+		Namespace: "rpc_client",
+		Subsystem: "request",
+		Name:      "duration_ms",
+		Help:      "rpc server requests duration(ms).",
+		Labels:    []string{"path"},
+	})
+	defer gaugeVec.close()
+	gv, _ := gaugeVec.(*promGaugeVec)
+	gv.Sub(-100, "/classroom")
+	gv.Sub(30, "/classroom")
+	r := testutil.ToFloat64(gv.gauge)
+	assert.Equal(t, float64(70), r)
+}
+
 func TestGaugeSet(t *testing.T) {
 	startAgent()
 	gaugeVec := NewGaugeVec(&GaugeVecOpts{

core/metric/histogram.go

@@ -3,24 +3,26 @@ package metric
 import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/zeromicro/go-zero/core/proc"
-	"github.com/zeromicro/go-zero/core/prometheus"
 )
 
 type (
 	// A HistogramVecOpts is a histogram vector options.
 	HistogramVecOpts struct {
-		Namespace string
-		Subsystem string
-		Name      string
-		Help      string
-		Labels    []string
-		Buckets   []float64
+		Namespace   string
+		Subsystem   string
+		Name        string
+		Help        string
+		Labels      []string
+		Buckets     []float64
+		ConstLabels map[string]string
 	}
 
 	// A HistogramVec interface represents a histogram vector.
 	HistogramVec interface {
 		// Observe adds observation v to labels.
 		Observe(v int64, labels ...string)
+		// ObserveFloat allow to observe float64 values.
+		ObserveFloat(v float64, labels ...string)
 		close() bool
 	}
 
@@ -36,11 +38,12 @@ func NewHistogramVec(cfg *HistogramVecOpts) HistogramVec {
 	}
 
 	vec := prom.NewHistogramVec(prom.HistogramOpts{
-		Namespace: cfg.Namespace,
-		Subsystem: cfg.Subsystem,
-		Name:      cfg.Name,
-		Help:      cfg.Help,
-		Buckets:   cfg.Buckets,
+		Namespace:   cfg.Namespace,
+		Subsystem:   cfg.Subsystem,
+		Name:        cfg.Name,
+		Help:        cfg.Help,
+		Buckets:     cfg.Buckets,
+		ConstLabels: cfg.ConstLabels,
 	}, cfg.Labels)
 	prom.MustRegister(vec)
 	hv := &promHistogramVec{
@@ -54,11 +57,15 @@ func NewHistogramVec(cfg *HistogramVecOpts) HistogramVec {
 }
 
 func (hv *promHistogramVec) Observe(v int64, labels ...string) {
-	if !prometheus.Enabled() {
-		return
-	}
+	update(func() {
+		hv.histogram.WithLabelValues(labels...).Observe(float64(v))
+	})
+}
 
-	hv.histogram.WithLabelValues(labels...).Observe(float64(v))
+func (hv *promHistogramVec) ObserveFloat(v float64, labels ...string) {
+	update(func() {
+		hv.histogram.WithLabelValues(labels...).Observe(v)
+	})
 }
 
 func (hv *promHistogramVec) close() bool {

core/metric/histogram_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
-	"github.com/zeromicro/go-zero/core/proc"
 )
 
 func TestNewHistogramVec(t *testing.T) {
@@ -15,7 +14,7 @@ func TestNewHistogramVec(t *testing.T) {
 		Help:    "rpc server requests duration(ms).",
 		Buckets: []float64{1, 2, 3},
 	})
-	defer histogramVec.close()
+	defer histogramVec.(*promHistogramVec).close()
 	histogramVecNil := NewHistogramVec(nil)
 	assert.NotNil(t, histogramVec)
 	assert.Nil(t, histogramVecNil)
@@ -29,9 +28,10 @@ func TestHistogramObserve(t *testing.T) {
 		Buckets: []float64{1, 2, 3},
 		Labels:  []string{"method"},
 	})
-	defer histogramVec.close()
+	defer histogramVec.(*promHistogramVec).close()
 	hv, _ := histogramVec.(*promHistogramVec)
 	hv.Observe(2, "/Users")
+	hv.ObserveFloat(1.1, "/Users")
 
 	metadata := `
 		# HELP counts rpc server requests duration(ms).
@@ -39,15 +39,13 @@ func TestHistogramObserve(t *testing.T) {
 `
 	val := `
 		counts_bucket{method="/Users",le="1"} 0
-		counts_bucket{method="/Users",le="2"} 1
-		counts_bucket{method="/Users",le="3"} 1
-		counts_bucket{method="/Users",le="+Inf"} 1
-		counts_sum{method="/Users"} 2
-        counts_count{method="/Users"} 1
+		counts_bucket{method="/Users",le="2"} 2
+		counts_bucket{method="/Users",le="3"} 2
+		counts_bucket{method="/Users",le="+Inf"} 2
+		counts_sum{method="/Users"} 3.1
+        counts_count{method="/Users"} 2
 `
 
 	err := testutil.CollectAndCompare(hv.histogram, strings.NewReader(metadata+val))
 	assert.Nil(t, err)
-
-	proc.Shutdown()
 }

core/metric/metric.go

@@ -1,5 +1,7 @@
 package metric
 
+import "github.com/zeromicro/go-zero/core/prometheus"
+
 // A VectorOpts is a general configuration.
 type VectorOpts struct {
 	Namespace string
@@ -8,3 +10,11 @@ type VectorOpts struct {
 	Help      string
 	Labels    []string
 }
+
+func update(fn func()) {
+	if !prometheus.Enabled() {
+		return
+	}
+
+	fn()
+}

core/metric/summary.go

@@ -0,0 +1,62 @@
+package metric
+
+import (
+	prom "github.com/prometheus/client_golang/prometheus"
+	"github.com/zeromicro/go-zero/core/proc"
+)
+
+type (
+	// A SummaryVecOpts is a summary vector options
+	SummaryVecOpts struct {
+		VecOpt     VectorOpts
+		Objectives map[float64]float64
+	}
+
+	// A SummaryVec interface represents a summary vector.
+	SummaryVec interface {
+		// Observe adds observation v to labels.
+		Observe(v float64, labels ...string)
+		close() bool
+	}
+
+	promSummaryVec struct {
+		summary *prom.SummaryVec
+	}
+)
+
+// NewSummaryVec return a SummaryVec
+func NewSummaryVec(cfg *SummaryVecOpts) SummaryVec {
+	if cfg == nil {
+		return nil
+	}
+
+	vec := prom.NewSummaryVec(
+		prom.SummaryOpts{
+			Namespace:  cfg.VecOpt.Namespace,
+			Subsystem:  cfg.VecOpt.Subsystem,
+			Name:       cfg.VecOpt.Name,
+			Help:       cfg.VecOpt.Help,
+			Objectives: cfg.Objectives,
+		},
+		cfg.VecOpt.Labels,
+	)
+	prom.MustRegister(vec)
+	sv := &promSummaryVec{
+		summary: vec,
+	}
+	proc.AddShutdownListener(func() {
+		sv.close()
+	})
+
+	return sv
+}
+
+func (sv *promSummaryVec) Observe(v float64, labels ...string) {
+	update(func() {
+		sv.summary.WithLabelValues(labels...).Observe(v)
+	})
+}
+
+func (sv *promSummaryVec) close() bool {
+	return prom.Unregister(sv.summary)
+}

core/metric/summary_test.go

@@ -0,0 +1,68 @@
+package metric
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/core/proc"
+)
+
+func TestNewSummaryVec(t *testing.T) {
+	summaryVec := NewSummaryVec(&SummaryVecOpts{
+		VecOpt: VectorOpts{
+			Namespace: "http_server",
+			Subsystem: "requests",
+			Name:      "duration_quantiles",
+			Help:      "rpc client requests duration(ms) φ quantiles ",
+			Labels:    []string{"method"},
+		},
+		Objectives: map[float64]float64{
+			0.5: 0.01,
+			0.9: 0.01,
+		},
+	})
+	defer summaryVec.close()
+	summaryVecNil := NewSummaryVec(nil)
+	assert.NotNil(t, summaryVec)
+	assert.Nil(t, summaryVecNil)
+}
+
+func TestSummaryObserve(t *testing.T) {
+	startAgent()
+	summaryVec := NewSummaryVec(&SummaryVecOpts{
+		VecOpt: VectorOpts{
+			Namespace: "http_server",
+			Subsystem: "requests",
+			Name:      "duration_quantiles",
+			Help:      "rpc client requests duration(ms) φ quantiles ",
+			Labels:    []string{"method"},
+		},
+		Objectives: map[float64]float64{
+			0.3: 0.01,
+			0.6: 0.01,
+			1:   0.01,
+		},
+	})
+	defer summaryVec.close()
+	sv := summaryVec.(*promSummaryVec)
+	sv.Observe(100, "GET")
+	sv.Observe(200, "GET")
+	sv.Observe(300, "GET")
+	metadata := `
+		# HELP http_server_requests_duration_quantiles rpc client requests duration(ms) φ quantiles 
+		# TYPE http_server_requests_duration_quantiles summary
+`
+	val := `
+		http_server_requests_duration_quantiles{method="GET",quantile="0.3"} 100
+		http_server_requests_duration_quantiles{method="GET",quantile="0.6"} 200
+		http_server_requests_duration_quantiles{method="GET",quantile="1"} 300
+		http_server_requests_duration_quantiles_sum{method="GET"} 600
+		http_server_requests_duration_quantiles_count{method="GET"} 3
+`
+
+	err := testutil.CollectAndCompare(sv.summary, strings.NewReader(metadata+val))
+	assert.Nil(t, err)
+	proc.Shutdown()
+}

core/mr/mapreduce_fuzz_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func FuzzMapReduce(f *testing.F) {
-	rand.Seed(time.Now().UnixNano())
+	rand.NewSource(time.Now().UnixNano())
 
 	f.Add(int64(10), runtime.NumCPU())
 	f.Fuzz(func(t *testing.T, n int64, workers int) {

core/mr/mapreduce_fuzzcase_test.go

@@ -20,7 +20,7 @@ import (
 // If Fuzz stuck, we don't know why, because it only returns hung or unexpected,
 // so we need to simulate the fuzz test in test mode.
 func TestMapReduceRandom(t *testing.T) {
-	rand.Seed(time.Now().UnixNano())
+	rand.NewSource(time.Now().UnixNano())
 
 	const (
 		times  = 10000

core/mr/mapreduce_test.go

@@ -3,7 +3,7 @@ package mr
 import (
 	"context"
 	"errors"
-	"io/ioutil"
+	"io"
 	"log"
 	"runtime"
 	"sync/atomic"
@@ -17,7 +17,7 @@ import (
 var errDummy = errors.New("dummy")
 
 func init() {
-	log.SetOutput(ioutil.Discard)
+	log.SetOutput(io.Discard)
 }
 
 func TestFinish(t *testing.T) {
@@ -574,6 +574,7 @@ func TestMapReduceWithContext(t *testing.T) {
 			cancel()
 		}
 		writer.Write(i)
+		time.Sleep(time.Millisecond)
 	}, func(pipe <-chan int, cancel func(error)) {
 		for item := range pipe {
 			i := item

core/proc/env_test.go

@@ -1,7 +1,6 @@
 package proc
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -21,13 +20,11 @@ func TestEnvInt(t *testing.T) {
 	val, ok := EnvInt("any")
 	assert.Equal(t, 0, val)
 	assert.False(t, ok)
-	err := os.Setenv("anyInt", "10")
-	assert.Nil(t, err)
+	t.Setenv("anyInt", "10")
 	val, ok = EnvInt("anyInt")
 	assert.Equal(t, 10, val)
 	assert.True(t, ok)
-	err = os.Setenv("anyString", "a")
-	assert.Nil(t, err)
+	t.Setenv("anyString", "a")
 	val, ok = EnvInt("anyString")
 	assert.Equal(t, 0, val)
 	assert.False(t, ok)

core/proc/goroutines+polyfill.go

@@ -1,6 +0,0 @@
-//go:build windows
-
-package proc
-
-func dumpGoroutines() {
-}

core/proc/goroutines.go

@@ -18,7 +18,11 @@ const (
 	debugLevel       = 2
 )
 
-func dumpGoroutines() {
+type creator interface {
+	Create(name string) (file *os.File, err error)
+}
+
+func dumpGoroutines(ctor creator) {
 	command := path.Base(os.Args[0])
 	pid := syscall.Getpid()
 	dumpFile := path.Join(os.TempDir(), fmt.Sprintf("%s-%d-goroutines-%s.dump",
@@ -26,10 +30,16 @@ func dumpGoroutines() {
 
 	logx.Infof("Got dump goroutine signal, printing goroutine profile to %s", dumpFile)
 
-	if f, err := os.Create(dumpFile); err != nil {
+	if f, err := ctor.Create(dumpFile); err != nil {
 		logx.Errorf("Failed to dump goroutine profile, error: %v", err)
 	} else {
 		defer f.Close()
 		pprof.Lookup(goroutineProfile).WriteTo(f, debugLevel)
 	}
 }
+
+type fileCreator struct{}
+
+func (fc fileCreator) Create(name string) (file *os.File, err error) {
+	return os.Create(name)
+}

core/proc/goroutines_test.go

@@ -1,23 +1,41 @@
+//go:build linux || darwin
+
 package proc
 
 import (
+	"errors"
+	"os"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/zeromicro/go-zero/core/logx"
+	"github.com/zeromicro/go-zero/core/logx/logtest"
 )
 
 func TestDumpGoroutines(t *testing.T) {
-	var buf strings.Builder
-	w := logx.NewWriter(&buf)
-	o := logx.Reset()
-	logx.SetWriter(w)
-	defer func() {
-		logx.Reset()
-		logx.SetWriter(o)
-	}()
+	t.Run("real file", func(t *testing.T) {
+		buf := logtest.NewCollector(t)
+		dumpGoroutines(fileCreator{})
+		assert.True(t, strings.Contains(buf.String(), ".dump"))
+	})
 
-	dumpGoroutines()
-	assert.True(t, strings.Contains(buf.String(), ".dump"))
+	t.Run("fake file", func(t *testing.T) {
+		const msg = "any message"
+		buf := logtest.NewCollector(t)
+		err := errors.New(msg)
+		dumpGoroutines(fakeCreator{
+			file: &os.File{},
+			err:  err,
+		})
+		assert.True(t, strings.Contains(buf.String(), msg))
+	})
+}
+
+type fakeCreator struct {
+	file *os.File
+	err  error
+}
+
+func (fc fakeCreator) Create(_ string) (file *os.File, err error) {
+	return fc.file, fc.err
 }