Compare commits

..

1 Commits

Author SHA1 Message Date
spectatorMrZ
304fb182bb Fix pg model generation without tag (#1407)
1. fix pg model struct haven't tag
2. add pg command test from datasource
2022-01-08 21:48:09 +08:00
1027 changed files with 19389 additions and 65318 deletions

View File

@@ -1,13 +1,3 @@
coverage:
status:
patch: true
project: false # disabled because project coverage is not stable
comment:
layout: "flags, files"
behavior: once
require_changes: true
comment: false
ignore:
- "tools"
- "**/mock"
- "**/*_mock.go"
- "**/*test"
- "tools"

3
.github/FUNDING.yml vendored
View File

@@ -9,5 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: # https://gitee.com/kevwan/static/raw/master/images/sponsor.jpg
ethereum: # 0x5052b7f6B937B02563996D23feb69b38D06Ca150 | kevwan
custom: https://gitee.com/kevwan/static/raw/master/images/sponsor.jpg

View File

@@ -1,15 +0,0 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "daily"
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/tools/goctl" # Location of package manifests
schedule:
interval: "daily"

View File

@@ -35,11 +35,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
uses: github/codeql-action/autobuild@v1
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@v1

View File

@@ -7,59 +7,32 @@ on:
branches: [ master ]
jobs:
test-linux:
name: Linux
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v4
with:
go-version: 1.19
check-latest: true
cache: true
id: go
- name: Set up Go 1.x
uses: actions/setup-go@v2
with:
go-version: ^1.14
id: go
- name: Get dependencies
run: |
go get -v -t -d ./...
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Lint
run: |
go vet -stdmethods=false $(go list ./...)
- name: Get dependencies
run: |
go get -v -t -d ./...
go mod tidy
if ! test -z "$(git status --porcelain)"; then
echo "Please run 'go mod tidy'"
exit 1
fi
- name: Lint
run: |
go vet -stdmethods=false $(go list ./...)
go install mvdan.cc/gofumpt@latest
test -z "$(gofumpt -s -l -extra .)" || echo "Please run 'gofumpt -l -w -extra .'"
- name: Test
run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
- name: Test
run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
- name: Codecov
uses: codecov/codecov-action@v3
test-win:
name: Windows
runs-on: windows-latest
steps:
- name: Checkout codebase
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v4
with:
# use 1.19 to guarantee Go 1.19 compatibility
go-version: 1.19
check-latest: true
cache: true
- name: Test
run: |
go mod verify
go mod download
go test ./...
cd tools/goctl && go build -v goctl.go
- name: Codecov
uses: codecov/codecov-action@v2

View File

@@ -1,18 +0,0 @@
name: 'issue-translator'
on:
issue_comment:
types: [created]
issues:
types: [opened]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: usthe/issues-translate-action@v2.7
with:
IS_MODIFY_TITLE: true
# not require, default false, . Decide whether to modify the issue title
# if true, the robot account @Issues-translate-bot must have modification permissions, invite @Issues-translate-bot to your project or use your custom bot.
CUSTOM_BOT_NOTE: Bot detected the issue body's language is not English, translate it automatically. 👯👭🏻🧑‍🤝‍🧑👫🧑🏿‍🤝‍🧑🏻👩🏾‍🤝‍👨🏿👬🏿
# not require. Customize the translation robot prefix message.

View File

@@ -7,10 +7,10 @@ jobs:
close-issues:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v8
- uses: actions/stale@v3
with:
days-before-issue-stale: 365
days-before-issue-close: 90
days-before-issue-stale: 30
days-before-issue-close: 14
stale-issue-label: "stale"
stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."

View File

@@ -1,28 +0,0 @@
on:
push:
tags:
- "tools/goctl/*"
jobs:
releases-matrix:
name: Release goctl binary
runs-on: ubuntu-latest
strategy:
matrix:
# build and publish in parallel: linux/386, linux/amd64, linux/arm64,
# windows/386, windows/amd64, windows/arm64, darwin/amd64, darwin/arm64
goos: [ linux, windows, darwin ]
goarch: [ "386", amd64, arm64 ]
exclude:
- goarch: "386"
goos: darwin
steps:
- uses: actions/checkout@v4
- uses: zeromicro/go-zero-release-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
goversion: "https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz"
project_path: "tools/goctl"
binary_name: "goctl"
extra_files: tools/goctl/readme.md tools/goctl/readme-cn.md

View File

@@ -5,7 +5,7 @@ jobs:
name: runner / staticcheck
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v2
- uses: reviewdog/action-staticcheck@v1
with:
github_token: ${{ secrets.github_token }}

9
.gitignore vendored
View File

@@ -11,20 +11,15 @@
!api
# ignore
**/.idea
**/.vscode
.idea
**/.DS_Store
**/logs
**/adhoc
**/coverage.txt
# for test purpose
go.work
go.work.sum
adhoc
# gitlab ci
.cache
.golangci.yml
# vim auto backup file
*~

View File

@@ -1,76 +1,102 @@
# 🚀 Contributing to go-zero
# Contributing
Welcome to the go-zero community! We're thrilled to have you here. Contributing to our project is a fantastic way to be a part of the go-zero journey. Let's make this guide exciting and fun!
Welcome to go-zero!
## 📜 Before You Dive In
- [Before you get started](#before-you-get-started)
- [Code of Conduct](#code-of-conduct)
- [Community Expectations](#community-expectations)
- [Getting started](#getting-started)
- [Your First Contribution](#your-first-contribution)
- [Find something to work on](#find-something-to-work-on)
- [Find a good first topic](#find-a-good-first-topic)
- [Work on an Issue](#work-on-an-issue)
- [File an Issue](#file-an-issue)
- [Contributor Workflow](#contributor-workflow)
- [Creating Pull Requests](#creating-pull-requests)
- [Code Review](#code-review)
- [Testing](#testing)
### 🤝 Code of Conduct
# Before you get started
Let's start on the right foot. Please take a moment to read and embrace our [Code of Conduct](/code-of-conduct.md). We're all about creating a welcoming and respectful environment.
## Code of Conduct
### 🌟 Community Expectations
Please make sure to read and observe our [Code of Conduct](/code-of-conduct.md).
At go-zero, we're like a close-knit family, and we believe in creating a healthy, friendly, and productive atmosphere. It's all about sharing knowledge and building amazing things together.
## Community Expectations
## 🚀 Getting Started
go-zero is a community project driven by its community which strives to promote a healthy, friendly and productive environment.
go-zero is a web and rpc framework written in Go. It's born to ensure the stability of the busy sites with resilient design. Builtin goctl greatly improves the development productivity.
Get your adventure rolling! Here's how to begin:
# Getting started
1. 🍴 **Fork the Repository**: Head over to the GitHub repository and fork it to your own space.
- Fork the repository on GitHub.
- Make your changes on your fork repository.
- Submit a PR.
2. 🛠️ **Make Your Magic**: Work your magic in your forked repository. Create new features, squash bugs, or improve documentation - it's your world to conquer!
3. 🚀 **Submit a PR (Pull Request)**: When you're ready to unveil your creation, submit a Pull Request. We can't wait to see your awesome work!
# Your First Contribution
## 🌟 Your First Contribution
We will help you to contribute in different areas like filing issues, developing features, fixing critical bugs and
getting your work reviewed and merged.
We're here to guide you on your quest to become a go-zero contributor. Whether you want to file issues, develop features, or tame some critical bugs, we've got you covered.
If you have questions about the development process,
feel free to [file an issue](https://github.com/tal-tech/go-zero/issues/new/choose).
If you have questions or need guidance at any stage, don't hesitate to [open an issue](https://github.com/zeromicro/go-zero/issues/new/choose).
## Find something to work on
## 🔍 Find Something to Work On
We are always in need of help, be it fixing documentation, reporting bugs or writing some code.
Look at places where you feel best coding practices aren't followed, code refactoring is needed or tests are missing.
Here is how you get started.
Ready to dive into the action? There are several ways to contribute:
### Find a good first topic
### 💼 Find a Good First Topic
[go-zero](https://github.com/tal-tech/go-zero) has beginner-friendly issues that provide a good first issue.
For example, [go-zero](https://github.com/tal-tech/go-zero) has
[help wanted](https://github.com/tal-tech/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) and
[good first issue](https://github.com/tal-tech/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
labels for issues that should not need deep knowledge of the system.
We can help new contributors who wish to work on such issues.
Discover easy-entry issues labeled as [help wanted](https://github.com/zeromicro/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) or [good first issue](https://github.com/zeromicro/go-zero/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). These issues are perfect for newcomers and don't require deep knowledge of the system. We're here to assist you with these tasks.
Another good way to contribute is to find a documentation improvement, such as a missing/broken link.
Please see [Contributing](#contributing) below for the workflow.
### 🪄 Work on an Issue
#### Work on an issue
Once you've picked an issue that excites you, let us know by commenting on it. Our maintainers will assign it to you, and you can embark on your mission!
When you are willing to take on an issue, just reply on the issue. The maintainer will assign it to you.
### 📢 File an Issue
### File an Issue
Reporting an issue is just as valuable as code contributions. If you discover a problem, don't hesitate to [open an issue](https://github.com/zeromicro/go-zero/issues/new/choose). Be sure to follow our guidelines when submitting an issue.
While we encourage everyone to contribute code, it is also appreciated when someone reports an issue.
## 🎯 Contributor Workflow
Please follow the prompted submission guidelines while opening an issue.
Here's a rough guide to your contributor journey:
# Contributor Workflow
1. 🌱 Create a New Branch: Start by creating a topic branch, usually based on the 'master' branch. This is where your contribution will grow.
Please do not ever hesitate to ask a question or send a pull request.
2. 💡 Make Commits: Commit your work in logical units. Each commit should tell a story.
This is a rough outline of what a contributor's workflow looks like:
3. 🚀 Push Changes: Push the changes in your topic branch to your personal fork of the repository.
- Create a topic branch from where to base the contribution. This is usually master.
- Make commits of logical units.
- Push changes in a topic branch to a personal fork of the repository.
- Submit a pull request to [go-zero](https://github.com/tal-tech/go-zero).
4. 📦 Submit a Pull Request: When your creation is complete, submit a Pull Request to the [go-zero repository](https://github.com/zeromicro/go-zero).
## Creating Pull Requests
## 🌠 Creating Pull Requests
Pull requests are often called simply "PR".
go-zero generally follows the standard [github pull request](https://help.github.com/articles/about-pull-requests/) process.
To submit a proposed change, please develop the code/fix and add new test cases.
After that, run these local verifications before submitting pull request to predict the pass or
fail of continuous integration.
Pull Requests (PRs) are your way of making a grand entrance with your contribution. Here's how to do it:
* Format the code with `gofmt`
* Run the test with data race enabled `go test -race ./...`
- 💼 Format Your Code: Ensure your code is beautifully formatted with `gofmt`.
- 🏃 Run Tests: Verify that your changes pass all the tests, including data race tests. Run `go test -race ./...` for the ultimate validation.
## Code Review
## 👁️‍🗨️ Code Review
To make it easier for your PR to receive reviews, consider the reviewers will need you to:
Getting your PR reviewed is the final step before your contribution becomes part of go-zero's magical world. To make the process smooth, keep these things in mind:
* follow [good coding guidelines](https://github.com/golang/go/wiki/CodeReviewComments).
* write [good commit messages](https://chris.beams.io/posts/git-commit/).
* break large changes into a logical series of smaller patches which individually make easily understandable changes, and in aggregate solve a broader issue.
- 🧙‍♀️ Follow Good Coding Practices: Stick to [good coding guidelines](https://github.com/golang/go/wiki/CodeReviewComments).
- 📝 Write Awesome Commit Messages: Craft [impressive commit messages](https://chris.beams.io/posts/git-commit/) - they're like spells in the wizard's book!
- 🔍 Break It Down: For larger changes, consider breaking them into a series of smaller, logical patches. Each patch should make an understandable and meaningful improvement.
Congratulations on your contribution journey! We're thrilled to have you as part of our go-zero community. Let's make amazing things together! 🌟
Now, go out there and start your adventure! If you have any more magical ideas to enhance this guide, please share them. 🔥

View File

@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2022 zeromicro
Copyright (c) 2020 xiaoheiban_server_go
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

28
ROADMAP.md Normal file
View File

@@ -0,0 +1,28 @@
# go-zero Roadmap
This document defines a high level roadmap for go-zero development and upcoming releases.
Community and contributor involvement is vital for successfully implementing all desired items for each release.
We hope that the items listed below will inspire further engagement from the community to keep go-zero progressing and shipping exciting and valuable features.
## 2021 Q2
- [x] Support service discovery through K8S client api
- [x] Log full sql statements for easier sql problem solving
## 2021 Q3
- [x] Support `goctl model pg` to support PostgreSQL code generation
- [x] Adapt builtin tracing mechanism to opentracing solutions
## 2021 Q4
- [x] Support `username/password` authentication in ETCD
- [x] Support `SSL/TLS` in ETCD
- [x] Support `SSL/TLS` in `zRPC`
- [x] Support `TLS` in redis connections
- [x] Support `goctl bug` to report bugs conveniently
## 2022
- [ ] Support `goctl mock` command to start a mocking server with given `.api` file
- [ ] Add `httpx.Client` with governance, like circuit breaker etc.
- [ ] Support `goctl doctor` command to report potential issues for given service
- [ ] Support `context` in redis related methods for timeout and tracing
- [ ] Support `context` in sql related methods for timeout and tracing
- [ ] Support `context` in mongodb related methods for timeout and tracing

View File

@@ -1,127 +1,76 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
Examples of behavior that contributes to creating a positive environment
include:
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the overall
community
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior include:
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery, and sexual attention or advances of
any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email address,
without their explicit permission
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
## Our Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
This Code of Conduct applies within all project spaces, and it also applies when
an individual is representing the project or its community in public spaces.
Examples of representing a project or community include using an official
project e-mail address, posting via an official social media account, or acting
as an appointed representative at an online or offline event. Representation of
a project may be further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
[INSERT CONTACT METHOD].
All complaints will be reviewed and investigated promptly and fairly.
reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq

View File

@@ -1,37 +1,35 @@
package bloom
import (
"context"
"errors"
"strconv"
"github.com/zeromicro/go-zero/core/hash"
"github.com/zeromicro/go-zero/core/stores/redis"
"github.com/tal-tech/go-zero/core/hash"
"github.com/tal-tech/go-zero/core/stores/redis"
)
// for detailed error rate table, see http://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html
// maps as k in the error rate table
const maps = 14
var (
// ErrTooLargeOffset indicates the offset is too large in bitset.
ErrTooLargeOffset = errors.New("too large offset")
setScript = redis.NewScript(`
const (
// for detailed error rate table, see http://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html
// maps as k in the error rate table
maps = 14
setScript = `
for _, offset in ipairs(ARGV) do
redis.call("setbit", KEYS[1], offset, 1)
end
`)
testScript = redis.NewScript(`
`
testScript = `
for _, offset in ipairs(ARGV) do
if tonumber(redis.call("getbit", KEYS[1], offset)) == 0 then
return false
end
end
return true
`)
`
)
// ErrTooLargeOffset indicates the offset is too large in bitset.
var ErrTooLargeOffset = errors.New("too large offset")
type (
// A Filter is a bloom filter.
Filter struct {
@@ -40,8 +38,8 @@ type (
}
bitSetProvider interface {
check(ctx context.Context, offsets []uint) (bool, error)
set(ctx context.Context, offsets []uint) error
check([]uint) (bool, error)
set([]uint) error
}
)
@@ -60,29 +58,22 @@ func New(store *redis.Redis, key string, bits uint) *Filter {
// Add adds data into f.
func (f *Filter) Add(data []byte) error {
return f.AddCtx(context.Background(), data)
}
// AddCtx adds data into f with context.
func (f *Filter) AddCtx(ctx context.Context, data []byte) error {
locations := f.getLocations(data)
return f.bitSet.set(ctx, locations)
return f.bitSet.set(locations)
}
// Exists checks if data is in f.
func (f *Filter) Exists(data []byte) (bool, error) {
return f.ExistsCtx(context.Background(), data)
}
// ExistsCtx checks if data is in f with context.
func (f *Filter) ExistsCtx(ctx context.Context, data []byte) (bool, error) {
locations := f.getLocations(data)
isSet, err := f.bitSet.check(ctx, locations)
isSet, err := f.bitSet.check(locations)
if err != nil {
return false, err
}
if !isSet {
return false, nil
}
return isSet, nil
return true, nil
}
func (f *Filter) getLocations(data []byte) []uint {
@@ -123,13 +114,13 @@ func (r *redisBitSet) buildOffsetArgs(offsets []uint) ([]string, error) {
return args, nil
}
func (r *redisBitSet) check(ctx context.Context, offsets []uint) (bool, error) {
func (r *redisBitSet) check(offsets []uint) (bool, error) {
args, err := r.buildOffsetArgs(offsets)
if err != nil {
return false, err
}
resp, err := r.store.ScriptRunCtx(ctx, testScript, []string{r.key}, args)
resp, err := r.store.Eval(testScript, []string{r.key}, args)
if err == redis.Nil {
return false, nil
} else if err != nil {
@@ -144,24 +135,22 @@ func (r *redisBitSet) check(ctx context.Context, offsets []uint) (bool, error) {
return exists == 1, nil
}
// del only use for testing.
func (r *redisBitSet) del() error {
_, err := r.store.Del(r.key)
return err
}
// expire only use for testing.
func (r *redisBitSet) expire(seconds int) error {
return r.store.Expire(r.key, seconds)
}
func (r *redisBitSet) set(ctx context.Context, offsets []uint) error {
func (r *redisBitSet) set(offsets []uint) error {
args, err := r.buildOffsetArgs(offsets)
if err != nil {
return err
}
_, err = r.store.ScriptRunCtx(ctx, setScript, []string{r.key}, args)
_, err = r.store.Eval(setScript, []string{r.key}, args)
if err == redis.Nil {
return nil
}

View File

@@ -1,31 +1,30 @@
package bloom
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/stores/redis/redistest"
"github.com/tal-tech/go-zero/core/stores/redis/redistest"
)
func TestRedisBitSet_New_Set_Test(t *testing.T) {
store := redistest.CreateRedis(t)
ctx := context.Background()
store, clean, err := redistest.CreateRedis()
assert.Nil(t, err)
defer clean()
bitSet := newRedisBitSet(store, "test_key", 1024)
isSetBefore, err := bitSet.check(ctx, []uint{0})
isSetBefore, err := bitSet.check([]uint{0})
if err != nil {
t.Fatal(err)
}
if isSetBefore {
t.Fatal("Bit should not be set")
}
err = bitSet.set(ctx, []uint{512})
err = bitSet.set([]uint{512})
if err != nil {
t.Fatal(err)
}
isSetAfter, err := bitSet.check(ctx, []uint{512})
isSetAfter, err := bitSet.check([]uint{512})
if err != nil {
t.Fatal(err)
}
@@ -43,7 +42,9 @@ func TestRedisBitSet_New_Set_Test(t *testing.T) {
}
func TestRedisBitSet_Add(t *testing.T) {
store := redistest.CreateRedis(t)
store, clean, err := redistest.CreateRedis()
assert.Nil(t, err)
defer clean()
filter := New(store, "test_key", 64)
assert.Nil(t, filter.Add([]byte("hello")))
@@ -52,51 +53,3 @@ func TestRedisBitSet_Add(t *testing.T) {
assert.Nil(t, err)
assert.True(t, ok)
}
func TestFilter_Exists(t *testing.T) {
store, clean := redistest.CreateRedisWithClean(t)
rbs := New(store, "test", 64)
_, err := rbs.Exists([]byte{0, 1, 2})
assert.NoError(t, err)
clean()
rbs = New(store, "test", 64)
_, err = rbs.Exists([]byte{0, 1, 2})
assert.Error(t, err)
}
func TestRedisBitSet_check(t *testing.T) {
store, clean := redistest.CreateRedisWithClean(t)
ctx := context.Background()
rbs := newRedisBitSet(store, "test", 0)
assert.Error(t, rbs.set(ctx, []uint{0, 1, 2}))
_, err := rbs.check(ctx, []uint{0, 1, 2})
assert.Error(t, err)
rbs = newRedisBitSet(store, "test", 64)
_, err = rbs.check(ctx, []uint{0, 1, 2})
assert.NoError(t, err)
clean()
rbs = newRedisBitSet(store, "test", 64)
_, err = rbs.check(ctx, []uint{0, 1, 2})
assert.Error(t, err)
}
func TestRedisBitSet_set(t *testing.T) {
logx.Disable()
store, clean := redistest.CreateRedisWithClean(t)
ctx := context.Background()
rbs := newRedisBitSet(store, "test", 0)
assert.Error(t, rbs.set(ctx, []uint{0, 1, 2}))
rbs = newRedisBitSet(store, "test", 64)
assert.NoError(t, rbs.set(ctx, []uint{0, 1, 2}))
clean()
rbs = newRedisBitSet(store, "test", 64)
assert.Error(t, rbs.set(ctx, []uint{0, 1, 2}))
}

View File

@@ -5,12 +5,12 @@ import (
"fmt"
"strings"
"sync"
"time"
"github.com/zeromicro/go-zero/core/mathx"
"github.com/zeromicro/go-zero/core/proc"
"github.com/zeromicro/go-zero/core/stat"
"github.com/zeromicro/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/mathx"
"github.com/tal-tech/go-zero/core/proc"
"github.com/tal-tech/go-zero/core/stat"
"github.com/tal-tech/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/timex"
)
const (
@@ -46,7 +46,7 @@ type (
// DoWithAcceptable returns an error instantly if the Breaker rejects the request.
// If a panic occurs in the request, the Breaker handles it as an error
// and causes the same panic again.
// acceptable checks if it's a successful call, even if the error is not nil.
// acceptable checks if it's a successful call, even if the err is not nil.
DoWithAcceptable(req func() error, acceptable Acceptable) error
// DoWithFallback runs the given request if the Breaker accepts it.
@@ -59,7 +59,7 @@ type (
// DoWithFallbackAcceptable runs the fallback if the Breaker rejects the request.
// If a panic occurs in the request, the Breaker handles it as an error
// and causes the same panic again.
// acceptable checks if it's a successful call, even if the error is not nil.
// acceptable checks if it's a successful call, even if the err is not nil.
DoWithFallbackAcceptable(req func() error, fallback func(err error) error, acceptable Acceptable) error
}
@@ -171,7 +171,7 @@ func (lt loggedThrottle) allow() (Promise, error) {
func (lt loggedThrottle) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
return lt.logError(lt.internalThrottle.doReq(req, fallback, func(err error) bool {
accept := acceptable(err)
if !accept && err != nil {
if !accept {
lt.errWin.add(err.Error())
}
return accept
@@ -179,7 +179,7 @@ func (lt loggedThrottle) doReq(req func() error, fallback func(err error) error,
}
func (lt loggedThrottle) logError(err error) error {
if errors.Is(err, ErrServiceUnavailable) {
if err == ErrServiceUnavailable {
// if circuit open, not possible to have empty error window
stat.Report(fmt.Sprintf(
"proc(%s/%d), callee: %s, breaker is open and requests dropped\nlast errors:\n%s",
@@ -198,7 +198,7 @@ type errorWindow struct {
func (ew *errorWindow) add(reason string) {
ew.lock.Lock()
ew.reasons[ew.index] = fmt.Sprintf("%s %s", time.Now().Format(timeFormat), reason)
ew.reasons[ew.index] = fmt.Sprintf("%s %s", timex.Time().Format(timeFormat), reason)
ew.index = (ew.index + 1) % numHistoryReasons
ew.count = mathx.MinInt(ew.count+1, numHistoryReasons)
ew.lock.Unlock()

View File

@@ -8,7 +8,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/stat"
"github.com/tal-tech/go-zero/core/stat"
)
func init() {

View File

@@ -59,7 +59,7 @@ func GetBreaker(name string) Breaker {
// NoBreakerFor disables the circuit breaker for the given name.
func NoBreakerFor(name string) {
lock.Lock()
breakers[name] = newNopBreaker()
breakers[name] = newNoOpBreaker()
lock.Unlock()
}

View File

@@ -6,7 +6,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/stat"
"github.com/tal-tech/go-zero/core/stat"
)
func init() {
@@ -30,7 +30,7 @@ func TestBreakersDoWithAcceptable(t *testing.T) {
assert.Equal(t, errDummy, GetBreaker("anyone").DoWithAcceptable(func() error {
return errDummy
}, func(err error) bool {
return err == nil || errors.Is(err, errDummy)
return err == nil || err == errDummy
}))
}
verify(t, func() bool {
@@ -45,12 +45,12 @@ func TestBreakersDoWithAcceptable(t *testing.T) {
}, func(err error) bool {
return err == nil
})
assert.True(t, errors.Is(err, errDummy) || errors.Is(err, ErrServiceUnavailable))
assert.True(t, err == errDummy || err == ErrServiceUnavailable)
}
verify(t, func() bool {
return errors.Is(Do("another", func() error {
return ErrServiceUnavailable == Do("another", func() error {
return nil
}), ErrServiceUnavailable)
})
})
}
@@ -75,12 +75,12 @@ func TestBreakersFallback(t *testing.T) {
}, func(err error) error {
return nil
})
assert.True(t, err == nil || errors.Is(err, errDummy))
assert.True(t, err == nil || err == errDummy)
}
verify(t, func() bool {
return errors.Is(Do("fallback", func() error {
return ErrServiceUnavailable == Do("fallback", func() error {
return nil
}), ErrServiceUnavailable)
})
})
}
@@ -94,12 +94,12 @@ func TestBreakersAcceptableFallback(t *testing.T) {
}, func(err error) bool {
return err == nil
})
assert.True(t, err == nil || errors.Is(err, errDummy))
assert.True(t, err == nil || err == errDummy)
}
verify(t, func() bool {
return errors.Is(Do("acceptablefallback", func() error {
return ErrServiceUnavailable == Do("acceptablefallback", func() error {
return nil
}), ErrServiceUnavailable)
})
})
}

View File

@@ -4,8 +4,8 @@ import (
"math"
"time"
"github.com/zeromicro/go-zero/core/collection"
"github.com/zeromicro/go-zero/core/mathx"
"github.com/tal-tech/go-zero/core/collection"
"github.com/tal-tech/go-zero/core/mathx"
)
const (

View File

@@ -7,9 +7,9 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/collection"
"github.com/zeromicro/go-zero/core/mathx"
"github.com/zeromicro/go-zero/core/stat"
"github.com/tal-tech/go-zero/core/collection"
"github.com/tal-tech/go-zero/core/mathx"
"github.com/tal-tech/go-zero/core/stat"
)
const (
@@ -95,7 +95,7 @@ func TestGoogleBreakerAcceptable(t *testing.T) {
assert.Equal(t, errAcceptable, b.doReq(func() error {
return errAcceptable
}, nil, func(err error) bool {
return errors.Is(err, errAcceptable)
return err == errAcceptable
}))
}
@@ -105,7 +105,7 @@ func TestGoogleBreakerNotAcceptable(t *testing.T) {
assert.Equal(t, errAcceptable, b.doReq(func() error {
return errAcceptable
}, nil, func(err error) bool {
return !errors.Is(err, errAcceptable)
return err != errAcceptable
}))
}

View File

@@ -1,35 +1,35 @@
package breaker
const nopBreakerName = "nopBreaker"
const noOpBreakerName = "nopBreaker"
type nopBreaker struct{}
type noOpBreaker struct{}
func newNopBreaker() Breaker {
return nopBreaker{}
func newNoOpBreaker() Breaker {
return noOpBreaker{}
}
func (b nopBreaker) Name() string {
return nopBreakerName
func (b noOpBreaker) Name() string {
return noOpBreakerName
}
func (b nopBreaker) Allow() (Promise, error) {
func (b noOpBreaker) Allow() (Promise, error) {
return nopPromise{}, nil
}
func (b nopBreaker) Do(req func() error) error {
func (b noOpBreaker) Do(req func() error) error {
return req()
}
func (b nopBreaker) DoWithAcceptable(req func() error, _ Acceptable) error {
func (b noOpBreaker) DoWithAcceptable(req func() error, acceptable Acceptable) error {
return req()
}
func (b nopBreaker) DoWithFallback(req func() error, _ func(err error) error) error {
func (b noOpBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
return req()
}
func (b nopBreaker) DoWithFallbackAcceptable(req func() error, _ func(err error) error,
_ Acceptable) error {
func (b noOpBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
acceptable Acceptable) error {
return req()
}
@@ -38,5 +38,5 @@ type nopPromise struct{}
func (p nopPromise) Accept() {
}
func (p nopPromise) Reject(_ string) {
func (p nopPromise) Reject(reason string) {
}

View File

@@ -8,8 +8,8 @@ import (
)
func TestNopBreaker(t *testing.T) {
b := newNopBreaker()
assert.Equal(t, nopBreakerName, b.Name())
b := newNoOpBreaker()
assert.Equal(t, noOpBreakerName, b.Name())
p, err := b.Allow()
assert.Nil(t, err)
p.Accept()

View File

@@ -8,8 +8,8 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/iox"
"github.com/zeromicro/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/iox"
"github.com/tal-tech/go-zero/core/lang"
)
func TestEnterToContinue(t *testing.T) {

View File

@@ -7,7 +7,7 @@ import (
"encoding/base64"
"errors"
"github.com/zeromicro/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/logx"
)
// ErrPaddingSize indicates bad padding size.
@@ -32,11 +32,9 @@ func NewECBEncrypter(b cipher.Block) cipher.BlockMode {
return (*ecbEncrypter)(newECB(b))
}
// BlockSize returns the mode's block size.
func (x *ecbEncrypter) BlockSize() int { return x.blockSize }
// CryptBlocks encrypts a number of blocks. The length of src must be a multiple of
// the block size. Dst and src must overlap entirely or not at all.
// why we don't return error is because cipher.BlockMode doesn't allow this
func (x *ecbEncrypter) CryptBlocks(dst, src []byte) {
if len(src)%x.blockSize != 0 {
logx.Error("crypto/cipher: input not full blocks")
@@ -61,13 +59,11 @@ func NewECBDecrypter(b cipher.Block) cipher.BlockMode {
return (*ecbDecrypter)(newECB(b))
}
// BlockSize returns the mode's block size.
func (x *ecbDecrypter) BlockSize() int {
return x.blockSize
}
// CryptBlocks decrypts a number of blocks. The length of src must be a multiple of
// the block size. Dst and src must overlap entirely or not at all.
// why we don't return error is because cipher.BlockMode doesn't allow this
func (x *ecbDecrypter) CryptBlocks(dst, src []byte) {
if len(src)%x.blockSize != 0 {
logx.Error("crypto/cipher: input not full blocks")

View File

@@ -1,7 +1,6 @@
package codec
import (
"crypto/aes"
"encoding/base64"
"testing"
@@ -11,8 +10,7 @@ import (
func TestAesEcb(t *testing.T) {
var (
key = []byte("q4t7w!z%C*F-JaNdRgUjXn2r5u8x/A?D")
val = []byte("helloworld")
valLong = []byte("helloworldlong..")
val = []byte("hello")
badKey1 = []byte("aaaaaaaaa")
// more than 32 chars
badKey2 = []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
@@ -33,39 +31,6 @@ func TestAesEcb(t *testing.T) {
src, err := EcbDecrypt(key, dst)
assert.Nil(t, err)
assert.Equal(t, val, src)
block, err := aes.NewCipher(key)
assert.NoError(t, err)
encrypter := NewECBEncrypter(block)
assert.Equal(t, 16, encrypter.BlockSize())
decrypter := NewECBDecrypter(block)
assert.Equal(t, 16, decrypter.BlockSize())
dst = make([]byte, 8)
encrypter.CryptBlocks(dst, val)
for _, b := range dst {
assert.Equal(t, byte(0), b)
}
dst = make([]byte, 8)
encrypter.CryptBlocks(dst, valLong)
for _, b := range dst {
assert.Equal(t, byte(0), b)
}
dst = make([]byte, 8)
decrypter.CryptBlocks(dst, val)
for _, b := range dst {
assert.Equal(t, byte(0), b)
}
dst = make([]byte, 8)
decrypter.CryptBlocks(dst, valLong)
for _, b := range dst {
assert.Equal(t, byte(0), b)
}
_, err = EcbEncryptBase64("cTR0N3dDKkYtSmFOZFJnVWpYbjJyNXU4eC9BP0QK", "aGVsbG93b3JsZGxvbmcuLgo=")
assert.Error(t, err)
}
func TestAesEcbBase64(t *testing.T) {

View File

@@ -80,17 +80,3 @@ func TestKeyBytes(t *testing.T) {
assert.Nil(t, err)
assert.True(t, len(key.Bytes()) > 0)
}
func TestDHOnErrors(t *testing.T) {
key, err := GenerateKey()
assert.Nil(t, err)
assert.NotEmpty(t, key.Bytes())
_, err = ComputeKey(key.PubKey, key.PriKey)
assert.NoError(t, err)
_, err = ComputeKey(nil, key.PriKey)
assert.Error(t, err)
_, err = ComputeKey(key.PubKey, nil)
assert.Error(t, err)
assert.NotNil(t, NewPublicKey([]byte("")))
}

View File

@@ -2,8 +2,6 @@ package codec
import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"testing"
@@ -23,45 +21,3 @@ func TestGzip(t *testing.T) {
assert.True(t, len(bs) < buf.Len())
assert.Equal(t, buf.Bytes(), actual)
}
func TestGunzip(t *testing.T) {
tests := []struct {
name string
input []byte
expected []byte
expectedErr error
}{
{
name: "valid input",
input: func() []byte {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
gz.Write([]byte("hello"))
gz.Close()
return buf.Bytes()
}(),
expected: []byte("hello"),
expectedErr: nil,
},
{
name: "invalid input",
input: []byte("invalid input"),
expected: nil,
expectedErr: gzip.ErrHeader,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := Gunzip(test.input)
if !bytes.Equal(result, test.expected) {
t.Errorf("unexpected result: %v", result)
}
if !errors.Is(err, test.expectedErr) {
t.Errorf("unexpected error: %v", err)
}
})
}
}

View File

@@ -7,7 +7,7 @@ import (
"encoding/base64"
"encoding/pem"
"errors"
"os"
"io/ioutil"
)
var (
@@ -48,7 +48,7 @@ type (
// NewRsaDecrypter returns a RsaDecrypter with the given file.
func NewRsaDecrypter(file string) (RsaDecrypter, error) {
content, err := os.ReadFile(file)
content, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}

View File

@@ -2,11 +2,10 @@ package codec
import (
"encoding/base64"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/fs"
"github.com/tal-tech/go-zero/core/fs"
)
const (
@@ -42,7 +41,6 @@ func TestCryption(t *testing.T) {
file, err := fs.TempFilenameWithText(priKey)
assert.Nil(t, err)
defer os.Remove(file)
dec, err := NewRsaDecrypter(file)
assert.Nil(t, err)
actual, err := dec.Decrypt(ret)

View File

@@ -6,9 +6,9 @@ import (
"sync/atomic"
"time"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/mathx"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/mathx"
"github.com/tal-tech/go-zero/core/syncx"
)
const (
@@ -26,11 +26,11 @@ type (
// CacheOption defines the method to customize a Cache.
CacheOption func(cache *Cache)
// A Cache object is an in-memory cache.
// A Cache object is a in-memory cache.
Cache struct {
name string
lock sync.Mutex
data map[string]any
data map[string]interface{}
expire time.Duration
timingWheel *TimingWheel
lruCache lru
@@ -43,7 +43,7 @@ type (
// NewCache returns a Cache with given expire.
func NewCache(expire time.Duration, opts ...CacheOption) (*Cache, error) {
cache := &Cache{
data: make(map[string]any),
data: make(map[string]interface{}),
expire: expire,
lruCache: emptyLruCache,
barrier: syncx.NewSingleFlight(),
@@ -59,7 +59,7 @@ func NewCache(expire time.Duration, opts ...CacheOption) (*Cache, error) {
}
cache.stats = newCacheStat(cache.name, cache.size)
timingWheel, err := NewTimingWheel(time.Second, slots, func(k, v any) {
timingWheel, err := NewTimingWheel(time.Second, slots, func(k, v interface{}) {
key, ok := k.(string)
if !ok {
return
@@ -85,7 +85,7 @@ func (c *Cache) Del(key string) {
}
// Get returns the item with the given key from c.
func (c *Cache) Get(key string) (any, bool) {
func (c *Cache) Get(key string) (interface{}, bool) {
value, ok := c.doGet(key)
if ok {
c.stats.IncrementHit()
@@ -97,19 +97,14 @@ func (c *Cache) Get(key string) (any, bool) {
}
// Set sets value into c with key.
func (c *Cache) Set(key string, value any) {
c.SetWithExpire(key, value, c.expire)
}
// SetWithExpire sets value into c with key and expire with the given value.
func (c *Cache) SetWithExpire(key string, value any, expire time.Duration) {
func (c *Cache) Set(key string, value interface{}) {
c.lock.Lock()
_, ok := c.data[key]
c.data[key] = value
c.lruCache.add(key)
c.lock.Unlock()
expiry := c.unstableExpiry.AroundDuration(expire)
expiry := c.unstableExpiry.AroundDuration(c.expire)
if ok {
c.timingWheel.MoveTimer(key, expiry)
} else {
@@ -120,14 +115,14 @@ func (c *Cache) SetWithExpire(key string, value any, expire time.Duration) {
// Take returns the item with the given key.
// If the item is in c, return it directly.
// If not, use fetch method to get the item, set into c and return it.
func (c *Cache) Take(key string, fetch func() (any, error)) (any, error) {
func (c *Cache) Take(key string, fetch func() (interface{}, error)) (interface{}, error) {
if val, ok := c.doGet(key); ok {
c.stats.IncrementHit()
return val, nil
}
var fresh bool
val, err := c.barrier.Do(key, func() (any, error) {
val, err := c.barrier.Do(key, func() (interface{}, error) {
// because O(1) on map search in memory, and fetch is an IO query
// so we do double check, cache might be taken by another call
if val, ok := c.doGet(key); ok {
@@ -157,7 +152,7 @@ func (c *Cache) Take(key string, fetch func() (any, error)) (any, error) {
return val, nil
}
func (c *Cache) doGet(key string) (any, bool) {
func (c *Cache) doGet(key string) (interface{}, bool) {
c.lock.Lock()
defer c.lock.Unlock()

View File

@@ -18,7 +18,7 @@ func TestCacheSet(t *testing.T) {
assert.Nil(t, err)
cache.Set("first", "first element")
cache.SetWithExpire("second", "second element", time.Second*3)
cache.Set("second", "second element")
value, ok := cache.Get("first")
assert.True(t, ok)
@@ -52,7 +52,7 @@ func TestCacheTake(t *testing.T) {
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
cache.Take("first", func() (any, error) {
cache.Take("first", func() (interface{}, error) {
atomic.AddInt32(&count, 1)
time.Sleep(time.Millisecond * 100)
return "first element", nil
@@ -76,7 +76,7 @@ func TestCacheTakeExists(t *testing.T) {
wg.Add(1)
go func() {
cache.Set("first", "first element")
cache.Take("first", func() (any, error) {
cache.Take("first", func() (interface{}, error) {
atomic.AddInt32(&count, 1)
time.Sleep(time.Millisecond * 100)
return "first element", nil
@@ -99,7 +99,7 @@ func TestCacheTakeError(t *testing.T) {
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
_, err := cache.Take("first", func() (any, error) {
_, err := cache.Take("first", func() (interface{}, error) {
atomic.AddInt32(&count, 1)
time.Sleep(time.Millisecond * 100)
return "", errDummy

View File

@@ -5,7 +5,7 @@ import "sync"
// A Queue is a FIFO queue.
type Queue struct {
lock sync.Mutex
elements []any
elements []interface{}
size int
head int
tail int
@@ -15,7 +15,7 @@ type Queue struct {
// NewQueue returns a Queue object.
func NewQueue(size int) *Queue {
return &Queue{
elements: make([]any, size),
elements: make([]interface{}, size),
size: size,
}
}
@@ -30,12 +30,12 @@ func (q *Queue) Empty() bool {
}
// Put puts element into q at the last position.
func (q *Queue) Put(element any) {
func (q *Queue) Put(element interface{}) {
q.lock.Lock()
defer q.lock.Unlock()
if q.head == q.tail && q.count > 0 {
nodes := make([]any, len(q.elements)+q.size)
nodes := make([]interface{}, len(q.elements)+q.size)
copy(nodes, q.elements[q.head:])
copy(nodes[len(q.elements)-q.head:], q.elements[:q.head])
q.head = 0
@@ -49,7 +49,7 @@ func (q *Queue) Put(element any) {
}
// Take takes the first element out of q if not empty.
func (q *Queue) Take() (any, bool) {
func (q *Queue) Take() (interface{}, bool) {
q.lock.Lock()
defer q.lock.Unlock()

View File

@@ -61,41 +61,3 @@ func TestPutMore(t *testing.T) {
assert.Equal(t, string(element), string(body.([]byte)))
}
}
func TestPutMoreWithHeaderNotZero(t *testing.T) {
elements := [][]byte{
[]byte("hello"),
[]byte("world"),
[]byte("again"),
}
queue := NewQueue(4)
for i := range elements {
queue.Put(elements[i])
}
// take 1
body, ok := queue.Take()
assert.True(t, ok)
element, ok := body.([]byte)
assert.True(t, ok)
assert.Equal(t, element, []byte("hello"))
// put more
queue.Put([]byte("b4"))
queue.Put([]byte("b5")) // will store in elements[0]
queue.Put([]byte("b6")) // cause expansion
results := [][]byte{
[]byte("world"),
[]byte("again"),
[]byte("b4"),
[]byte("b5"),
[]byte("b6"),
}
for _, element := range results {
body, ok := queue.Take()
assert.True(t, ok)
assert.Equal(t, string(element), string(body.([]byte)))
}
}

View File

@@ -4,9 +4,9 @@ import "sync"
// A Ring can be used as fixed size ring.
type Ring struct {
elements []any
elements []interface{}
index int
lock sync.RWMutex
lock sync.Mutex
}
// NewRing returns a Ring object with the given size n.
@@ -16,44 +16,36 @@ func NewRing(n int) *Ring {
}
return &Ring{
elements: make([]any, n),
elements: make([]interface{}, n),
}
}
// Add adds v into r.
func (r *Ring) Add(v any) {
func (r *Ring) Add(v interface{}) {
r.lock.Lock()
defer r.lock.Unlock()
rlen := len(r.elements)
r.elements[r.index%rlen] = v
r.elements[r.index%len(r.elements)] = v
r.index++
// prevent ring index overflow
if r.index >= rlen<<1 {
r.index -= rlen
}
}
// Take takes all items from r.
func (r *Ring) Take() []any {
r.lock.RLock()
defer r.lock.RUnlock()
func (r *Ring) Take() []interface{} {
r.lock.Lock()
defer r.lock.Unlock()
var size int
var start int
rlen := len(r.elements)
if r.index > rlen {
size = rlen
start = r.index % rlen
if r.index > len(r.elements) {
size = len(r.elements)
start = r.index % len(r.elements)
} else {
size = r.index
}
elements := make([]any, size)
elements := make([]interface{}, size)
for i := 0; i < size; i++ {
elements[i] = r.elements[(start+i)%rlen]
elements[i] = r.elements[(start+i)%len(r.elements)]
}
return elements

View File

@@ -19,7 +19,7 @@ func TestRingLess(t *testing.T) {
ring.Add(i)
}
elements := ring.Take()
assert.ElementsMatch(t, []any{0, 1, 2}, elements)
assert.ElementsMatch(t, []interface{}{0, 1, 2}, elements)
}
func TestRingMore(t *testing.T) {
@@ -28,7 +28,7 @@ func TestRingMore(t *testing.T) {
ring.Add(i)
}
elements := ring.Take()
assert.ElementsMatch(t, []any{6, 7, 8, 9, 10}, elements)
assert.ElementsMatch(t, []interface{}{6, 7, 8, 9, 10}, elements)
}
func TestRingAdd(t *testing.T) {

View File

@@ -4,7 +4,7 @@ import (
"sync"
"time"
"github.com/zeromicro/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/timex"
)
type (

View File

@@ -6,7 +6,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/stringx"
)
const duration = time.Millisecond * 50

View File

@@ -14,23 +14,21 @@ type SafeMap struct {
lock sync.RWMutex
deletionOld int
deletionNew int
dirtyOld map[any]any
dirtyNew map[any]any
dirtyOld map[interface{}]interface{}
dirtyNew map[interface{}]interface{}
}
// NewSafeMap returns a SafeMap.
func NewSafeMap() *SafeMap {
return &SafeMap{
dirtyOld: make(map[any]any),
dirtyNew: make(map[any]any),
dirtyOld: make(map[interface{}]interface{}),
dirtyNew: make(map[interface{}]interface{}),
}
}
// Del deletes the value with the given key from m.
func (m *SafeMap) Del(key any) {
func (m *SafeMap) Del(key interface{}) {
m.lock.Lock()
defer m.lock.Unlock()
if _, ok := m.dirtyOld[key]; ok {
delete(m.dirtyOld, key)
m.deletionOld++
@@ -44,20 +42,21 @@ func (m *SafeMap) Del(key any) {
}
m.dirtyOld = m.dirtyNew
m.deletionOld = m.deletionNew
m.dirtyNew = make(map[any]any)
m.dirtyNew = make(map[interface{}]interface{})
m.deletionNew = 0
}
if m.deletionNew >= maxDeletion && len(m.dirtyNew) < copyThreshold {
for k, v := range m.dirtyNew {
m.dirtyOld[k] = v
}
m.dirtyNew = make(map[any]any)
m.dirtyNew = make(map[interface{}]interface{})
m.deletionNew = 0
}
m.lock.Unlock()
}
// Get gets the value with the given key from m.
func (m *SafeMap) Get(key any) (any, bool) {
func (m *SafeMap) Get(key interface{}) (interface{}, bool) {
m.lock.RLock()
defer m.lock.RUnlock()
@@ -69,29 +68,9 @@ func (m *SafeMap) Get(key any) (any, bool) {
return val, ok
}
// Range calls f sequentially for each key and value present in the map.
// If f returns false, range stops the iteration.
func (m *SafeMap) Range(f func(key, val any) bool) {
m.lock.RLock()
defer m.lock.RUnlock()
for k, v := range m.dirtyOld {
if !f(k, v) {
return
}
}
for k, v := range m.dirtyNew {
if !f(k, v) {
return
}
}
}
// Set sets the value into m with the given key.
func (m *SafeMap) Set(key, value any) {
func (m *SafeMap) Set(key, value interface{}) {
m.lock.Lock()
defer m.lock.Unlock()
if m.deletionOld <= maxDeletion {
if _, ok := m.dirtyNew[key]; ok {
delete(m.dirtyNew, key)
@@ -105,6 +84,7 @@ func (m *SafeMap) Set(key, value any) {
}
m.dirtyNew[key] = value
}
m.lock.Unlock()
}
// Size returns the size of m.

View File

@@ -1,11 +1,10 @@
package collection
import (
"sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/stringx"
)
func TestSafeMap(t *testing.T) {
@@ -108,104 +107,3 @@ func testSafeMapWithParameters(t *testing.T, size, exception int) {
}
}
}
func TestSafeMap_Range(t *testing.T) {
const (
size = 100000
exception1 = 5
exception2 = 500
)
m := NewSafeMap()
newMap := NewSafeMap()
for i := 0; i < size; i++ {
m.Set(i, i)
}
for i := 0; i < size; i++ {
if i%exception1 == 0 {
m.Del(i)
}
}
for i := size; i < size<<1; i++ {
m.Set(i, i)
}
for i := size; i < size<<1; i++ {
if i%exception2 != 0 {
m.Del(i)
}
}
var count int32
m.Range(func(k, v any) bool {
atomic.AddInt32(&count, 1)
newMap.Set(k, v)
return true
})
assert.Equal(t, int(atomic.LoadInt32(&count)), m.Size())
assert.Equal(t, m.dirtyNew, newMap.dirtyNew)
assert.Equal(t, m.dirtyOld, newMap.dirtyOld)
}
func TestSetManyTimes(t *testing.T) {
const iteration = maxDeletion * 2
m := NewSafeMap()
for i := 0; i < iteration; i++ {
m.Set(i, i)
if i%3 == 0 {
m.Del(i / 2)
}
}
var count int
m.Range(func(k, v any) bool {
count++
return count < maxDeletion/2
})
assert.Equal(t, maxDeletion/2, count)
for i := 0; i < iteration; i++ {
m.Set(i, i)
if i%3 == 0 {
m.Del(i / 2)
}
}
for i := 0; i < iteration; i++ {
m.Set(i, i)
if i%3 == 0 {
m.Del(i / 2)
}
}
for i := 0; i < iteration; i++ {
m.Set(i, i)
if i%3 == 0 {
m.Del(i / 2)
}
}
count = 0
m.Range(func(k, v any) bool {
count++
return count < maxDeletion
})
assert.Equal(t, maxDeletion, count)
}
func TestSetManyTimesNew(t *testing.T) {
m := NewSafeMap()
for i := 0; i < maxDeletion*3; i++ {
m.Set(i, i)
}
for i := 0; i < maxDeletion*2; i++ {
m.Del(i)
}
for i := 0; i < maxDeletion*3; i++ {
m.Set(i+maxDeletion*3, i+maxDeletion*3)
}
for i := 0; i < maxDeletion*2; i++ {
m.Del(i + maxDeletion*2)
}
for i := 0; i < maxDeletion-copyThreshold+1; i++ {
m.Del(i + maxDeletion*2)
}
assert.Equal(t, 0, len(m.dirtyNew))
}

View File

@@ -1,8 +1,8 @@
package collection
import (
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/logx"
)
const (
@@ -17,28 +17,28 @@ const (
// Set is not thread-safe, for concurrent use, make sure to use it with synchronization.
type Set struct {
data map[any]lang.PlaceholderType
data map[interface{}]lang.PlaceholderType
tp int
}
// NewSet returns a managed Set, can only put the values with the same type.
func NewSet() *Set {
return &Set{
data: make(map[any]lang.PlaceholderType),
data: make(map[interface{}]lang.PlaceholderType),
tp: untyped,
}
}
// NewUnmanagedSet returns an unmanaged Set, which can put values with different types.
// NewUnmanagedSet returns a unmanaged Set, which can put values with different types.
func NewUnmanagedSet() *Set {
return &Set{
data: make(map[any]lang.PlaceholderType),
data: make(map[interface{}]lang.PlaceholderType),
tp: unmanaged,
}
}
// Add adds i into s.
func (s *Set) Add(i ...any) {
func (s *Set) Add(i ...interface{}) {
for _, each := range i {
s.add(each)
}
@@ -80,7 +80,7 @@ func (s *Set) AddStr(ss ...string) {
}
// Contains checks if i is in s.
func (s *Set) Contains(i any) bool {
func (s *Set) Contains(i interface{}) bool {
if len(s.data) == 0 {
return false
}
@@ -91,8 +91,8 @@ func (s *Set) Contains(i any) bool {
}
// Keys returns the keys in s.
func (s *Set) Keys() []any {
var keys []any
func (s *Set) Keys() []interface{} {
var keys []interface{}
for key := range s.data {
keys = append(keys, key)
@@ -167,7 +167,7 @@ func (s *Set) KeysStr() []string {
}
// Remove removes i from s.
func (s *Set) Remove(i any) {
func (s *Set) Remove(i interface{}) {
s.validate(i)
delete(s.data, i)
}
@@ -177,7 +177,7 @@ func (s *Set) Count() int {
return len(s.data)
}
func (s *Set) add(i any) {
func (s *Set) add(i interface{}) {
switch s.tp {
case unmanaged:
// do nothing
@@ -189,7 +189,7 @@ func (s *Set) add(i any) {
s.data[i] = lang.Placeholder
}
func (s *Set) setType(i any) {
func (s *Set) setType(i interface{}) {
// s.tp can only be untyped here
switch i.(type) {
case int:
@@ -205,7 +205,7 @@ func (s *Set) setType(i any) {
}
}
func (s *Set) validate(i any) {
func (s *Set) validate(i interface{}) {
if s.tp == unmanaged {
return
}
@@ -213,23 +213,23 @@ func (s *Set) validate(i any) {
switch i.(type) {
case int:
if s.tp != intType {
logx.Errorf("element is int, but set contains elements with type %d", s.tp)
logx.Errorf("Error: element is int, but set contains elements with type %d", s.tp)
}
case int64:
if s.tp != int64Type {
logx.Errorf("element is int64, but set contains elements with type %d", s.tp)
logx.Errorf("Error: element is int64, but set contains elements with type %d", s.tp)
}
case uint:
if s.tp != uintType {
logx.Errorf("element is uint, but set contains elements with type %d", s.tp)
logx.Errorf("Error: element is uint, but set contains elements with type %d", s.tp)
}
case uint64:
if s.tp != uint64Type {
logx.Errorf("element is uint64, but set contains elements with type %d", s.tp)
logx.Errorf("Error: element is uint64, but set contains elements with type %d", s.tp)
}
case string:
if s.tp != stringType {
logx.Errorf("element is string, but set contains elements with type %d", s.tp)
logx.Errorf("Error: element is string, but set contains elements with type %d", s.tp)
}
}
}

View File

@@ -5,7 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/logx"
)
func init() {
@@ -13,7 +13,7 @@ func init() {
}
func BenchmarkRawSet(b *testing.B) {
m := make(map[any]struct{})
m := make(map[interface{}]struct{})
for i := 0; i < b.N; i++ {
m[i] = struct{}{}
_ = m[i]
@@ -39,7 +39,7 @@ func BenchmarkSet(b *testing.B) {
func TestAdd(t *testing.T) {
// given
set := NewUnmanagedSet()
values := []any{1, 2, 3}
values := []interface{}{1, 2, 3}
// when
set.Add(values...)
@@ -135,7 +135,7 @@ func TestContainsUnmanagedWithoutElements(t *testing.T) {
func TestRemove(t *testing.T) {
// given
set := NewSet()
set.Add([]any{1, 2, 3}...)
set.Add([]interface{}{1, 2, 3}...)
// when
set.Remove(2)
@@ -147,7 +147,7 @@ func TestRemove(t *testing.T) {
func TestCount(t *testing.T) {
// given
set := NewSet()
set.Add([]any{1, 2, 3}...)
set.Add([]interface{}{1, 2, 3}...)
// then
assert.Equal(t, set.Count(), 3)
@@ -198,5 +198,5 @@ func TestSetType(t *testing.T) {
set.add(1)
set.add("2")
vals := set.Keys()
assert.ElementsMatch(t, []any{1, "2"}, vals)
assert.ElementsMatch(t, []interface{}{1, "2"}, vals)
}

View File

@@ -2,25 +2,19 @@ package collection
import (
"container/list"
"errors"
"fmt"
"time"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/threading"
"github.com/zeromicro/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/threading"
"github.com/tal-tech/go-zero/core/timex"
)
const drainWorkers = 8
var (
ErrClosed = errors.New("TimingWheel is closed already")
ErrArgument = errors.New("incorrect task argument")
)
type (
// Execute defines the method to execute the task.
Execute func(key, value any)
Execute func(key, value interface{})
// A TimingWheel is a timing wheel object to schedule tasks.
TimingWheel struct {
@@ -33,14 +27,14 @@ type (
execute Execute
setChannel chan timingEntry
moveChannel chan baseEntry
removeChannel chan any
drainChannel chan func(key, value any)
removeChannel chan interface{}
drainChannel chan func(key, value interface{})
stopChannel chan lang.PlaceholderType
}
timingEntry struct {
baseEntry
value any
value interface{}
circle int
diff int
removed bool
@@ -48,7 +42,7 @@ type (
baseEntry struct {
delay time.Duration
key any
key interface{}
}
positionEntry struct {
@@ -57,24 +51,22 @@ type (
}
timingTask struct {
key any
value any
key interface{}
value interface{}
}
)
// NewTimingWheel returns a TimingWheel.
func NewTimingWheel(interval time.Duration, numSlots int, execute Execute) (*TimingWheel, error) {
if interval <= 0 || numSlots <= 0 || execute == nil {
return nil, fmt.Errorf("interval: %v, slots: %d, execute: %p",
interval, numSlots, execute)
return nil, fmt.Errorf("interval: %v, slots: %d, execute: %p", interval, numSlots, execute)
}
return NewTimingWheelWithTicker(interval, numSlots, execute, timex.NewTicker(interval))
return newTimingWheelWithClock(interval, numSlots, execute, timex.NewTicker(interval))
}
// NewTimingWheelWithTicker returns a TimingWheel with the given ticker.
func NewTimingWheelWithTicker(interval time.Duration, numSlots int, execute Execute,
ticker timex.Ticker) (*TimingWheel, error) {
func newTimingWheelWithClock(interval time.Duration, numSlots int, execute Execute, ticker timex.Ticker) (
*TimingWheel, error) {
tw := &TimingWheel{
interval: interval,
ticker: ticker,
@@ -85,8 +77,8 @@ func NewTimingWheelWithTicker(interval time.Duration, numSlots int, execute Exec
numSlots: numSlots,
setChannel: make(chan timingEntry),
moveChannel: make(chan baseEntry),
removeChannel: make(chan any),
drainChannel: make(chan func(key, value any)),
removeChannel: make(chan interface{}),
drainChannel: make(chan func(key, value interface{})),
stopChannel: make(chan lang.PlaceholderType),
}
@@ -97,72 +89,52 @@ func NewTimingWheelWithTicker(interval time.Duration, numSlots int, execute Exec
}
// Drain drains all items and executes them.
func (tw *TimingWheel) Drain(fn func(key, value any)) error {
select {
case tw.drainChannel <- fn:
return nil
case <-tw.stopChannel:
return ErrClosed
}
func (tw *TimingWheel) Drain(fn func(key, value interface{})) {
tw.drainChannel <- fn
}
// MoveTimer moves the task with the given key to the given delay.
func (tw *TimingWheel) MoveTimer(key any, delay time.Duration) error {
func (tw *TimingWheel) MoveTimer(key interface{}, delay time.Duration) {
if delay <= 0 || key == nil {
return ErrArgument
return
}
select {
case tw.moveChannel <- baseEntry{
tw.moveChannel <- baseEntry{
delay: delay,
key: key,
}:
return nil
case <-tw.stopChannel:
return ErrClosed
}
}
// RemoveTimer removes the task with the given key.
func (tw *TimingWheel) RemoveTimer(key any) error {
func (tw *TimingWheel) RemoveTimer(key interface{}) {
if key == nil {
return ErrArgument
return
}
select {
case tw.removeChannel <- key:
return nil
case <-tw.stopChannel:
return ErrClosed
}
tw.removeChannel <- key
}
// SetTimer sets the task value with the given key to the delay.
func (tw *TimingWheel) SetTimer(key, value any, delay time.Duration) error {
func (tw *TimingWheel) SetTimer(key, value interface{}, delay time.Duration) {
if delay <= 0 || key == nil {
return ErrArgument
return
}
select {
case tw.setChannel <- timingEntry{
tw.setChannel <- timingEntry{
baseEntry: baseEntry{
delay: delay,
key: key,
},
value: value,
}:
return nil
case <-tw.stopChannel:
return ErrClosed
}
}
// Stop stops tw. No more actions after stopping a TimingWheel.
// Stop stops tw.
func (tw *TimingWheel) Stop() {
close(tw.stopChannel)
}
func (tw *TimingWheel) drainAll(fn func(key, value any)) {
func (tw *TimingWheel) drainAll(fn func(key, value interface{})) {
runner := threading.NewTaskRunner(drainWorkers)
for _, slot := range tw.slots {
for e := slot.Front(); e != nil; {
@@ -232,7 +204,7 @@ func (tw *TimingWheel) onTick() {
tw.scanAndRunTasks(l)
}
func (tw *TimingWheel) removeTask(key any) {
func (tw *TimingWheel) removeTask(key interface{}) {
val, ok := tw.timers.Get(key)
if !ok {
return

View File

@@ -8,10 +8,10 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/stringx"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/zeromicro/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/syncx"
"github.com/tal-tech/go-zero/core/timex"
)
const (
@@ -20,14 +20,15 @@ const (
)
func TestNewTimingWheel(t *testing.T) {
_, err := NewTimingWheel(0, 10, func(key, value any) {})
_, err := NewTimingWheel(0, 10, func(key, value interface{}) {})
assert.NotNil(t, err)
}
func TestTimingWheel_Drain(t *testing.T) {
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {
tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
}, ticker)
defer tw.Stop()
tw.SetTimer("first", 3, testStep*4)
tw.SetTimer("second", 5, testStep*7)
tw.SetTimer("third", 7, testStep*7)
@@ -36,7 +37,7 @@ func TestTimingWheel_Drain(t *testing.T) {
var lock sync.Mutex
var wg sync.WaitGroup
wg.Add(3)
tw.Drain(func(key, value any) {
tw.Drain(func(key, value interface{}) {
lock.Lock()
defer lock.Unlock()
keys = append(keys, key.(string))
@@ -50,19 +51,17 @@ func TestTimingWheel_Drain(t *testing.T) {
assert.EqualValues(t, []string{"first", "second", "third"}, keys)
assert.EqualValues(t, []int{3, 5, 7}, vals)
var count int
tw.Drain(func(key, value any) {
tw.Drain(func(key, value interface{}) {
count++
})
time.Sleep(time.Millisecond * 100)
assert.Equal(t, 0, count)
tw.Stop()
assert.Equal(t, ErrClosed, tw.Drain(func(key, value any) {}))
}
func TestTimingWheel_SetTimerSoon(t *testing.T) {
run := syncx.NewAtomicBool()
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {
tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
assert.True(t, run.CompareAndSwap(false, true))
assert.Equal(t, "any", k)
assert.Equal(t, 3, v.(int))
@@ -78,7 +77,7 @@ func TestTimingWheel_SetTimerSoon(t *testing.T) {
func TestTimingWheel_SetTimerTwice(t *testing.T) {
run := syncx.NewAtomicBool()
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {
tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
assert.True(t, run.CompareAndSwap(false, true))
assert.Equal(t, "any", k)
assert.Equal(t, 5, v.(int))
@@ -96,29 +95,23 @@ func TestTimingWheel_SetTimerTwice(t *testing.T) {
func TestTimingWheel_SetTimerWrongDelay(t *testing.T) {
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {}, ticker)
tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {}, ticker)
defer tw.Stop()
assert.NotPanics(t, func() {
tw.SetTimer("any", 3, -testStep)
})
}
func TestTimingWheel_SetTimerAfterClose(t *testing.T) {
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {}, ticker)
tw.Stop()
assert.Equal(t, ErrClosed, tw.SetTimer("any", 3, testStep))
}
func TestTimingWheel_MoveTimer(t *testing.T) {
run := syncx.NewAtomicBool()
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 3, func(k, v any) {
tw, _ := newTimingWheelWithClock(testStep, 3, func(k, v interface{}) {
assert.True(t, run.CompareAndSwap(false, true))
assert.Equal(t, "any", k)
assert.Equal(t, 3, v.(int))
ticker.Done()
}, ticker)
defer tw.Stop()
tw.SetTimer("any", 3, testStep*4)
tw.MoveTimer("any", testStep*7)
tw.MoveTimer("any", -testStep)
@@ -132,14 +125,12 @@ func TestTimingWheel_MoveTimer(t *testing.T) {
}
assert.Nil(t, ticker.Wait(waitTime))
assert.True(t, run.True())
tw.Stop()
assert.Equal(t, ErrClosed, tw.MoveTimer("any", time.Millisecond))
}
func TestTimingWheel_MoveTimerSoon(t *testing.T) {
run := syncx.NewAtomicBool()
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 3, func(k, v any) {
tw, _ := newTimingWheelWithClock(testStep, 3, func(k, v interface{}) {
assert.True(t, run.CompareAndSwap(false, true))
assert.Equal(t, "any", k)
assert.Equal(t, 3, v.(int))
@@ -155,7 +146,7 @@ func TestTimingWheel_MoveTimerSoon(t *testing.T) {
func TestTimingWheel_MoveTimerEarlier(t *testing.T) {
run := syncx.NewAtomicBool()
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {
tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
assert.True(t, run.CompareAndSwap(false, true))
assert.Equal(t, "any", k)
assert.Equal(t, 3, v.(int))
@@ -173,7 +164,7 @@ func TestTimingWheel_MoveTimerEarlier(t *testing.T) {
func TestTimingWheel_RemoveTimer(t *testing.T) {
ticker := timex.NewFakeTicker()
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {}, ticker)
tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {}, ticker)
tw.SetTimer("any", 3, testStep)
assert.NotPanics(t, func() {
tw.RemoveTimer("any")
@@ -184,7 +175,6 @@ func TestTimingWheel_RemoveTimer(t *testing.T) {
ticker.Tick()
}
tw.Stop()
assert.Equal(t, ErrClosed, tw.RemoveTimer("any"))
}
func TestTimingWheel_SetTimer(t *testing.T) {
@@ -236,7 +226,7 @@ func TestTimingWheel_SetTimer(t *testing.T) {
}
var actual int32
done := make(chan lang.PlaceholderType)
tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value any) {
tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
assert.Equal(t, 1, key.(int))
assert.Equal(t, 2, value.(int))
actual = atomic.LoadInt32(&count)
@@ -317,7 +307,7 @@ func TestTimingWheel_SetAndMoveThenStart(t *testing.T) {
}
var actual int32
done := make(chan lang.PlaceholderType)
tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value any) {
tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
actual = atomic.LoadInt32(&count)
close(done)
}, ticker)
@@ -405,7 +395,7 @@ func TestTimingWheel_SetAndMoveTwice(t *testing.T) {
}
var actual int32
done := make(chan lang.PlaceholderType)
tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value any) {
tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
actual = atomic.LoadInt32(&count)
close(done)
}, ticker)
@@ -486,7 +476,7 @@ func TestTimingWheel_ElapsedAndSet(t *testing.T) {
}
var actual int32
done := make(chan lang.PlaceholderType)
tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value any) {
tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
actual = atomic.LoadInt32(&count)
close(done)
}, ticker)
@@ -577,7 +567,7 @@ func TestTimingWheel_ElapsedAndSetThenMove(t *testing.T) {
}
var actual int32
done := make(chan lang.PlaceholderType)
tw, err := NewTimingWheelWithTicker(testStep, test.slots, func(key, value any) {
tw, err := newTimingWheelWithClock(testStep, test.slots, func(key, value interface{}) {
actual = atomic.LoadInt32(&count)
close(done)
}, ticker)
@@ -612,7 +602,7 @@ func TestMoveAndRemoveTask(t *testing.T) {
}
}
var keys []int
tw, _ := NewTimingWheelWithTicker(testStep, 10, func(k, v any) {
tw, _ := newTimingWheelWithClock(testStep, 10, func(k, v interface{}) {
assert.Equal(t, "any", k)
assert.Equal(t, 3, v.(int))
keys = append(keys, v.(int))
@@ -632,7 +622,7 @@ func TestMoveAndRemoveTask(t *testing.T) {
func BenchmarkTimingWheel(b *testing.B) {
b.ReportAllocs()
tw, _ := NewTimingWheel(time.Second, 100, func(k, v any) {})
tw, _ := NewTimingWheel(time.Second, 100, func(k, v interface{}) {})
for i := 0; i < b.N; i++ {
tw.SetTimer(i, i, time.Second)
tw.SetTimer(b.N+i, b.N+i, time.Second)

View File

@@ -1,73 +0,0 @@
package color
import "github.com/fatih/color"
const (
// NoColor is no color for both foreground and background.
NoColor Color = iota
// FgBlack is the foreground color black.
FgBlack
// FgRed is the foreground color red.
FgRed
// FgGreen is the foreground color green.
FgGreen
// FgYellow is the foreground color yellow.
FgYellow
// FgBlue is the foreground color blue.
FgBlue
// FgMagenta is the foreground color magenta.
FgMagenta
// FgCyan is the foreground color cyan.
FgCyan
// FgWhite is the foreground color white.
FgWhite
// BgBlack is the background color black.
BgBlack
// BgRed is the background color red.
BgRed
// BgGreen is the background color green.
BgGreen
// BgYellow is the background color yellow.
BgYellow
// BgBlue is the background color blue.
BgBlue
// BgMagenta is the background color magenta.
BgMagenta
// BgCyan is the background color cyan.
BgCyan
// BgWhite is the background color white.
BgWhite
)
var colors = map[Color][]color.Attribute{
FgBlack: {color.FgBlack, color.Bold},
FgRed: {color.FgRed, color.Bold},
FgGreen: {color.FgGreen, color.Bold},
FgYellow: {color.FgYellow, color.Bold},
FgBlue: {color.FgBlue, color.Bold},
FgMagenta: {color.FgMagenta, color.Bold},
FgCyan: {color.FgCyan, color.Bold},
FgWhite: {color.FgWhite, color.Bold},
BgBlack: {color.BgBlack, color.FgHiWhite, color.Bold},
BgRed: {color.BgRed, color.FgHiWhite, color.Bold},
BgGreen: {color.BgGreen, color.FgHiWhite, color.Bold},
BgYellow: {color.BgHiYellow, color.FgHiBlack, color.Bold},
BgBlue: {color.BgBlue, color.FgHiWhite, color.Bold},
BgMagenta: {color.BgMagenta, color.FgHiWhite, color.Bold},
BgCyan: {color.BgCyan, color.FgHiWhite, color.Bold},
BgWhite: {color.BgHiWhite, color.FgHiBlack, color.Bold},
}
type Color uint32
// WithColor returns a string with the given color applied.
func WithColor(text string, colour Color) string {
c := color.New(colors[colour]...)
return c.Sprint(text)
}
// WithColorPadding returns a string with the given color applied with leading and trailing spaces.
func WithColorPadding(text string, colour Color) string {
return WithColor(" "+text+" ", colour)
}

View File

@@ -1,17 +0,0 @@
package color
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestWithColor(t *testing.T) {
output := WithColor("Hello", BgRed)
assert.Equal(t, "Hello", output)
}
func TestWithColorPadding(t *testing.T) {
output := WithColorPadding("Hello", BgRed)
assert.Equal(t, " Hello ", output)
}

View File

@@ -2,53 +2,28 @@ package conf
import (
"fmt"
"io/ioutil"
"log"
"os"
"path"
"reflect"
"strings"
"github.com/zeromicro/go-zero/core/jsonx"
"github.com/zeromicro/go-zero/core/mapping"
"github.com/zeromicro/go-zero/internal/encoding"
"github.com/tal-tech/go-zero/core/mapping"
)
const (
jsonTagKey = "json"
jsonTagSep = ','
)
var (
fillDefaultUnmarshaler = mapping.NewUnmarshaler(jsonTagKey, mapping.WithDefault())
loaders = map[string]func([]byte, any) error{
".json": LoadFromJsonBytes,
".toml": LoadFromTomlBytes,
".yaml": LoadFromYamlBytes,
".yml": LoadFromYamlBytes,
}
)
// children and mapField should not be both filled.
// named fields and map cannot be bound to the same field name.
type fieldInfo struct {
children map[string]*fieldInfo
mapField *fieldInfo
var loaders = map[string]func([]byte, interface{}) error{
".json": LoadConfigFromJsonBytes,
".yaml": LoadConfigFromYamlBytes,
".yml": LoadConfigFromYamlBytes,
}
// FillDefault fills the default values for the given v,
// and the premise is that the value of v must be guaranteed to be empty.
func FillDefault(v any) error {
return fillDefaultUnmarshaler.Unmarshal(map[string]any{}, v)
}
// Load loads config into v from file, .json, .yaml and .yml are acceptable.
func Load(file string, v any, opts ...Option) error {
content, err := os.ReadFile(file)
// LoadConfig loads config into v from file, .json, .yaml and .yml are acceptable.
func LoadConfig(file string, v interface{}, opts ...Option) error {
content, err := ioutil.ReadFile(file)
if err != nil {
return err
}
loader, ok := loaders[strings.ToLower(path.Ext(file))]
loader, ok := loaders[path.Ext(file)]
if !ok {
return fmt.Errorf("unrecognized file type: %s", file)
}
@@ -65,297 +40,19 @@ func Load(file string, v any, opts ...Option) error {
return loader(content, v)
}
// LoadConfig loads config into v from file, .json, .yaml and .yml are acceptable.
// Deprecated: use Load instead.
func LoadConfig(file string, v any, opts ...Option) error {
return Load(file, v, opts...)
}
// LoadFromJsonBytes loads config into v from content json bytes.
func LoadFromJsonBytes(content []byte, v any) error {
info, err := buildFieldsInfo(reflect.TypeOf(v), "")
if err != nil {
return err
}
var m map[string]any
if err = jsonx.Unmarshal(content, &m); err != nil {
return err
}
lowerCaseKeyMap := toLowerCaseKeyMap(m, info)
return mapping.UnmarshalJsonMap(lowerCaseKeyMap, v, mapping.WithCanonicalKeyFunc(toLowerCase))
}
// LoadConfigFromJsonBytes loads config into v from content json bytes.
// Deprecated: use LoadFromJsonBytes instead.
func LoadConfigFromJsonBytes(content []byte, v any) error {
return LoadFromJsonBytes(content, v)
}
// LoadFromTomlBytes loads config into v from content toml bytes.
func LoadFromTomlBytes(content []byte, v any) error {
b, err := encoding.TomlToJson(content)
if err != nil {
return err
}
return LoadFromJsonBytes(b, v)
}
// LoadFromYamlBytes loads config into v from content yaml bytes.
func LoadFromYamlBytes(content []byte, v any) error {
b, err := encoding.YamlToJson(content)
if err != nil {
return err
}
return LoadFromJsonBytes(b, v)
func LoadConfigFromJsonBytes(content []byte, v interface{}) error {
return mapping.UnmarshalJsonBytes(content, v)
}
// LoadConfigFromYamlBytes loads config into v from content yaml bytes.
// Deprecated: use LoadFromYamlBytes instead.
func LoadConfigFromYamlBytes(content []byte, v any) error {
return LoadFromYamlBytes(content, v)
func LoadConfigFromYamlBytes(content []byte, v interface{}) error {
return mapping.UnmarshalYamlBytes(content, v)
}
// MustLoad loads config into v from path, exits on error.
func MustLoad(path string, v any, opts ...Option) {
if err := Load(path, v, opts...); err != nil {
func MustLoad(path string, v interface{}, opts ...Option) {
if err := LoadConfig(path, v, opts...); err != nil {
log.Fatalf("error: config file %s, %s", path, err.Error())
}
}
func addOrMergeFields(info *fieldInfo, key string, child *fieldInfo, fullName string) error {
if prev, ok := info.children[key]; ok {
if child.mapField != nil {
return newConflictKeyError(fullName)
}
if err := mergeFields(prev, key, child.children, fullName); err != nil {
return err
}
} else {
info.children[key] = child
}
return nil
}
func buildAnonymousFieldInfo(info *fieldInfo, lowerCaseName string, ft reflect.Type, fullName string) error {
switch ft.Kind() {
case reflect.Struct:
fields, err := buildFieldsInfo(ft, fullName)
if err != nil {
return err
}
for k, v := range fields.children {
if err = addOrMergeFields(info, k, v, fullName); err != nil {
return err
}
}
case reflect.Map:
elemField, err := buildFieldsInfo(mapping.Deref(ft.Elem()), fullName)
if err != nil {
return err
}
if _, ok := info.children[lowerCaseName]; ok {
return newConflictKeyError(fullName)
}
info.children[lowerCaseName] = &fieldInfo{
children: make(map[string]*fieldInfo),
mapField: elemField,
}
default:
if _, ok := info.children[lowerCaseName]; ok {
return newConflictKeyError(fullName)
}
info.children[lowerCaseName] = &fieldInfo{
children: make(map[string]*fieldInfo),
}
}
return nil
}
func buildFieldsInfo(tp reflect.Type, fullName string) (*fieldInfo, error) {
tp = mapping.Deref(tp)
switch tp.Kind() {
case reflect.Struct:
return buildStructFieldsInfo(tp, fullName)
case reflect.Array, reflect.Slice:
return buildFieldsInfo(mapping.Deref(tp.Elem()), fullName)
case reflect.Chan, reflect.Func:
return nil, fmt.Errorf("unsupported type: %s", tp.Kind())
default:
return &fieldInfo{
children: make(map[string]*fieldInfo),
}, nil
}
}
func buildNamedFieldInfo(info *fieldInfo, lowerCaseName string, ft reflect.Type, fullName string) error {
var finfo *fieldInfo
var err error
switch ft.Kind() {
case reflect.Struct:
finfo, err = buildFieldsInfo(ft, fullName)
if err != nil {
return err
}
case reflect.Array, reflect.Slice:
finfo, err = buildFieldsInfo(ft.Elem(), fullName)
if err != nil {
return err
}
case reflect.Map:
elemInfo, err := buildFieldsInfo(mapping.Deref(ft.Elem()), fullName)
if err != nil {
return err
}
finfo = &fieldInfo{
children: make(map[string]*fieldInfo),
mapField: elemInfo,
}
default:
finfo, err = buildFieldsInfo(ft, fullName)
if err != nil {
return err
}
}
return addOrMergeFields(info, lowerCaseName, finfo, fullName)
}
func buildStructFieldsInfo(tp reflect.Type, fullName string) (*fieldInfo, error) {
info := &fieldInfo{
children: make(map[string]*fieldInfo),
}
for i := 0; i < tp.NumField(); i++ {
field := tp.Field(i)
if !field.IsExported() {
continue
}
name := getTagName(field)
lowerCaseName := toLowerCase(name)
ft := mapping.Deref(field.Type)
// flatten anonymous fields
if field.Anonymous {
if err := buildAnonymousFieldInfo(info, lowerCaseName, ft,
getFullName(fullName, lowerCaseName)); err != nil {
return nil, err
}
} else if err := buildNamedFieldInfo(info, lowerCaseName, ft,
getFullName(fullName, lowerCaseName)); err != nil {
return nil, err
}
}
return info, nil
}
// getTagName get the tag name of the given field, if no tag name, use file.Name.
// field.Name is returned on tags like `json:""` and `json:",optional"`.
func getTagName(field reflect.StructField) string {
if tag, ok := field.Tag.Lookup(jsonTagKey); ok {
if pos := strings.IndexByte(tag, jsonTagSep); pos >= 0 {
tag = tag[:pos]
}
tag = strings.TrimSpace(tag)
if len(tag) > 0 {
return tag
}
}
return field.Name
}
func mergeFields(prev *fieldInfo, key string, children map[string]*fieldInfo, fullName string) error {
if len(prev.children) == 0 || len(children) == 0 {
return newConflictKeyError(fullName)
}
// merge fields
for k, v := range children {
if _, ok := prev.children[k]; ok {
return newConflictKeyError(fullName)
}
prev.children[k] = v
}
return nil
}
func toLowerCase(s string) string {
return strings.ToLower(s)
}
func toLowerCaseInterface(v any, info *fieldInfo) any {
switch vv := v.(type) {
case map[string]any:
return toLowerCaseKeyMap(vv, info)
case []any:
var arr []any
for _, vvv := range vv {
arr = append(arr, toLowerCaseInterface(vvv, info))
}
return arr
default:
return v
}
}
func toLowerCaseKeyMap(m map[string]any, info *fieldInfo) map[string]any {
res := make(map[string]any)
for k, v := range m {
ti, ok := info.children[k]
if ok {
res[k] = toLowerCaseInterface(v, ti)
continue
}
lk := toLowerCase(k)
if ti, ok = info.children[lk]; ok {
res[lk] = toLowerCaseInterface(v, ti)
} else if info.mapField != nil {
res[k] = toLowerCaseInterface(v, info.mapField)
} else {
res[k] = v
}
}
return res
}
type conflictKeyError struct {
key string
}
func newConflictKeyError(key string) conflictKeyError {
return conflictKeyError{key: key}
}
func (e conflictKeyError) Error() string {
return fmt.Sprintf("conflict key %s, pay attention to anonymous fields", e.key)
}
func getFullName(parent, child string) string {
if len(parent) == 0 {
return child
}
return strings.Join([]string{parent, child}, ".")
}

File diff suppressed because it is too large Load Diff

View File

@@ -7,11 +7,12 @@ import (
"strings"
"sync"
"github.com/zeromicro/go-zero/core/iox"
"github.com/tal-tech/go-zero/core/iox"
)
// PropertyError represents a configuration error message.
type PropertyError struct {
error
message string
}

View File

@@ -5,7 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/fs"
"github.com/tal-tech/go-zero/core/fs"
)
func TestProperties(t *testing.T) {
@@ -45,7 +45,8 @@ func TestPropertiesEnv(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(tmpfile)
t.Setenv("FOO", "2")
os.Setenv("FOO", "2")
defer os.Unsetenv("FOO")
props, err := LoadProperties(tmpfile, UseEnv())
assert.Nil(t, err)

View File

@@ -1,58 +0,0 @@
## How to use
1. Define a config structure, like below:
```go
type RestfulConf struct {
ServiceName string `json:",env=SERVICE_NAME"` // read from env automatically
Host string `json:",default=0.0.0.0"`
Port int
LogMode string `json:",options=[file,console]"`
Verbose bool `json:",optional"`
MaxConns int `json:",default=10000"`
MaxBytes int64 `json:",default=1048576"`
Timeout time.Duration `json:",default=3s"`
CpuThreshold int64 `json:",default=900,range=[0:1000]"`
}
```
2. Write the yaml, toml or json config file:
- yaml example
```yaml
# most fields are optional or have default values
port: 8080
logMode: console
# you can use env settings
maxBytes: ${MAX_BYTES}
```
- toml example
```toml
# most fields are optional or have default values
port = 8_080
logMode = "console"
# you can use env settings
maxBytes = "${MAX_BYTES}"
```
3. Load the config from a file:
```go
// exit on error
var config RestfulConf
conf.MustLoad(configFile, &config)
// or handle the error on your own
var config RestfulConf
if err := conf.Load(configFile, &config); err != nil {
log.Fatal(err)
}
// enable reading from environments
var config RestfulConf
conf.MustLoad(configFile, &config, conf.UseEnv())
```

View File

@@ -3,7 +3,7 @@ package contextx
import (
"context"
"github.com/zeromicro/go-zero/core/mapping"
"github.com/tal-tech/go-zero/core/mapping"
)
const contextTagKey = "ctx"
@@ -14,13 +14,13 @@ type contextValuer struct {
context.Context
}
func (cv contextValuer) Value(key string) (any, bool) {
func (cv contextValuer) Value(key string) (interface{}, bool) {
v := cv.Context.Value(key)
return v, v != nil
}
// For unmarshals ctx into v.
func For(ctx context.Context, v any) error {
func For(ctx context.Context, v interface{}) error {
return unmarshaler.UnmarshalValuer(contextValuer{
Context: ctx,
}, v)

View File

@@ -1,6 +1,6 @@
package discov
import "github.com/zeromicro/go-zero/core/discov/internal"
import "github.com/tal-tech/go-zero/core/discov/internal"
// RegisterAccount registers the username/password to the given etcd cluster.
func RegisterAccount(endpoints []string, user, pass string) {

View File

@@ -4,8 +4,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/discov/internal"
"github.com/zeromicro/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/discov/internal"
"github.com/tal-tech/go-zero/core/stringx"
)
func TestRegisterAccount(t *testing.T) {

View File

@@ -4,7 +4,7 @@ import (
"fmt"
"strings"
"github.com/zeromicro/go-zero/core/discov/internal"
"github.com/tal-tech/go-zero/core/discov/internal"
)
const (

View File

@@ -5,7 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/discov/internal"
"github.com/tal-tech/go-zero/core/discov/internal"
)
var mockLock sync.Mutex

View File

@@ -2,18 +2,10 @@ package discov
import "errors"
var (
// errEmptyEtcdHosts indicates that etcd hosts are empty.
errEmptyEtcdHosts = errors.New("empty etcd hosts")
// errEmptyEtcdKey indicates that etcd key is empty.
errEmptyEtcdKey = errors.New("empty etcd key")
)
// EtcdConf is the config item with the given key on etcd.
type EtcdConf struct {
Hosts []string
Key string
ID int64 `json:",optional"`
User string `json:",optional"`
Pass string `json:",optional"`
CertFile string `json:",optional"`
@@ -27,11 +19,6 @@ func (c EtcdConf) HasAccount() bool {
return len(c.User) > 0 && len(c.Pass) > 0
}
// HasID returns if ID provided.
func (c EtcdConf) HasID() bool {
return c.ID > 0
}
// HasTLS returns if TLS CertFile/CertKeyFile/CACertFile are provided.
func (c EtcdConf) HasTLS() bool {
return len(c.CertFile) > 0 && len(c.CertKeyFile) > 0 && len(c.CACertFile) > 0
@@ -40,9 +27,9 @@ func (c EtcdConf) HasTLS() bool {
// Validate validates c.
func (c EtcdConf) Validate() error {
if len(c.Hosts) == 0 {
return errEmptyEtcdHosts
return errors.New("empty etcd hosts")
} else if len(c.Key) == 0 {
return errEmptyEtcdKey
return errors.New("empty etcd key")
} else {
return nil
}

View File

@@ -80,90 +80,3 @@ func TestEtcdConf_HasAccount(t *testing.T) {
assert.Equal(t, test.hasAccount, test.EtcdConf.HasAccount())
}
}
func TestEtcdConf_HasID(t *testing.T) {
tests := []struct {
EtcdConf
hasServerID bool
}{
{
EtcdConf: EtcdConf{
Hosts: []string{"any"},
ID: -1,
},
hasServerID: false,
},
{
EtcdConf: EtcdConf{
Hosts: []string{"any"},
ID: 0,
},
hasServerID: false,
},
{
EtcdConf: EtcdConf{
Hosts: []string{"any"},
ID: 10000,
},
hasServerID: true,
},
}
for _, test := range tests {
assert.Equal(t, test.hasServerID, test.EtcdConf.HasID())
}
}
func TestEtcdConf_HasTLS(t *testing.T) {
tests := []struct {
name string
conf EtcdConf
want bool
}{
{
name: "empty config",
conf: EtcdConf{},
want: false,
},
{
name: "missing CertFile",
conf: EtcdConf{
CertKeyFile: "key",
CACertFile: "ca",
},
want: false,
},
{
name: "missing CertKeyFile",
conf: EtcdConf{
CertFile: "cert",
CACertFile: "ca",
},
want: false,
},
{
name: "missing CACertFile",
conf: EtcdConf{
CertFile: "cert",
CertKeyFile: "key",
},
want: false,
},
{
name: "valid config",
conf: EtcdConf{
CertFile: "cert",
CertKeyFile: "key",
CACertFile: "ca",
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.conf.HasTLS()
assert.Equal(t, tt.want, got)
})
}
}

View File

@@ -3,7 +3,7 @@ package internal
import (
"crypto/tls"
"crypto/x509"
"os"
"io/ioutil"
"sync"
)
@@ -37,7 +37,7 @@ func AddTLS(endpoints []string, certFile, certKeyFile, caFile string, insecureSk
return err
}
caData, err := os.ReadFile(caFile)
caData, err := ioutil.ReadFile(caFile)
if err != nil {
return err
}

View File

@@ -1,83 +1,10 @@
package internal
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/stringx"
)
const (
certContent = `-----BEGIN CERTIFICATE-----
MIIDazCCAlOgAwIBAgIUEg9GVO2oaPn+YSmiqmFIuAo10WIwDQYJKoZIhvcNAQEM
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMzAzMTExMzIxMjNaGA8yMTIz
MDIxNTEzMjEyM1owRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN
AQEBBQADggEPADCCAQoCggEBALplXlWsIf0O/IgnIplmiZHKGnxyfyufyE2FBRNk
OofRqbKuPH8GNqbkvZm7N29fwTDAQ+mViAggCkDht4hOzoWJMA7KYJt8JnTSWL48
M1lcrpc9DL2gszC/JF/FGvyANbBtLklkZPFBGdHUX14pjrT937wqPtm+SqUHSvRT
B7bmwmm2drRcmhpVm98LSlV7uQ2EgnJgsLjBPITKUejLmVLHfgX0RwQ2xIpX9pS4
FCe1BTacwl2gGp7Mje7y4Mfv3o0ArJW6Tuwbjx59ZXwb1KIP71b7bT04AVS8ZeYO
UMLKKuB5UR9x9Rn6cLXOTWBpcMVyzDgrAFLZjnE9LPUolZMCAwEAAaNRME8wHwYD
VR0jBBgwFoAUeW8w8pmhncbRgTsl48k4/7wnfx8wCQYDVR0TBAIwADALBgNVHQ8E
BAMCBPAwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBDAUAA4IBAQAI
y9xaoS88CLPBsX6mxfcTAFVfGNTRW9VN9Ng1cCnUR+YGoXGM/l+qP4f7p8ocdGwK
iYZErVTzXYIn+D27//wpY3klJk3gAnEUBT3QRkStBw7XnpbeZ2oPBK+cmDnCnZPS
BIF1wxPX7vIgaxs5Zsdqwk3qvZ4Djr2wP7LabNWTLSBKgQoUY45Liw6pffLwcGF9
UKlu54bvGze2SufISCR3ib+I+FLvqpvJhXToZWYb/pfI/HccuCL1oot1x8vx6DQy
U+TYxlZsKS5mdNxAX3dqEkEMsgEi+g/tzDPXJImfeCGGBhIOXLm8SRypiuGdEbc9
xkWYxRPegajuEZGvCqVs
-----END CERTIFICATE-----`
keyContent = `-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAumVeVawh/Q78iCcimWaJkcoafHJ/K5/ITYUFE2Q6h9Gpsq48
fwY2puS9mbs3b1/BMMBD6ZWICCAKQOG3iE7OhYkwDspgm3wmdNJYvjwzWVyulz0M
vaCzML8kX8Ua/IA1sG0uSWRk8UEZ0dRfXimOtP3fvCo+2b5KpQdK9FMHtubCabZ2
tFyaGlWb3wtKVXu5DYSCcmCwuME8hMpR6MuZUsd+BfRHBDbEilf2lLgUJ7UFNpzC
XaAansyN7vLgx+/ejQCslbpO7BuPHn1lfBvUog/vVvttPTgBVLxl5g5Qwsoq4HlR
H3H1Gfpwtc5NYGlwxXLMOCsAUtmOcT0s9SiVkwIDAQABAoIBAD5meTJNMgO55Kjg
ESExxpRcCIno+tHr5+6rvYtEXqPheOIsmmwb9Gfi4+Z3WpOaht5/Pz0Ppj6yGzyl
U//6AgGKb+BDuBvVcDpjwPnOxZIBCSHwejdxeQu0scSuA97MPS0XIAvJ5FEv7ijk
5Bht6SyGYURpECltHygoTNuGgGqmO+McCJRLE9L09lTBI6UQ/JQwWJqSr7wx6iPU
M1Ze/srIV+7cyEPu6i0DGjS1gSQKkX68Lqn1w6oE290O+OZvleO0gZ02fLDWCZke
aeD9+EU/Pw+rqm3H6o0szOFIpzhRp41FUdW9sybB3Yp3u7c/574E+04Z/e30LMKs
TCtE1QECgYEA3K7KIpw0NH2HXL5C3RHcLmr204xeBfS70riBQQuVUgYdmxak2ima
80RInskY8hRhSGTg0l+VYIH8cmjcUyqMSOELS5XfRH99r4QPiK8AguXg80T4VumY
W3Pf+zEC2ssgP/gYthV0g0Xj5m2QxktOF9tRw5nkg739ZR4dI9lm/iECgYEA2Dnf
uwEDGqHiQRF6/fh5BG/nGVMvrefkqx6WvTJQ3k/M/9WhxB+lr/8yH46TuS8N2b29
FoTf3Mr9T7pr/PWkOPzoY3P56nYbKU8xSwCim9xMzhBMzj8/N9ukJvXy27/VOz56
eQaKqnvdXNGtPJrIMDGHps2KKWlKLyAlapzjVTMCgYAA/W++tACv85g13EykfT4F
n0k4LbsGP9DP4zABQLIMyiY72eAncmRVjwrcW36XJ2xATOONTgx3gF3HjZzfaqNy
eD/6uNNllUTVEryXGmHgNHPL45VRnn6memCY2eFvZdXhM5W4y2PYaunY0MkDercA
+GTngbs6tBF88KOk04bYwQKBgFl68cRgsdkmnwwQYNaTKfmVGYzYaQXNzkqmWPko
xmCJo6tHzC7ubdG8iRCYHzfmahPuuj6EdGPZuSRyYFgJi5Ftz/nAN+84OxtIQ3zn
YWOgskQgaLh9YfsKsQ7Sf1NDOsnOnD5TX7UXl07fEpLe9vNCvAFiU8e5Y9LGudU5
4bYTAoGBAMdX3a3bXp4cZvXNBJ/QLVyxC6fP1Q4haCR1Od3m+T00Jth2IX2dk/fl
p6xiJT1av5JtYabv1dFKaXOS5s1kLGGuCCSKpkvFZm826aQ2AFm0XGqEQDLeei5b
A52Kpy/YJ+RkG4BTFtAooFq6DmA0cnoP6oPvG2h6XtDJwDTPInJb
-----END RSA PRIVATE KEY-----`
caContent = `-----BEGIN CERTIFICATE-----
MIIDbTCCAlWgAwIBAgIUBJvFoCowKich7MMfseJ+DYzzirowDQYJKoZIhvcNAQEM
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMzAzMTExMzIxMDNaGA8yMTIz
MDIxNTEzMjEwM1owRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN
AQEBBQADggEPADCCAQoCggEBAO4to2YMYj0bxgr2FCiweSTSFuPx33zSw2x/s9Wf
OR41bm2DFsyYT5f3sOIKlXZEdLmOKty2e3ho3yC0EyNpVHdykkkHT3aDI17quZax
kYi/URqqtl1Z08A22txolc04hAZisg2BypGi3vql81UW1t3zyloGnJoIAeXR9uca
ljP6Bk3bwsxoVBLi1JtHrO0hHLQaeHmKhAyrys06X0LRdn7Px48yRZlt6FaLSa8X
YiRM0G44bVy/h6BkoQjMYGwVmCVk6zjJ9U7ZPFqdnDMNxAfR+hjDnYodqdLDMTTR
1NPVrnEnNwFx0AMLvgt/ba/45vZCEAmSZnFXFAJJcM7ai9ECAwEAAaNTMFEwHQYD
VR0OBBYEFHlvMPKZoZ3G0YE7JePJOP+8J38fMB8GA1UdIwQYMBaAFHlvMPKZoZ3G
0YE7JePJOP+8J38fMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggEB
AMX8dNulADOo9uQgBMyFb9TVra7iY0zZjzv4GY5XY7scd52n6CnfAPvYBBDnTr/O
BgNp5jaujb4+9u/2qhV3f9n+/3WOb2CmPehBgVSzlXqHeQ9lshmgwZPeem2T+8Tm
Nnc/xQnsUfCFszUDxpkr55+aLVM22j02RWqcZ4q7TAaVYL+kdFVMc8FoqG/0ro6A
BjE/Qn0Nn7ciX1VUjDt8l+k7ummPJTmzdi6i6E4AwO9dzrGNgGJ4aWL8cC6xYcIX
goVIRTFeONXSDno/oPjWHpIPt7L15heMpKBHNuzPkKx2YVqPHE5QZxWfS+Lzgx+Q
E2oTTM0rYKOZ8p6000mhvKI=
-----END CERTIFICATE-----`
"github.com/tal-tech/go-zero/core/stringx"
)
func TestAccount(t *testing.T) {
@@ -105,34 +32,3 @@ func TestAccount(t *testing.T) {
assert.Equal(t, username, account.User)
assert.Equal(t, anotherPassword, account.Pass)
}
func TestTLSMethods(t *testing.T) {
certFile := createTempFile(t, []byte(certContent))
defer os.Remove(certFile)
keyFile := createTempFile(t, []byte(keyContent))
defer os.Remove(keyFile)
caFile := createTempFile(t, []byte(caContent))
defer os.Remove(caFile)
assert.NoError(t, AddTLS([]string{"foo"}, certFile, keyFile, caFile, false))
cfg, ok := GetTLS([]string{"foo"})
assert.True(t, ok)
assert.NotNil(t, cfg)
assert.Error(t, AddTLS([]string{"bar"}, "bad-file", keyFile, caFile, false))
assert.Error(t, AddTLS([]string{"bar"}, certFile, keyFile, "bad-file", false))
}
func createTempFile(t *testing.T, body []byte) string {
tmpFile, err := os.CreateTemp(os.TempDir(), "go-unit-*.tmp")
if err != nil {
t.Fatal(err)
}
tmpFile.Close()
if err = os.WriteFile(tmpFile.Name(), body, os.ModePerm); err != nil {
t.Fatal(err)
}
return tmpFile.Name()
}

View File

@@ -81,7 +81,7 @@ func (mr *MockEtcdClientMockRecorder) Ctx() *gomock.Call {
// Get mocks base method
func (m *MockEtcdClient) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
m.ctrl.T.Helper()
varargs := []any{ctx, key}
varargs := []interface{}{ctx, key}
for _, a := range opts {
varargs = append(varargs, a)
}
@@ -92,9 +92,9 @@ func (m *MockEtcdClient) Get(ctx context.Context, key string, opts ...clientv3.O
}
// Get indicates an expected call of Get
func (mr *MockEtcdClientMockRecorder) Get(ctx, key any, opts ...any) *gomock.Call {
func (mr *MockEtcdClientMockRecorder) Get(ctx, key interface{}, opts ...interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{ctx, key}, opts...)
varargs := append([]interface{}{ctx, key}, opts...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockEtcdClient)(nil).Get), varargs...)
}
@@ -108,7 +108,7 @@ func (m *MockEtcdClient) Grant(ctx context.Context, ttl int64) (*clientv3.LeaseG
}
// Grant indicates an expected call of Grant
func (mr *MockEtcdClientMockRecorder) Grant(ctx, ttl any) *gomock.Call {
func (mr *MockEtcdClientMockRecorder) Grant(ctx, ttl interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Grant", reflect.TypeOf((*MockEtcdClient)(nil).Grant), ctx, ttl)
}
@@ -123,7 +123,7 @@ func (m *MockEtcdClient) KeepAlive(ctx context.Context, id clientv3.LeaseID) (<-
}
// KeepAlive indicates an expected call of KeepAlive
func (mr *MockEtcdClientMockRecorder) KeepAlive(ctx, id any) *gomock.Call {
func (mr *MockEtcdClientMockRecorder) KeepAlive(ctx, id interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAlive", reflect.TypeOf((*MockEtcdClient)(nil).KeepAlive), ctx, id)
}
@@ -131,7 +131,7 @@ func (mr *MockEtcdClientMockRecorder) KeepAlive(ctx, id any) *gomock.Call {
// Put mocks base method
func (m *MockEtcdClient) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
m.ctrl.T.Helper()
varargs := []any{ctx, key, val}
varargs := []interface{}{ctx, key, val}
for _, a := range opts {
varargs = append(varargs, a)
}
@@ -142,9 +142,9 @@ func (m *MockEtcdClient) Put(ctx context.Context, key, val string, opts ...clien
}
// Put indicates an expected call of Put
func (mr *MockEtcdClientMockRecorder) Put(ctx, key, val any, opts ...any) *gomock.Call {
func (mr *MockEtcdClientMockRecorder) Put(ctx, key, val interface{}, opts ...interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{ctx, key, val}, opts...)
varargs := append([]interface{}{ctx, key, val}, opts...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockEtcdClient)(nil).Put), varargs...)
}
@@ -158,7 +158,7 @@ func (m *MockEtcdClient) Revoke(ctx context.Context, id clientv3.LeaseID) (*clie
}
// Revoke indicates an expected call of Revoke
func (mr *MockEtcdClientMockRecorder) Revoke(ctx, id any) *gomock.Call {
func (mr *MockEtcdClientMockRecorder) Revoke(ctx, id interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Revoke", reflect.TypeOf((*MockEtcdClient)(nil).Revoke), ctx, id)
}
@@ -166,7 +166,7 @@ func (mr *MockEtcdClientMockRecorder) Revoke(ctx, id any) *gomock.Call {
// Watch mocks base method
func (m *MockEtcdClient) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {
m.ctrl.T.Helper()
varargs := []any{ctx, key}
varargs := []interface{}{ctx, key}
for _, a := range opts {
varargs = append(varargs, a)
}
@@ -176,8 +176,8 @@ func (m *MockEtcdClient) Watch(ctx context.Context, key string, opts ...clientv3
}
// Watch indicates an expected call of Watch
func (mr *MockEtcdClientMockRecorder) Watch(ctx, key any, opts ...any) *gomock.Call {
func (mr *MockEtcdClientMockRecorder) Watch(ctx, key interface{}, opts ...interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{ctx, key}, opts...)
varargs := append([]interface{}{ctx, key}, opts...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockEtcdClient)(nil).Watch), varargs...)
}

View File

@@ -2,7 +2,6 @@ package internal
import (
"context"
"errors"
"fmt"
"io"
"sort"
@@ -10,12 +9,11 @@ import (
"sync"
"time"
"github.com/zeromicro/go-zero/core/contextx"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/zeromicro/go-zero/core/threading"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
"github.com/tal-tech/go-zero/core/contextx"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/syncx"
"github.com/tal-tech/go-zero/core/threading"
clientv3 "go.etcd.io/etcd/client/v3"
)
@@ -24,7 +22,6 @@ var (
clusters: make(map[string]*cluster),
}
connManager = syncx.NewResourceManager()
errClosed = errors.New("etcd monitor chan has been closed")
)
// A Registry is a registry that manages the etcd client connections.
@@ -194,11 +191,9 @@ func (c *cluster) handleWatchEvents(key string, events []*clientv3.Event) {
})
}
case clientv3.EventTypeDelete:
c.lock.Lock()
if vals, ok := c.values[key]; ok {
delete(vals, string(ev.Kv.Key))
}
c.lock.Unlock()
for _, l := range listeners {
l.OnDelete(KV{
Key: string(ev.Kv.Key),
@@ -211,7 +206,7 @@ func (c *cluster) handleWatchEvents(key string, events []*clientv3.Event) {
}
}
func (c *cluster) load(cli EtcdClient, key string) int64 {
func (c *cluster) load(cli EtcdClient, key string) {
var resp *clientv3.GetResponse
for {
var err error
@@ -235,8 +230,6 @@ func (c *cluster) load(cli EtcdClient, key string) int64 {
}
c.handleChanges(key, kvs)
return resp.Header.Revision
}
func (c *cluster) monitor(key string, l UpdateListener) error {
@@ -249,9 +242,9 @@ func (c *cluster) monitor(key string, l UpdateListener) error {
return err
}
rev := c.load(cli, key)
c.load(cli, key)
c.watchGroup.Run(func() {
c.watch(cli, key, rev)
c.watch(cli, key)
})
return nil
@@ -283,55 +276,41 @@ func (c *cluster) reload(cli EtcdClient) {
for _, key := range keys {
k := key
c.watchGroup.Run(func() {
rev := c.load(cli, k)
c.watch(cli, k, rev)
c.load(cli, k)
c.watch(cli, k)
})
}
}
func (c *cluster) watch(cli EtcdClient, key string, rev int64) {
func (c *cluster) watch(cli EtcdClient, key string) {
for {
err := c.watchStream(cli, key, rev)
if err == nil {
if c.watchStream(cli, key) {
return
}
if rev != 0 && errors.Is(err, rpctypes.ErrCompacted) {
logx.Errorf("etcd watch stream has been compacted, try to reload, rev %d", rev)
rev = c.load(cli, key)
}
// log the error and retry
logx.Error(err)
}
}
func (c *cluster) watchStream(cli EtcdClient, key string, rev int64) error {
var rch clientv3.WatchChan
if rev != 0 {
rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key),
clientv3.WithPrefix(), clientv3.WithRev(rev+1))
} else {
rch = cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key),
clientv3.WithPrefix())
}
func (c *cluster) watchStream(cli EtcdClient, key string) bool {
rch := cli.Watch(clientv3.WithRequireLeader(c.context(cli)), makeKeyPrefix(key), clientv3.WithPrefix())
for {
select {
case wresp, ok := <-rch:
if !ok {
return errClosed
logx.Error("etcd monitor chan has been closed")
return false
}
if wresp.Canceled {
return fmt.Errorf("etcd monitor chan has been canceled, error: %w", wresp.Err())
logx.Errorf("etcd monitor chan has been canceled, error: %v", wresp.Err())
return false
}
if wresp.Err() != nil {
return fmt.Errorf("etcd monitor chan error: %w", wresp.Err())
logx.Error(fmt.Sprintf("etcd monitor chan error: %v", wresp.Err()))
return false
}
c.handleWatchEvents(key, wresp.Events)
case <-c.done:
return nil
return true
}
}
}
@@ -347,11 +326,12 @@ func (c *cluster) watchConnState(cli EtcdClient) {
// DialClient dials an etcd cluster with given endpoints.
func DialClient(endpoints []string) (EtcdClient, error) {
cfg := clientv3.Config{
Endpoints: endpoints,
AutoSyncInterval: autoSyncInterval,
DialTimeout: DialTimeout,
RejectOldCluster: true,
PermitWithoutStream: true,
Endpoints: endpoints,
AutoSyncInterval: autoSyncInterval,
DialTimeout: DialTimeout,
DialKeepAliveTime: dialKeepAliveTime,
DialKeepAliveTimeout: DialTimeout,
RejectOldCluster: true,
}
if account, ok := GetAccount(endpoints); ok {
cfg.Username = account.User

View File

@@ -2,21 +2,17 @@ package internal
import (
"context"
"os"
"sync"
"testing"
"time"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/contextx"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/stringx"
"go.etcd.io/etcd/api/v3/etcdserverpb"
"github.com/tal-tech/go-zero/core/contextx"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/stringx"
"go.etcd.io/etcd/api/v3/mvccpb"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/mock/mockserver"
)
var mockLock sync.Mutex
@@ -116,7 +112,6 @@ func TestCluster_Load(t *testing.T) {
restore := setMockClient(cli)
defer restore()
cli.EXPECT().Get(gomock.Any(), "any/", gomock.Any()).Return(&clientv3.GetResponse{
Header: &etcdserverpb.ResponseHeader{},
Kvs: []*mvccpb.KeyValue{
{
Key: []byte("hello"),
@@ -170,10 +165,10 @@ func TestCluster_Watch(t *testing.T) {
assert.Equal(t, "world", kv.Val)
wg.Done()
}).MaxTimes(1)
listener.EXPECT().OnDelete(gomock.Any()).Do(func(_ any) {
listener.EXPECT().OnDelete(gomock.Any()).Do(func(_ interface{}) {
wg.Done()
}).MaxTimes(1)
go c.watch(cli, "any", 0)
go c.watch(cli, "any")
ch <- clientv3.WatchResponse{
Events: []*clientv3.Event{
{
@@ -217,7 +212,7 @@ func TestClusterWatch_RespFailures(t *testing.T) {
ch <- resp
close(c.done)
}()
c.watch(cli, "any", 0)
c.watch(cli, "any")
})
}
}
@@ -237,7 +232,7 @@ func TestClusterWatch_CloseChan(t *testing.T) {
close(ch)
close(c.done)
}()
c.watch(cli, "any", 0)
c.watch(cli, "any")
}
func TestValueOnlyContext(t *testing.T) {
@@ -245,58 +240,3 @@ func TestValueOnlyContext(t *testing.T) {
ctx.Done()
assert.Nil(t, ctx.Err())
}
func TestDialClient(t *testing.T) {
svr, err := mockserver.StartMockServers(1)
assert.NoError(t, err)
svr.StartAt(0)
certFile := createTempFile(t, []byte(certContent))
defer os.Remove(certFile)
keyFile := createTempFile(t, []byte(keyContent))
defer os.Remove(keyFile)
caFile := createTempFile(t, []byte(caContent))
defer os.Remove(caFile)
endpoints := []string{svr.Servers[0].Address}
AddAccount(endpoints, "foo", "bar")
assert.NoError(t, AddTLS(endpoints, certFile, keyFile, caFile, false))
old := DialTimeout
DialTimeout = time.Millisecond
defer func() {
DialTimeout = old
}()
_, err = DialClient(endpoints)
assert.Error(t, err)
}
func TestRegistry_Monitor(t *testing.T) {
svr, err := mockserver.StartMockServers(1)
assert.NoError(t, err)
svr.StartAt(0)
endpoints := []string{svr.Servers[0].Address}
GetRegistry().lock.Lock()
GetRegistry().clusters = map[string]*cluster{
getClusterKey(endpoints): {
listeners: map[string][]UpdateListener{},
values: map[string]map[string]string{
"foo": {
"bar": "baz",
},
},
},
}
GetRegistry().lock.Unlock()
assert.Error(t, GetRegistry().Monitor(endpoints, "foo", new(mockListener)))
}
type mockListener struct {
}
func (m *mockListener) OnAdd(_ KV) {
}
func (m *mockListener) OnDelete(_ KV) {
}

View File

@@ -58,7 +58,7 @@ func (m *MocketcdConn) WaitForStateChange(ctx context.Context, sourceState conne
}
// WaitForStateChange indicates an expected call of WaitForStateChange
func (mr *MocketcdConnMockRecorder) WaitForStateChange(ctx, sourceState any) *gomock.Call {
func (mr *MocketcdConnMockRecorder) WaitForStateChange(ctx, sourceState interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForStateChange", reflect.TypeOf((*MocketcdConn)(nil).WaitForStateChange), ctx, sourceState)
}

View File

@@ -40,7 +40,7 @@ func (m *MockUpdateListener) OnAdd(kv KV) {
}
// OnAdd indicates an expected call of OnAdd
func (mr *MockUpdateListenerMockRecorder) OnAdd(kv any) *gomock.Call {
func (mr *MockUpdateListenerMockRecorder) OnAdd(kv interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnAdd", reflect.TypeOf((*MockUpdateListener)(nil).OnAdd), kv)
}
@@ -52,7 +52,7 @@ func (m *MockUpdateListener) OnDelete(kv KV) {
}
// OnDelete indicates an expected call of OnDelete
func (mr *MockUpdateListenerMockRecorder) OnDelete(kv any) *gomock.Call {
func (mr *MockUpdateListenerMockRecorder) OnDelete(kv interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnDelete", reflect.TypeOf((*MockUpdateListener)(nil).OnDelete), kv)
}

View File

@@ -9,6 +9,7 @@ const (
autoSyncInterval = time.Minute
coolDownInterval = time.Second
dialTimeout = 5 * time.Second
dialKeepAliveTime = 5 * time.Second
requestTimeout = 3 * time.Second
endpointsSeparator = ","
)

View File

@@ -1,14 +1,12 @@
package discov
import (
"time"
"github.com/zeromicro/go-zero/core/discov/internal"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/proc"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/zeromicro/go-zero/core/threading"
"github.com/tal-tech/go-zero/core/discov/internal"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/proc"
"github.com/tal-tech/go-zero/core/syncx"
"github.com/tal-tech/go-zero/core/threading"
clientv3 "go.etcd.io/etcd/client/v3"
)
@@ -53,7 +51,12 @@ func NewPublisher(endpoints []string, key, value string, opts ...PubOption) *Pub
// KeepAlive keeps key:value alive.
func (p *Publisher) KeepAlive() error {
cli, err := p.doRegister()
cli, err := internal.GetRegistry().GetConn(p.endpoints)
if err != nil {
return err
}
p.lease, err = p.register(cli)
if err != nil {
return err
}
@@ -80,43 +83,6 @@ func (p *Publisher) Stop() {
p.quit.Close()
}
func (p *Publisher) doKeepAlive() error {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for range ticker.C {
select {
case <-p.quit.Done():
return nil
default:
cli, err := p.doRegister()
if err != nil {
logx.Errorf("etcd publisher doRegister: %s", err.Error())
break
}
if err := p.keepAliveAsync(cli); err != nil {
logx.Errorf("etcd publisher keepAliveAsync: %s", err.Error())
break
}
return nil
}
}
return nil
}
func (p *Publisher) doRegister() (internal.EtcdClient, error) {
cli, err := internal.GetRegistry().GetConn(p.endpoints)
if err != nil {
return nil, err
}
p.lease, err = p.register(cli)
return cli, err
}
func (p *Publisher) keepAliveAsync(cli internal.EtcdClient) error {
ch, err := cli.KeepAlive(cli.Ctx(), p.lease)
if err != nil {
@@ -129,8 +95,8 @@ func (p *Publisher) keepAliveAsync(cli internal.EtcdClient) error {
case _, ok := <-ch:
if !ok {
p.revoke(cli)
if err := p.doKeepAlive(); err != nil {
logx.Errorf("etcd publisher KeepAlive: %s", err.Error())
if err := p.KeepAlive(); err != nil {
logx.Errorf("KeepAlive: %s", err.Error())
}
return
}
@@ -139,8 +105,8 @@ func (p *Publisher) keepAliveAsync(cli internal.EtcdClient) error {
p.revoke(cli)
select {
case <-p.resumeChan:
if err := p.doKeepAlive(); err != nil {
logx.Errorf("etcd publisher KeepAlive: %s", err.Error())
if err := p.KeepAlive(); err != nil {
logx.Errorf("KeepAlive: %s", err.Error())
}
return
case <-p.quit.Done():
@@ -175,7 +141,7 @@ func (p *Publisher) register(client internal.EtcdClient) (clientv3.LeaseID, erro
func (p *Publisher) revoke(cli internal.EtcdClient) {
if _, err := cli.Revoke(cli.Ctx(), p.lease); err != nil {
logx.Errorf("etcd publisher revoke: %s", err.Error())
logx.Error(err)
}
}

View File

@@ -1,98 +1,18 @@
package discov
import (
"context"
"errors"
"net"
"os"
"sync"
"testing"
"time"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/discov/internal"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/discov/internal"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/stringx"
clientv3 "go.etcd.io/etcd/client/v3"
"golang.org/x/net/http2"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/resolver/manual"
)
const (
certContent = `-----BEGIN CERTIFICATE-----
MIIDazCCAlOgAwIBAgIUEg9GVO2oaPn+YSmiqmFIuAo10WIwDQYJKoZIhvcNAQEM
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMzAzMTExMzIxMjNaGA8yMTIz
MDIxNTEzMjEyM1owRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN
AQEBBQADggEPADCCAQoCggEBALplXlWsIf0O/IgnIplmiZHKGnxyfyufyE2FBRNk
OofRqbKuPH8GNqbkvZm7N29fwTDAQ+mViAggCkDht4hOzoWJMA7KYJt8JnTSWL48
M1lcrpc9DL2gszC/JF/FGvyANbBtLklkZPFBGdHUX14pjrT937wqPtm+SqUHSvRT
B7bmwmm2drRcmhpVm98LSlV7uQ2EgnJgsLjBPITKUejLmVLHfgX0RwQ2xIpX9pS4
FCe1BTacwl2gGp7Mje7y4Mfv3o0ArJW6Tuwbjx59ZXwb1KIP71b7bT04AVS8ZeYO
UMLKKuB5UR9x9Rn6cLXOTWBpcMVyzDgrAFLZjnE9LPUolZMCAwEAAaNRME8wHwYD
VR0jBBgwFoAUeW8w8pmhncbRgTsl48k4/7wnfx8wCQYDVR0TBAIwADALBgNVHQ8E
BAMCBPAwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBDAUAA4IBAQAI
y9xaoS88CLPBsX6mxfcTAFVfGNTRW9VN9Ng1cCnUR+YGoXGM/l+qP4f7p8ocdGwK
iYZErVTzXYIn+D27//wpY3klJk3gAnEUBT3QRkStBw7XnpbeZ2oPBK+cmDnCnZPS
BIF1wxPX7vIgaxs5Zsdqwk3qvZ4Djr2wP7LabNWTLSBKgQoUY45Liw6pffLwcGF9
UKlu54bvGze2SufISCR3ib+I+FLvqpvJhXToZWYb/pfI/HccuCL1oot1x8vx6DQy
U+TYxlZsKS5mdNxAX3dqEkEMsgEi+g/tzDPXJImfeCGGBhIOXLm8SRypiuGdEbc9
xkWYxRPegajuEZGvCqVs
-----END CERTIFICATE-----`
keyContent = `-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAumVeVawh/Q78iCcimWaJkcoafHJ/K5/ITYUFE2Q6h9Gpsq48
fwY2puS9mbs3b1/BMMBD6ZWICCAKQOG3iE7OhYkwDspgm3wmdNJYvjwzWVyulz0M
vaCzML8kX8Ua/IA1sG0uSWRk8UEZ0dRfXimOtP3fvCo+2b5KpQdK9FMHtubCabZ2
tFyaGlWb3wtKVXu5DYSCcmCwuME8hMpR6MuZUsd+BfRHBDbEilf2lLgUJ7UFNpzC
XaAansyN7vLgx+/ejQCslbpO7BuPHn1lfBvUog/vVvttPTgBVLxl5g5Qwsoq4HlR
H3H1Gfpwtc5NYGlwxXLMOCsAUtmOcT0s9SiVkwIDAQABAoIBAD5meTJNMgO55Kjg
ESExxpRcCIno+tHr5+6rvYtEXqPheOIsmmwb9Gfi4+Z3WpOaht5/Pz0Ppj6yGzyl
U//6AgGKb+BDuBvVcDpjwPnOxZIBCSHwejdxeQu0scSuA97MPS0XIAvJ5FEv7ijk
5Bht6SyGYURpECltHygoTNuGgGqmO+McCJRLE9L09lTBI6UQ/JQwWJqSr7wx6iPU
M1Ze/srIV+7cyEPu6i0DGjS1gSQKkX68Lqn1w6oE290O+OZvleO0gZ02fLDWCZke
aeD9+EU/Pw+rqm3H6o0szOFIpzhRp41FUdW9sybB3Yp3u7c/574E+04Z/e30LMKs
TCtE1QECgYEA3K7KIpw0NH2HXL5C3RHcLmr204xeBfS70riBQQuVUgYdmxak2ima
80RInskY8hRhSGTg0l+VYIH8cmjcUyqMSOELS5XfRH99r4QPiK8AguXg80T4VumY
W3Pf+zEC2ssgP/gYthV0g0Xj5m2QxktOF9tRw5nkg739ZR4dI9lm/iECgYEA2Dnf
uwEDGqHiQRF6/fh5BG/nGVMvrefkqx6WvTJQ3k/M/9WhxB+lr/8yH46TuS8N2b29
FoTf3Mr9T7pr/PWkOPzoY3P56nYbKU8xSwCim9xMzhBMzj8/N9ukJvXy27/VOz56
eQaKqnvdXNGtPJrIMDGHps2KKWlKLyAlapzjVTMCgYAA/W++tACv85g13EykfT4F
n0k4LbsGP9DP4zABQLIMyiY72eAncmRVjwrcW36XJ2xATOONTgx3gF3HjZzfaqNy
eD/6uNNllUTVEryXGmHgNHPL45VRnn6memCY2eFvZdXhM5W4y2PYaunY0MkDercA
+GTngbs6tBF88KOk04bYwQKBgFl68cRgsdkmnwwQYNaTKfmVGYzYaQXNzkqmWPko
xmCJo6tHzC7ubdG8iRCYHzfmahPuuj6EdGPZuSRyYFgJi5Ftz/nAN+84OxtIQ3zn
YWOgskQgaLh9YfsKsQ7Sf1NDOsnOnD5TX7UXl07fEpLe9vNCvAFiU8e5Y9LGudU5
4bYTAoGBAMdX3a3bXp4cZvXNBJ/QLVyxC6fP1Q4haCR1Od3m+T00Jth2IX2dk/fl
p6xiJT1av5JtYabv1dFKaXOS5s1kLGGuCCSKpkvFZm826aQ2AFm0XGqEQDLeei5b
A52Kpy/YJ+RkG4BTFtAooFq6DmA0cnoP6oPvG2h6XtDJwDTPInJb
-----END RSA PRIVATE KEY-----`
caContent = `-----BEGIN CERTIFICATE-----
MIIDbTCCAlWgAwIBAgIUBJvFoCowKich7MMfseJ+DYzzirowDQYJKoZIhvcNAQEM
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMzAzMTExMzIxMDNaGA8yMTIz
MDIxNTEzMjEwM1owRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN
AQEBBQADggEPADCCAQoCggEBAO4to2YMYj0bxgr2FCiweSTSFuPx33zSw2x/s9Wf
OR41bm2DFsyYT5f3sOIKlXZEdLmOKty2e3ho3yC0EyNpVHdykkkHT3aDI17quZax
kYi/URqqtl1Z08A22txolc04hAZisg2BypGi3vql81UW1t3zyloGnJoIAeXR9uca
ljP6Bk3bwsxoVBLi1JtHrO0hHLQaeHmKhAyrys06X0LRdn7Px48yRZlt6FaLSa8X
YiRM0G44bVy/h6BkoQjMYGwVmCVk6zjJ9U7ZPFqdnDMNxAfR+hjDnYodqdLDMTTR
1NPVrnEnNwFx0AMLvgt/ba/45vZCEAmSZnFXFAJJcM7ai9ECAwEAAaNTMFEwHQYD
VR0OBBYEFHlvMPKZoZ3G0YE7JePJOP+8J38fMB8GA1UdIwQYMBaAFHlvMPKZoZ3G
0YE7JePJOP+8J38fMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggEB
AMX8dNulADOo9uQgBMyFb9TVra7iY0zZjzv4GY5XY7scd52n6CnfAPvYBBDnTr/O
BgNp5jaujb4+9u/2qhV3f9n+/3WOb2CmPehBgVSzlXqHeQ9lshmgwZPeem2T+8Tm
Nnc/xQnsUfCFszUDxpkr55+aLVM22j02RWqcZ4q7TAaVYL+kdFVMc8FoqG/0ro6A
BjE/Qn0Nn7ciX1VUjDt8l+k7ummPJTmzdi6i6E4AwO9dzrGNgGJ4aWL8cC6xYcIX
goVIRTFeONXSDno/oPjWHpIPt7L15heMpKBHNuzPkKx2YVqPHE5QZxWfS+Lzgx+Q
E2oTTM0rYKOZ8p6000mhvKI=
-----END CERTIFICATE-----`
)
func init() {
@@ -117,7 +37,7 @@ func TestPublisher_register(t *testing.T) {
assert.Nil(t, err)
}
func TestPublisher_registerWithOptions(t *testing.T) {
func TestPublisher_registerWithId(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
const id = 2
@@ -129,15 +49,7 @@ func TestPublisher_registerWithOptions(t *testing.T) {
ID: 1,
}, nil)
cli.EXPECT().Put(gomock.Any(), makeEtcdKey("thekey", id), "thevalue", gomock.Any())
certFile := createTempFile(t, []byte(certContent))
defer os.Remove(certFile)
keyFile := createTempFile(t, []byte(keyContent))
defer os.Remove(keyFile)
caFile := createTempFile(t, []byte(caContent))
defer os.Remove(caFile)
pub := NewPublisher(nil, "thekey", "thevalue", WithId(id),
WithPubEtcdTLS(certFile, keyFile, caFile, true))
pub := NewPublisher(nil, "thekey", "thevalue", WithId(id))
_, err := pub.register(cli)
assert.Nil(t, err)
}
@@ -213,7 +125,7 @@ func TestPublisher_keepAliveAsyncQuit(t *testing.T) {
cli.EXPECT().KeepAlive(gomock.Any(), id)
var wg sync.WaitGroup
wg.Add(1)
cli.EXPECT().Revoke(gomock.Any(), id).Do(func(_, _ any) {
cli.EXPECT().Revoke(gomock.Any(), id).Do(func(_, _ interface{}) {
wg.Done()
})
pub := NewPublisher(nil, "thekey", "thevalue")
@@ -235,7 +147,7 @@ func TestPublisher_keepAliveAsyncPause(t *testing.T) {
pub := NewPublisher(nil, "thekey", "thevalue")
var wg sync.WaitGroup
wg.Add(1)
cli.EXPECT().Revoke(gomock.Any(), id).Do(func(_, _ any) {
cli.EXPECT().Revoke(gomock.Any(), id).Do(func(_, _ interface{}) {
pub.Stop()
wg.Done()
})
@@ -257,92 +169,3 @@ func TestPublisher_Resume(t *testing.T) {
}()
<-publisher.resumeChan
}
func TestPublisher_keepAliveAsync(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
const id clientv3.LeaseID = 1
conn := createMockConn(t)
defer conn.Close()
cli := internal.NewMockEtcdClient(ctrl)
cli.EXPECT().ActiveConnection().Return(conn).AnyTimes()
cli.EXPECT().Close()
defer cli.Close()
cli.ActiveConnection()
restore := setMockClient(cli)
defer restore()
cli.EXPECT().Ctx().AnyTimes()
cli.EXPECT().KeepAlive(gomock.Any(), id)
cli.EXPECT().Grant(gomock.Any(), timeToLive).Return(&clientv3.LeaseGrantResponse{
ID: 1,
}, nil)
cli.EXPECT().Put(gomock.Any(), makeEtcdKey("thekey", int64(id)), "thevalue", gomock.Any())
var wg sync.WaitGroup
wg.Add(1)
cli.EXPECT().Revoke(gomock.Any(), id).Do(func(_, _ any) {
wg.Done()
})
pub := NewPublisher([]string{"the-endpoint"}, "thekey", "thevalue")
pub.lease = id
assert.Nil(t, pub.KeepAlive())
pub.Stop()
wg.Wait()
}
func createMockConn(t *testing.T) *grpc.ClientConn {
lis, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("Error while listening. Err: %v", err)
}
defer lis.Close()
lisAddr := resolver.Address{Addr: lis.Addr().String()}
lisDone := make(chan struct{})
dialDone := make(chan struct{})
// 1st listener accepts the connection and then does nothing
go func() {
defer close(lisDone)
conn, err := lis.Accept()
if err != nil {
t.Errorf("Error while accepting. Err: %v", err)
return
}
framer := http2.NewFramer(conn, conn)
if err := framer.WriteSettings(http2.Setting{}); err != nil {
t.Errorf("Error while writing settings. Err: %v", err)
return
}
<-dialDone // Close conn only after dial returns.
}()
r := manual.NewBuilderWithScheme("whatever")
r.InitialState(resolver.State{Addresses: []resolver.Address{lisAddr}})
client, err := grpc.DialContext(context.Background(), r.Scheme()+":///test.server",
grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
close(dialDone)
if err != nil {
t.Fatalf("Dial failed. Err: %v", err)
}
timeout := time.After(1 * time.Second)
select {
case <-timeout:
t.Fatal("timed out waiting for server to finish")
case <-lisDone:
}
return client
}
func createTempFile(t *testing.T, body []byte) string {
tmpFile, err := os.CreateTemp(os.TempDir(), "go-unit-*.tmp")
if err != nil {
t.Fatal(err)
}
tmpFile.Close()
if err = os.WriteFile(tmpFile.Name(), body, os.ModePerm); err != nil {
t.Fatal(err)
}
return tmpFile.Name()
}

View File

@@ -4,16 +4,16 @@ import (
"sync"
"sync/atomic"
"github.com/zeromicro/go-zero/core/discov/internal"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/tal-tech/go-zero/core/discov/internal"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/syncx"
)
type (
// SubOption defines the method to customize a Subscriber.
SubOption func(sub *Subscriber)
// A Subscriber is used to subscribe the given key on an etcd cluster.
// A Subscriber is used to subscribe the given key on a etcd cluster.
Subscriber struct {
endpoints []string
exclusive bool

View File

@@ -5,8 +5,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/discov/internal"
"github.com/zeromicro/go-zero/core/stringx"
"github.com/tal-tech/go-zero/core/discov/internal"
"github.com/tal-tech/go-zero/core/stringx"
)
const (

View File

@@ -11,12 +11,10 @@ type (
errorArray []error
)
// Add adds errs to be, nil errors are ignored.
func (be *BatchError) Add(errs ...error) {
for _, err := range errs {
if err != nil {
be.errs = append(be.errs, err)
}
// Add adds err to be.
func (be *BatchError) Add(err error) {
if err != nil {
be.errs = append(be.errs, err)
}
}

View File

@@ -1,21 +0,0 @@
package errorx
import "fmt"
// Wrap returns an error that wraps err with given message.
func Wrap(err error, message string) error {
if err == nil {
return nil
}
return fmt.Errorf("%s: %w", message, err)
}
// Wrapf returns an error that wraps err with given format and args.
func Wrapf(err error, format string, args ...any) error {
if err == nil {
return nil
}
return fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), err)
}

View File

@@ -1,24 +0,0 @@
package errorx
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
)
func TestWrap(t *testing.T) {
assert.Nil(t, Wrap(nil, "test"))
assert.Equal(t, "foo: bar", Wrap(errors.New("bar"), "foo").Error())
err := errors.New("foo")
assert.True(t, errors.Is(Wrap(err, "bar"), err))
}
func TestWrapf(t *testing.T) {
assert.Nil(t, Wrapf(nil, "%s", "test"))
assert.Equal(t, "foo bar: quz", Wrapf(errors.New("quz"), "foo %s", "bar").Error())
err := errors.New("foo")
assert.True(t, errors.Is(Wrapf(err, "foo %s", "bar"), err))
}

View File

@@ -42,7 +42,7 @@ func NewBulkExecutor(execute Execute, opts ...BulkOption) *BulkExecutor {
}
// Add adds task into be.
func (be *BulkExecutor) Add(task any) error {
func (be *BulkExecutor) Add(task interface{}) error {
be.executor.Add(task)
return nil
}
@@ -79,22 +79,22 @@ func newBulkOptions() bulkOptions {
}
type bulkContainer struct {
tasks []any
tasks []interface{}
execute Execute
maxTasks int
}
func (bc *bulkContainer) AddTask(task any) bool {
func (bc *bulkContainer) AddTask(task interface{}) bool {
bc.tasks = append(bc.tasks, task)
return len(bc.tasks) >= bc.maxTasks
}
func (bc *bulkContainer) Execute(tasks any) {
vals := tasks.([]any)
func (bc *bulkContainer) Execute(tasks interface{}) {
vals := tasks.([]interface{})
bc.execute(vals)
}
func (bc *bulkContainer) RemoveAll() any {
func (bc *bulkContainer) RemoveAll() interface{} {
tasks := bc.tasks
bc.tasks = nil
return tasks

View File

@@ -12,7 +12,7 @@ func TestBulkExecutor(t *testing.T) {
var values []int
var lock sync.Mutex
executor := NewBulkExecutor(func(items []any) {
executor := NewBulkExecutor(func(items []interface{}) {
lock.Lock()
values = append(values, len(items))
lock.Unlock()
@@ -40,7 +40,7 @@ func TestBulkExecutorFlushInterval(t *testing.T) {
var wait sync.WaitGroup
wait.Add(1)
executor := NewBulkExecutor(func(items []any) {
executor := NewBulkExecutor(func(items []interface{}) {
assert.Equal(t, size, len(items))
wait.Done()
}, WithBulkTasks(caches), WithBulkInterval(time.Millisecond*100))
@@ -53,7 +53,7 @@ func TestBulkExecutorFlushInterval(t *testing.T) {
}
func TestBulkExecutorEmpty(t *testing.T) {
NewBulkExecutor(func(items []any) {
NewBulkExecutor(func(items []interface{}) {
assert.Fail(t, "should not called")
}, WithBulkTasks(10), WithBulkInterval(time.Millisecond))
time.Sleep(time.Millisecond * 100)
@@ -67,7 +67,7 @@ func TestBulkExecutorFlush(t *testing.T) {
var wait sync.WaitGroup
wait.Add(1)
be := NewBulkExecutor(func(items []any) {
be := NewBulkExecutor(func(items []interface{}) {
assert.Equal(t, tasks, len(items))
wait.Done()
}, WithBulkTasks(caches), WithBulkInterval(time.Minute))
@@ -78,11 +78,11 @@ func TestBulkExecutorFlush(t *testing.T) {
wait.Wait()
}
func TestBulkExecutorFlushSlowTasks(t *testing.T) {
func TestBuldExecutorFlushSlowTasks(t *testing.T) {
const total = 1500
lock := new(sync.Mutex)
result := make([]any, 0, 10000)
exec := NewBulkExecutor(func(tasks []any) {
result := make([]interface{}, 0, 10000)
exec := NewBulkExecutor(func(tasks []interface{}) {
time.Sleep(time.Millisecond * 100)
lock.Lock()
defer lock.Unlock()
@@ -100,7 +100,7 @@ func TestBulkExecutorFlushSlowTasks(t *testing.T) {
func BenchmarkBulkExecutor(b *testing.B) {
b.ReportAllocs()
be := NewBulkExecutor(func(tasks []any) {
be := NewBulkExecutor(func(tasks []interface{}) {
time.Sleep(time.Millisecond * time.Duration(len(tasks)))
})
for i := 0; i < b.N; i++ {

View File

@@ -42,7 +42,7 @@ func NewChunkExecutor(execute Execute, opts ...ChunkOption) *ChunkExecutor {
}
// Add adds task with given chunk size into ce.
func (ce *ChunkExecutor) Add(task any, size int) error {
func (ce *ChunkExecutor) Add(task interface{}, size int) error {
ce.executor.Add(chunk{
val: task,
size: size,
@@ -82,25 +82,25 @@ func newChunkOptions() chunkOptions {
}
type chunkContainer struct {
tasks []any
tasks []interface{}
execute Execute
size int
maxChunkSize int
}
func (bc *chunkContainer) AddTask(task any) bool {
func (bc *chunkContainer) AddTask(task interface{}) bool {
ck := task.(chunk)
bc.tasks = append(bc.tasks, ck.val)
bc.size += ck.size
return bc.size >= bc.maxChunkSize
}
func (bc *chunkContainer) Execute(tasks any) {
vals := tasks.([]any)
func (bc *chunkContainer) Execute(tasks interface{}) {
vals := tasks.([]interface{})
bc.execute(vals)
}
func (bc *chunkContainer) RemoveAll() any {
func (bc *chunkContainer) RemoveAll() interface{} {
tasks := bc.tasks
bc.tasks = nil
bc.size = 0
@@ -108,6 +108,6 @@ func (bc *chunkContainer) RemoveAll() any {
}
type chunk struct {
val any
val interface{}
size int
}

View File

@@ -12,7 +12,7 @@ func TestChunkExecutor(t *testing.T) {
var values []int
var lock sync.Mutex
executor := NewChunkExecutor(func(items []any) {
executor := NewChunkExecutor(func(items []interface{}) {
lock.Lock()
values = append(values, len(items))
lock.Unlock()
@@ -40,7 +40,7 @@ func TestChunkExecutorFlushInterval(t *testing.T) {
var wait sync.WaitGroup
wait.Add(1)
executor := NewChunkExecutor(func(items []any) {
executor := NewChunkExecutor(func(items []interface{}) {
assert.Equal(t, size, len(items))
wait.Done()
}, WithChunkBytes(caches), WithFlushInterval(time.Millisecond*100))
@@ -53,11 +53,10 @@ func TestChunkExecutorFlushInterval(t *testing.T) {
}
func TestChunkExecutorEmpty(t *testing.T) {
executor := NewChunkExecutor(func(items []any) {
NewChunkExecutor(func(items []interface{}) {
assert.Fail(t, "should not called")
}, WithChunkBytes(10), WithFlushInterval(time.Millisecond))
time.Sleep(time.Millisecond * 100)
executor.Wait()
}
func TestChunkExecutorFlush(t *testing.T) {
@@ -68,7 +67,7 @@ func TestChunkExecutorFlush(t *testing.T) {
var wait sync.WaitGroup
wait.Add(1)
be := NewChunkExecutor(func(items []any) {
be := NewChunkExecutor(func(items []interface{}) {
assert.Equal(t, tasks, len(items))
wait.Done()
}, WithChunkBytes(caches), WithFlushInterval(time.Minute))
@@ -82,7 +81,7 @@ func TestChunkExecutorFlush(t *testing.T) {
func BenchmarkChunkExecutor(b *testing.B) {
b.ReportAllocs()
be := NewChunkExecutor(func(tasks []any) {
be := NewChunkExecutor(func(tasks []interface{}) {
time.Sleep(time.Millisecond * time.Duration(len(tasks)))
})
for i := 0; i < b.N; i++ {

View File

@@ -4,7 +4,7 @@ import (
"sync"
"time"
"github.com/zeromicro/go-zero/core/threading"
"github.com/tal-tech/go-zero/core/threading"
)
// A DelayExecutor delays a tasks on given delay interval.

View File

@@ -3,8 +3,8 @@ package executors
import (
"time"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/zeromicro/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/syncx"
"github.com/tal-tech/go-zero/core/timex"
)
// A LessExecutor is an executor to limit execution once within given time interval.

View File

@@ -5,7 +5,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/timex"
)
func TestLessExecutor_DoOrDiscard(t *testing.T) {

View File

@@ -6,11 +6,11 @@ import (
"sync/atomic"
"time"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/proc"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/zeromicro/go-zero/core/threading"
"github.com/zeromicro/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/proc"
"github.com/tal-tech/go-zero/core/syncx"
"github.com/tal-tech/go-zero/core/threading"
"github.com/tal-tech/go-zero/core/timex"
)
const idleRound = 10
@@ -21,16 +21,16 @@ type (
TaskContainer interface {
// AddTask adds the task into the container.
// Returns true if the container needs to be flushed after the addition.
AddTask(task any) bool
AddTask(task interface{}) bool
// Execute handles the collected tasks by the container when flushing.
Execute(tasks any)
Execute(tasks interface{})
// RemoveAll removes the contained tasks, and return them.
RemoveAll() any
RemoveAll() interface{}
}
// A PeriodicalExecutor is an executor that periodically execute tasks.
PeriodicalExecutor struct {
commander chan any
commander chan interface{}
interval time.Duration
container TaskContainer
waitGroup sync.WaitGroup
@@ -48,7 +48,7 @@ type (
func NewPeriodicalExecutor(interval time.Duration, container TaskContainer) *PeriodicalExecutor {
executor := &PeriodicalExecutor{
// buffer 1 to let the caller go quickly
commander: make(chan any, 1),
commander: make(chan interface{}, 1),
interval: interval,
container: container,
confirmChan: make(chan lang.PlaceholderType),
@@ -64,7 +64,7 @@ func NewPeriodicalExecutor(interval time.Duration, container TaskContainer) *Per
}
// Add adds tasks into pe.
func (pe *PeriodicalExecutor) Add(task any) {
func (pe *PeriodicalExecutor) Add(task interface{}) {
if vals, ok := pe.addAndCheck(task); ok {
pe.commander <- vals
<-pe.confirmChan
@@ -74,14 +74,14 @@ func (pe *PeriodicalExecutor) Add(task any) {
// Flush forces pe to execute tasks.
func (pe *PeriodicalExecutor) Flush() bool {
pe.enterExecution()
return pe.executeTasks(func() any {
return pe.executeTasks(func() interface{} {
pe.lock.Lock()
defer pe.lock.Unlock()
return pe.container.RemoveAll()
}())
}
// Sync lets caller run fn thread-safe with pe, especially for the underlying container.
// Sync lets caller to run fn thread-safe with pe, especially for the underlying container.
func (pe *PeriodicalExecutor) Sync(fn func()) {
pe.lock.Lock()
defer pe.lock.Unlock()
@@ -96,7 +96,7 @@ func (pe *PeriodicalExecutor) Wait() {
})
}
func (pe *PeriodicalExecutor) addAndCheck(task any) (any, bool) {
func (pe *PeriodicalExecutor) addAndCheck(task interface{}) (interface{}, bool) {
pe.lock.Lock()
defer func() {
if !pe.guarded {
@@ -116,7 +116,7 @@ func (pe *PeriodicalExecutor) addAndCheck(task any) (any, bool) {
}
func (pe *PeriodicalExecutor) backgroundFlush() {
go func() {
threading.GoSafe(func() {
// flush before quit goroutine to avoid missing tasks
defer pe.Flush()
@@ -144,7 +144,7 @@ func (pe *PeriodicalExecutor) backgroundFlush() {
}
}
}
}()
})
}
func (pe *PeriodicalExecutor) doneExecution() {
@@ -157,20 +157,18 @@ func (pe *PeriodicalExecutor) enterExecution() {
})
}
func (pe *PeriodicalExecutor) executeTasks(tasks any) bool {
func (pe *PeriodicalExecutor) executeTasks(tasks interface{}) bool {
defer pe.doneExecution()
ok := pe.hasTasks(tasks)
if ok {
threading.RunSafe(func() {
pe.container.Execute(tasks)
})
pe.container.Execute(tasks)
}
return ok
}
func (pe *PeriodicalExecutor) hasTasks(tasks any) bool {
func (pe *PeriodicalExecutor) hasTasks(tasks interface{}) bool {
if tasks == nil {
return false
}

View File

@@ -8,8 +8,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/proc"
"github.com/zeromicro/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/timex"
)
const threshold = 10
@@ -17,22 +16,22 @@ const threshold = 10
type container struct {
interval time.Duration
tasks []int
execute func(tasks any)
execute func(tasks interface{})
}
func newContainer(interval time.Duration, execute func(tasks any)) *container {
func newContainer(interval time.Duration, execute func(tasks interface{})) *container {
return &container{
interval: interval,
execute: execute,
}
}
func (c *container) AddTask(task any) bool {
func (c *container) AddTask(task interface{}) bool {
c.tasks = append(c.tasks, task.(int))
return len(c.tasks) > threshold
}
func (c *container) Execute(tasks any) {
func (c *container) Execute(tasks interface{}) {
if c.execute != nil {
c.execute(tasks)
} else {
@@ -40,7 +39,7 @@ func (c *container) Execute(tasks any) {
}
}
func (c *container) RemoveAll() any {
func (c *container) RemoveAll() interface{} {
tasks := c.tasks
c.tasks = nil
return tasks
@@ -68,7 +67,6 @@ func TestPeriodicalExecutor_QuitGoroutine(t *testing.T) {
ticker.Tick()
ticker.Wait(time.Millisecond * idleRound)
assert.Equal(t, routines, runtime.NumGoroutine())
proc.Shutdown()
}
func TestPeriodicalExecutor_Bulk(t *testing.T) {
@@ -76,7 +74,7 @@ func TestPeriodicalExecutor_Bulk(t *testing.T) {
var vals []int
// avoid data race
var lock sync.Mutex
exec := NewPeriodicalExecutor(time.Millisecond, newContainer(time.Millisecond, func(tasks any) {
exec := NewPeriodicalExecutor(time.Millisecond, newContainer(time.Millisecond, func(tasks interface{}) {
t := tasks.([]int)
for _, each := range t {
lock.Lock()
@@ -108,83 +106,25 @@ func TestPeriodicalExecutor_Bulk(t *testing.T) {
lock.Unlock()
}
func TestPeriodicalExecutor_Panic(t *testing.T) {
// avoid data race
var lock sync.Mutex
ticker := timex.NewFakeTicker()
var (
executedTasks []int
expected []int
)
executor := NewPeriodicalExecutor(time.Millisecond, newContainer(time.Millisecond, func(tasks any) {
tt := tasks.([]int)
lock.Lock()
executedTasks = append(executedTasks, tt...)
lock.Unlock()
if tt[0] == 0 {
panic("test")
}
}))
executor.newTicker = func(duration time.Duration) timex.Ticker {
return ticker
}
for i := 0; i < 30; i++ {
executor.Add(i)
expected = append(expected, i)
}
ticker.Tick()
ticker.Tick()
time.Sleep(time.Millisecond)
lock.Lock()
assert.Equal(t, expected, executedTasks)
lock.Unlock()
}
func TestPeriodicalExecutor_FlushPanic(t *testing.T) {
var (
executedTasks []int
expected []int
lock sync.Mutex
)
executor := NewPeriodicalExecutor(time.Millisecond, newContainer(time.Millisecond, func(tasks any) {
tt := tasks.([]int)
lock.Lock()
executedTasks = append(executedTasks, tt...)
lock.Unlock()
if tt[0] == 0 {
panic("flush panic")
}
}))
for i := 0; i < 8; i++ {
executor.Add(i)
expected = append(expected, i)
}
executor.Flush()
lock.Lock()
assert.Equal(t, expected, executedTasks)
lock.Unlock()
}
func TestPeriodicalExecutor_Wait(t *testing.T) {
var lock sync.Mutex
executor := NewBulkExecutor(func(tasks []any) {
executer := NewBulkExecutor(func(tasks []interface{}) {
lock.Lock()
defer lock.Unlock()
time.Sleep(10 * time.Millisecond)
}, WithBulkTasks(1), WithBulkInterval(time.Second))
for i := 0; i < 10; i++ {
executor.Add(1)
executer.Add(1)
}
executor.Flush()
executor.Wait()
executer.Flush()
executer.Wait()
}
func TestPeriodicalExecutor_WaitFast(t *testing.T) {
const total = 3
var cnt int
var lock sync.Mutex
executor := NewBulkExecutor(func(tasks []any) {
executer := NewBulkExecutor(func(tasks []interface{}) {
defer func() {
cnt++
}()
@@ -193,15 +133,15 @@ func TestPeriodicalExecutor_WaitFast(t *testing.T) {
time.Sleep(10 * time.Millisecond)
}, WithBulkTasks(1), WithBulkInterval(10*time.Millisecond))
for i := 0; i < total; i++ {
executor.Add(2)
executer.Add(2)
}
executor.Flush()
executor.Wait()
executer.Flush()
executer.Wait()
assert.Equal(t, total, cnt)
}
func TestPeriodicalExecutor_Deadlock(t *testing.T) {
executor := NewBulkExecutor(func(tasks []any) {
executor := NewBulkExecutor(func(tasks []interface{}) {
}, WithBulkTasks(1), WithBulkInterval(time.Millisecond))
for i := 0; i < 1e5; i++ {
executor.Add(1)
@@ -209,7 +149,13 @@ func TestPeriodicalExecutor_Deadlock(t *testing.T) {
}
func TestPeriodicalExecutor_hasTasks(t *testing.T) {
ticker := timex.NewFakeTicker()
defer ticker.Stop()
exec := NewPeriodicalExecutor(time.Millisecond, newContainer(time.Millisecond, nil))
exec.newTicker = func(d time.Duration) timex.Ticker {
return ticker
}
assert.False(t, exec.hasTasks(nil))
assert.True(t, exec.hasTasks(1))
}

View File

@@ -5,4 +5,4 @@ import "time"
const defaultFlushInterval = time.Second
// Execute defines the method to execute tasks.
type Execute func(tasks []any)
type Execute func(tasks []interface{})

View File

@@ -5,7 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/fs"
"github.com/tal-tech/go-zero/core/fs"
)
const (
@@ -74,11 +74,6 @@ func TestFirstLineShort(t *testing.T) {
assert.Equal(t, "first line", val)
}
func TestFirstLineError(t *testing.T) {
_, err := FirstLine("/tmp/does-not-exist")
assert.Error(t, err)
}
func TestLastLine(t *testing.T) {
filename, err := fs.TempFilenameWithText(text)
assert.Nil(t, err)
@@ -118,8 +113,3 @@ func TestLastLineWithLastNewlineShort(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, "last line", val)
}
func TestLastLineError(t *testing.T) {
_, err := LastLine("/tmp/does-not-exist")
assert.Error(t, err)
}

View File

@@ -5,7 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/fs"
"github.com/tal-tech/go-zero/core/fs"
)
func TestSplitLineChunks(t *testing.T) {

View File

@@ -5,9 +5,6 @@ import (
"os"
)
// errExceedFileSize indicates that the file size is exceeded.
var errExceedFileSize = errors.New("exceed file size")
// A RangeReader is used to read a range of content from a file.
type RangeReader struct {
file *os.File
@@ -32,7 +29,7 @@ func (rr *RangeReader) Read(p []byte) (n int, err error) {
}
if rr.stop < rr.start || rr.start >= stat.Size() {
return 0, errExceedFileSize
return 0, errors.New("exceed file size")
}
if rr.stop-rr.start < int64(len(p)) {

View File

@@ -5,7 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/fs"
"github.com/tal-tech/go-zero/core/fs"
)
func TestRangeReader(t *testing.T) {

View File

@@ -1,4 +1,5 @@
//go:build windows
// +build windows
package fs

View File

@@ -1,4 +1,5 @@
//go:build linux || darwin
// +build linux darwin
package fs

View File

@@ -1,15 +0,0 @@
package fs
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCloseOnExec(t *testing.T) {
file := os.NewFile(0, os.DevNull)
assert.NotPanics(t, func() {
CloseOnExec(file)
})
}

View File

@@ -1,9 +1,10 @@
package fs
import (
"io/ioutil"
"os"
"github.com/zeromicro/go-zero/core/hash"
"github.com/tal-tech/go-zero/core/hash"
)
// TempFileWithText creates the temporary file with the given content,
@@ -11,29 +12,29 @@ import (
// The file is kept as open, the caller should close the file handle,
// and remove the file by name.
func TempFileWithText(text string) (*os.File, error) {
tmpFile, err := os.CreateTemp(os.TempDir(), hash.Md5Hex([]byte(text)))
tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text)))
if err != nil {
return nil, err
}
if err := os.WriteFile(tmpFile.Name(), []byte(text), os.ModeTemporary); err != nil {
if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
return nil, err
}
return tmpFile, nil
return tmpfile, nil
}
// TempFilenameWithText creates the file with the given content,
// and returns the filename (full path).
// The caller should remove the file after use.
func TempFilenameWithText(text string) (string, error) {
tmpFile, err := TempFileWithText(text)
tmpfile, err := TempFileWithText(text)
if err != nil {
return "", err
}
filename := tmpFile.Name()
if err = tmpFile.Close(); err != nil {
filename := tmpfile.Name()
if err = tmpfile.Close(); err != nil {
return "", err
}

View File

@@ -1,49 +0,0 @@
package fs
import (
"io"
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestTempFileWithText(t *testing.T) {
f, err := TempFileWithText("test")
if err != nil {
t.Error(err)
}
if f == nil {
t.Error("TempFileWithText returned nil")
}
if f.Name() == "" {
t.Error("TempFileWithText returned empty file name")
}
defer os.Remove(f.Name())
bs, err := io.ReadAll(f)
assert.Nil(t, err)
if len(bs) != 4 {
t.Error("TempFileWithText returned wrong file size")
}
if f.Close() != nil {
t.Error("TempFileWithText returned error on close")
}
}
func TestTempFilenameWithText(t *testing.T) {
f, err := TempFilenameWithText("test")
if err != nil {
t.Error(err)
}
if f == "" {
t.Error("TempFilenameWithText returned empty file name")
}
defer os.Remove(f)
bs, err := os.ReadFile(f)
assert.Nil(t, err)
if len(bs) != 4 {
t.Error("TempFilenameWithText returned wrong file size")
}
}

View File

@@ -1,6 +1,6 @@
package fx
import "github.com/zeromicro/go-zero/core/threading"
import "github.com/tal-tech/go-zero/core/threading"
// Parallel runs fns parallelly and waits for done.
func Parallel(fns ...func()) {

View File

@@ -1,11 +1,6 @@
package fx
import (
"context"
"time"
"github.com/zeromicro/go-zero/core/errorx"
)
import "github.com/tal-tech/go-zero/core/errorx"
const defaultRetryTimes = 3
@@ -14,70 +9,23 @@ type (
RetryOption func(*retryOptions)
retryOptions struct {
times int
interval time.Duration
timeout time.Duration
times int
}
)
// DoWithRetry runs fn, and retries if failed. Default to retry 3 times.
// Note that if the fn function accesses global variables outside the function
// and performs modification operations, it is best to lock them,
// otherwise there may be data race issues
func DoWithRetry(fn func() error, opts ...RetryOption) error {
return retry(context.Background(), func(errChan chan error, retryCount int) {
errChan <- fn()
}, opts...)
}
// DoWithRetryCtx runs fn, and retries if failed. Default to retry 3 times.
// fn retryCount indicates the current number of retries, starting from 0
// Note that if the fn function accesses global variables outside the function
// and performs modification operations, it is best to lock them,
// otherwise there may be data race issues
func DoWithRetryCtx(ctx context.Context, fn func(ctx context.Context, retryCount int) error,
opts ...RetryOption) error {
return retry(ctx, func(errChan chan error, retryCount int) {
errChan <- fn(ctx, retryCount)
}, opts...)
}
func retry(ctx context.Context, fn func(errChan chan error, retryCount int), opts ...RetryOption) error {
options := newRetryOptions()
for _, opt := range opts {
opt(options)
}
var berr errorx.BatchError
var cancelFunc context.CancelFunc
if options.timeout > 0 {
ctx, cancelFunc = context.WithTimeout(ctx, options.timeout)
defer cancelFunc()
}
errChan := make(chan error, 1)
for i := 0; i < options.times; i++ {
go fn(errChan, i)
select {
case err := <-errChan:
if err != nil {
berr.Add(err)
} else {
return nil
}
case <-ctx.Done():
berr.Add(ctx.Err())
return berr.Err()
}
if options.interval > 0 {
select {
case <-ctx.Done():
berr.Add(ctx.Err())
return berr.Err()
case <-time.After(options.interval):
}
if err := fn(); err != nil {
berr.Add(err)
} else {
return nil
}
}
@@ -91,18 +39,6 @@ func WithRetry(times int) RetryOption {
}
}
func WithInterval(interval time.Duration) RetryOption {
return func(options *retryOptions) {
options.interval = interval
}
}
func WithTimeout(timeout time.Duration) RetryOption {
return func(options *retryOptions) {
options.timeout = timeout
}
}
func newRetryOptions() *retryOptions {
return &retryOptions{
times: defaultRetryTimes,

View File

@@ -1,10 +1,8 @@
package fx
import (
"context"
"errors"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
@@ -14,135 +12,31 @@ func TestRetry(t *testing.T) {
return errors.New("any")
}))
times1 := 0
var times int
assert.Nil(t, DoWithRetry(func() error {
times1++
if times1 == defaultRetryTimes {
times++
if times == defaultRetryTimes {
return nil
}
return errors.New("any")
}))
times2 := 0
times = 0
assert.NotNil(t, DoWithRetry(func() error {
times2++
if times2 == defaultRetryTimes+1 {
times++
if times == defaultRetryTimes+1 {
return nil
}
return errors.New("any")
}))
total := 2 * defaultRetryTimes
times3 := 0
times = 0
assert.Nil(t, DoWithRetry(func() error {
times3++
if times3 == total {
times++
if times == total {
return nil
}
return errors.New("any")
}, WithRetry(total)))
}
func TestRetryWithTimeout(t *testing.T) {
assert.Nil(t, DoWithRetry(func() error {
return nil
}, WithTimeout(time.Millisecond*500)))
times1 := 0
assert.Nil(t, DoWithRetry(func() error {
times1++
if times1 == 1 {
return errors.New("any ")
}
time.Sleep(time.Millisecond * 150)
return nil
}, WithTimeout(time.Millisecond*250)))
total := defaultRetryTimes
times2 := 0
assert.Nil(t, DoWithRetry(func() error {
times2++
if times2 == total {
return nil
}
time.Sleep(time.Millisecond * 50)
return errors.New("any")
}, WithTimeout(time.Millisecond*50*(time.Duration(total)+2))))
assert.NotNil(t, DoWithRetry(func() error {
return errors.New("any")
}, WithTimeout(time.Millisecond*250)))
}
func TestRetryWithInterval(t *testing.T) {
times1 := 0
assert.NotNil(t, DoWithRetry(func() error {
times1++
if times1 == 1 {
return errors.New("any")
}
time.Sleep(time.Millisecond * 150)
return nil
}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
times2 := 0
assert.NotNil(t, DoWithRetry(func() error {
times2++
if times2 == 2 {
return nil
}
time.Sleep(time.Millisecond * 150)
return errors.New("any ")
}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
}
func TestRetryCtx(t *testing.T) {
t.Run("with timeout", func(t *testing.T) {
assert.NotNil(t, DoWithRetryCtx(context.Background(), func(ctx context.Context, retryCount int) error {
if retryCount == 0 {
return errors.New("any")
}
time.Sleep(time.Millisecond * 150)
return nil
}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
assert.NotNil(t, DoWithRetryCtx(context.Background(), func(ctx context.Context, retryCount int) error {
if retryCount == 1 {
return nil
}
time.Sleep(time.Millisecond * 150)
return errors.New("any ")
}, WithTimeout(time.Millisecond*250), WithInterval(time.Millisecond*150)))
})
t.Run("with deadline exceeded", func(t *testing.T) {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond*250))
defer cancel()
var times int
assert.Error(t, DoWithRetryCtx(ctx, func(ctx context.Context, retryCount int) error {
times++
time.Sleep(time.Millisecond * 150)
return errors.New("any")
}, WithInterval(time.Millisecond*150)))
assert.Equal(t, 1, times)
})
t.Run("with deadline not exceeded", func(t *testing.T) {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond*250))
defer cancel()
var times int
assert.NoError(t, DoWithRetryCtx(ctx, func(ctx context.Context, retryCount int) error {
times++
if times == defaultRetryTimes {
return nil
}
time.Sleep(time.Millisecond * 50)
return errors.New("any")
}))
assert.Equal(t, defaultRetryTimes, times)
})
}

View File

@@ -4,9 +4,9 @@ import (
"sort"
"sync"
"github.com/zeromicro/go-zero/core/collection"
"github.com/zeromicro/go-zero/core/lang"
"github.com/zeromicro/go-zero/core/threading"
"github.com/tal-tech/go-zero/core/collection"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/threading"
)
const (
@@ -21,31 +21,31 @@ type (
}
// FilterFunc defines the method to filter a Stream.
FilterFunc func(item any) bool
FilterFunc func(item interface{}) bool
// ForAllFunc defines the method to handle all elements in a Stream.
ForAllFunc func(pipe <-chan any)
ForAllFunc func(pipe <-chan interface{})
// ForEachFunc defines the method to handle each element in a Stream.
ForEachFunc func(item any)
ForEachFunc func(item interface{})
// GenerateFunc defines the method to send elements into a Stream.
GenerateFunc func(source chan<- any)
GenerateFunc func(source chan<- interface{})
// KeyFunc defines the method to generate keys for the elements in a Stream.
KeyFunc func(item any) any
KeyFunc func(item interface{}) interface{}
// LessFunc defines the method to compare the elements in a Stream.
LessFunc func(a, b any) bool
LessFunc func(a, b interface{}) bool
// MapFunc defines the method to map each element to another object in a Stream.
MapFunc func(item any) any
MapFunc func(item interface{}) interface{}
// Option defines the method to customize a Stream.
Option func(opts *rxOptions)
// ParallelFunc defines the method to handle elements parallelly.
ParallelFunc func(item any)
ParallelFunc func(item interface{})
// ReduceFunc defines the method to reduce all the elements in a Stream.
ReduceFunc func(pipe <-chan any) (any, error)
ReduceFunc func(pipe <-chan interface{}) (interface{}, error)
// WalkFunc defines the method to walk through all the elements in a Stream.
WalkFunc func(item any, pipe chan<- any)
WalkFunc func(item interface{}, pipe chan<- interface{})
// A Stream is a stream that can be used to do stream processing.
Stream struct {
source <-chan any
source <-chan interface{}
}
)
@@ -56,7 +56,7 @@ func Concat(s Stream, others ...Stream) Stream {
// From constructs a Stream from the given GenerateFunc.
func From(generate GenerateFunc) Stream {
source := make(chan any)
source := make(chan interface{})
threading.GoSafe(func() {
defer close(source)
@@ -67,8 +67,8 @@ func From(generate GenerateFunc) Stream {
}
// Just converts the given arbitrary items to a Stream.
func Just(items ...any) Stream {
source := make(chan any, len(items))
func Just(items ...interface{}) Stream {
source := make(chan interface{}, len(items))
for _, item := range items {
source <- item
}
@@ -78,7 +78,7 @@ func Just(items ...any) Stream {
}
// Range converts the given channel to a Stream.
func Range(source <-chan any) Stream {
func Range(source <-chan interface{}) Stream {
return Stream{
source: source,
}
@@ -87,7 +87,7 @@ func Range(source <-chan any) Stream {
// AllMach returns whether all elements of this stream match the provided predicate.
// May not evaluate the predicate on all elements if not necessary for determining the result.
// If the stream is empty then true is returned and the predicate is not evaluated.
func (s Stream) AllMach(predicate func(item any) bool) bool {
func (s Stream) AllMach(predicate func(item interface{}) bool) bool {
for item := range s.source {
if !predicate(item) {
// make sure the former goroutine not block, and current func returns fast.
@@ -102,7 +102,7 @@ func (s Stream) AllMach(predicate func(item any) bool) bool {
// AnyMach returns whether any elements of this stream match the provided predicate.
// May not evaluate the predicate on all elements if not necessary for determining the result.
// If the stream is empty then false is returned and the predicate is not evaluated.
func (s Stream) AnyMach(predicate func(item any) bool) bool {
func (s Stream) AnyMach(predicate func(item interface{}) bool) bool {
for item := range s.source {
if predicate(item) {
// make sure the former goroutine not block, and current func returns fast.
@@ -121,7 +121,7 @@ func (s Stream) Buffer(n int) Stream {
n = 0
}
source := make(chan any, n)
source := make(chan interface{}, n)
go func() {
for item := range s.source {
source <- item
@@ -134,7 +134,7 @@ func (s Stream) Buffer(n int) Stream {
// Concat returns a Stream that concatenated other streams
func (s Stream) Concat(others ...Stream) Stream {
source := make(chan any)
source := make(chan interface{})
go func() {
group := threading.NewRoutineGroup()
@@ -170,12 +170,12 @@ func (s Stream) Count() (count int) {
// Distinct removes the duplicated items base on the given KeyFunc.
func (s Stream) Distinct(fn KeyFunc) Stream {
source := make(chan any)
source := make(chan interface{})
threading.GoSafe(func() {
defer close(source)
keys := make(map[any]lang.PlaceholderType)
keys := make(map[interface{}]lang.PlaceholderType)
for item := range s.source {
key := fn(item)
if _, ok := keys[key]; !ok {
@@ -195,7 +195,7 @@ func (s Stream) Done() {
// Filter filters the items by the given FilterFunc.
func (s Stream) Filter(fn FilterFunc, opts ...Option) Stream {
return s.Walk(func(item any, pipe chan<- any) {
return s.Walk(func(item interface{}, pipe chan<- interface{}) {
if fn(item) {
pipe <- item
}
@@ -203,7 +203,7 @@ func (s Stream) Filter(fn FilterFunc, opts ...Option) Stream {
}
// First returns the first item, nil if no items.
func (s Stream) First() any {
func (s Stream) First() interface{} {
for item := range s.source {
// make sure the former goroutine not block, and current func returns fast.
go drain(s.source)
@@ -229,13 +229,13 @@ func (s Stream) ForEach(fn ForEachFunc) {
// Group groups the elements into different groups based on their keys.
func (s Stream) Group(fn KeyFunc) Stream {
groups := make(map[any][]any)
groups := make(map[interface{}][]interface{})
for item := range s.source {
key := fn(item)
groups[key] = append(groups[key], item)
}
source := make(chan any)
source := make(chan interface{})
go func() {
for _, group := range groups {
source <- group
@@ -252,7 +252,7 @@ func (s Stream) Head(n int64) Stream {
panic("n must be greater than 0")
}
source := make(chan any)
source := make(chan interface{})
go func() {
for item := range s.source {
@@ -279,7 +279,7 @@ func (s Stream) Head(n int64) Stream {
}
// Last returns the last item, or nil if no items.
func (s Stream) Last() (item any) {
func (s Stream) Last() (item interface{}) {
for item = range s.source {
}
return
@@ -287,53 +287,29 @@ func (s Stream) Last() (item any) {
// Map converts each item to another corresponding item, which means it's a 1:1 model.
func (s Stream) Map(fn MapFunc, opts ...Option) Stream {
return s.Walk(func(item any, pipe chan<- any) {
return s.Walk(func(item interface{}, pipe chan<- interface{}) {
pipe <- fn(item)
}, opts...)
}
// Max returns the maximum item from the underlying source.
func (s Stream) Max(less LessFunc) any {
var max any
for item := range s.source {
if max == nil || less(max, item) {
max = item
}
}
return max
}
// Merge merges all the items into a slice and generates a new stream.
func (s Stream) Merge() Stream {
var items []any
var items []interface{}
for item := range s.source {
items = append(items, item)
}
source := make(chan any, 1)
source := make(chan interface{}, 1)
source <- items
close(source)
return Range(source)
}
// Min returns the minimum item from the underlying source.
func (s Stream) Min(less LessFunc) any {
var min any
for item := range s.source {
if min == nil || less(item, min) {
min = item
}
}
return min
}
// NoneMatch returns whether all elements of this stream don't match the provided predicate.
// May not evaluate the predicate on all elements if not necessary for determining the result.
// If the stream is empty then true is returned and the predicate is not evaluated.
func (s Stream) NoneMatch(predicate func(item any) bool) bool {
func (s Stream) NoneMatch(predicate func(item interface{}) bool) bool {
for item := range s.source {
if predicate(item) {
// make sure the former goroutine not block, and current func returns fast.
@@ -347,19 +323,19 @@ func (s Stream) NoneMatch(predicate func(item any) bool) bool {
// Parallel applies the given ParallelFunc to each item concurrently with given number of workers.
func (s Stream) Parallel(fn ParallelFunc, opts ...Option) {
s.Walk(func(item any, pipe chan<- any) {
s.Walk(func(item interface{}, pipe chan<- interface{}) {
fn(item)
}, opts...).Done()
}
// Reduce is an utility method to let the caller deal with the underlying channel.
func (s Stream) Reduce(fn ReduceFunc) (any, error) {
// Reduce is a utility method to let the caller deal with the underlying channel.
func (s Stream) Reduce(fn ReduceFunc) (interface{}, error) {
return fn(s.source)
}
// Reverse reverses the elements in the stream.
func (s Stream) Reverse() Stream {
var items []any
var items []interface{}
for item := range s.source {
items = append(items, item)
}
@@ -381,7 +357,7 @@ func (s Stream) Skip(n int64) Stream {
return s
}
source := make(chan any)
source := make(chan interface{})
go func() {
for item := range s.source {
@@ -400,7 +376,7 @@ func (s Stream) Skip(n int64) Stream {
// Sort sorts the items from the underlying source.
func (s Stream) Sort(less LessFunc) Stream {
var items []any
var items []interface{}
for item := range s.source {
items = append(items, item)
}
@@ -418,9 +394,9 @@ func (s Stream) Split(n int) Stream {
panic("n should be greater than 0")
}
source := make(chan any)
source := make(chan interface{})
go func() {
var chunk []any
var chunk []interface{}
for item := range s.source {
chunk = append(chunk, item)
if len(chunk) == n {
@@ -443,7 +419,7 @@ func (s Stream) Tail(n int64) Stream {
panic("n should be greater than 0")
}
source := make(chan any)
source := make(chan interface{})
go func() {
ring := collection.NewRing(int(n))
@@ -470,7 +446,7 @@ func (s Stream) Walk(fn WalkFunc, opts ...Option) Stream {
}
func (s Stream) walkLimited(fn WalkFunc, option *rxOptions) Stream {
pipe := make(chan any, option.workers)
pipe := make(chan interface{}, option.workers)
go func() {
var wg sync.WaitGroup
@@ -501,7 +477,7 @@ func (s Stream) walkLimited(fn WalkFunc, option *rxOptions) Stream {
}
func (s Stream) walkUnlimited(fn WalkFunc, option *rxOptions) Stream {
pipe := make(chan any, option.workers)
pipe := make(chan interface{}, option.workers)
go func() {
var wg sync.WaitGroup
@@ -553,7 +529,7 @@ func buildOptions(opts ...Option) *rxOptions {
}
// drain drains the given channel.
func drain(channel <-chan any) {
func drain(channel <-chan interface{}) {
for range channel {
}
}

View File

@@ -1,7 +1,7 @@
package fx
import (
"io"
"io/ioutil"
"log"
"math/rand"
"reflect"
@@ -13,8 +13,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/stringx"
"go.uber.org/goleak"
"github.com/tal-tech/go-zero/core/stringx"
)
func TestBuffer(t *testing.T) {
@@ -23,7 +22,7 @@ func TestBuffer(t *testing.T) {
var count int32
var wait sync.WaitGroup
wait.Add(1)
From(func(source chan<- any) {
From(func(source chan<- interface{}) {
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
@@ -36,7 +35,7 @@ func TestBuffer(t *testing.T) {
return
}
}
}).Buffer(N).ForAll(func(pipe <-chan any) {
}).Buffer(N).ForAll(func(pipe <-chan interface{}) {
wait.Wait()
// why N+1, because take one more to wait for sending into the channel
assert.Equal(t, int32(N+1), atomic.LoadInt32(&count))
@@ -47,7 +46,7 @@ func TestBuffer(t *testing.T) {
func TestBufferNegative(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4).Buffer(-1).Reduce(func(pipe <-chan any) (any, error) {
Just(1, 2, 3, 4).Buffer(-1).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -61,22 +60,22 @@ func TestCount(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
tests := []struct {
name string
elements []any
elements []interface{}
}{
{
name: "no elements with nil",
},
{
name: "no elements",
elements: []any{},
elements: []interface{}{},
},
{
name: "1 element",
elements: []any{1},
elements: []interface{}{1},
},
{
name: "multiple elements",
elements: []any{1, 2, 3},
elements: []interface{}{1, 2, 3},
},
}
@@ -92,7 +91,7 @@ func TestCount(t *testing.T) {
func TestDone(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var count int32
Just(1, 2, 3).Walk(func(item any, pipe chan<- any) {
Just(1, 2, 3).Walk(func(item interface{}, pipe chan<- interface{}) {
time.Sleep(time.Millisecond * 100)
atomic.AddInt32(&count, int32(item.(int)))
}).Done()
@@ -103,7 +102,7 @@ func TestDone(t *testing.T) {
func TestJust(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4).Reduce(func(pipe <-chan any) (any, error) {
Just(1, 2, 3, 4).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -116,9 +115,9 @@ func TestJust(t *testing.T) {
func TestDistinct(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(4, 1, 3, 2, 3, 4).Distinct(func(item any) any {
Just(4, 1, 3, 2, 3, 4).Distinct(func(item interface{}) interface{} {
return item
}).Reduce(func(pipe <-chan any) (any, error) {
}).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -131,9 +130,9 @@ func TestDistinct(t *testing.T) {
func TestFilter(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4).Filter(func(item any) bool {
Just(1, 2, 3, 4).Filter(func(item interface{}) bool {
return item.(int)%2 == 0
}).Reduce(func(pipe <-chan any) (any, error) {
}).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -154,9 +153,9 @@ func TestFirst(t *testing.T) {
func TestForAll(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4).Filter(func(item any) bool {
Just(1, 2, 3, 4).Filter(func(item interface{}) bool {
return item.(int)%2 == 0
}).ForAll(func(pipe <-chan any) {
}).ForAll(func(pipe <-chan interface{}) {
for item := range pipe {
result += item.(int)
}
@@ -168,11 +167,11 @@ func TestForAll(t *testing.T) {
func TestGroup(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var groups [][]int
Just(10, 11, 20, 21).Group(func(item any) any {
Just(10, 11, 20, 21).Group(func(item interface{}) interface{} {
v := item.(int)
return v / 10
}).ForEach(func(item any) {
v := item.([]any)
}).ForEach(func(item interface{}) {
v := item.([]interface{})
var group []int
for _, each := range v {
group = append(group, each.(int))
@@ -191,7 +190,7 @@ func TestGroup(t *testing.T) {
func TestHead(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4).Head(2).Reduce(func(pipe <-chan any) (any, error) {
Just(1, 2, 3, 4).Head(2).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -204,7 +203,7 @@ func TestHead(t *testing.T) {
func TestHeadZero(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
assert.Panics(t, func() {
Just(1, 2, 3, 4).Head(0).Reduce(func(pipe <-chan any) (any, error) {
Just(1, 2, 3, 4).Head(0).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
return nil, nil
})
})
@@ -214,7 +213,7 @@ func TestHeadZero(t *testing.T) {
func TestHeadMore(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4).Head(6).Reduce(func(pipe <-chan any) (any, error) {
Just(1, 2, 3, 4).Head(6).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -238,21 +237,21 @@ func TestLast(t *testing.T) {
func TestMap(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
log.SetOutput(io.Discard)
log.SetOutput(ioutil.Discard)
tests := []struct {
mapper MapFunc
expect int
}{
{
mapper: func(item any) any {
mapper: func(item interface{}) interface{} {
v := item.(int)
return v * v
},
expect: 30,
},
{
mapper: func(item any) any {
mapper: func(item interface{}) interface{} {
v := item.(int)
if v%2 == 0 {
return 0
@@ -262,7 +261,7 @@ func TestMap(t *testing.T) {
expect: 10,
},
{
mapper: func(item any) any {
mapper: func(item interface{}) interface{} {
v := item.(int)
if v%2 == 0 {
panic(v)
@@ -283,12 +282,12 @@ func TestMap(t *testing.T) {
} else {
workers = runtime.NumCPU()
}
From(func(source chan<- any) {
From(func(source chan<- interface{}) {
for i := 1; i < 5; i++ {
source <- i
}
}).Map(test.mapper, WithWorkers(workers)).Reduce(
func(pipe <-chan any) (any, error) {
func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -303,8 +302,8 @@ func TestMap(t *testing.T) {
func TestMerge(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
Just(1, 2, 3, 4).Merge().ForEach(func(item any) {
assert.ElementsMatch(t, []any{1, 2, 3, 4}, item.([]any))
Just(1, 2, 3, 4).Merge().ForEach(func(item interface{}) {
assert.ElementsMatch(t, []interface{}{1, 2, 3, 4}, item.([]interface{}))
})
})
}
@@ -312,7 +311,7 @@ func TestMerge(t *testing.T) {
func TestParallelJust(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var count int32
Just(1, 2, 3).Parallel(func(item any) {
Just(1, 2, 3).Parallel(func(item interface{}) {
time.Sleep(time.Millisecond * 100)
atomic.AddInt32(&count, int32(item.(int)))
}, UnlimitedWorkers())
@@ -322,8 +321,8 @@ func TestParallelJust(t *testing.T) {
func TestReverse(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
Just(1, 2, 3, 4).Reverse().Merge().ForEach(func(item any) {
assert.ElementsMatch(t, []any{4, 3, 2, 1}, item.([]any))
Just(1, 2, 3, 4).Reverse().Merge().ForEach(func(item interface{}) {
assert.ElementsMatch(t, []interface{}{4, 3, 2, 1}, item.([]interface{}))
})
})
}
@@ -331,9 +330,9 @@ func TestReverse(t *testing.T) {
func TestSort(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var prev int
Just(5, 3, 7, 1, 9, 6, 4, 8, 2).Sort(func(a, b any) bool {
Just(5, 3, 7, 1, 9, 6, 4, 8, 2).Sort(func(a, b interface{}) bool {
return a.(int) < b.(int)
}).ForEach(func(item any) {
}).ForEach(func(item interface{}) {
next := item.(int)
assert.True(t, prev < next)
prev = next
@@ -346,12 +345,12 @@ func TestSplit(t *testing.T) {
assert.Panics(t, func() {
Just(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).Split(0).Done()
})
var chunks [][]any
Just(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).Split(4).ForEach(func(item any) {
chunk := item.([]any)
var chunks [][]interface{}
Just(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).Split(4).ForEach(func(item interface{}) {
chunk := item.([]interface{})
chunks = append(chunks, chunk)
})
assert.EqualValues(t, [][]any{
assert.EqualValues(t, [][]interface{}{
{1, 2, 3, 4},
{5, 6, 7, 8},
{9, 10},
@@ -362,7 +361,7 @@ func TestSplit(t *testing.T) {
func TestTail(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4).Tail(2).Reduce(func(pipe <-chan any) (any, error) {
Just(1, 2, 3, 4).Tail(2).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
for item := range pipe {
result += item.(int)
}
@@ -375,7 +374,7 @@ func TestTail(t *testing.T) {
func TestTailZero(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
assert.Panics(t, func() {
Just(1, 2, 3, 4).Tail(0).Reduce(func(pipe <-chan any) (any, error) {
Just(1, 2, 3, 4).Tail(0).Reduce(func(pipe <-chan interface{}) (interface{}, error) {
return nil, nil
})
})
@@ -385,11 +384,11 @@ func TestTailZero(t *testing.T) {
func TestWalk(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
var result int
Just(1, 2, 3, 4, 5).Walk(func(item any, pipe chan<- any) {
Just(1, 2, 3, 4, 5).Walk(func(item interface{}, pipe chan<- interface{}) {
if item.(int)%2 != 0 {
pipe <- item
}
}, UnlimitedWorkers()).ForEach(func(item any) {
}, UnlimitedWorkers()).ForEach(func(item interface{}) {
result += item.(int)
})
assert.Equal(t, 9, result)
@@ -398,16 +397,16 @@ func TestWalk(t *testing.T) {
func TestStream_AnyMach(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item any) bool {
assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
return item.(int) == 4
}))
assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item any) bool {
assetEqual(t, false, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
return item.(int) == 0
}))
assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item any) bool {
assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
return item.(int) == 2
}))
assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item any) bool {
assetEqual(t, true, Just(1, 2, 3).AnyMach(func(item interface{}) bool {
return item.(int) == 2
}))
})
@@ -416,17 +415,17 @@ func TestStream_AnyMach(t *testing.T) {
func TestStream_AllMach(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
assetEqual(
t, true, Just(1, 2, 3).AllMach(func(item any) bool {
t, true, Just(1, 2, 3).AllMach(func(item interface{}) bool {
return true
}),
)
assetEqual(
t, false, Just(1, 2, 3).AllMach(func(item any) bool {
t, false, Just(1, 2, 3).AllMach(func(item interface{}) bool {
return false
}),
)
assetEqual(
t, false, Just(1, 2, 3).AllMach(func(item any) bool {
t, false, Just(1, 2, 3).AllMach(func(item interface{}) bool {
return item.(int) == 1
}),
)
@@ -436,17 +435,17 @@ func TestStream_AllMach(t *testing.T) {
func TestStream_NoneMatch(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
assetEqual(
t, true, Just(1, 2, 3).NoneMatch(func(item any) bool {
t, true, Just(1, 2, 3).NoneMatch(func(item interface{}) bool {
return false
}),
)
assetEqual(
t, false, Just(1, 2, 3).NoneMatch(func(item any) bool {
t, false, Just(1, 2, 3).NoneMatch(func(item interface{}) bool {
return true
}),
)
assetEqual(
t, true, Just(1, 2, 3).NoneMatch(func(item any) bool {
t, true, Just(1, 2, 3).NoneMatch(func(item interface{}) bool {
return item.(int) == 4
}),
)
@@ -455,19 +454,19 @@ func TestStream_NoneMatch(t *testing.T) {
func TestConcat(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
a1 := []any{1, 2, 3}
a2 := []any{4, 5, 6}
a1 := []interface{}{1, 2, 3}
a2 := []interface{}{4, 5, 6}
s1 := Just(a1...)
s2 := Just(a2...)
stream := Concat(s1, s2)
var items []any
var items []interface{}
for item := range stream.source {
items = append(items, item)
}
sort.Slice(items, func(i, j int) bool {
return items[i].(int) < items[j].(int)
})
ints := make([]any, 0)
ints := make([]interface{}, 0)
ints = append(ints, a1...)
ints = append(ints, a2...)
assetEqual(t, ints, items)
@@ -479,7 +478,7 @@ func TestStream_Skip(t *testing.T) {
assetEqual(t, 3, Just(1, 2, 3, 4).Skip(1).Count())
assetEqual(t, 1, Just(1, 2, 3, 4).Skip(3).Count())
assetEqual(t, 4, Just(1, 2, 3, 4).Skip(0).Count())
equal(t, Just(1, 2, 3, 4).Skip(3), []any{4})
equal(t, Just(1, 2, 3, 4).Skip(3), []interface{}{4})
assert.Panics(t, func() {
Just(1, 2, 3, 4).Skip(-1)
})
@@ -489,104 +488,27 @@ func TestStream_Skip(t *testing.T) {
func TestStream_Concat(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
stream := Just(1).Concat(Just(2), Just(3))
var items []any
var items []interface{}
for item := range stream.source {
items = append(items, item)
}
sort.Slice(items, func(i, j int) bool {
return items[i].(int) < items[j].(int)
})
assetEqual(t, []any{1, 2, 3}, items)
assetEqual(t, []interface{}{1, 2, 3}, items)
just := Just(1)
equal(t, just.Concat(just), []any{1})
})
}
func TestStream_Max(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
tests := []struct {
name string
elements []any
max any
}{
{
name: "no elements with nil",
},
{
name: "no elements",
elements: []any{},
max: nil,
},
{
name: "1 element",
elements: []any{1},
max: 1,
},
{
name: "multiple elements",
elements: []any{1, 2, 9, 5, 8},
max: 9,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
val := Just(test.elements...).Max(func(a, b any) bool {
return a.(int) < b.(int)
})
assetEqual(t, test.max, val)
})
}
})
}
func TestStream_Min(t *testing.T) {
runCheckedTest(t, func(t *testing.T) {
tests := []struct {
name string
elements []any
min any
}{
{
name: "no elements with nil",
min: nil,
},
{
name: "no elements",
elements: []any{},
min: nil,
},
{
name: "1 element",
elements: []any{1},
min: 1,
},
{
name: "multiple elements",
elements: []any{-1, 1, 2, 9, 5, 8},
min: -1,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
val := Just(test.elements...).Min(func(a, b any) bool {
return a.(int) < b.(int)
})
assetEqual(t, test.min, val)
})
}
equal(t, just.Concat(just), []interface{}{1})
})
}
func BenchmarkParallelMapReduce(b *testing.B) {
b.ReportAllocs()
mapper := func(v any) any {
mapper := func(v interface{}) interface{} {
return v.(int64) * v.(int64)
}
reducer := func(input <-chan any) (any, error) {
reducer := func(input <-chan interface{}) (interface{}, error) {
var result int64
for v := range input {
result += v.(int64)
@@ -594,7 +516,7 @@ func BenchmarkParallelMapReduce(b *testing.B) {
return result, nil
}
b.ResetTimer()
From(func(input chan<- any) {
From(func(input chan<- interface{}) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
input <- int64(rand.Int())
@@ -606,10 +528,10 @@ func BenchmarkParallelMapReduce(b *testing.B) {
func BenchmarkMapReduce(b *testing.B) {
b.ReportAllocs()
mapper := func(v any) any {
mapper := func(v interface{}) interface{} {
return v.(int64) * v.(int64)
}
reducer := func(input <-chan any) (any, error) {
reducer := func(input <-chan interface{}) (interface{}, error) {
var result int64
for v := range input {
result += v.(int64)
@@ -617,21 +539,21 @@ func BenchmarkMapReduce(b *testing.B) {
return result, nil
}
b.ResetTimer()
From(func(input chan<- any) {
From(func(input chan<- interface{}) {
for i := 0; i < b.N; i++ {
input <- int64(rand.Int())
}
}).Map(mapper).Reduce(reducer)
}
func assetEqual(t *testing.T, except, data any) {
func assetEqual(t *testing.T, except, data interface{}) {
if !reflect.DeepEqual(except, data) {
t.Errorf(" %v, want %v", data, except)
}
}
func equal(t *testing.T, stream Stream, data []any) {
items := make([]any, 0)
func equal(t *testing.T, stream Stream, data []interface{}) {
items := make([]interface{}, 0)
for item := range stream.source {
items = append(items, item)
}
@@ -641,6 +563,9 @@ func equal(t *testing.T, stream Stream, data []any) {
}
func runCheckedTest(t *testing.T, fn func(t *testing.T)) {
defer goleak.VerifyNone(t)
goroutines := runtime.NumGoroutine()
fn(t)
// let scheduler schedule first
time.Sleep(time.Millisecond)
assert.True(t, runtime.NumGoroutine() <= goroutines)
}

View File

@@ -29,7 +29,7 @@ func DoWithTimeout(fn func() error, timeout time.Duration, opts ...DoOption) err
// create channel with buffer size 1 to avoid goroutine leak
done := make(chan error, 1)
panicChan := make(chan any, 1)
panicChan := make(chan interface{}, 1)
go func() {
defer func() {
if p := recover(); p != nil {

View File

@@ -6,7 +6,8 @@ import (
"strconv"
"sync"
"github.com/zeromicro/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/lang"
"github.com/tal-tech/go-zero/core/mapping"
)
const (
@@ -26,7 +27,7 @@ type (
hashFunc Func
replicas int
keys []uint64
ring map[uint64][]any
ring map[uint64][]interface{}
nodes map[string]lang.PlaceholderType
lock sync.RWMutex
}
@@ -50,21 +51,21 @@ func NewCustomConsistentHash(replicas int, fn Func) *ConsistentHash {
return &ConsistentHash{
hashFunc: fn,
replicas: replicas,
ring: make(map[uint64][]any),
ring: make(map[uint64][]interface{}),
nodes: make(map[string]lang.PlaceholderType),
}
}
// Add adds the node with the number of h.replicas,
// the later call will overwrite the replicas of the former calls.
func (h *ConsistentHash) Add(node any) {
func (h *ConsistentHash) Add(node interface{}) {
h.AddWithReplicas(node, h.replicas)
}
// AddWithReplicas adds the node with the number of replicas,
// replicas will be truncated to h.replicas if it's larger than h.replicas,
// the later call will overwrite the replicas of the former calls.
func (h *ConsistentHash) AddWithReplicas(node any, replicas int) {
func (h *ConsistentHash) AddWithReplicas(node interface{}, replicas int) {
h.Remove(node)
if replicas > h.replicas {
@@ -89,7 +90,7 @@ func (h *ConsistentHash) AddWithReplicas(node any, replicas int) {
// AddWithWeight adds the node with weight, the weight can be 1 to 100, indicates the percent,
// the later call will overwrite the replicas of the former calls.
func (h *ConsistentHash) AddWithWeight(node any, weight int) {
func (h *ConsistentHash) AddWithWeight(node interface{}, weight int) {
// don't need to make sure weight not larger than TopWeight,
// because AddWithReplicas makes sure replicas cannot be larger than h.replicas
replicas := h.replicas * weight / TopWeight
@@ -97,7 +98,7 @@ func (h *ConsistentHash) AddWithWeight(node any, weight int) {
}
// Get returns the corresponding node from h base on the given v.
func (h *ConsistentHash) Get(v any) (any, bool) {
func (h *ConsistentHash) Get(v interface{}) (interface{}, bool) {
h.lock.RLock()
defer h.lock.RUnlock()
@@ -124,7 +125,7 @@ func (h *ConsistentHash) Get(v any) (any, bool) {
}
// Remove removes the given node from h.
func (h *ConsistentHash) Remove(node any) {
func (h *ConsistentHash) Remove(node interface{}) {
nodeRepr := repr(node)
h.lock.Lock()
@@ -177,10 +178,10 @@ func (h *ConsistentHash) removeNode(nodeRepr string) {
delete(h.nodes, nodeRepr)
}
func innerRepr(node any) string {
func innerRepr(node interface{}) string {
return fmt.Sprintf("%d:%v", prime, node)
}
func repr(node any) string {
return lang.Repr(node)
func repr(node interface{}) string {
return mapping.Repr(node)
}

Some files were not shown because too many files have changed in this diff Show More