zlei6/go-zero
1
0
Fork 0
mirror of https://github.com/zeromicro/go-zero.git synced 2026-05-10 16:30:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

optimize: improve breaker algorithm on recovery time (#4077)

Browse Source
This commit is contained in:
Kevin Wan
2024-04-18 22:33:25 +08:00
committed by GitHub
parent 95b32b5779
commit 1540bdc4c9
10 changed files with 311 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
core/load/adaptiveshedder.go
View File

@@ -76,8 +76,8 @@ type (
avgFlyingLock syncx.SpinLock
overloadTime *syncx.AtomicDuration
droppedRecently *syncx.AtomicBool
passCounter *collection.RollingWindow
rtCounter *collection.RollingWindow
passCounter *collection.RollingWindow[int64, *collection.Bucket[int64]]
rtCounter *collection.RollingWindow[int64, *collection.Bucket[int64]]
}
)
@@ -107,15 +107,16 @@ func NewAdaptiveShedder(opts ...ShedderOption) Shedder {
opt(&options)
}
bucketDuration := options.window / time.Duration(options.buckets)
newBucket := func() *collection.Bucket[int64] {
return new(collection.Bucket[int64])
}
return &adaptiveShedder{
cpuThreshold: options.cpuThreshold,
windowScale: float64(time.Second) / float64(bucketDuration) / millisecondsPerSecond,
overloadTime: syncx.NewAtomicDuration(),
droppedRecently: syncx.NewAtomicBool(),
passCounter: collection.NewRollingWindow(options.buckets, bucketDuration,
collection.IgnoreCurrentBucket()),
rtCounter: collection.NewRollingWindow(options.buckets, bucketDuration,
collection.IgnoreCurrentBucket()),
passCounter: collection.NewRollingWindow[int64, *collection.Bucket[int64]](newBucket, options.buckets, bucketDuration, collection.IgnoreCurrentBucket[int64, *collection.Bucket[int64]]()),
rtCounter: collection.NewRollingWindow[int64, *collection.Bucket[int64]](newBucket, options.buckets, bucketDuration, collection.IgnoreCurrentBucket[int64, *collection.Bucket[int64]]()),
}
}
@@ -167,15 +168,15 @@ func (as *adaptiveShedder) maxFlight() float64 {
}
func (as *adaptiveShedder) maxPass() int64 {
var result float64 = 1
var result int64 = 1
as.passCounter.Reduce(func(b *collection.Bucket) {
as.passCounter.Reduce(func(b *collection.Bucket[int64]) {
if b.Sum > result {
result = b.Sum
}
})
return int64(result)
return result
}
func (as *adaptiveShedder) minRt() float64 {
@@ -183,12 +184,12 @@ func (as *adaptiveShedder) minRt() float64 {
// its a reasonable large value to avoid dropping requests.
result := defaultMinRt
as.rtCounter.Reduce(func(b *collection.Bucket) {
as.rtCounter.Reduce(func(b *collection.Bucket[int64]) {
if b.Count <= 0 {
return
}
avg := math.Round(b.Sum / float64(b.Count))
avg := math.Round(float64(b.Sum) / float64(b.Count))
if avg < result {
result = avg
}
@@ -283,6 +284,6 @@ func (p *promise) Fail() {
func (p *promise) Pass() {
rt := float64(timex.Since(p.start)) / float64(time.Millisecond)
p.shedder.addFlying(-1)
p.shedder.rtCounter.Add(math.Ceil(rt))
p.shedder.rtCounter.Add(int64(math.Ceil(rt)))
p.shedder.passCounter.Add(1)
}

26
core/load/adaptiveshedder_test.go
View File

@@ -58,7 +58,7 @@ func TestAdaptiveShedder(t *testing.T) {
func TestAdaptiveShedderMaxPass(t *testing.T) {
passCounter := newRollingWindow()
for i := 1; i <= 10; i++ {
passCounter.Add(float64(i * 100))
passCounter.Add(int64(i * 100))
time.Sleep(bucketDuration)
}
shedder := &adaptiveShedder{
@@ -83,7 +83,7 @@ func TestAdaptiveShedderMinRt(t *testing.T) {
time.Sleep(bucketDuration)
}
for j := i*10 + 1; j <= i*10+10; j++ {
rtCounter.Add(float64(j))
rtCounter.Add(int64(j))
}
}
shedder := &adaptiveShedder{
@@ -107,9 +107,9 @@ func TestAdaptiveShedderMaxFlight(t *testing.T) {
if i > 0 {
time.Sleep(bucketDuration)
}
passCounter.Add(float64((i + 1) * 100))
passCounter.Add(int64((i + 1) * 100))
for j := i*10 + 1; j <= i*10+10; j++ {
rtCounter.Add(float64(j))
rtCounter.Add(int64(j))
}
}
shedder := &adaptiveShedder{
@@ -129,9 +129,9 @@ func TestAdaptiveShedderShouldDrop(t *testing.T) {
if i > 0 {
time.Sleep(bucketDuration)
}
passCounter.Add(float64((i + 1) * 100))
passCounter.Add(int64((i + 1) * 100))
for j := i*10 + 1; j <= i*10+10; j++ {
rtCounter.Add(float64(j))
rtCounter.Add(int64(j))
}
}
shedder := &adaptiveShedder{
@@ -184,9 +184,9 @@ func TestAdaptiveShedderStillHot(t *testing.T) {
if i > 0 {
time.Sleep(bucketDuration)
}
passCounter.Add(float64((i + 1) * 100))
passCounter.Add(int64((i + 1) * 100))
for j := i*10 + 1; j <= i*10+10; j++ {
rtCounter.Add(float64(j))
rtCounter.Add(int64(j))
}
}
shedder := &adaptiveShedder{
@@ -248,9 +248,9 @@ func BenchmarkMaxFlight(b *testing.B) {
if i > 0 {
time.Sleep(bucketDuration)
}
passCounter.Add(float64((i + 1) * 100))
passCounter.Add(int64((i + 1) * 100))
for j := i*10 + 1; j <= i*10+10; j++ {
rtCounter.Add(float64(j))
rtCounter.Add(int64(j))
}
}
shedder := &adaptiveShedder{
@@ -265,6 +265,8 @@ func BenchmarkMaxFlight(b *testing.B) {
}
}
func newRollingWindow() *collection.RollingWindow {
return collection.NewRollingWindow(buckets, bucketDuration, collection.IgnoreCurrentBucket())
func newRollingWindow() *collection.RollingWindow[int64, *collection.Bucket[int64]] {
return collection.NewRollingWindow[int64, *collection.Bucket[int64]](func() *collection.Bucket[int64] {
return new(collection.Bucket[int64])
}, buckets, bucketDuration, collection.IgnoreCurrentBucket[int64, *collection.Bucket[int64]]())
}