backoff.go 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181
  1. /*
  2. Copyright 2015 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package flowcontrol
  14. import (
  15. "math/rand"
  16. "sync"
  17. "time"
  18. "k8s.io/utils/clock"
  19. testingclock "k8s.io/utils/clock/testing"
  20. "k8s.io/utils/integer"
  21. )
  22. type backoffEntry struct {
  23. backoff time.Duration
  24. lastUpdate time.Time
  25. }
  26. type Backoff struct {
  27. sync.RWMutex
  28. Clock clock.Clock
  29. defaultDuration time.Duration
  30. maxDuration time.Duration
  31. perItemBackoff map[string]*backoffEntry
  32. rand *rand.Rand
  33. // maxJitterFactor adds jitter to the exponentially backed off delay.
  34. // if maxJitterFactor is zero, no jitter is added to the delay in
  35. // order to maintain current behavior.
  36. maxJitterFactor float64
  37. }
  38. func NewFakeBackOff(initial, max time.Duration, tc *testingclock.FakeClock) *Backoff {
  39. return newBackoff(tc, initial, max, 0.0)
  40. }
  41. func NewBackOff(initial, max time.Duration) *Backoff {
  42. return NewBackOffWithJitter(initial, max, 0.0)
  43. }
  44. func NewFakeBackOffWithJitter(initial, max time.Duration, tc *testingclock.FakeClock, maxJitterFactor float64) *Backoff {
  45. return newBackoff(tc, initial, max, maxJitterFactor)
  46. }
  47. func NewBackOffWithJitter(initial, max time.Duration, maxJitterFactor float64) *Backoff {
  48. clock := clock.RealClock{}
  49. return newBackoff(clock, initial, max, maxJitterFactor)
  50. }
  51. func newBackoff(clock clock.Clock, initial, max time.Duration, maxJitterFactor float64) *Backoff {
  52. var random *rand.Rand
  53. if maxJitterFactor > 0 {
  54. random = rand.New(rand.NewSource(clock.Now().UnixNano()))
  55. }
  56. return &Backoff{
  57. perItemBackoff: map[string]*backoffEntry{},
  58. Clock: clock,
  59. defaultDuration: initial,
  60. maxDuration: max,
  61. maxJitterFactor: maxJitterFactor,
  62. rand: random,
  63. }
  64. }
  65. // Get the current backoff Duration
  66. func (p *Backoff) Get(id string) time.Duration {
  67. p.RLock()
  68. defer p.RUnlock()
  69. var delay time.Duration
  70. entry, ok := p.perItemBackoff[id]
  71. if ok {
  72. delay = entry.backoff
  73. }
  74. return delay
  75. }
  76. // move backoff to the next mark, capping at maxDuration
  77. func (p *Backoff) Next(id string, eventTime time.Time) {
  78. p.Lock()
  79. defer p.Unlock()
  80. entry, ok := p.perItemBackoff[id]
  81. if !ok || hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
  82. entry = p.initEntryUnsafe(id)
  83. entry.backoff += p.jitter(entry.backoff)
  84. } else {
  85. delay := entry.backoff * 2 // exponential
  86. delay += p.jitter(entry.backoff) // add some jitter to the delay
  87. entry.backoff = time.Duration(integer.Int64Min(int64(delay), int64(p.maxDuration)))
  88. }
  89. entry.lastUpdate = p.Clock.Now()
  90. }
  91. // Reset forces clearing of all backoff data for a given key.
  92. func (p *Backoff) Reset(id string) {
  93. p.Lock()
  94. defer p.Unlock()
  95. delete(p.perItemBackoff, id)
  96. }
  97. // Returns True if the elapsed time since eventTime is smaller than the current backoff window
  98. func (p *Backoff) IsInBackOffSince(id string, eventTime time.Time) bool {
  99. p.RLock()
  100. defer p.RUnlock()
  101. entry, ok := p.perItemBackoff[id]
  102. if !ok {
  103. return false
  104. }
  105. if hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
  106. return false
  107. }
  108. return p.Clock.Since(eventTime) < entry.backoff
  109. }
  110. // Returns True if time since lastupdate is less than the current backoff window.
  111. func (p *Backoff) IsInBackOffSinceUpdate(id string, eventTime time.Time) bool {
  112. p.RLock()
  113. defer p.RUnlock()
  114. entry, ok := p.perItemBackoff[id]
  115. if !ok {
  116. return false
  117. }
  118. if hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
  119. return false
  120. }
  121. return eventTime.Sub(entry.lastUpdate) < entry.backoff
  122. }
  123. // Garbage collect records that have aged past maxDuration. Backoff users are expected
  124. // to invoke this periodically.
  125. func (p *Backoff) GC() {
  126. p.Lock()
  127. defer p.Unlock()
  128. now := p.Clock.Now()
  129. for id, entry := range p.perItemBackoff {
  130. if now.Sub(entry.lastUpdate) > p.maxDuration*2 {
  131. // GC when entry has not been updated for 2*maxDuration
  132. delete(p.perItemBackoff, id)
  133. }
  134. }
  135. }
  136. func (p *Backoff) DeleteEntry(id string) {
  137. p.Lock()
  138. defer p.Unlock()
  139. delete(p.perItemBackoff, id)
  140. }
  141. // Take a lock on *Backoff, before calling initEntryUnsafe
  142. func (p *Backoff) initEntryUnsafe(id string) *backoffEntry {
  143. entry := &backoffEntry{backoff: p.defaultDuration}
  144. p.perItemBackoff[id] = entry
  145. return entry
  146. }
  147. func (p *Backoff) jitter(delay time.Duration) time.Duration {
  148. if p.rand == nil {
  149. return 0
  150. }
  151. return time.Duration(p.rand.Float64() * p.maxJitterFactor * float64(delay))
  152. }
  153. // After 2*maxDuration we restart the backoff factor to the beginning
  154. func hasExpired(eventTime time.Time, lastUpdate time.Time, maxDuration time.Duration) bool {
  155. return eventTime.Sub(lastUpdate) > maxDuration*2 // consider stable if it's ok for twice the maxDuration
  156. }