You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2024/05/29 14:21:55 UTC
(skywalking-banyandb) 06/08: Limit flush interval
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch topn-val
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit dd08b6458112fab086e30a916b45e723e33b3629
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed May 29 04:12:04 2024 +0000
Limit flush interval
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
banyand/measure/topn.go | 7 ++++++-
pkg/flow/streaming/sliding_window.go | 24 ++++++++++++++++++------
pkg/flow/streaming/sliding_window_test.go | 2 +-
pkg/flow/streaming/streaming_test.go | 4 ++--
pkg/flow/streaming/topn.go | 18 ++++++++++--------
5 files changed, 37 insertions(+), 18 deletions(-)
diff --git a/banyand/measure/topn.go b/banyand/measure/topn.go
index 54525614..85460d5d 100644
--- a/banyand/measure/topn.go
+++ b/banyand/measure/topn.go
@@ -52,6 +52,7 @@ import (
const (
timeBucketFormat = "200601021504"
resultPersistencyTimeout = 10 * time.Second
+ maxFlushInterval = time.Minute
)
var (
@@ -264,7 +265,11 @@ func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum i
}
func (t *topNStreamingProcessor) start() *topNStreamingProcessor {
- t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval)).
+ flushInterval := t.interval
+ if flushInterval > maxFlushInterval {
+ flushInterval = maxFlushInterval
+ }
+ t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval, flushInterval)).
AllowedMaxWindows(int(t.topNSchema.GetLruSize())).
TopN(int(t.topNSchema.GetCountersNumber()),
streaming.WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
diff --git a/pkg/flow/streaming/sliding_window.go b/pkg/flow/streaming/sliding_window.go
index 0164f413..27b11275 100644
--- a/pkg/flow/streaming/sliding_window.go
+++ b/pkg/flow/streaming/sliding_window.go
@@ -85,6 +85,7 @@ type tumblingTimeWindows struct {
windowCount int
currentWatermark int64
timerMu sync.Mutex
+ maxFlushInterval int64
l *logger.Logger
}
@@ -103,7 +104,9 @@ func (s *tumblingTimeWindows) Setup(_ context.Context) (err error) {
}
s.snapshots, err = lru.NewWithEvict(s.windowCount, func(key interface{}, value interface{}) {
flushed := s.flushSnapshot(key.(timeWindow), value.(flow.AggregationOp))
- s.l.Info().Stringer("window", key.(timeWindow)).Bool("flushed", flushed).Msg("evict window on lru cache is full")
+ if e := s.l.Debug(); e.Enabled() {
+ e.Stringer("window", key.(timeWindow)).Bool("flushed", flushed).Msg("evict window on lru cache is full")
+ }
})
if err != nil {
return err
@@ -127,7 +130,9 @@ func (s *tumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.Aggregat
func (s *tumblingTimeWindows) flushWindow(w timeWindow) {
if snapshot, ok := s.snapshots.Get(w); ok {
flushed := s.flushSnapshot(w, snapshot.(flow.AggregationOp))
- s.l.Info().Stringer("window", w).Bool("flushed", flushed).Msg("flush window")
+ if e := s.l.Debug(); e.Enabled() {
+ e.Stringer("window", w).Bool("flushed", flushed).Msg("flush window")
+ }
}
}
@@ -178,7 +183,9 @@ func (s *tumblingTimeWindows) receive() {
newAggr := s.aggregationFactory()
newAggr.Add([]flow.StreamRecord{elem})
s.snapshots.Add(tw, newAggr)
- s.l.Info().Stringer("window", tw).Msg("create new window")
+ if e := s.l.Debug(); e.Enabled() {
+ e.Stringer("window", tw).Msg("create new window")
+ }
}
result := ctx.OnElement(elem)
@@ -206,8 +213,12 @@ func (s *tumblingTimeWindows) receive() {
// | 40% | 40% | 20% |
// | flush flush |
// |---------------------------------|
- // TODO: how to determine the threshold
- if previousWaterMark > 0 && float64(pastDur) > float64(s.windowSize)*0.4 {
+ // the max flush interval is 1 minute.
+ it := float64(s.windowSize) * 0.4
+ if it > float64(s.maxFlushInterval) {
+ it = float64(s.maxFlushInterval)
+ }
+ if previousWaterMark > 0 && float64(pastDur) > it {
s.flushDirtyWindows()
}
}
@@ -235,7 +246,7 @@ func (s *tumblingTimeWindows) Exec(downstream flow.Inlet) {
}
// NewTumblingTimeWindows return tumbling-time windows.
-func NewTumblingTimeWindows(size time.Duration) flow.WindowAssigner {
+func NewTumblingTimeWindows(size time.Duration, maxFlushInterval time.Duration) flow.WindowAssigner {
return &tumblingTimeWindows{
windowSize: size.Milliseconds(),
timerHeap: flow.NewPriorityQueue(func(a, b interface{}) int {
@@ -244,6 +255,7 @@ func NewTumblingTimeWindows(size time.Duration) flow.WindowAssigner {
in: make(chan flow.StreamRecord),
out: make(chan flow.StreamRecord),
currentWatermark: 0,
+ maxFlushInterval: maxFlushInterval.Milliseconds(),
}
}
diff --git a/pkg/flow/streaming/sliding_window_test.go b/pkg/flow/streaming/sliding_window_test.go
index 54436c0c..cee7973d 100644
--- a/pkg/flow/streaming/sliding_window_test.go
+++ b/pkg/flow/streaming/sliding_window_test.go
@@ -73,7 +73,7 @@ var _ = g.Describe("Sliding Window", func() {
g.JustBeforeEach(func() {
snk = newSlice()
- slidingWindows = NewTumblingTimeWindows(time.Second * 15).(*tumblingTimeWindows)
+ slidingWindows = NewTumblingTimeWindows(time.Second*15, time.Second*15).(*tumblingTimeWindows)
slidingWindows.aggregationFactory = aggrFactory
slidingWindows.windowCount = 2
slidingWindows.l = logger.GetLogger("tumblingTimeWindows")
diff --git a/pkg/flow/streaming/streaming_test.go b/pkg/flow/streaming/streaming_test.go
index 0a7129c5..796022e5 100644
--- a/pkg/flow/streaming/streaming_test.go
+++ b/pkg/flow/streaming/streaming_test.go
@@ -148,7 +148,7 @@ var _ = g.Describe("Streaming", func() {
// groupBy
return flow.Data{item.(*record).service, int64(item.(*record).value)}
})).
- Window(NewTumblingTimeWindows(15*time.Second)).
+ Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
return record.Data().(flow.Data)[1].(int64)
}), OrderBy(ASC), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
@@ -211,7 +211,7 @@ var _ = g.Describe("Streaming", func() {
// groupBy
return flow.Data{item.(*record).service, int64(item.(*record).value)}
})).
- Window(NewTumblingTimeWindows(15*time.Second)).
+ Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
return record.Data().(flow.Data)[1].(int64)
}), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
diff --git a/pkg/flow/streaming/topn.go b/pkg/flow/streaming/topn.go
index 180e73a7..54a2af16 100644
--- a/pkg/flow/streaming/topn.go
+++ b/pkg/flow/streaming/topn.go
@@ -148,15 +148,17 @@ func (t *topNAggregatorGroup) Snapshot() interface{} {
groupRanks[group] = items
}
if len(groupRanks) > 0 {
- sb := strings.Builder{}
- for g, item := range groupRanks {
- sb.WriteString("{")
- sb.WriteString(g)
- sb.WriteString(":")
- sb.WriteString(strconv.Itoa(len(item)))
- sb.WriteString("}")
+ if e := t.l.Debug(); e.Enabled() {
+ sb := strings.Builder{}
+ for g, item := range groupRanks {
+ sb.WriteString("{")
+ sb.WriteString(g)
+ sb.WriteString(":")
+ sb.WriteString(strconv.Itoa(len(item)))
+ sb.WriteString("}")
+ }
+ t.l.Debug().Interface("snapshot", sb.String()).Msg("taken a topN snapshot")
}
- t.l.Info().Interface("snapshot", sb.String()).Msg("taken a topN snapshot")
}
return groupRanks
}