You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2024/05/29 14:21:55 UTC

(skywalking-banyandb) 06/08: Limit flush interval

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch topn-val
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit dd08b6458112fab086e30a916b45e723e33b3629
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed May 29 04:12:04 2024 +0000

    Limit flush interval
    
    Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
 banyand/measure/topn.go                   |  7 ++++++-
 pkg/flow/streaming/sliding_window.go      | 24 ++++++++++++++++++------
 pkg/flow/streaming/sliding_window_test.go |  2 +-
 pkg/flow/streaming/streaming_test.go      |  4 ++--
 pkg/flow/streaming/topn.go                | 18 ++++++++++--------
 5 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/banyand/measure/topn.go b/banyand/measure/topn.go
index 54525614..85460d5d 100644
--- a/banyand/measure/topn.go
+++ b/banyand/measure/topn.go
@@ -52,6 +52,7 @@ import (
 const (
 	timeBucketFormat         = "200601021504"
 	resultPersistencyTimeout = 10 * time.Second
+	maxFlushInterval         = time.Minute
 )
 
 var (
@@ -264,7 +265,11 @@ func (t *topNStreamingProcessor) locate(tagValues []*modelv1.TagValue, rankNum i
 }
 
 func (t *topNStreamingProcessor) start() *topNStreamingProcessor {
-	t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval)).
+	flushInterval := t.interval
+	if flushInterval > maxFlushInterval {
+		flushInterval = maxFlushInterval
+	}
+	t.errCh = t.streamingFlow.Window(streaming.NewTumblingTimeWindows(t.interval, flushInterval)).
 		AllowedMaxWindows(int(t.topNSchema.GetLruSize())).
 		TopN(int(t.topNSchema.GetCountersNumber()),
 			streaming.WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
diff --git a/pkg/flow/streaming/sliding_window.go b/pkg/flow/streaming/sliding_window.go
index 0164f413..27b11275 100644
--- a/pkg/flow/streaming/sliding_window.go
+++ b/pkg/flow/streaming/sliding_window.go
@@ -85,6 +85,7 @@ type tumblingTimeWindows struct {
 	windowCount      int
 	currentWatermark int64
 	timerMu          sync.Mutex
+	maxFlushInterval int64
 	l                *logger.Logger
 }
 
@@ -103,7 +104,9 @@ func (s *tumblingTimeWindows) Setup(_ context.Context) (err error) {
 		}
 		s.snapshots, err = lru.NewWithEvict(s.windowCount, func(key interface{}, value interface{}) {
 			flushed := s.flushSnapshot(key.(timeWindow), value.(flow.AggregationOp))
-			s.l.Info().Stringer("window", key.(timeWindow)).Bool("flushed", flushed).Msg("evict window on lru cache is full")
+			if e := s.l.Debug(); e.Enabled() {
+				e.Stringer("window", key.(timeWindow)).Bool("flushed", flushed).Msg("evict window on lru cache is full")
+			}
 		})
 		if err != nil {
 			return err
@@ -127,7 +130,9 @@ func (s *tumblingTimeWindows) flushSnapshot(w timeWindow, snapshot flow.Aggregat
 func (s *tumblingTimeWindows) flushWindow(w timeWindow) {
 	if snapshot, ok := s.snapshots.Get(w); ok {
 		flushed := s.flushSnapshot(w, snapshot.(flow.AggregationOp))
-		s.l.Info().Stringer("window", w).Bool("flushed", flushed).Msg("flush window")
+		if e := s.l.Debug(); e.Enabled() {
+			e.Stringer("window", w).Bool("flushed", flushed).Msg("flush window")
+		}
 	}
 }
 
@@ -178,7 +183,9 @@ func (s *tumblingTimeWindows) receive() {
 				newAggr := s.aggregationFactory()
 				newAggr.Add([]flow.StreamRecord{elem})
 				s.snapshots.Add(tw, newAggr)
-				s.l.Info().Stringer("window", tw).Msg("create new window")
+				if e := s.l.Debug(); e.Enabled() {
+					e.Stringer("window", tw).Msg("create new window")
+				}
 			}
 
 			result := ctx.OnElement(elem)
@@ -206,8 +213,12 @@ func (s *tumblingTimeWindows) receive() {
 			// |    40%     |    40%     |  20%  |
 			// |          flush        flush     |
 			// |---------------------------------|
-			// TODO: how to determine the threshold
-			if previousWaterMark > 0 && float64(pastDur) > float64(s.windowSize)*0.4 {
+			// the max flush interval is 1 minute.
+			it := float64(s.windowSize) * 0.4
+			if it > float64(s.maxFlushInterval) {
+				it = float64(s.maxFlushInterval)
+			}
+			if previousWaterMark > 0 && float64(pastDur) > it {
 				s.flushDirtyWindows()
 			}
 		}
@@ -235,7 +246,7 @@ func (s *tumblingTimeWindows) Exec(downstream flow.Inlet) {
 }
 
 // NewTumblingTimeWindows return tumbling-time windows.
-func NewTumblingTimeWindows(size time.Duration) flow.WindowAssigner {
+func NewTumblingTimeWindows(size time.Duration, maxFlushInterval time.Duration) flow.WindowAssigner {
 	return &tumblingTimeWindows{
 		windowSize: size.Milliseconds(),
 		timerHeap: flow.NewPriorityQueue(func(a, b interface{}) int {
@@ -244,6 +255,7 @@ func NewTumblingTimeWindows(size time.Duration) flow.WindowAssigner {
 		in:               make(chan flow.StreamRecord),
 		out:              make(chan flow.StreamRecord),
 		currentWatermark: 0,
+		maxFlushInterval: maxFlushInterval.Milliseconds(),
 	}
 }
 
diff --git a/pkg/flow/streaming/sliding_window_test.go b/pkg/flow/streaming/sliding_window_test.go
index 54436c0c..cee7973d 100644
--- a/pkg/flow/streaming/sliding_window_test.go
+++ b/pkg/flow/streaming/sliding_window_test.go
@@ -73,7 +73,7 @@ var _ = g.Describe("Sliding Window", func() {
 	g.JustBeforeEach(func() {
 		snk = newSlice()
 
-		slidingWindows = NewTumblingTimeWindows(time.Second * 15).(*tumblingTimeWindows)
+		slidingWindows = NewTumblingTimeWindows(time.Second*15, time.Second*15).(*tumblingTimeWindows)
 		slidingWindows.aggregationFactory = aggrFactory
 		slidingWindows.windowCount = 2
 		slidingWindows.l = logger.GetLogger("tumblingTimeWindows")
diff --git a/pkg/flow/streaming/streaming_test.go b/pkg/flow/streaming/streaming_test.go
index 0a7129c5..796022e5 100644
--- a/pkg/flow/streaming/streaming_test.go
+++ b/pkg/flow/streaming/streaming_test.go
@@ -148,7 +148,7 @@ var _ = g.Describe("Streaming", func() {
 					// groupBy
 					return flow.Data{item.(*record).service, int64(item.(*record).value)}
 				})).
-				Window(NewTumblingTimeWindows(15*time.Second)).
+				Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
 				TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
 					return record.Data().(flow.Data)[1].(int64)
 				}), OrderBy(ASC), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
@@ -211,7 +211,7 @@ var _ = g.Describe("Streaming", func() {
 					// groupBy
 					return flow.Data{item.(*record).service, int64(item.(*record).value)}
 				})).
-				Window(NewTumblingTimeWindows(15*time.Second)).
+				Window(NewTumblingTimeWindows(15*time.Second, 15*time.Second)).
 				TopN(3, WithSortKeyExtractor(func(record flow.StreamRecord) int64 {
 					return record.Data().(flow.Data)[1].(int64)
 				}), WithGroupKeyExtractor(func(record flow.StreamRecord) string {
diff --git a/pkg/flow/streaming/topn.go b/pkg/flow/streaming/topn.go
index 180e73a7..54a2af16 100644
--- a/pkg/flow/streaming/topn.go
+++ b/pkg/flow/streaming/topn.go
@@ -148,15 +148,17 @@ func (t *topNAggregatorGroup) Snapshot() interface{} {
 		groupRanks[group] = items
 	}
 	if len(groupRanks) > 0 {
-		sb := strings.Builder{}
-		for g, item := range groupRanks {
-			sb.WriteString("{")
-			sb.WriteString(g)
-			sb.WriteString(":")
-			sb.WriteString(strconv.Itoa(len(item)))
-			sb.WriteString("}")
+		if e := t.l.Debug(); e.Enabled() {
+			sb := strings.Builder{}
+			for g, item := range groupRanks {
+				sb.WriteString("{")
+				sb.WriteString(g)
+				sb.WriteString(":")
+				sb.WriteString(strconv.Itoa(len(item)))
+				sb.WriteString("}")
+			}
+			t.l.Debug().Interface("snapshot", sb.String()).Msg("taken a topN snapshot")
 		}
-		t.l.Info().Interface("snapshot", sb.String()).Msg("taken a topN snapshot")
 	}
 	return groupRanks
 }